class GeoCode(PropertyHolder):
""" Property holder for a latitude and longitude
"""
latitude = StringProperty(title="Latitude", default='')
longitude = StringProperty(title="Longitude", default='')
radius = StringProperty(title="Radius (miles)", default='')
class AWSCreds(PropertyHolder):
aws_access_key_id = StringProperty(
title="Access Key ID", default="", allow_none=False)
aws_secret_access_key = StringProperty(
title="Secret Access Key", default="", allow_none=False)
aws_session_token = StringProperty(
title="Session Token", default="", allow_none=True)
class AuthCreds(PropertyHolder):
access_key_id = StringProperty(title="Access Key ID",
default="",
allow_none=True)
secret_key = StringProperty(title="Secret Key",
default="",
allow_none=True)
class Lifx(Block):
version = VersionProperty('0.1.0')
mac = StringProperty(title='MAC address', default='[[LIFX_MAC]]')
ip = StringProperty(title='IP Address', default='[[LIFX_IP]]')
power = IntProperty(title='1 for on 0 for off', default=0)
hue = IntProperty(title='Hue (0-65535)', default=0)
sat = IntProperty(title='Saturation (0-65535)', default=0)
bri = IntProperty(title='Brightness (0-65535)', default=65535)
kelvin = IntProperty(title='Kelvin (2500-9000)', default=3500)
kill_switch = BoolProperty(title='Turn off Light at Service Stop?',
default=True, advanced=True)
def configure(self, context):
super().configure(context)
self.bulb = Light(self.mac(), self.ip())
def process_signals(self, signals):
for signal in signals:
if self.power(signal) == 0:
brightness = 0
else:
brightness = self.bri(signal)
self.bulb.set_power(True)
self.bulb.set_color([self.hue(signal),
self.sat(signal),
brightness,
self.kelvin(signal)])
pass
self.notify_signals(signals)
def stop(self):
if self.kill_switch():
self.bulb.set_power(False)
super().stop()
class GetEncodingFromFile(Block):
image_paths = ListProperty(StringType, title='Image Path', default=[])
uid = StringProperty(title='User ID', defult='')
sname = StringProperty(title='Save Name', default='')
version = VersionProperty("2.1.0")
def save_encoding(self, file_path, save_name, user_id):
serialized_encoding = []
for f in file_path:
image = face_recognition.load_image_file(f)
face_encoding = face_recognition.face_encodings(image)[0]
serialized_encoding.append(
base64.b64encode(pickle.dumps(face_encoding)).decode())
entry = {
'user_id': user_id,
'name': save_name,
'encoding': serialized_encoding
}
return entry
def process_signals(self, signals):
add_face_signals = []
for signal in signals:
confirmation = self.save_encoding(self.image_paths(signal),
self.sname(signal),
self.uid(signal))
add_face_signals.append(Signal(confirmation))
self.notify_signals(add_face_signals)
class HarperDBBulkData(HarperDBBase, Block, EnrichSignals):
version = VersionProperty('0.1.0')
operation = SelectProperty(Operation,
title='Data Source',
default=Operation.DATA,
order=2)
schema = StringProperty(title='Schema', default='dev', order=3)
table = StringProperty(title='Table', default='dog', order=4)
data = Property(title='Data (escaped string, file, or url)',
default='{{ $data }}',
order=5)
def process_signals(self, signals):
out_sigs = []
for signal in signals:
payload = {
'schema': self.schema(),
'table': self.table(),
'action': 'insert',
'operation': self.operation().value,
}
if self.operation() is Operation.DATA:
payload['data'] = self.data(signal).replace('\\n', '\n')
if self.operation() is Operation.FILE:
payload['file_path'] = self.data(signal)
if self.operation() is Operation.URL:
payload['csv_url'] = self.data(signal)
result = self.sendQuery(payload)
job_result = self.get_job_result(result["message"].replace(
"Starting job with id ", ""))
out_sigs.append(self.get_output_signal(job_result, signal))
self.notify_signals(out_sigs)
class RethinkDBBase(LimitLock, Retry, Block):
"""
A block for communicating with a RethinkDB server.
Properties:
host (str): server host to connect to
port (int): port on the server host, default rethink port is 28015
database_name (str): database name to access
connect_timeout (interval): time to wait for a successful connection
"""
version = VersionProperty('1.0.0')
host = StringProperty(title='Host', default='[[RETHINKDB_HOST]]')
port = IntProperty(title='Port', default='[[RETHINKDB_PORT]]')
database_name = StringProperty(title='DB name', default='test')
connect_timeout = TimeDeltaProperty(title="Connect timeout",
default={"seconds": 20},
visible=False)
def process_signals(self, signals):
self.execute_with_lock(self._locked_process_signals,
10,
signals=signals)
def _locked_process_signals(self, signals):
pass
class AquaCheck(Block):
signalName = StringProperty(title='Signal Name', default='default')
portNumber = StringProperty(title='UART Port', default='/dev/ttymxc4')
sendMarking = BoolProperty(default=False, title='Send Marking')
rs485 = BoolProperty(default=False, title='Hardware RS485 Port')
version = VersionProperty('0.0.1')
def configure(self, context):
super().configure(context)
self.logger.debug("Got here with {}".format(self.portNumber()))
self.AQ = SDI12AquaCheck(self.portNumber(),
sendMarking=self.sendMarking(),
rs485=self.rs485())
def process_signals(self, signals):
for signal in signals:
if self.AQ.pollProbe(0) == 0:
#TODO: Add polling for temperature
value = self.AQ.moistureData
results = {self.signalName(): value}
self.logger.debug("Got results: {}".format(results))
try:
self.notify_signals([Signal(results)])
except:
self.logger.exception("Signal is not valid:"
" {}".format(results))
class TCPClient(Block):
host = StringProperty(title='IP Address', default='127.0.0.1')
message = StringProperty(title='Message', default='GET / HTTP/1.1\n')
port = IntProperty(title='Port', default=50001)
expect_response = BoolProperty(title='Expect response?', default=True)
version = VersionProperty('0.0.1')
def process_signals(self, signals):
for signal in signals:
message = self.message(signal).encode('utf-8')
response = self.send_message(message)
if response:
signal.response = response
self.notify_signals(signals)
def send_message(self, message):
response = None
buffer_size = 8192
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((self.host(), self.port()))
s.send(message)
if self.expect_response():
response = s.recv(buffer_size)
s.shutdown(2)
s.close()
return response
class FirebaseAuthProperty(PropertyHolder):
""" This is a property holder that holds Firebase authentication details.
It also contains a method that returns an Authenticator object that
can be passed to the firebase SDK to perform authentication.
"""
apiKey = StringProperty(title='API Key',
default='[[FIREBASE_API_KEY]]')
databaseURL = StringProperty(title='Database URL',
default='[[FIREBASE_DATABASE_URL]]')
projectId = StringProperty(title='Firebase Project ID',
default='[[FIREBASE_PROJECT_ID]]')
def get_auth_object(self):
""" Return an Authenticator that can be passed to the firebase SDK
Returns:
FirebaseAuthenticator if authentication details were provided,
None otherwise. Note that None is a valid argument to the firebase
SDK's authenticator argument, it just won't use authentication.
"""
apiKey = self.apiKey()
databaseURL = self.databaseURL()
projectId = self.projectId()
if apiKey and databaseURL and projectId:
config = {
"apiKey": apiKey,
"authDomain": "{}.firebaseapp.com".format(projectId),
"databaseURL": databaseURL,
"storageBucket": "{}.appspot.com".format(projectId)
}
return config
else:
return None
class HarperDBHashSearch(HarperDBBase, Block, EnrichSignals):
version = VersionProperty('0.1.0')
schema = StringProperty(title='Schema', default='dev', order=2)
table = StringProperty(title='Table', default='dog', order=3)
hash_attribute = StringProperty(title='Hash Attribute', default='id', order=4)
hash_values = Property(title='Hash Values', default='1, 3', order=5)
get_attributes = StringProperty(title='Get Attributes', default='name, breed', order=6)
def process_signals(self, signals):
out_sigs = []
for signal in signals:
payload = {
'schema': self.schema(),
'table': self.table(),
'operation': 'search_by_hash',
'hash_attribute': self.hash_attribute(signal),
'hash_values': self.hash_values(signal).replace(" ","").split(","),
'get_attributes': self.get_attributes(signal).replace(" ","").split(",")
}
result = self.sendQuery(payload)
for r in result:
out_sigs.append(self.get_output_signal(r, signal))
self.notify_signals(out_sigs)
class Credentials(PropertyHolder):
""" User credentials
Properties:
username (str): User to connect as
password (str): User's password
"""
username = StringProperty(title='User to connect as', default="")
password = StringProperty(title='Password to connect with', default="")
class Creds(PropertyHolder):
""" Property holder for Facebook credentials.
"""
consumer_key = StringProperty(title='App ID',
default='[[FACEBOOK_APP_ID]]')
app_secret = StringProperty(title='App Secret',
default='[[FACEBOOK_APP_SECRET]]')
class FacebookCreds(PropertyHolder):
""" Property holder for Facebook OAuth credentials.
"""
consumer_key = StringProperty(title='Consumer Key',
default='[[FACEBOOK_CONSUMER_KEY]]')
app_secret = StringProperty(title='App Secret',
default='[[FACEBOOK_APP_SECRET]]')
class PortConfig(PropertyHolder):
baudrate = IntProperty(title='Baud Rate', default=19200, order=21)
parity = StringProperty(title='Parity (N, E, O)', default='N', order=23)
bytesize = IntProperty(title='Byte Size', default=8, order=22)
stopbits = IntProperty(title='Stop Bits', default=1, order=24)
port = StringProperty(title='Serial Port',
default='/dev/ttyUSB0',
order=20)
class Join(EnrichSignals, GroupBy, Block):
""" Join block.
Group a list of signals into one signal.
The output signal will contain an attribute for each
evaluated *key* and the value of that attribute will
be a list with an item of *value* for each matching signal.
If *one_value* is True, the Signal attributes will be just a
single matching value instead of a list of all matching values.
If multiple matches, then the last signal processed will be the
value used.
"""
key = StringProperty(title='Key', default="{{ $key }}")
value = Property(title='Value', default="{{ $value }}", allow_none=True)
group_attr = StringProperty(title="Group Attribute Name",
default="group",
visible=False)
one_value = BoolProperty(title="One Value Per Key", default=False)
version = VersionProperty("1.0.0")
def process_signals(self, signals, input_id='default'):
self.notify_signals(
self.for_each_group(self._get_hash_from_group, signals))
def _get_hash_from_group(self, signals, group):
self.logger.debug("Processing group {} of {} signals".format(
group, len(signals)))
out_sig = self._perform_hash(signals)
if out_sig:
setattr(out_sig, self.group_attr(), group)
return out_sig
def _perform_hash(self, signals):
hash_dict = defaultdict(None) if self.one_value() \
else defaultdict(list)
for signal in signals:
sig_key = self.key(signal)
sig_value = self.value(signal)
# Add sig_value to the proper hash key
try:
if sig_key is not None:
if self.one_value():
hash_dict[sig_key] = sig_value
else:
hash_dict[sig_key].append(sig_value)
else:
self.logger.debug("Skipping key: {}".format(sig_key))
except:
self.logger.exception(
"Failed to add value {} to key {}".format(
sig_value, sig_key))
if len(hash_dict):
return self.get_output_signal(hash_dict, signals[-1])
class LocalPublisher(PubSubConnectivity, TerminatorBlock):
""" A block for publishing to a local nio communication channel.
Functions regardless of communication module implementation.
Unlike the regular Publisher block, the one does not need data to be json
"""
version = VersionProperty("1.1.1")
topic = StringProperty(title="Topic", default="")
local_identifier = StringProperty(title='Local Identifier',
default='[[INSTANCE_ID]]',
advanced=True)
def __init__(self):
super().__init__()
self._publisher = None
def configure(self, context):
super().configure(context)
topic = self.topic()
# If a local identifier was included use it as a prefix
if self.local_identifier():
topic = "{}.{}".format(self.local_identifier(), topic)
self._publisher = NioPublisher(topic=topic)
try:
self._publisher.open(on_connected=self.conn_on_connected,
on_disconnected=self.conn_on_disconnected)
except TypeError as e:
self.logger.warning(
"Connecting to an outdated communication module")
# try previous interface
self._publisher.open()
# no need to configure connectivity if not supported
return
self.conn_configure(self._publisher.is_connected)
def stop(self):
""" Stop the block by closing the underlying publisher """
self._publisher.close()
super().stop()
def process_signals(self, signals):
""" Publish each list of signals """
try:
signals = pickle.dumps(signals)
signals = [Signal({"signals": b64encode(signals)})]
self._publisher.send(signals)
except pickle.PicklingError:
self.logger.exception("Pickling based pickle error")
except TypeError:
self.logger.exception("Unable to encode pickled signals")
except PublisherError:
self.logger.exception("Error publishing signals")
except:
self.logger.exception("Error processing signals")
class MojioCreds(PropertyHolder):
username = StringProperty(title="Moj.io Username",
default="[[MOJIO_USERNAME]]")
password = StringProperty(title="Moj.io Password",
default="[[MOJIO_PASSWORD]]")
client_id = StringProperty(title="Moj.io Client ID",
default="[[MOJIO_CLIENT_ID]]")
client_secret = StringProperty(title="Moj.io Client Secret",
default="[[MOJIO_CLIENT_SECRET]]")
class AWSCreds(PropertyHolder):
region_name = SelectProperty(
Regions, title="Region Name", default=Regions.us_east_2)
aws_access_key_id = StringProperty(
title="Access Key ID", default="", allow_none=False)
aws_secret_access_key = StringProperty(
title="Secret Access Key", default="", allow_none=False)
aws_session_token = StringProperty(
title="Session Token", default="", allow_none=True)
class GoogleIoTMQTTBase(object):
"""The base block for Google IoT. This block is responsible for connecting
to the cloud broker via MQTT paho."""
project_id = StringProperty(title="Project Id", order=10)
project_region = StringProperty(title="Project Region", order=11,
default="")
registry_id = StringProperty(title="Registry Id", order=12)
device_id = StringProperty(title="Device Id", order=13)
private_key_path = FileProperty(
title="Private Key Path", order=14,
default="[[PROJECT_ROOT]]/etc/google_iot_rsa_private.pem")
cert_path = FileProperty(
title="Certificate Path", order=15,
default="[[PROJECT_ROOT]]/etc/google_iot_cert.pem")
keep_alive = IntProperty(title="Keep Alive", default=10, advanced=True,
order=21)
def __init__(self):
super().__init__()
self._client = None
def configure(self, context):
"""set up google client properties"""
super().configure(context)
self._client = IoTCoreClient(
self.project_id(),
self.project_region(),
self.registry_id(),
self.device_id(),
self.private_key_path().value,
self.cert_path().value,
self.keep_alive(),
self.logger,
on_message=self.on_message
)
self._client.connect()
def stop(self):
self.disconnect()
super().stop()
def connect(self):
self.logger.debug("Connecting...")
self.client.connect()
def disconnect(self):
self.logger.debug("Disconnecting...")
self._client.disconnect()
def on_message(self, client, user_data, message):
self.logger.debug("on_message, client: {}, message: {}, user data: {}".
format(client, message, user_data))
class AuthCreds(PropertyHolder):
username = StringProperty(title="Username",
default="",
allow_none=False,
order=0)
password = StringProperty(title="Password",
default="",
allow_none=False,
order=1)
class DatabaseCredentials(PropertyHolder):
userid = StringProperty(title='User ID',
default='HDB_ADMIN',
allow_none=True,
order=5)
password = StringProperty(title="Password",
default='password',
allow_none=True,
order=6)
class LocalSubscriber(PubSubConnectivity, GeneratorBlock):
""" A block for subscribing to a local nio communication channel.
Functions regardless of communication module implementation.
Unlike the regular Subscriber block, the one does not need data to be json
"""
version = VersionProperty("1.1.0")
topic = StringProperty(title='Topic', default="")
local_identifier = StringProperty(title='Local Identifier',
default='[[INSTANCE_ID]]',
visible=False)
def __init__(self):
super().__init__()
self._subscriber = None
def configure(self, context):
super().configure(context)
self._subscriber = NioSubscriber(self._subscriber_handler,
topic="{}.{}".format(
self.local_identifier(),
self.topic()))
try:
self._subscriber.open(on_connected=self.conn_on_connected,
on_disconnected=self.conn_on_disconnected)
except TypeError as e:
self.logger.warning(
"Connecting to an outdated communication module")
# try previous interface
self._subscriber.open()
# no need to configure connectivity if not supported
return
# let connectivity configure
self.conn_configure(self._subscriber.is_connected)
def stop(self):
""" Stop the block by closing the underlying subscriber """
self._subscriber.close()
super().stop()
def _subscriber_handler(self, signals):
try:
signals = b64decode(signals[0].signals)
signals = pickle.loads(signals)
except pickle.UnpicklingError:
self.logger.exception("Unpickling based pickle error")
except TypeError:
self.logger.exception("Unable to decode pickled signals")
except:
self.logger.exception("Error handling signals")
else:
self.notify_signals(signals)
class XeroUpdateInvoice(Block):
version = VersionProperty("0.1.3")
consumer_key = StringProperty(title='Xero Consumer Key',
default='[[XERO_CONSUMER_KEY]]',
allow_none=False)
contact_name = StringProperty(title='Contact Name (Stripe customerID)',
default='{{ $customer }}')
payment_amount = FloatProperty(title='Amount Paid',
default='{{ $amount }}')
invoice_account_code = IntProperty(title='Invoice Account Code',
default=310)
def __init__(self):
self.xero = None
self.credentials = None
super().__init__()
def configure(self, context):
super().configure(context)
con_key = self.consumer_key()
with open('blocks/xero/keys/privatekey.pem') as keyfile:
rsa_private_key = keyfile.read()
self.credentials = PrivateCredentials(con_key, rsa_private_key)
self.xero = Xero(self.credentials)
def start(self):
super().start()
def process_signals(self, signals):
response_signal = []
for signal in signals:
invoice_resp_signal = self.xero.invoices.filter(
Contact_Name=self.contact_name(signal),
Status='AUTHORISED',
order='UpdatedDateUTC DESC')[0]
invoice_id = invoice_resp_signal['InvoiceID']
response_signal.append(
Signal(
self.xero.payments.put({
'Invoice': {
'InvoiceID': invoice_id
},
'Account': {
'Code': self.invoice_account_code()
},
'Amount':
self.payment_amount(signal)
})[0]))
self.notify_signals(response_signal)
class LineItems(PropertyHolder):
description = StringProperty(title='Line Item Description',
default='Invoice Description')
quantity = IntProperty(title='Quantity', default=1)
unit_amount = FloatProperty(title='Unit Amount', default='{{ $amount }}')
tax_amount = FloatProperty(title='Tax Amount', default='{{ $sales_tax }}')
invoice_type = StringProperty(title='Invoice Type',
default='ACCREC',
allow_none=False)
invoice_account_code = IntProperty(title='Invoice Account Code',
default=100)
class TwitterCreds(PropertyHolder):
""" Property holder for Twitter OAuth credentials.
"""
consumer_key = StringProperty(title='Consumer Key',
default='[[TWITTER_CONSUMER_KEY]]')
app_secret = StringProperty(title='App Secret',
default='[[TWITTER_APP_SECRET]]')
oauth_token = StringProperty(title='OAuth Token',
default='[[TWITTER_OAUTH_TOKEN]]')
oauth_token_secret = StringProperty(
title='OAuth Token Secret',
default='[[TWITTER_OAUTH_TOKEN_SECRET]]')
class MSSQLTabledBase(MSSQLBase):
table = StringProperty(title='Table', default='TableName', order=1)
def __init__(self):
super().__init__()
# maintains a LUT index by table containing a list of columns for it
self._table_lut = {}
def validate_column(self, column, table, cursor):
""" Makes sure column belongs to table
Args:
column (str): column in question
table (str): table name
cursor (pyodbc.Cursor): active cursor
Returns:
True/False
"""
columns = self._table_lut.get(table)
if columns is None:
columns = tuple(column.column_name
for column in cursor.columns(table))
self._table_lut[table] = columns
if column not in columns:
cursor.close()
raise ValueError(
'\"{}\" is not a valid column in table \"{}\".'.format(
column, table))
return column
class SQSSendMessage(SQSBase):
"""Send message over Amazon SQS
User needs to specify a queue url and message body"""
message_body = StringProperty(
title="Message Body", default="Hello Mr. SQS", allow_none=False)
delay_seconds = IntProperty(
title="Delay sending message", default=0, allow_none=True)
def process_signals(self, signals):
new_signals = []
for signal in signals:
try:
self.logger.debug("Sending message via {} queue".format(
self.queue_url(signal)))
response = self.client.send_message(QueueUrl=self.queue_url(signal),
DelaySeconds=self.delay_seconds(signal),
MessageBody=self.message_body(signal))
new_signals.append(Signal(response))
except:
self.logger.exception("Message failed to send")
self.notify_signals(new_signals)
class JWTRefresh(EnrichSignals, JWTBase):
version = VersionProperty('0.1.0')
input = StringProperty(
title='Token Value',
default='{{ $headers.get(\'Authorization\').split()[1] }}',
order=3)
exp_minutes = Property(title='Valid For Minutes (exp claim)',
order=4,
allow_none=True)
def process_signal(self, signal, input_id=None):
_token = self.input(signal)
_key = self.key(signal)
_algorithm = self.algorithm(signal)
_exp_minutes = self.exp_minutes(signal)
try:
_claims = jwt.decode(_token, _key, algorithms=[_algorithm.value])
if isinstance(_exp_minutes, int):
_claims['exp'] = self.set_new_exp_time(_exp_minutes)
else:
try:
del _claims['exp']
except KeyError:
pass
_token = jwt.encode(_claims, _key, algorithm=_algorithm.value)
return self.get_output_signal({'token': _token.decode('UTF-8')},
signal)
except PyJWTError as e:
self.notify_signals(
self.get_output_signal({'message': e.args[0]}, signal),
'error')
class Replicator(Block):
"""Each incoming signal is replicated x times, where x
is the length of list. Each output signal with have a
new attribute, title, with the value of the list.
"""
version = VersionProperty("1.1.0")
list = Property(title='List', default='', order=0)
title = StringProperty(title='Attribute Title', default='', order=1)
def process_signals(self, signals):
return_signals = []
for signal in signals:
try:
values = self.list(signal)
except Exception:
values = [None]
self.logger.exception("Failed to evaluate list")
values = [None] if not values else values
for value in values:
sig = Signal(signal.to_dict())
setattr(sig, self.title(), value)
return_signals.append(sig)
if return_signals:
self.notify_signals(return_signals)