def test_block_expressions(self, mock_driver):
"""Set an attribute value from the specified path, host, and value"""
drvr = mock_driver.return_value
config = {
'host': 'dummyhost',
'path': {
'class_id': '{{ $class_id }}',
'instance_num': '{{ $instance_num }}',
'attribute_num': '{{ $attribute_num }}',
},
'value': '{{ $value }}'
}
blk = EIPSetAttribute()
self.configure_block(blk, config)
blk.start()
drvr.open.assert_called_once_with('dummyhost')
incoming_signal = Signal({
'class_id': 8,
'instance_num': 6,
'attribute_num': 7,
'value': bytes([5, 3, 0, 9])
})
blk.process_signals([incoming_signal])
drvr.set_attribute_single.assert_called_once_with(
bytes([5, 3, 0, 9]), 8, 6, 7)
blk.stop()
drvr.close.assert_called_once_with()
self.assert_last_signal_notified(
Signal({
'host': 'dummyhost',
'path': [8, 6, 7],
'value': bytes([5, 3, 0, 9])
}))
def test_subscriber(self):
subscriber = Subscriber()
topic = "test_topic"
instance_id = "id"
with patch(Subscriber.__module__ + '.NioSubscriber') as communication:
self.configure_block(subscriber, {
"topic": topic,
"local_identifier": instance_id
})
communication.assert_called_once_with(ANY,
topic="{}.{}".format(
instance_id, topic))
subscriber.start()
communication.return_value.open.assert_called_once_with()
# call the subscriber handler with a signal and then check that
# signals are notified for decoded, unpickled result
communication.call_args[0][0]([
Signal({
"signals":
b64encode(pickle.dumps([Signal({"a": "signal"})]))
})
])
self.assert_num_signals_notified(1)
self.assertEqual(
[s.to_dict() for s in self.last_notified[DEFAULT_TERMINAL]],
[{
"a": "signal"
}])
subscriber.stop()
communication.return_value.close.assert_called_once_with()
def test_advanced_configuration(self):
""" Signal attribute, millisecond, and enrichment options."""
format = '%Y-%m-%dT%H:%M:%SZ'
blk = AddTimestamp()
config = {
'enrich': {
'exclude_existing': True,
},
'milliseconds': False,
'output_attr': '{{ $custom_attr }}',
}
self.configure_block(blk, config)
# process a signal
blk.start()
blk.process_signals([
Signal({
'custom_attr': 'custom',
}),
])
blk.stop()
# check output
self.validate_timestamps(format, attr='custom')
self.assert_last_signal_list_notified([
Signal({
'custom': ANY,
}),
])
def test_block_expressions(self, mock_driver):
"""Get an attribute from the specified path and host"""
drvr = mock_driver.return_value
drvr.get_attribute_single.return_value = 5309
config = {
'host': 'dummyhost',
'path': {
'class_id': '{{ $class_id }}',
'instance_num': '{{ $instance_num }}',
'attribute_num': '{{ $attribute_num }}',
},
}
blk = EIPGetAttribute()
self.configure_block(blk, config)
blk.start()
drvr.open.assert_called_once_with('dummyhost')
incoming_signal = Signal({
'class_id': 8,
'instance_num': 6,
'attribute_num': 7
})
blk.process_signals([incoming_signal])
drvr.get_attribute_single.assert_called_once_with(8, 6, 7)
blk.stop()
drvr.close.assert_called_once_with()
self.assert_last_signal_notified(
Signal({
'host': 'dummyhost',
'path': [8, 6, 7],
'value': 5309
}))
def test_creating_multiple_publishers(self, pub):
publisher = DynamicPublisher()
topic = "topic.{{ $sig }}"
self.configure_block(publisher, {"topic": topic})
publisher.start()
self.assertEqual(pub.call_count, 0)
signals = [Signal(dict(sig="foo"))]
publisher.process_signals(signals)
pub.assert_called_once_with(topic="topic.foo")
self.assertEqual(pub.return_value.open.call_count, 1)
pub.return_value.send.assert_called_once_with(signals)
pub.reset_mock()
signals = [Signal(dict(sig="bar"))]
publisher.process_signals(signals)
pub.assert_called_once_with(topic="topic.bar")
self.assertEqual(pub.return_value.open.call_count, 1)
pub.return_value.send.assert_called_once_with(signals)
publisher.stop()
self.assertEqual(pub.return_value.close.call_count, 2)
def test_face_locations(self, mock_face):
""" For every input frame, faces are located and added to
.the signal
"""
dummy_frame = Mock()
dummy_locations = [('T', 'R', 'B', 'L')]
mock_face.face_locations.return_value = dummy_locations
blk = FindFace()
self.configure_block(blk, {
'upsample': 7,
})
blk.start()
blk.process_signals([
Signal({'frame': dummy_frame}),
])
blk.stop()
mock_face.face_locations.assert_called_once_with(dummy_frame, 7)
self.assert_num_signals_notified(1)
self.assert_last_signal_list_notified([
Signal({
'frame': dummy_frame,
'faces': [('L', 'T', 'R', 'B')]
}),
])
def test_retry_connection_before_retry_request(self, mock_driver):
"""When a request fails, the connection is retried first."""
drvr = mock_driver.return_value
drvr.get_attribute_single.side_effect = [
CustomException, CustomException, 42
]
blk = EIPGetAttribute()
config = {
'retry_options': {
'max_retry': 2, # make three total attempts
'multiplier': 0, # don't wait while testing
},
}
self.configure_block(blk, config)
self.assertEqual(drvr.open.call_count, 1)
self.assertEqual(blk.cnxn, drvr)
blk.start()
blk.process_signals([Signal()])
self.assertEqual(drvr.get_attribute_single.call_count, 3)
# Before each retry to get_attribute_single() the connection is
# retried and get_attribute_single works on the third attempt
self.assertEqual(drvr.close.call_count, 2)
self.assertEqual(drvr.open.call_count, 3)
blk.stop()
self.assertEqual(drvr.close.call_count, 3)
self.assert_last_signal_notified(
Signal({
'host': 'localhost',
'path': [1, 1],
'value': 42
}))
def test_publisher(self):
publisher = Publisher()
topic = "test_topic"
instance_id = "id"
with patch(Publisher.__module__ + '.NioPublisher') as communication:
self.configure_block(publisher, {
"topic": topic,
"local_identifier": instance_id
})
communication.assert_called_once_with(
topic="{}.{}".format(instance_id, topic))
communication.return_value.open.assert_called_once_with()
publisher.start()
# Each list of processed signals publishes one new signal with the
# pickled list of signals
signals = [Signal({"a": "signal"})]
publisher.process_signals(signals)
self.assertEqual(1, communication.return_value.send.call_count)
self.assertIsInstance(
communication.return_value.send.call_args[0][0][0], Signal)
self.assertEqual([
s.to_dict()
for s in communication.return_value.send.call_args[0][0]
], [{
"signals": b64encode(pickle.dumps([Signal({"a": "signal"})]))
}])
publisher.stop()
communication.return_value.close.assert_called_once_with()
def test_ping_with_timeout(self, mock_monotonic, mock_call):
""" If they provide a timeout add the -W flag """
mock_call.return_value = 0
mock_monotonic.side_effect = [0, 0.123321]
blk = Ping()
self.configure_block(blk, {
'hostname': '{{ $host }}',
'timeout': 3.14,
'enrich': {'exclude_existing': False},
})
blk.start()
blk.process_signals([
Signal({'host': 'foo'}),
])
blk.stop()
self.assertEqual(mock_call.call_count, 1)
self.assertEqual(
mock_call.call_args_list[0][0],
('ping -c 1 -W 3.14 foo',)
)
self.assert_last_signal_list_notified([
Signal({
'ping_response': True,
'ping_time_ms': 123.3,
'host': 'foo',
})
])
def test_default_config(self, mock_datetime):
""" UTC timestamp is added to each signal in a list, `Z` timezone."""
format = '%Y-%m-%dT%H:%M:%S.%fZ'
# set up mocks for assertions,
# return a real datetime object for simplicity in testing
mock_datetime.utcnow = Mock(return_value=datetime.utcnow())
blk = AddTimestamp()
config = {}
self.configure_block(blk, config)
# process a list of signals
blk.start()
blk.process_signals([
Signal({'foo': 'bar'}),
Signal({'foo': 'baz'}),
])
blk.stop()
# check calls
mock_datetime.now.assert_not_called()
mock_datetime.utcnow.assert_called_once_with()
# check output, enriched by default
self.validate_timestamps(format)
self.assert_last_signal_list_notified([
Signal({
'foo': 'bar',
'timestamp': ANY,
}),
Signal({
'foo': 'baz',
'timestamp': ANY,
}),
])
def test_default_process_signals(self):
blk = Logger()
self.configure_block(blk, {})
blk.logger = MagicMock()
signal = Signal({"I <3": "n.io"})
blk.process_signals([signal])
blk.logger.info.assert_called_once_with(json.dumps(signal.to_dict()))
self.assertEqual(blk.logger.error.call_count, 0)
def test_process_signals(self):
"""Signals pass through block unmodified."""
blk = CvColorChange()
self.configure_block(blk, {})
blk.start()
blk.process_signals([Signal({'hello': 'nio'})])
blk.stop()
self.assert_num_signals_notified(1)
self.assert_last_signal_notified(Signal({'hello': 'nio'}))
def test_exception_on_logging(self):
blk = Logger()
self.configure_block(blk, {})
blk.logger = MagicMock()
blk.logger.info.side_effect = Exception()
signal = Signal({"I <3": "n.io"})
blk.process_signals([signal])
blk.logger.info.assert_called_once_with(json.dumps(signal.to_dict()))
blk.logger.exception.assert_called_once_with("Failed to log signal")
def test_list_logging(self):
blk = Logger()
self.configure_block(blk, {"log_as_list": True})
blk.logger = MagicMock()
signal = Signal({"I <3": "n.io"})
blk.process_signals([signal, signal])
blk.logger.info.assert_called_once_with(
[json.dumps(signal.to_dict()),
json.dumps(signal.to_dict())])
self.assertEqual(blk.logger.error.call_count, 0)
def test_log_hidden_attributes(self):
blk = Logger()
self.configure_block(blk, {"log_hidden_attributes": True})
blk.logger = MagicMock()
signal = Signal({"_hidden": "hidden!", "not_hidden": "not hidden!"})
blk.process_signals([signal])
blk.logger.info.assert_called_once_with(
json.dumps(signal.to_dict(True), sort_keys=True))
self.assertEqual(blk.logger.error.call_count, 0)
self.assertEqual(len(signal.to_dict(True)), 2)
def test_list_process_signals(self):
blk = Logger()
self.configure_block(blk, {})
blk.logger = MagicMock()
signal = Signal({"I <3": "n.io"})
blk.process_signals([signal, signal])
blk.logger.info.assert_has_calls([
call(json.dumps(signal.to_dict())),
call(json.dumps(signal.to_dict())),
])
self.assertEqual(blk.logger.error.call_count, 0)
def _get_next_signal(self, _random=False):
""" Get the next individual signal from the file.
If configured to do so, this will pull randomly from the file.
Otherwise, it will return the next one in line.
"""
if _random:
return Signal(random.choice(self._json_signals))
else:
sig = Signal(self._json_signals[self._index])
self._increment_index()
return sig
def test_signal_enrighment(self):
"""Use enrich signal mixin"""
blk = UnpackBytes()
self.configure_block(blk, {'enrich': {'exclude_existing': False}})
blk.start()
blk.process_signals([Signal({'key': 'one', 'value': b'\x00\x01'})])
blk.stop()
self.assert_num_signals_notified(1)
self.assert_last_signal_notified(
Signal({
'one': 1,
'key': 'one',
'value': b'\x00\x01'
}))
def test_interval_default(self):
''' Testing if interval trigger notifies all signals when total signals is not specified. '''
interval = SampleIntervalBlock()
self.configure_block(interval, {'interval': {'seconds': 1}})
returns = [Signal(), Signal()]
interval.generate_signals = MagicMock(return_value=returns)
interval.start()
# Give it enough time for two notifications
# (one immediately, one after a second)
sleep(1.5)
interval.stop()
self.assert_num_signals_notified(4)
def test_compass(self, mockSenseHat):
"""Signals are enriched with compass data."""
mock_hat = Mock()
mock_hat.get_compass_raw.return_value = {
'x': -1,
'y': 0,
'z': 1,
}
mockSenseHat.return_value = mock_hat
blk = SenseHAT()
self.configure_block(
blk, {
'env': {
'press': False,
'rh': False,
'temp': False,
},
'imu': {
'accel': False,
'compass': True,
'gyro': False,
},
'enrich': {
'exclude_existing': False,
},
})
blk.start()
mockSenseHat.assert_called_once_with()
mock_hat.set_imu_config.assert_called_once_with(False, True, False)
blk.process_signals([Signal({'pi': 3.14})])
mock_hat.get_accelerometer_raw.assert_not_called()
mock_hat.get_compass_raw.assert_called_once_with()
mock_hat.get_gyroscope_raw.assert_not_called()
blk.stop()
self.assert_num_signals_notified(1)
self.assert_last_signal_list_notified([
Signal({
'compass': {
'x': -1,
'y': 0,
'z': 1,
},
'pi': 3.14,
}),
])
def test_signal_enrichment(self, mock_driver):
"""Incoming signals are enriched new data"""
drvr = mock_driver.return_value
config = {'enrich': {'exclude_existing': False}}
blk = EIPSetAttribute()
self.configure_block(blk, config)
blk.start()
blk.process_signals([Signal({'foo': 'bar'})])
blk.stop()
self.assert_last_signal_notified(
Signal({
'foo': 'bar',
'host': 'localhost',
'path': [1, 1],
'value': b'\x00\x00'
}))
def _delimited_reader(self):
thread_id = current_thread().name
self.logger.debug('Reader thread {} spawned'.format(thread_id))
buffer = []
while not self._kill:
try:
new_byte = self.file_descriptor.read(1)
except Exception:
if not self.status.is_set(RunnerStatus.warning):
self.set_status('warning')
self.logger.exception('Read operation from HID Device failed')
self._disconnect()
self._connect()
break
if new_byte == self.delimiter:
try:
barcode = self._decode_buffer(buffer)
except Exception:
self.logger.exception('Failed to decode barcode!')
barcode = None
signal_dict = {'barcode': barcode}
self.notify_signals([Signal(signal_dict)])
buffer = []
continue
buffer.append(new_byte)
self.logger.debug('Reader thread {} completed'.format(thread_id))
def test_base(self):
blk = FirebaseInsert()
self.configure_block(
blk, {
"application": MY_APPLICATION,
"auth": MY_AUTH,
"collection": "integration_tests/blah",
"enrich": {
"exclude_existing": False,
"enrich_field": "output"
}
})
blk.start()
blk.process_signals([Signal({"test_key": "test_val"})])
# We should get an output signal from the insert
self.assert_num_signals_notified(1)
# TODO: Move this to the framework block unit test
sigs_notified = self.last_notified[
Terminal.get_default_terminal_on_class(FirebaseInsert,
TerminalType.input).id]
out_sig = sigs_notified[0]
# Make sure the details of our input signal were notified
self.assertEqual(out_sig.test_key, "test_val")
# Make sure the ID/name of the saved signal was included on the
# output signal under the "name" field
self.assertIsNotNone(out_sig.output["name"])
blk.stop()
def run(self):
while self._running:
try:
self.logger.debug('loading inference model ...')
with CameraInference(
image_classification.model()) as inference:
self.logger.debug('running inference ...')
for result in inference.run():
predictions = image_classification.get_classes(
result,
top_k=self.top_k_predictions(),
threshold=0)
outgoing_signals = []
for label, score in predictions:
signal_dict = {
'label': label,
'score': score,
}
outgoing_signal = Signal(signal_dict)
outgoing_signals.append(outgoing_signal)
if not self._running:
break
self.notify_signals(outgoing_signals)
except:
self.logger.exception('failed to get inference result!')
self.reset_camera()
self.release_camera()
def run(self):
while self._running:
try:
self.logger.debug('loading inference model ...')
with CameraInference(face_detection.model()) as inference:
self.logger.debug('running inference ...')
for result in inference.run():
faces = face_detection.get_faces(result)
if faces:
self.logger.debug('found {} faces'.format(
len(faces)))
outgoing_signals = []
for face in faces:
signal_dict = {
'bounding_box': face.bounding_box,
'face_score': face.face_score,
'joy_score': face.joy_score,
}
outgoing_signal = Signal(signal_dict)
outgoing_signals.append(outgoing_signal)
if not self._running:
break
self.notify_signals(outgoing_signals)
except:
self.logger.exception('failed to get inference result!')
self.reset_camera()
self.release_camera()
def process_signals(self, in_signals):
""" Publish each group of signals """
ttl = self.ttl()
groups = defaultdict(list)
for signal in in_signals:
try:
topic = self.topic(signal)
if self._is_local and self._local_id:
topic = '{}.{}'.format(self._local_id, topic)
except Exception:
self.logger.exception('topic expression failed, ignoring signal')
continue
groups[topic].append(signal)
for topic, out_signals in groups.items():
try:
if self._is_local:
out_signals = [Signal({"signals": b64encode(pickle.dumps(out_signals))})]
self.__get_publisher(topic, ttl).send(out_signals)
except pickle.PicklingError:
self.logger.exception("Pickling based pickle error")
except TypeError:
self.logger.exception("Unable to encode pickled signals")
except PublisherError: # pragma no cover
self.logger.exception('Error publishing {:n} signals to "{}"'.format(len(out_signals), topic))
except:
self.logger.exception("Error processing signals")
def test_insert(self):
blk = FirebaseInsert()
fbase_imp = self.get_absolute_import('firebase_base.pyrebase')
with patch(fbase_imp) as mock_fbase:
self.configure_block(
blk, {
"collection": "COLLECTION",
"userEmail": "USER_EMAIL",
"userPassword": "******",
"config": {
"apiKey": "API_KEY",
"databaseURL": "DATABASE_URL",
"projectId": "PROJECT_ID"
},
"enrich": {
"exclude_existing": False
}
})
# We will simulate the post returning a name with a fake ID
mock_fbase.initialize_app.return_value.database.return_value.\
child.return_value.push.return_value = {"name": "fake id"}
blk.start()
blk.process_signals([Signal({"test_key": "test_val"})])
# We should get an output signal from the insert
self.assert_num_signals_notified(1)
out_sig = self.last_notified["__default_terminal_value"][0]
# Make sure the details of our input signal were notified
self.assertEqual(out_sig.test_key, "test_val")
# Make sure the ID/name of the saved signal was included on the
# output signal under the "name" field
self.assertEqual(out_sig.name, "fake id")
blk.stop()
def test_multiple_values(self):
"""Unpack two integers from one byte array"""
blk = UnpackBytes()
self.configure_block(
blk, {
'new_attributes': [{
'key': 'one',
'value': '{{ $value[0:2] }}'
}, {
'key': 'two',
'value': '{{ $value[2:4] }}'
}]
})
blk.start()
blk.process_signals([Signal({'value': b'\x00\x01\x00\x02'})])
blk.stop()
self.assert_num_signals_notified(1)
self.assert_last_signal_notified(Signal({'one': 1, 'two': 2}))
def test_process_signals(self):
"""Signals pass through block unmodified."""
blk = SpreadsheetLookUp()
self.configure_block(
blk, {
'source':
'tests/test.xlsx',
'match': [
{
'key': '{{ $host }}',
'value': 'MOXA HOST',
},
{
'key': '{{ $channel }}',
'value': 'CHANNEL',
},
],
'output_structure': [
{
'key': 'flavor',
'value': 'FLAVOR',
},
{
'key': 'position',
'value': 'POSITION',
},
],
})
blk.start()
blk.process_signals([
Signal({
'host': '101',
'channel': '0',
}),
])
blk.stop()
self.assert_num_signals_notified(1)
self.assert_last_signal_list_notified([
Signal({
'flavor': 'House Roast',
'position': '0',
}),
])
def test_signal_lists(self, mock_driver):
"""Outgoing signal lists have the same length as incoming"""
config = {}
blk = EIPGetAttribute()
self.configure_block(blk, config)
blk.start()
blk.process_signals([Signal()] * 3)
blk.stop()
self.assertEqual(len(self.notified_signals[DEFAULT_TERMINAL]), 1)
self.assertEqual(len(self.notified_signals[DEFAULT_TERMINAL][0]), 3)
