def test_block_expressions(self, mock_driver): """Set an attribute value from the specified path, host, and value""" drvr = mock_driver.return_value config = { 'host': 'dummyhost', 'path': { 'class_id': '{{ $class_id }}', 'instance_num': '{{ $instance_num }}', 'attribute_num': '{{ $attribute_num }}', }, 'value': '{{ $value }}' } blk = EIPSetAttribute() self.configure_block(blk, config) blk.start() drvr.open.assert_called_once_with('dummyhost') incoming_signal = Signal({ 'class_id': 8, 'instance_num': 6, 'attribute_num': 7, 'value': bytes([5, 3, 0, 9]) }) blk.process_signals([incoming_signal]) drvr.set_attribute_single.assert_called_once_with( bytes([5, 3, 0, 9]), 8, 6, 7) blk.stop() drvr.close.assert_called_once_with() self.assert_last_signal_notified( Signal({ 'host': 'dummyhost', 'path': [8, 6, 7], 'value': bytes([5, 3, 0, 9]) }))
def test_subscriber(self): subscriber = Subscriber() topic = "test_topic" instance_id = "id" with patch(Subscriber.__module__ + '.NioSubscriber') as communication: self.configure_block(subscriber, { "topic": topic, "local_identifier": instance_id }) communication.assert_called_once_with(ANY, topic="{}.{}".format( instance_id, topic)) subscriber.start() communication.return_value.open.assert_called_once_with() # call the subscriber handler with a signal and then check that # signals are notified for decoded, unpickled result communication.call_args[0][0]([ Signal({ "signals": b64encode(pickle.dumps([Signal({"a": "signal"})])) }) ]) self.assert_num_signals_notified(1) self.assertEqual( [s.to_dict() for s in self.last_notified[DEFAULT_TERMINAL]], [{ "a": "signal" }]) subscriber.stop() communication.return_value.close.assert_called_once_with()
def test_advanced_configuration(self): """ Signal attribute, millisecond, and enrichment options.""" format = '%Y-%m-%dT%H:%M:%SZ' blk = AddTimestamp() config = { 'enrich': { 'exclude_existing': True, }, 'milliseconds': False, 'output_attr': '{{ $custom_attr }}', } self.configure_block(blk, config) # process a signal blk.start() blk.process_signals([ Signal({ 'custom_attr': 'custom', }), ]) blk.stop() # check output self.validate_timestamps(format, attr='custom') self.assert_last_signal_list_notified([ Signal({ 'custom': ANY, }), ])
def test_block_expressions(self, mock_driver): """Get an attribute from the specified path and host""" drvr = mock_driver.return_value drvr.get_attribute_single.return_value = 5309 config = { 'host': 'dummyhost', 'path': { 'class_id': '{{ $class_id }}', 'instance_num': '{{ $instance_num }}', 'attribute_num': '{{ $attribute_num }}', }, } blk = EIPGetAttribute() self.configure_block(blk, config) blk.start() drvr.open.assert_called_once_with('dummyhost') incoming_signal = Signal({ 'class_id': 8, 'instance_num': 6, 'attribute_num': 7 }) blk.process_signals([incoming_signal]) drvr.get_attribute_single.assert_called_once_with(8, 6, 7) blk.stop() drvr.close.assert_called_once_with() self.assert_last_signal_notified( Signal({ 'host': 'dummyhost', 'path': [8, 6, 7], 'value': 5309 }))
def test_creating_multiple_publishers(self, pub): publisher = DynamicPublisher() topic = "topic.{{ $sig }}" self.configure_block(publisher, {"topic": topic}) publisher.start() self.assertEqual(pub.call_count, 0) signals = [Signal(dict(sig="foo"))] publisher.process_signals(signals) pub.assert_called_once_with(topic="topic.foo") self.assertEqual(pub.return_value.open.call_count, 1) pub.return_value.send.assert_called_once_with(signals) pub.reset_mock() signals = [Signal(dict(sig="bar"))] publisher.process_signals(signals) pub.assert_called_once_with(topic="topic.bar") self.assertEqual(pub.return_value.open.call_count, 1) pub.return_value.send.assert_called_once_with(signals) publisher.stop() self.assertEqual(pub.return_value.close.call_count, 2)
def test_face_locations(self, mock_face): """ For every input frame, faces are located and added to .the signal """ dummy_frame = Mock() dummy_locations = [('T', 'R', 'B', 'L')] mock_face.face_locations.return_value = dummy_locations blk = FindFace() self.configure_block(blk, { 'upsample': 7, }) blk.start() blk.process_signals([ Signal({'frame': dummy_frame}), ]) blk.stop() mock_face.face_locations.assert_called_once_with(dummy_frame, 7) self.assert_num_signals_notified(1) self.assert_last_signal_list_notified([ Signal({ 'frame': dummy_frame, 'faces': [('L', 'T', 'R', 'B')] }), ])
def test_retry_connection_before_retry_request(self, mock_driver): """When a request fails, the connection is retried first.""" drvr = mock_driver.return_value drvr.get_attribute_single.side_effect = [ CustomException, CustomException, 42 ] blk = EIPGetAttribute() config = { 'retry_options': { 'max_retry': 2, # make three total attempts 'multiplier': 0, # don't wait while testing }, } self.configure_block(blk, config) self.assertEqual(drvr.open.call_count, 1) self.assertEqual(blk.cnxn, drvr) blk.start() blk.process_signals([Signal()]) self.assertEqual(drvr.get_attribute_single.call_count, 3) # Before each retry to get_attribute_single() the connection is # retried and get_attribute_single works on the third attempt self.assertEqual(drvr.close.call_count, 2) self.assertEqual(drvr.open.call_count, 3) blk.stop() self.assertEqual(drvr.close.call_count, 3) self.assert_last_signal_notified( Signal({ 'host': 'localhost', 'path': [1, 1], 'value': 42 }))
def test_publisher(self): publisher = Publisher() topic = "test_topic" instance_id = "id" with patch(Publisher.__module__ + '.NioPublisher') as communication: self.configure_block(publisher, { "topic": topic, "local_identifier": instance_id }) communication.assert_called_once_with( topic="{}.{}".format(instance_id, topic)) communication.return_value.open.assert_called_once_with() publisher.start() # Each list of processed signals publishes one new signal with the # pickled list of signals signals = [Signal({"a": "signal"})] publisher.process_signals(signals) self.assertEqual(1, communication.return_value.send.call_count) self.assertIsInstance( communication.return_value.send.call_args[0][0][0], Signal) self.assertEqual([ s.to_dict() for s in communication.return_value.send.call_args[0][0] ], [{ "signals": b64encode(pickle.dumps([Signal({"a": "signal"})])) }]) publisher.stop() communication.return_value.close.assert_called_once_with()
def test_ping_with_timeout(self, mock_monotonic, mock_call): """ If they provide a timeout add the -W flag """ mock_call.return_value = 0 mock_monotonic.side_effect = [0, 0.123321] blk = Ping() self.configure_block(blk, { 'hostname': '{{ $host }}', 'timeout': 3.14, 'enrich': {'exclude_existing': False}, }) blk.start() blk.process_signals([ Signal({'host': 'foo'}), ]) blk.stop() self.assertEqual(mock_call.call_count, 1) self.assertEqual( mock_call.call_args_list[0][0], ('ping -c 1 -W 3.14 foo',) ) self.assert_last_signal_list_notified([ Signal({ 'ping_response': True, 'ping_time_ms': 123.3, 'host': 'foo', }) ])
def test_default_config(self, mock_datetime): """ UTC timestamp is added to each signal in a list, `Z` timezone.""" format = '%Y-%m-%dT%H:%M:%S.%fZ' # set up mocks for assertions, # return a real datetime object for simplicity in testing mock_datetime.utcnow = Mock(return_value=datetime.utcnow()) blk = AddTimestamp() config = {} self.configure_block(blk, config) # process a list of signals blk.start() blk.process_signals([ Signal({'foo': 'bar'}), Signal({'foo': 'baz'}), ]) blk.stop() # check calls mock_datetime.now.assert_not_called() mock_datetime.utcnow.assert_called_once_with() # check output, enriched by default self.validate_timestamps(format) self.assert_last_signal_list_notified([ Signal({ 'foo': 'bar', 'timestamp': ANY, }), Signal({ 'foo': 'baz', 'timestamp': ANY, }), ])
def test_default_process_signals(self): blk = Logger() self.configure_block(blk, {}) blk.logger = MagicMock() signal = Signal({"I <3": "n.io"}) blk.process_signals([signal]) blk.logger.info.assert_called_once_with(json.dumps(signal.to_dict())) self.assertEqual(blk.logger.error.call_count, 0)
def test_process_signals(self): """Signals pass through block unmodified.""" blk = CvColorChange() self.configure_block(blk, {}) blk.start() blk.process_signals([Signal({'hello': 'nio'})]) blk.stop() self.assert_num_signals_notified(1) self.assert_last_signal_notified(Signal({'hello': 'nio'}))
def test_exception_on_logging(self): blk = Logger() self.configure_block(blk, {}) blk.logger = MagicMock() blk.logger.info.side_effect = Exception() signal = Signal({"I <3": "n.io"}) blk.process_signals([signal]) blk.logger.info.assert_called_once_with(json.dumps(signal.to_dict())) blk.logger.exception.assert_called_once_with("Failed to log signal")
def test_list_logging(self): blk = Logger() self.configure_block(blk, {"log_as_list": True}) blk.logger = MagicMock() signal = Signal({"I <3": "n.io"}) blk.process_signals([signal, signal]) blk.logger.info.assert_called_once_with( [json.dumps(signal.to_dict()), json.dumps(signal.to_dict())]) self.assertEqual(blk.logger.error.call_count, 0)
def test_log_hidden_attributes(self): blk = Logger() self.configure_block(blk, {"log_hidden_attributes": True}) blk.logger = MagicMock() signal = Signal({"_hidden": "hidden!", "not_hidden": "not hidden!"}) blk.process_signals([signal]) blk.logger.info.assert_called_once_with( json.dumps(signal.to_dict(True), sort_keys=True)) self.assertEqual(blk.logger.error.call_count, 0) self.assertEqual(len(signal.to_dict(True)), 2)
def test_list_process_signals(self): blk = Logger() self.configure_block(blk, {}) blk.logger = MagicMock() signal = Signal({"I <3": "n.io"}) blk.process_signals([signal, signal]) blk.logger.info.assert_has_calls([ call(json.dumps(signal.to_dict())), call(json.dumps(signal.to_dict())), ]) self.assertEqual(blk.logger.error.call_count, 0)
def _get_next_signal(self, _random=False): """ Get the next individual signal from the file. If configured to do so, this will pull randomly from the file. Otherwise, it will return the next one in line. """ if _random: return Signal(random.choice(self._json_signals)) else: sig = Signal(self._json_signals[self._index]) self._increment_index() return sig
def test_signal_enrighment(self): """Use enrich signal mixin""" blk = UnpackBytes() self.configure_block(blk, {'enrich': {'exclude_existing': False}}) blk.start() blk.process_signals([Signal({'key': 'one', 'value': b'\x00\x01'})]) blk.stop() self.assert_num_signals_notified(1) self.assert_last_signal_notified( Signal({ 'one': 1, 'key': 'one', 'value': b'\x00\x01' }))
def test_interval_default(self): ''' Testing if interval trigger notifies all signals when total signals is not specified. ''' interval = SampleIntervalBlock() self.configure_block(interval, {'interval': {'seconds': 1}}) returns = [Signal(), Signal()] interval.generate_signals = MagicMock(return_value=returns) interval.start() # Give it enough time for two notifications # (one immediately, one after a second) sleep(1.5) interval.stop() self.assert_num_signals_notified(4)
def test_compass(self, mockSenseHat): """Signals are enriched with compass data.""" mock_hat = Mock() mock_hat.get_compass_raw.return_value = { 'x': -1, 'y': 0, 'z': 1, } mockSenseHat.return_value = mock_hat blk = SenseHAT() self.configure_block( blk, { 'env': { 'press': False, 'rh': False, 'temp': False, }, 'imu': { 'accel': False, 'compass': True, 'gyro': False, }, 'enrich': { 'exclude_existing': False, }, }) blk.start() mockSenseHat.assert_called_once_with() mock_hat.set_imu_config.assert_called_once_with(False, True, False) blk.process_signals([Signal({'pi': 3.14})]) mock_hat.get_accelerometer_raw.assert_not_called() mock_hat.get_compass_raw.assert_called_once_with() mock_hat.get_gyroscope_raw.assert_not_called() blk.stop() self.assert_num_signals_notified(1) self.assert_last_signal_list_notified([ Signal({ 'compass': { 'x': -1, 'y': 0, 'z': 1, }, 'pi': 3.14, }), ])
def test_signal_enrichment(self, mock_driver): """Incoming signals are enriched new data""" drvr = mock_driver.return_value config = {'enrich': {'exclude_existing': False}} blk = EIPSetAttribute() self.configure_block(blk, config) blk.start() blk.process_signals([Signal({'foo': 'bar'})]) blk.stop() self.assert_last_signal_notified( Signal({ 'foo': 'bar', 'host': 'localhost', 'path': [1, 1], 'value': b'\x00\x00' }))
def _delimited_reader(self): thread_id = current_thread().name self.logger.debug('Reader thread {} spawned'.format(thread_id)) buffer = [] while not self._kill: try: new_byte = self.file_descriptor.read(1) except Exception: if not self.status.is_set(RunnerStatus.warning): self.set_status('warning') self.logger.exception('Read operation from HID Device failed') self._disconnect() self._connect() break if new_byte == self.delimiter: try: barcode = self._decode_buffer(buffer) except Exception: self.logger.exception('Failed to decode barcode!') barcode = None signal_dict = {'barcode': barcode} self.notify_signals([Signal(signal_dict)]) buffer = [] continue buffer.append(new_byte) self.logger.debug('Reader thread {} completed'.format(thread_id))
def test_base(self): blk = FirebaseInsert() self.configure_block( blk, { "application": MY_APPLICATION, "auth": MY_AUTH, "collection": "integration_tests/blah", "enrich": { "exclude_existing": False, "enrich_field": "output" } }) blk.start() blk.process_signals([Signal({"test_key": "test_val"})]) # We should get an output signal from the insert self.assert_num_signals_notified(1) # TODO: Move this to the framework block unit test sigs_notified = self.last_notified[ Terminal.get_default_terminal_on_class(FirebaseInsert, TerminalType.input).id] out_sig = sigs_notified[0] # Make sure the details of our input signal were notified self.assertEqual(out_sig.test_key, "test_val") # Make sure the ID/name of the saved signal was included on the # output signal under the "name" field self.assertIsNotNone(out_sig.output["name"]) blk.stop()
def run(self): while self._running: try: self.logger.debug('loading inference model ...') with CameraInference( image_classification.model()) as inference: self.logger.debug('running inference ...') for result in inference.run(): predictions = image_classification.get_classes( result, top_k=self.top_k_predictions(), threshold=0) outgoing_signals = [] for label, score in predictions: signal_dict = { 'label': label, 'score': score, } outgoing_signal = Signal(signal_dict) outgoing_signals.append(outgoing_signal) if not self._running: break self.notify_signals(outgoing_signals) except: self.logger.exception('failed to get inference result!') self.reset_camera() self.release_camera()
def run(self): while self._running: try: self.logger.debug('loading inference model ...') with CameraInference(face_detection.model()) as inference: self.logger.debug('running inference ...') for result in inference.run(): faces = face_detection.get_faces(result) if faces: self.logger.debug('found {} faces'.format( len(faces))) outgoing_signals = [] for face in faces: signal_dict = { 'bounding_box': face.bounding_box, 'face_score': face.face_score, 'joy_score': face.joy_score, } outgoing_signal = Signal(signal_dict) outgoing_signals.append(outgoing_signal) if not self._running: break self.notify_signals(outgoing_signals) except: self.logger.exception('failed to get inference result!') self.reset_camera() self.release_camera()
def process_signals(self, in_signals): """ Publish each group of signals """ ttl = self.ttl() groups = defaultdict(list) for signal in in_signals: try: topic = self.topic(signal) if self._is_local and self._local_id: topic = '{}.{}'.format(self._local_id, topic) except Exception: self.logger.exception('topic expression failed, ignoring signal') continue groups[topic].append(signal) for topic, out_signals in groups.items(): try: if self._is_local: out_signals = [Signal({"signals": b64encode(pickle.dumps(out_signals))})] self.__get_publisher(topic, ttl).send(out_signals) except pickle.PicklingError: self.logger.exception("Pickling based pickle error") except TypeError: self.logger.exception("Unable to encode pickled signals") except PublisherError: # pragma no cover self.logger.exception('Error publishing {:n} signals to "{}"'.format(len(out_signals), topic)) except: self.logger.exception("Error processing signals")
def test_insert(self): blk = FirebaseInsert() fbase_imp = self.get_absolute_import('firebase_base.pyrebase') with patch(fbase_imp) as mock_fbase: self.configure_block( blk, { "collection": "COLLECTION", "userEmail": "USER_EMAIL", "userPassword": "******", "config": { "apiKey": "API_KEY", "databaseURL": "DATABASE_URL", "projectId": "PROJECT_ID" }, "enrich": { "exclude_existing": False } }) # We will simulate the post returning a name with a fake ID mock_fbase.initialize_app.return_value.database.return_value.\ child.return_value.push.return_value = {"name": "fake id"} blk.start() blk.process_signals([Signal({"test_key": "test_val"})]) # We should get an output signal from the insert self.assert_num_signals_notified(1) out_sig = self.last_notified["__default_terminal_value"][0] # Make sure the details of our input signal were notified self.assertEqual(out_sig.test_key, "test_val") # Make sure the ID/name of the saved signal was included on the # output signal under the "name" field self.assertEqual(out_sig.name, "fake id") blk.stop()
def test_multiple_values(self): """Unpack two integers from one byte array""" blk = UnpackBytes() self.configure_block( blk, { 'new_attributes': [{ 'key': 'one', 'value': '{{ $value[0:2] }}' }, { 'key': 'two', 'value': '{{ $value[2:4] }}' }] }) blk.start() blk.process_signals([Signal({'value': b'\x00\x01\x00\x02'})]) blk.stop() self.assert_num_signals_notified(1) self.assert_last_signal_notified(Signal({'one': 1, 'two': 2}))
def test_process_signals(self): """Signals pass through block unmodified.""" blk = SpreadsheetLookUp() self.configure_block( blk, { 'source': 'tests/test.xlsx', 'match': [ { 'key': '{{ $host }}', 'value': 'MOXA HOST', }, { 'key': '{{ $channel }}', 'value': 'CHANNEL', }, ], 'output_structure': [ { 'key': 'flavor', 'value': 'FLAVOR', }, { 'key': 'position', 'value': 'POSITION', }, ], }) blk.start() blk.process_signals([ Signal({ 'host': '101', 'channel': '0', }), ]) blk.stop() self.assert_num_signals_notified(1) self.assert_last_signal_list_notified([ Signal({ 'flavor': 'House Roast', 'position': '0', }), ])
def test_signal_lists(self, mock_driver): """Outgoing signal lists have the same length as incoming""" config = {} blk = EIPGetAttribute() self.configure_block(blk, config) blk.start() blk.process_signals([Signal()] * 3) blk.stop() self.assertEqual(len(self.notified_signals[DEFAULT_TERMINAL]), 1) self.assertEqual(len(self.notified_signals[DEFAULT_TERMINAL][0]), 3)