def test_function_filter(self): """Verify the function filter class """ s = ValueListSensor(1, value_stream) st = SensorAsOutputThing(s) captured_list_ref = [[]] got_completed_ref = [ False, ] def on_next(self, x): captured_list_ref[0].append(x.val) self._dispatch_next(x) def on_completed(self): got_completed_ref[0] = True self._dispatch_completed() ff = FunctionFilter(st, on_next=on_next, on_completed=on_completed) vo = ValidationInputThing(value_stream, self.test_function_filter) ff.connect(vo) st.print_downstream() scheduler = Scheduler(asyncio.get_event_loop()) scheduler.schedule_periodic(st, 0.5) # sample twice every second scheduler.run_forever() self.assertTrue( vo.completed, "Schedule exited before validation observer completed") self.assertEqual(value_stream, captured_list_ref[0]) self.assertTrue(got_completed_ref[0]) print("That's all folks")
def test_influx_output(): loop = asyncio.get_event_loop() s = ValueListSensor(1, value_stream) p = SensorPub(s) b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor', fields=['val', 'ts'], tags=['sensor_id']), generate_timestamp=False) p.subscribe(b) scheduler = Scheduler(loop) scheduler.schedule_periodic(p, 0.2) # sample five times every second scheduler.run_forever() # Now play back c = InfluxDBClient(database='antevents') rs = c.query('SELECT * FROM Sensor;').get_points() for d in rs: print(d) # Play back using a publisher p = InfluxDBReader('SELECT * FROM Sensor;') p.subscribe(CallableAsSubscriber(print)) scheduler = Scheduler(loop) scheduler.schedule_periodic(p, 0.2) # sample five times every second scheduler.run_forever() print("That's all folks")
def test_influx_output(self): loop = asyncio.get_event_loop() s = ValueListSensor(1, value_stream) p = SensorAsOutputThing(s) b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor', fields=['val', 'ts'], tags=['sensor_id']), generate_timestamp=False, username=INFLUXDB_USER, password=INFLUXDB_PASSWORD, database=INFLUXDB_DATABASE) p.connect(b) scheduler = Scheduler(loop) scheduler.schedule_periodic(p, 0.2) # sample five times every second scheduler.run_forever() # Now play back rs = self.c.query('SELECT * FROM Sensor;').get_points() for d in rs: print(d) # Play back using an output thing p = InfluxDBReader('SELECT * FROM Sensor;', database=INFLUXDB_DATABASE, username=INFLUXDB_USER, password=INFLUXDB_PASSWORD) p.connect(CallableAsInputThing(print)) scheduler = Scheduler(loop) scheduler.schedule_periodic(p, 0.2) # sample five times every second scheduler.run_forever() print("That's all folks")
def test_function_filter_error_handling(self): """Verify the error handling functionality of the function filter. We do this by connecting two downstream paths to the sensor. The first includes a function filter that throws an error when it encouters a sensor reading of 120. This should disconnect th stream at this point. The second is a normal validation input thing. It is connected directly to the sensor, and thus should not see any errors. """ s = ValueListSensor(1, value_stream) st = SensorAsOutputThing(s) captured_list_ref = [[]] got_completed_ref = [ False, ] got_on_error_ref = [ False, ] def on_next_throw_exc(self, x): if x.val == 120: raise Exception("expected exc") else: captured_list_ref[0].append(x.val) self._dispatch_next(x) def on_completed(self): got_completed_ref[0] = True self._dispatch_completed() def on_error(self, e): got_on_error_ref[0] = True self._dispatch_error(e) ff = FunctionFilter(st, on_next=on_next_throw_exc, on_completed=on_completed, on_error=on_error) ct = CaptureInputThing(expecting_error=True) ff.map(lambda x: x.val).connect(ct) vo = ValidationInputThing(value_stream, self.test_function_filter_error_handling) st.connect(vo) st.print_downstream() scheduler = Scheduler(asyncio.get_event_loop()) scheduler.schedule_periodic(st, 0.5) # sample twice every second scheduler.run_forever() self.assertTrue( vo.completed, "Schedule exited before validation observer completed") self.assertFalse(ct.completed, "Capture thing should not have completed") self.assertTrue(ct.errored, "Capture thing should have seen an error") self.assertFalse(got_completed_ref[0]) self.assertTrue(got_on_error_ref[0]) self.assertEqual([20, 30, 100], ct.events, "Capture thing event mismatch") self.assertEqual([20, 30, 100], captured_list_ref[0], "captured_list_ref mismatch") print("That's all folks")
def test_gpio(self): import thingflow.adapters.rpi.gpio o = thingflow.adapters.rpi.gpio.GpioPinOut() sensor_thing = SensorAsOutputThing(ValueListSensor("sensor-1", values)) sensor_thing.map(lambda evt: evt.val > 0).passthrough( output()).connect(o) s = Scheduler(asyncio.get_event_loop()) s.schedule_periodic(sensor_thing, 1.0) s.run_forever()
def test_tracing(self): s = ValueListSensor(1, values) p = SensorAsOutputThing(s) v = ValidationInputThing([v+1 for v in values], self, extract_value_fn=lambda x:x) p.passthrough(output).map(lambda x : x.val+1).passthrough(output).connect(v) p.trace_downstream() scheduler = Scheduler(asyncio.get_event_loop()) scheduler.schedule_periodic(p, 0.5) # sample twice every second scheduler.run_forever()
def test_schedule_sensor(self): """In this version, we pass the sensor directly to the scheduler and use a functional style to compose the filters""" s = ValueListSensor(1, value_stream) vo = ValidationSubscriber(expected_stream, self.test_schedule_sensor) scheduler = Scheduler(asyncio.get_event_loop()) scheduler.schedule_sensor(s, 0.5, where(predicate), passthrough(vo), output()) scheduler.run_forever() self.assertTrue( vo.completed, "Schedule exited before validation observer completed") print("That's all folks")
def test_bokeh_output(): loop = asyncio.get_event_loop() s = ValueListSensor(1, value_stream) p = SensorPub(s) b = BokehStreamer([ SensorEvent(ts=0,val=10,sensor_id="temp" ) ], io_loop=loop) p.subscribe(b) scheduler = Scheduler(loop) scheduler.schedule_periodic(p, 0.5) # sample twice every second scheduler.run_forever() self.assertTrue(vo.completed, "Schedule exited before validation observer completed") print("That's all folks")
def test_client_only(self): SENSOR_ID = 'sensor-1' TOPIC = get_topic_name(self) sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES)) td = sensor.transduce(PeriodicMedianTransducer(period=3)) qw = QueueWriter(td, URL, TOPIC, self.sched) qw.output() self.sched.schedule_periodic(sensor, 0.5) self.sched.run_forever() self.assertFalse( qw.has_pending_requests(), "QueueWriter has pending requests: %s" % qw.dump_state()) print("test_client_only completed")
def test_passthrough_as_a_method(self): """Verify that, when passthrough is used as a method, it can still take thunks. """ scheduler = Scheduler(asyncio.get_event_loop()) luxpub = SensorAsOutputThing(ValueListSensor('lux-2', lux_data)) vs1 = ValidationInputThing([450, 600], self) vs2 = ValidationInputThing(lux_data, self) luxpub.passthrough(compose(where(lambda evt: evt.val > 300), vs1)).connect(vs2) scheduler.schedule_periodic(luxpub, 0.5) scheduler.run_forever() self.assertTrue(vs1.completed) self.assertTrue(vs2.completed)
def test_bokeh_manager(): loop = asyncio.get_event_loop() s1 = ValueListSensor(1, value_stream) p1 = SensorPub(s1) s2 = ValueListSensor(1, value_stream2) p2 = SensorPub(s2) bm = BokehPlotManager() bplot1 = BokehPlot('Sensor1', y_axis_label='value') bplot2 = BokehPlot('Sensor2', y_axis_label='value') bm.register(bplot1) bm.register(bplot2) p1.map(lambda v: ('Sensor1', v) ).subscribe(bm) p2.map(lambda v: ('Sensor2', v) ).subscribe(bm) bm.start() scheduler = Scheduler(loop) scheduler.schedule_periodic(p1, 1.0) # sample every second scheduler.schedule_periodic(p2, 0.5) # sample twice every second scheduler.run_forever() # self.assertTrue(vo.completed, # "Schedule exited before validation observer completed") print("That's all folks")
def test_thunk_builder_handling(self): """We have logic where we can pass a thunk builder into a combinator and it will do the right thing. Check that it works""" scheduler = Scheduler(asyncio.get_event_loop()) lux = ValueListSensor('lux-2', lux_data) vs = ValidationInputThing(lux_data, self) scheduler.schedule_sensor( lux, 0.5, passthrough(output()), # output() evaluates to a thunk passthrough(output), # output is a thunk builder passthrough(output(sys.stdout)), # output can take an arg vs, print_downstream=True) scheduler.run_forever() self.assertTrue(vs.completed)
def test_where(self): """In this version, we create a publisher and use method chaining to compose the filters""" s = ValueListSensor(1, value_stream) p = SensorPub(s) w = p.where(predicate) w.output() vo = ValidationSubscriber(expected_stream, self.test_where) w.subscribe(vo) scheduler = Scheduler(asyncio.get_event_loop()) scheduler.schedule_periodic(p, 0.5) # sample twice every second p.print_downstream() scheduler.run_forever() self.assertTrue( vo.completed, "Schedule exited before validation observer completed") print("That's all folks")
def test_publish_and_subscribe(self): sensor = ValueListSensor(1, sensor_values) scheduler = Scheduler(asyncio.get_event_loop()) pg = PostgresWriter(scheduler, self.connect_string, self.mapping) capture = CaptureInputThing() scheduler.schedule_sensor(sensor, 0.5, parallel(pg, output, capture)) scheduler.run_forever() print("finish writing to the database") row_source = PostgresReader(self.connect_string, self.mapping) row_source.output() validate = SensorEventValidationInputThing(capture.seq, self) row_source.connect(validate) scheduler.schedule_recurring(row_source) scheduler.run_forever() self.assertTrue(validate.completed) print("finished reading rows")
def send_and_recv_body(self, sleep_timeout): SENSOR_ID = 'sensor-1' TOPIC = get_topic_name(self) sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES)) td = sensor.transduce(PeriodicMedianTransducer(period=3)) qw = QueueWriter(td, URL, TOPIC, self.sched) qw.output() qr = QueueReader(URL, TOPIC, self.sched, timeout=sleep_timeout) self.sched.schedule_periodic(sensor, 0.5) stop_qr = self.sched.schedule_on_main_event_loop(qr) vs = ValidateAndStopInputThing(EXPECTED, self, stop_qr) qr.select(msg_to_event).connect(vs) self.sched.run_forever() self.assertFalse( qw.has_pending_requests(), "QueueWriter has pending requests: %s" % qw.dump_state()) self.assertEqual(qr.state, QueueReader.FINAL_STATE) self.assertEqual(vs.next_idx, len(EXPECTED)) print("send_and_recv_bod(%s) completed" % sleep_timeout)
def test_pandas(self): s = ValueListSensor(1, value_stream) p = SensorAsOutputThing(s) import thingflow.adapters.pandas import numpy w = thingflow.adapters.pandas.PandasSeriesWriter() p.connect(w) sch = Scheduler(asyncio.get_event_loop()) sch.schedule_recurring(p) sch.run_forever() self.assertTrue(w.result is not None, "Result of pandas never set") # now we verify each element for (i, v) in enumerate(value_stream): pv = w.result[i] self.assertTrue( isinstance(pv, numpy.int64), "Expecting pandas value '%s' to be numpy.int64, but instead was %s" % (pv, repr(type(pv)))) self.assertEqual( v, pv, "Pandas value '%s' not equal to original value '%s'" % (repr(pv), repr(v))) print("Validate pandas array")
def test_complex_workflow(self): THRESHOLD = 300 lux = ValueListSensor('lux-1', lux_data) scheduler = Scheduler(asyncio.get_event_loop()) vs1 = ValidationInputThing(lux_data, self) vs2 = ValidationInputThing( [False, False, False, True, False, False, True], self, extract_value_fn=lambda v: v) scheduler.schedule_sensor( lux, 0.5, passthrough(output()), passthrough(vs1), map(lambda event: event.val > THRESHOLD), passthrough(lambda v: print('ON' if v else 'OFF')), vs2, print_downstream=True) scheduler.run_forever() self.assertTrue(vs1.completed) self.assertTrue(vs2.completed) print("That's all folks")
def setUp(self): self.scheduler = Scheduler(asyncio.get_event_loop()) self.sensor = ValueListSensor(1, value_stream)