def test_function_filter(self):
"""Verify the function filter class
"""
s = ValueListSensor(1, value_stream)
st = SensorAsOutputThing(s)
captured_list_ref = [[]]
got_completed_ref = [
False,
]
def on_next(self, x):
captured_list_ref[0].append(x.val)
self._dispatch_next(x)
def on_completed(self):
got_completed_ref[0] = True
self._dispatch_completed()
ff = FunctionFilter(st, on_next=on_next, on_completed=on_completed)
vo = ValidationInputThing(value_stream, self.test_function_filter)
ff.connect(vo)
st.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(st, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertEqual(value_stream, captured_list_ref[0])
self.assertTrue(got_completed_ref[0])
print("That's all folks")
def test_influx_output():
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor',
fields=['val', 'ts'],
tags=['sensor_id']),
generate_timestamp=False)
p.subscribe(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
# Now play back
c = InfluxDBClient(database='antevents')
rs = c.query('SELECT * FROM Sensor;').get_points()
for d in rs:
print(d)
# Play back using a publisher
p = InfluxDBReader('SELECT * FROM Sensor;')
p.subscribe(CallableAsSubscriber(print))
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
print("That's all folks")
def test_influx_output(self):
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorAsOutputThing(s)
b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor',
fields=['val', 'ts'],
tags=['sensor_id']),
generate_timestamp=False,
username=INFLUXDB_USER,
password=INFLUXDB_PASSWORD,
database=INFLUXDB_DATABASE)
p.connect(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
# Now play back
rs = self.c.query('SELECT * FROM Sensor;').get_points()
for d in rs:
print(d)
# Play back using an output thing
p = InfluxDBReader('SELECT * FROM Sensor;',
database=INFLUXDB_DATABASE,
username=INFLUXDB_USER,
password=INFLUXDB_PASSWORD)
p.connect(CallableAsInputThing(print))
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
print("That's all folks")
def test_function_filter_error_handling(self):
"""Verify the error handling functionality of the function filter. We do
this by connecting two downstream paths to the sensor. The first includes
a function filter that throws an error when it encouters a sensor reading
of 120. This should disconnect th stream at this point. The second is
a normal validation input thing. It is connected directly to the sensor,
and thus should not see any errors.
"""
s = ValueListSensor(1, value_stream)
st = SensorAsOutputThing(s)
captured_list_ref = [[]]
got_completed_ref = [
False,
]
got_on_error_ref = [
False,
]
def on_next_throw_exc(self, x):
if x.val == 120:
raise Exception("expected exc")
else:
captured_list_ref[0].append(x.val)
self._dispatch_next(x)
def on_completed(self):
got_completed_ref[0] = True
self._dispatch_completed()
def on_error(self, e):
got_on_error_ref[0] = True
self._dispatch_error(e)
ff = FunctionFilter(st,
on_next=on_next_throw_exc,
on_completed=on_completed,
on_error=on_error)
ct = CaptureInputThing(expecting_error=True)
ff.map(lambda x: x.val).connect(ct)
vo = ValidationInputThing(value_stream,
self.test_function_filter_error_handling)
st.connect(vo)
st.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(st, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertFalse(ct.completed,
"Capture thing should not have completed")
self.assertTrue(ct.errored, "Capture thing should have seen an error")
self.assertFalse(got_completed_ref[0])
self.assertTrue(got_on_error_ref[0])
self.assertEqual([20, 30, 100], ct.events,
"Capture thing event mismatch")
self.assertEqual([20, 30, 100], captured_list_ref[0],
"captured_list_ref mismatch")
print("That's all folks")
def test_gpio(self):
import thingflow.adapters.rpi.gpio
o = thingflow.adapters.rpi.gpio.GpioPinOut()
sensor_thing = SensorAsOutputThing(ValueListSensor("sensor-1", values))
sensor_thing.map(lambda evt: evt.val > 0).passthrough(
output()).connect(o)
s = Scheduler(asyncio.get_event_loop())
s.schedule_periodic(sensor_thing, 1.0)
s.run_forever()
def test_tracing(self):
s = ValueListSensor(1, values)
p = SensorAsOutputThing(s)
v = ValidationInputThing([v+1 for v in values], self,
extract_value_fn=lambda x:x)
p.passthrough(output).map(lambda x : x.val+1).passthrough(output).connect(v)
p.trace_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(p, 0.5) # sample twice every second
scheduler.run_forever()
def test_schedule_sensor(self):
"""In this version, we pass the sensor directly to the scheduler and use
a functional style to compose the filters"""
s = ValueListSensor(1, value_stream)
vo = ValidationSubscriber(expected_stream, self.test_schedule_sensor)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_sensor(s, 0.5, where(predicate), passthrough(vo),
output())
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_bokeh_output():
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
b = BokehStreamer([ SensorEvent(ts=0,val=10,sensor_id="temp" ) ], io_loop=loop)
p.subscribe(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_client_only(self):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
self.sched.schedule_periodic(sensor, 0.5)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
print("test_client_only completed")
def test_passthrough_as_a_method(self):
"""Verify that, when passthrough is used as a method, it can still take
thunks.
"""
scheduler = Scheduler(asyncio.get_event_loop())
luxpub = SensorAsOutputThing(ValueListSensor('lux-2', lux_data))
vs1 = ValidationInputThing([450, 600], self)
vs2 = ValidationInputThing(lux_data, self)
luxpub.passthrough(compose(where(lambda evt: evt.val > 300),
vs1)).connect(vs2)
scheduler.schedule_periodic(luxpub, 0.5)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
def test_bokeh_manager():
loop = asyncio.get_event_loop()
s1 = ValueListSensor(1, value_stream)
p1 = SensorPub(s1)
s2 = ValueListSensor(1, value_stream2)
p2 = SensorPub(s2)
bm = BokehPlotManager()
bplot1 = BokehPlot('Sensor1', y_axis_label='value')
bplot2 = BokehPlot('Sensor2', y_axis_label='value')
bm.register(bplot1)
bm.register(bplot2)
p1.map(lambda v: ('Sensor1', v) ).subscribe(bm)
p2.map(lambda v: ('Sensor2', v) ).subscribe(bm)
bm.start()
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p1, 1.0) # sample every second
scheduler.schedule_periodic(p2, 0.5) # sample twice every second
scheduler.run_forever()
# self.assertTrue(vo.completed,
# "Schedule exited before validation observer completed")
print("That's all folks")
def test_thunk_builder_handling(self):
"""We have logic where we can pass a thunk builder into a combinator and
it will do the right thing. Check that it works"""
scheduler = Scheduler(asyncio.get_event_loop())
lux = ValueListSensor('lux-2', lux_data)
vs = ValidationInputThing(lux_data, self)
scheduler.schedule_sensor(
lux,
0.5,
passthrough(output()), # output() evaluates to a thunk
passthrough(output), # output is a thunk builder
passthrough(output(sys.stdout)), # output can take an arg
vs,
print_downstream=True)
scheduler.run_forever()
self.assertTrue(vs.completed)
def test_where(self):
"""In this version, we create a publisher and use method chaining to
compose the filters"""
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
w = p.where(predicate)
w.output()
vo = ValidationSubscriber(expected_stream, self.test_where)
w.subscribe(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(p, 0.5) # sample twice every second
p.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_publish_and_subscribe(self):
sensor = ValueListSensor(1, sensor_values)
scheduler = Scheduler(asyncio.get_event_loop())
pg = PostgresWriter(scheduler, self.connect_string, self.mapping)
capture = CaptureInputThing()
scheduler.schedule_sensor(sensor, 0.5,
parallel(pg, output, capture))
scheduler.run_forever()
print("finish writing to the database")
row_source = PostgresReader(self.connect_string, self.mapping)
row_source.output()
validate = SensorEventValidationInputThing(capture.seq, self)
row_source.connect(validate)
scheduler.schedule_recurring(row_source)
scheduler.run_forever()
self.assertTrue(validate.completed)
print("finished reading rows")
def send_and_recv_body(self, sleep_timeout):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
qr = QueueReader(URL, TOPIC, self.sched, timeout=sleep_timeout)
self.sched.schedule_periodic(sensor, 0.5)
stop_qr = self.sched.schedule_on_main_event_loop(qr)
vs = ValidateAndStopInputThing(EXPECTED, self, stop_qr)
qr.select(msg_to_event).connect(vs)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
self.assertEqual(qr.state, QueueReader.FINAL_STATE)
self.assertEqual(vs.next_idx, len(EXPECTED))
print("send_and_recv_bod(%s) completed" % sleep_timeout)
def test_pandas(self):
s = ValueListSensor(1, value_stream)
p = SensorAsOutputThing(s)
import thingflow.adapters.pandas
import numpy
w = thingflow.adapters.pandas.PandasSeriesWriter()
p.connect(w)
sch = Scheduler(asyncio.get_event_loop())
sch.schedule_recurring(p)
sch.run_forever()
self.assertTrue(w.result is not None, "Result of pandas never set")
# now we verify each element
for (i, v) in enumerate(value_stream):
pv = w.result[i]
self.assertTrue(
isinstance(pv, numpy.int64),
"Expecting pandas value '%s' to be numpy.int64, but instead was %s"
% (pv, repr(type(pv))))
self.assertEqual(
v, pv, "Pandas value '%s' not equal to original value '%s'" %
(repr(pv), repr(v)))
print("Validate pandas array")
def test_complex_workflow(self):
THRESHOLD = 300
lux = ValueListSensor('lux-1', lux_data)
scheduler = Scheduler(asyncio.get_event_loop())
vs1 = ValidationInputThing(lux_data, self)
vs2 = ValidationInputThing(
[False, False, False, True, False, False, True],
self,
extract_value_fn=lambda v: v)
scheduler.schedule_sensor(
lux,
0.5,
passthrough(output()),
passthrough(vs1),
map(lambda event: event.val > THRESHOLD),
passthrough(lambda v: print('ON' if v else 'OFF')),
vs2,
print_downstream=True)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
print("That's all folks")
def setUp(self):
self.scheduler = Scheduler(asyncio.get_event_loop())
self.sensor = ValueListSensor(1, value_stream)