def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Distributed lux example, data capture process')
parser.add_argument('-i',
'--interval',
type=float,
default=5.0,
help="Sample interval in seconds")
parser.add_argument(
'-t',
'--threshold',
type=float,
default=25.0,
help="Threshold lux level above which light should be turned on")
parser.add_argument('broker',
metavar="BROKER",
type=str,
help="hostname or ip address of mqtt broker")
parsed_args = parser.parse_args(argv)
(lux, led) = setup(parsed_args.broker, parsed_args.threshold)
scheduler = Scheduler(asyncio.get_event_loop())
stop = scheduler.schedule_periodic_on_separate_thread(
lux, parsed_args.interval)
print("starting run...")
try:
scheduler.run_forever()
except KeyboardInterrupt:
led.on_completed()
stop()
return 0
def setUp(self):
# Creating a new event loop each test case does not seem to work.
# I think it is due to hbmqtt not cleaning up some state in the asyncio
# layer.
#self.loop = asyncio.new_event_loop()
self.loop = asyncio.get_event_loop()
self.sched = Scheduler(self.loop)
def test_file_write_read(self):
tf = NamedTemporaryFile(mode='w', delete=False)
tf.close()
try:
sensor = make_test_publisher(1, stop_after_events=NUM_EVENTS)
capture = CaptureSubscriber()
sensor.subscribe(capture)
sensor.csv_writer(tf.name)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
print("Writing sensor events to temp file")
scheduler.run_forever()
self.assertTrue(capture.completed,
"CaptureSubscriber did not complete")
self.assertEqual(
len(capture.events), NUM_EVENTS,
"number of events captured did not match generated events")
reader = CsvReader(tf.name)
vs = SensorEventValidationSubscriber(capture.events, self)
reader.subscribe(vs)
scheduler.schedule_recurring(reader)
print("reading sensor events back from temp file")
scheduler.run_forever()
self.assertTrue(vs.completed,
"ValidationSubscriber did not complete")
finally:
os.remove(tf.name)
def test_blocking_publisher(self):
o = BlockingPublisher()
o.output()
scheduler = Scheduler(asyncio.get_event_loop())
c = scheduler.schedule_periodic_on_separate_thread(o, 1)
vs = ValidationSubscriber([i+1 for i in range(EVENTS)], self,
extract_value_fn=lambda v:v)
o.subscribe(vs)
o.subscribe(StopLoopAfter(EVENTS, c))
o.print_downstream()
scheduler.run_forever()
print("that's it")
class TestCase(unittest.TestCase):
def setUp(self):
# Creating a new event loop each test case does not seem to work.
# I think it is due to hbmqtt not cleaning up some state in the asyncio
# layer.
#self.loop = asyncio.new_event_loop()
self.loop = asyncio.get_event_loop()
self.sched = Scheduler(self.loop)
def tearDown(self):
pass
#self.loop.stop()
#self.loop.close()
def test_client_only(self):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorPub(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
self.sched.schedule_periodic(sensor, 0.5)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
print("test_client_only completed")
def send_and_recv_body(self, sleep_timeout):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorPub(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
qr = QueueReader(URL, TOPIC, self.sched, timeout=sleep_timeout)
self.sched.schedule_periodic(sensor, 0.5)
stop_qr = self.sched.schedule_on_main_event_loop(qr)
vs = ValidateAndStopSubscriber(EXPECTED, self, stop_qr)
qr.select(msg_to_event).subscribe(vs)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
self.assertEqual(qr.state, QueueReader.FINAL_STATE)
self.assertEqual(vs.next_idx, len(EXPECTED))
print("send_and_recv_bod(%s) completed" % sleep_timeout)
def test_short_timeout(self):
self.send_and_recv_body(0.1)
def test_long_timeout(self):
self.send_and_recv_body(3.0)
def test_case(self):
"""Just run the reader in its own event loop. We stop everything after 4
events.
"""
s = Scheduler(asyncio.get_event_loop())
m = MQTTReader("localhost",
topics=[
('bogus/bogus', 0),
],
mock_class=MockMQTTClient)
m.output()
c = s.schedule_on_private_event_loop(m)
m.subscribe(StopLoopAfter(4, c))
m.print_downstream()
s.run_forever()
print("that's it")
def test_publish_and_subscribe(self):
sensor = ValueListSensor(1, sensor_values)
scheduler = Scheduler(asyncio.get_event_loop())
pg = PostgresWriter(scheduler, self.connect_string, self.mapping)
capture = CaptureSubscriber()
scheduler.schedule_sensor(sensor, 0.5,
parallel(pg, output, capture))
scheduler.run_forever()
print("finish writing to the database")
row_source = PostgresReader(self.connect_string, self.mapping)
row_source.output()
validate = SensorEventValidationSubscriber(capture.seq, self)
row_source.subscribe(validate)
scheduler.schedule_recurring(row_source)
scheduler.run_forever()
self.assertTrue(validate.completed)
print("finished reading rows")
def test_mqtt(self):
loop = asyncio.get_event_loop()
s = Scheduler(loop)
sensor = make_test_publisher_from_vallist(1, sensor_data)
mqtt_writer = MQTTWriter('localhost', topics=[
('bogus/bogus', 0),
])
sensor.to_json().subscribe(mqtt_writer)
s.schedule_periodic(sensor, 0.5)
mqtt_reader = MQTTReader("localhost", topics=[
('bogus/bogus', 0),
])
vs = ValidationSubscriber(sensor_data, self)
mqtt_reader.take(5).select(mqtt_msg_to_unicode).from_json(constructor=SensorEvent) \
.output().subscribe(vs)
c = s.schedule_on_private_event_loop(mqtt_reader)
stop = StopLoopAfter(5, c)
mqtt_reader.subscribe(stop)
mqtt_reader.print_downstream()
sensor.print_downstream()
s.run_forever()
loop.stop()
self.assertTrue(vs.completed)
print("that's it")
def setup(host):
mqtt = MQTTReader(host, topics=[('bogus/bogus', 2)])
decoded = mqtt.select(lambda m:(m.payload).decode("utf-8")) \
.from_json(constructor=SensorEvent)
scheduler = Scheduler(asyncio.get_event_loop())
decoded.subscribe(PostgresWriter(scheduler, connect_string, mapping))
decoded.output()
mqtt.print_downstream()
return mqtt, scheduler
def main(argv=sys.argv[1:]):
if len(argv) != 2:
print("%s threshold interval" % sys.argv[0])
return 1
threshold = float(argv[0])
interval = float(argv[1])
print("%f seconds interval and an led threshold of %f lux" %
(interval, threshold))
(lux, led) = setup(threshold)
scheduler = Scheduler(asyncio.get_event_loop())
stop = scheduler.schedule_periodic_on_separate_thread(lux, interval)
print("starting run...")
try:
scheduler.run_forever()
except KeyboardInterrupt:
led.on_completed()
stop()
return 0
def test_dispatch(self):
"""Test a scenario where we dispatch to one of several writers
depending on the sensor id.
"""
def generator():
for e in EVENTS2:
yield e
sensor = IterableAsPublisher(generator(), name='sensor')
dispatcher = sensor.dispatch(dispatch_rules)
for s in sensor_ids:
dispatcher.rolling_csv_writer('.', s, sub_topic=s)
dispatcher.subscribe(
lambda x: self.assertTrue(False, "bad dispatch of %s" % x))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
scheduler.run_forever()
for f in FILES2:
self.assertTrue(os.path.exists(f), 'did not find file %s' % f)
cnt = 0
with open(f, 'r') as fobj:
for line in fobj:
cnt += 1
self.assertEqual(2, cnt, "File %s did not have 2 lines" % f)
print("found log file %s" % f)
def main_linear():
obs_stream = from_iterable(
iter([[[[1.0, 1.0], [2.0, 2.0]], [1.0, 2.0]],
[[[6.0, 6.0], [9.0, 9.0]], [6.0, 9.0]]]))
pred_stream = from_iterable(iter([[3.0, 3.0]]))
model = LinearRegressionModel()
obs_stream.subscribe(model, topic_mapping=('default', 'train'))
obs_stream.subscribe(print)
pred_stream.subscribe(model, topic_mapping=('default', 'observe'))
model.subscribe(print, topic_mapping=('predict', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(obs_stream, 1)
scheduler.schedule_periodic(pred_stream, 5)
scheduler.run_forever()
def test(self):
scheduler = Scheduler(asyncio.get_event_loop())
sensor = make_test_publisher_from_vallist(1, values)
scheduler.schedule_periodic(sensor, 1)
blocking_subscriber = TestSubscriber(scheduler, values, self)
sensor.subscribe(blocking_subscriber)
scheduler.run_forever()
self.assertTrue(blocking_subscriber.completed)
def test_blocking_sensor(self):
s = BlockingSensor(1, stop_after=EVENTS)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_sensor_on_separate_thread(s, 1,
passthrough(output()),
ValidationSubscriber([i+1 for i in range(EVENTS)], self,
extract_value_fn=lambda v:v),
make_event_fn=lambda s, v: v)
scheduler.run_forever()
print("that's it")
def test_schedule_sensor(self):
"""In this version, we pass the sensor directly to the scheduler and use
a functional style to compose the filters"""
s = ValueListSensor(1, value_stream)
vo = ValidationSubscriber(expected_stream, self.test_schedule_sensor)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_sensor(s, 0.5, where(predicate), passthrough(vo),
output())
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
class TestCase(unittest.TestCase):
def setUp(self):
self.scheduler = Scheduler(asyncio.get_event_loop())
self.sensor = ValueListSensor(1, value_stream)
def test_sensor_event_sliding_window(self):
vs = ValidationSubscriber(mean_stream, self)
self.scheduler.schedule_sensor(self.sensor, 0.1,
transduce(SensorSlidingMean(4)),
parallel(vs, output()))
self.scheduler.run_forever()
self.assertTrue(vs.completed)
def test_periodic_median_transducer(self):
vs = ValidationSubscriber(periodic_median_stream, self)
self.scheduler.schedule_sensor(self.sensor, 0.1,
transduce(PeriodicMedianTransducer(3)),
parallel(vs, output()))
self.scheduler.run_forever()
self.assertTrue(vs.completed)
def test_bokeh_output():
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
b = BokehStreamer([ SensorEvent(ts=0,val=10,sensor_id="temp" ) ], io_loop=loop)
p.subscribe(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_passthrough_as_a_method(self):
"""Verify that, when passthrough is used as a method, it can still take
thunks.
"""
scheduler = Scheduler(asyncio.get_event_loop())
luxpub = SensorPub(ValueListSensor('lux-2', lux_data))
vs1 = ValidationSubscriber([450, 600], self)
vs2 = ValidationSubscriber(lux_data, self)
luxpub.passthrough(compose(where(lambda evt: evt.val > 300),
vs1)).subscribe(vs2)
scheduler.schedule_periodic(luxpub, 0.5)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
def test_where(self):
s = make_test_publisher_from_vallist(1, value_stream)
w = s.where(predicate)
w.output()
vo = ValidationSubscriber(expected_stream, self)
w.subscribe(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(s, 0.5) # sample twice every second
s.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertTrue(vo.completed)
print("That's all folks")
def test_rollover(self):
def generator():
for e in EVENTS:
yield e
sensor = IterableAsPublisher(generator(), name='sensor')
sensor.rolling_csv_writer('.', 'dining-room')
vs = SensorEventValidationSubscriber(EVENTS, self)
sensor.subscribe(vs)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
scheduler.run_forever()
for f in FILES:
self.assertTrue(os.path.exists(f), 'did not find file %s' % f)
print("found log file %s" % f)
def test_thunk_builder_handling(self):
"""We have logic where we can pass a thunk builder into a combinator and
it will do the right thing. Check that it works"""
scheduler = Scheduler(asyncio.get_event_loop())
lux = ValueListSensor('lux-2', lux_data)
vs = ValidationSubscriber(lux_data, self)
scheduler.schedule_sensor(
lux,
0.5,
passthrough(output()), # output() evaluates to a thunk
passthrough(output), # output is a thunk builder
passthrough(output(sys.stdout)), # output can take an arg
vs,
print_downstream=True)
scheduler.run_forever()
self.assertTrue(vs.completed)
def test_where(self):
"""In this version, we create a publisher and use method chaining to
compose the filters"""
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
w = p.where(predicate)
w.output()
vo = ValidationSubscriber(expected_stream, self.test_where)
w.subscribe(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(p, 0.5) # sample twice every second
p.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_case(self):
sensor = make_test_publisher(1, stop_after_events=10)
split = SplitPublisher()
sensor.subscribe(split)
split.subscribe(lambda x: print("above:%s" % x),
topic_mapping=('above', 'default'))
split.subscribe(lambda x: print("below:%s" % x),
topic_mapping=('below', 'default'))
split.subscribe(lambda x: print("within:%s" % x),
topic_mapping=('within', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
print("that's all")
def test_supplying_event_on_timeout(self):
"""In this testcase, we drop every other event.
We set the timeout to a bit longer than the event interval of
one second. It then supplies the previous event. The resulting
output stream will show every other value repeated twice.
"""
sensor = make_test_publisher_from_vallist(1, sensor_values)
drop = DropPeriodic(sensor)
scheduler = Scheduler(asyncio.get_event_loop())
vo = ValidationSubscriber(expected_values, self)
drop.supply_event_when_timeout(EventWatcher(), scheduler,
1.1).output().subscribe(vo)
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
def test_multiple_timeouts(self):
"""In this testcase, we pass two events, drop two events, etc.
We set the timeout to a bit longer than the event interval. The last
good value is supplied when the timeout expires. Thus, we should see
two good events, two repeats of the first event, two good events, etc.
"""
sensor = make_test_publisher_from_vallist(1, sensor_values)
drop = DropPeriodic(sensor, N=2)
scheduler = Scheduler(asyncio.get_event_loop())
vo = ValidationSubscriber(expected_values_multiple_timeouts, self)
drop.supply_event_when_timeout(EventWatcher(), scheduler,
1.1).output().subscribe(vo)
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
def run_example():
sensor = SensorPub(DummyTempSensor('temp-1', input_sequence))
sensor.output() # let us see the raw values
dispatcher = sensor.transduce(RunningAvg(4))\
.passthrough(lambda evt:
print("Running avg temp: %s" %
round(evt.val, 2))) \
.dispatch([(lambda v: v[2]>=T_high, 't_high'),
(lambda v: v[2]<=T_low, 't_low')])
controller = Controller()
dispatcher.subscribe(controller, topic_mapping=('t_high', 't_high'))
dispatcher.subscribe(controller, topic_mapping=('t_low', 't_low'))
dispatcher.subscribe(controller, topic_mapping=('default', 'between'))
controller.subscribe(BypassValveActuator())
sensor.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 0.5)
scheduler.run_forever()
print("got to the end")
def test_complex_workflow(self):
THRESHOLD = 300
lux = ValueListSensor('lux-1', lux_data)
scheduler = Scheduler(asyncio.get_event_loop())
vs1 = ValidationSubscriber(lux_data, self)
vs2 = ValidationSubscriber(
[False, False, False, True, False, False, True],
self,
extract_value_fn=lambda v: v)
scheduler.schedule_sensor(
lux,
0.5,
passthrough(output()),
passthrough(vs1),
map(lambda event: event.val > THRESHOLD),
passthrough(lambda v: print('ON' if v else 'OFF')),
vs2,
print_downstream=True)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
print("That's all folks")
def test_bokeh_manager():
loop = asyncio.get_event_loop()
s1 = ValueListSensor(1, value_stream)
p1 = SensorPub(s1)
s2 = ValueListSensor(1, value_stream2)
p2 = SensorPub(s2)
bm = BokehPlotManager()
bplot1 = BokehPlot('Sensor1', y_axis_label='value')
bplot2 = BokehPlot('Sensor2', y_axis_label='value')
bm.register(bplot1)
bm.register(bplot2)
p1.map(lambda v: ('Sensor1', v) ).subscribe(bm)
p2.map(lambda v: ('Sensor2', v) ).subscribe(bm)
bm.start()
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p1, 1.0) # sample every second
scheduler.schedule_periodic(p2, 0.5) # sample twice every second
scheduler.run_forever()
# self.assertTrue(vo.completed,
# "Schedule exited before validation observer completed")
print("That's all folks")
def main_kalman():
dim_x = 2
dim_u = 1
dim_z = 1
initial_state_mean = np.array([[1.0], [0.0]])
initial_state_covariance = 1000 * np.eye(dim_x)
F = np.array([[1., 1.], [0., 1.]])
B = np.zeros((2, 1))
Q = Q_discrete_white_noise(dim=2, dt=0.1, var=0.13)
H = np.array([[1., 0.]])
R = 5 * np.eye(1)
model = KalmanFilterModel(dim_x, dim_u, dim_z, initial_state_mean,
initial_state_covariance, F, B, Q, H, R)
measurement_stream = from_iterable(iter([[1.0], [0.0]]))
# measurement_stream = from_iterable(iter([ np.array([ [1.0, 1.0] ]) ]))
measurement_stream.subscribe(model, topic_mapping=('default', 'observe'))
model.subscribe(print, topic_mapping=('predict', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(measurement_stream, 1)
scheduler.run_forever()
if self.events_since_last == self.period:
val = median(self.samples)
event = SensorEvent(sensor_id=v.sensor_id, ts=v.ts, val=val)
self.events_since_last = 0
return event
else:
self.last_event = v # save in case we complete before completing a period
return None
def complete(self):
if self.events_since_last > 0:
# if we have some partial state, we emit one final event that
# averages whatever we saw since the last emission.
return SensorEvent(sensor_id=self.last_event.sensor_id,
ts=self.last_event.ts,
val=median(
self.samples[0:self.events_since_last]))
SENSOR_ID = 'sensor-1'
scheduler = Scheduler(asyncio.get_event_loop())
sensor = SensorPub(
RandomSensor(SENSOR_ID, mean=10, stddev=5, stop_after_events=12))
sensor.csv_writer('raw_data.csv').subscribe(
lambda x: print("raw data: %s" % repr(x)))
sensor.transduce(PeriodicMedianTransducer()).mqtt_async_send(
URL, SENSOR_ID, scheduler).output()
scheduler.schedule_periodic(sensor, 0.5)
scheduler.run_forever()
print("that's all folks")