def test_mqtt(self):
loop = asyncio.get_event_loop()
s = Scheduler(loop)
sensor = make_test_publisher_from_vallist(1, sensor_data)
mqtt_writer = MQTTWriter('localhost', topics=[
('bogus/bogus', 0),
])
sensor.to_json().subscribe(mqtt_writer)
s.schedule_periodic(sensor, 0.5)
mqtt_reader = MQTTReader("localhost", topics=[
('bogus/bogus', 0),
])
vs = ValidationSubscriber(sensor_data, self)
mqtt_reader.take(5).select(mqtt_msg_to_unicode).from_json(constructor=SensorEvent) \
.output().subscribe(vs)
c = s.schedule_on_private_event_loop(mqtt_reader)
stop = StopLoopAfter(5, c)
mqtt_reader.subscribe(stop)
mqtt_reader.print_downstream()
sensor.print_downstream()
s.run_forever()
loop.stop()
self.assertTrue(vs.completed)
print("that's it")
def test_dispatch(self):
"""Test a scenario where we dispatch to one of several writers
depending on the sensor id.
"""
def generator():
for e in EVENTS2:
yield e
sensor = IterableAsPublisher(generator(), name='sensor')
dispatcher = sensor.dispatch(dispatch_rules)
for s in sensor_ids:
dispatcher.rolling_csv_writer('.', s, sub_topic=s)
dispatcher.subscribe(
lambda x: self.assertTrue(False, "bad dispatch of %s" % x))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
scheduler.run_forever()
for f in FILES2:
self.assertTrue(os.path.exists(f), 'did not find file %s' % f)
cnt = 0
with open(f, 'r') as fobj:
for line in fobj:
cnt += 1
self.assertEqual(2, cnt, "File %s did not have 2 lines" % f)
print("found log file %s" % f)
def test_influx_output():
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor',
fields=['val', 'ts'],
tags=['sensor_id']),
generate_timestamp=False)
p.subscribe(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
# Now play back
c = InfluxDBClient(database='antevents')
rs = c.query('SELECT * FROM Sensor;').get_points()
for d in rs:
print(d)
# Play back using a publisher
p = InfluxDBReader('SELECT * FROM Sensor;')
p.subscribe(CallableAsSubscriber(print))
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
print("That's all folks")
def test_file_write_read(self):
tf = NamedTemporaryFile(mode='w', delete=False)
tf.close()
try:
sensor = make_test_publisher(1, stop_after_events=NUM_EVENTS)
capture = CaptureSubscriber()
sensor.subscribe(capture)
sensor.csv_writer(tf.name)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
print("Writing sensor events to temp file")
scheduler.run_forever()
self.assertTrue(capture.completed,
"CaptureSubscriber did not complete")
self.assertEqual(
len(capture.events), NUM_EVENTS,
"number of events captured did not match generated events")
reader = CsvReader(tf.name)
vs = SensorEventValidationSubscriber(capture.events, self)
reader.subscribe(vs)
scheduler.schedule_recurring(reader)
print("reading sensor events back from temp file")
scheduler.run_forever()
self.assertTrue(vs.completed,
"ValidationSubscriber did not complete")
finally:
os.remove(tf.name)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Distributed lux example, data capture process')
parser.add_argument('-i',
'--interval',
type=float,
default=5.0,
help="Sample interval in seconds")
parser.add_argument(
'-t',
'--threshold',
type=float,
default=25.0,
help="Threshold lux level above which light should be turned on")
parser.add_argument('broker',
metavar="BROKER",
type=str,
help="hostname or ip address of mqtt broker")
parsed_args = parser.parse_args(argv)
(lux, led) = setup(parsed_args.broker, parsed_args.threshold)
scheduler = Scheduler(asyncio.get_event_loop())
stop = scheduler.schedule_periodic_on_separate_thread(
lux, parsed_args.interval)
print("starting run...")
try:
scheduler.run_forever()
except KeyboardInterrupt:
led.on_completed()
stop()
return 0
def test(self):
scheduler = Scheduler(asyncio.get_event_loop())
sensor = make_test_publisher_from_vallist(1, values)
scheduler.schedule_periodic(sensor, 1)
blocking_subscriber = TestSubscriber(scheduler, values, self)
sensor.subscribe(blocking_subscriber)
scheduler.run_forever()
self.assertTrue(blocking_subscriber.completed)
def test_blocking_sensor(self):
s = BlockingSensor(1, stop_after=EVENTS)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_sensor_on_separate_thread(s, 1,
passthrough(output()),
ValidationSubscriber([i+1 for i in range(EVENTS)], self,
extract_value_fn=lambda v:v),
make_event_fn=lambda s, v: v)
scheduler.run_forever()
print("that's it")
class TestCase(unittest.TestCase):
def setUp(self):
# Creating a new event loop each test case does not seem to work.
# I think it is due to hbmqtt not cleaning up some state in the asyncio
# layer.
#self.loop = asyncio.new_event_loop()
self.loop = asyncio.get_event_loop()
self.sched = Scheduler(self.loop)
def tearDown(self):
pass
#self.loop.stop()
#self.loop.close()
def test_client_only(self):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorPub(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
self.sched.schedule_periodic(sensor, 0.5)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
print("test_client_only completed")
def send_and_recv_body(self, sleep_timeout):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorPub(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
qr = QueueReader(URL, TOPIC, self.sched, timeout=sleep_timeout)
self.sched.schedule_periodic(sensor, 0.5)
stop_qr = self.sched.schedule_on_main_event_loop(qr)
vs = ValidateAndStopSubscriber(EXPECTED, self, stop_qr)
qr.select(msg_to_event).subscribe(vs)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
self.assertEqual(qr.state, QueueReader.FINAL_STATE)
self.assertEqual(vs.next_idx, len(EXPECTED))
print("send_and_recv_bod(%s) completed" % sleep_timeout)
def test_short_timeout(self):
self.send_and_recv_body(0.1)
def test_long_timeout(self):
self.send_and_recv_body(3.0)
def test_blocking_publisher(self):
o = BlockingPublisher()
o.output()
scheduler = Scheduler(asyncio.get_event_loop())
c = scheduler.schedule_periodic_on_separate_thread(o, 1)
vs = ValidationSubscriber([i+1 for i in range(EVENTS)], self,
extract_value_fn=lambda v:v)
o.subscribe(vs)
o.subscribe(StopLoopAfter(EVENTS, c))
o.print_downstream()
scheduler.run_forever()
print("that's it")
def test_schedule_sensor(self):
"""In this version, we pass the sensor directly to the scheduler and use
a functional style to compose the filters"""
s = ValueListSensor(1, value_stream)
vo = ValidationSubscriber(expected_stream, self.test_schedule_sensor)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_sensor(s, 0.5, where(predicate), passthrough(vo),
output())
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_bokeh_output():
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
b = BokehStreamer([ SensorEvent(ts=0,val=10,sensor_id="temp" ) ], io_loop=loop)
p.subscribe(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_passthrough_as_a_method(self):
"""Verify that, when passthrough is used as a method, it can still take
thunks.
"""
scheduler = Scheduler(asyncio.get_event_loop())
luxpub = SensorPub(ValueListSensor('lux-2', lux_data))
vs1 = ValidationSubscriber([450, 600], self)
vs2 = ValidationSubscriber(lux_data, self)
luxpub.passthrough(compose(where(lambda evt: evt.val > 300),
vs1)).subscribe(vs2)
scheduler.schedule_periodic(luxpub, 0.5)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
def main_linear():
obs_stream = from_iterable(
iter([[[[1.0, 1.0], [2.0, 2.0]], [1.0, 2.0]],
[[[6.0, 6.0], [9.0, 9.0]], [6.0, 9.0]]]))
pred_stream = from_iterable(iter([[3.0, 3.0]]))
model = LinearRegressionModel()
obs_stream.subscribe(model, topic_mapping=('default', 'train'))
obs_stream.subscribe(print)
pred_stream.subscribe(model, topic_mapping=('default', 'observe'))
model.subscribe(print, topic_mapping=('predict', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(obs_stream, 1)
scheduler.schedule_periodic(pred_stream, 5)
scheduler.run_forever()
def test_rollover(self):
def generator():
for e in EVENTS:
yield e
sensor = IterableAsPublisher(generator(), name='sensor')
sensor.rolling_csv_writer('.', 'dining-room')
vs = SensorEventValidationSubscriber(EVENTS, self)
sensor.subscribe(vs)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
scheduler.run_forever()
for f in FILES:
self.assertTrue(os.path.exists(f), 'did not find file %s' % f)
print("found log file %s" % f)
def test_where(self):
s = make_test_publisher_from_vallist(1, value_stream)
w = s.where(predicate)
w.output()
vo = ValidationSubscriber(expected_stream, self)
w.subscribe(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(s, 0.5) # sample twice every second
s.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertTrue(vo.completed)
print("That's all folks")
def test_case(self):
"""Just run the reader in its own event loop. We stop everything after 4
events.
"""
s = Scheduler(asyncio.get_event_loop())
m = MQTTReader("localhost",
topics=[
('bogus/bogus', 0),
],
mock_class=MockMQTTClient)
m.output()
c = s.schedule_on_private_event_loop(m)
m.subscribe(StopLoopAfter(4, c))
m.print_downstream()
s.run_forever()
print("that's it")
def test_thunk_builder_handling(self):
"""We have logic where we can pass a thunk builder into a combinator and
it will do the right thing. Check that it works"""
scheduler = Scheduler(asyncio.get_event_loop())
lux = ValueListSensor('lux-2', lux_data)
vs = ValidationSubscriber(lux_data, self)
scheduler.schedule_sensor(
lux,
0.5,
passthrough(output()), # output() evaluates to a thunk
passthrough(output), # output is a thunk builder
passthrough(output(sys.stdout)), # output can take an arg
vs,
print_downstream=True)
scheduler.run_forever()
self.assertTrue(vs.completed)
def test_publish_and_subscribe(self):
sensor = ValueListSensor(1, sensor_values)
scheduler = Scheduler(asyncio.get_event_loop())
pg = PostgresWriter(scheduler, self.connect_string, self.mapping)
capture = CaptureSubscriber()
scheduler.schedule_sensor(sensor, 0.5,
parallel(pg, output, capture))
scheduler.run_forever()
print("finish writing to the database")
row_source = PostgresReader(self.connect_string, self.mapping)
row_source.output()
validate = SensorEventValidationSubscriber(capture.seq, self)
row_source.subscribe(validate)
scheduler.schedule_recurring(row_source)
scheduler.run_forever()
self.assertTrue(validate.completed)
print("finished reading rows")
def test_where(self):
"""In this version, we create a publisher and use method chaining to
compose the filters"""
s = ValueListSensor(1, value_stream)
p = SensorPub(s)
w = p.where(predicate)
w.output()
vo = ValidationSubscriber(expected_stream, self.test_where)
w.subscribe(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(p, 0.5) # sample twice every second
p.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_case(self):
sensor = make_test_publisher(1, stop_after_events=10)
split = SplitPublisher()
sensor.subscribe(split)
split.subscribe(lambda x: print("above:%s" % x),
topic_mapping=('above', 'default'))
split.subscribe(lambda x: print("below:%s" % x),
topic_mapping=('below', 'default'))
split.subscribe(lambda x: print("within:%s" % x),
topic_mapping=('within', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
print("that's all")
def test_supplying_event_on_timeout(self):
"""In this testcase, we drop every other event.
We set the timeout to a bit longer than the event interval of
one second. It then supplies the previous event. The resulting
output stream will show every other value repeated twice.
"""
sensor = make_test_publisher_from_vallist(1, sensor_values)
drop = DropPeriodic(sensor)
scheduler = Scheduler(asyncio.get_event_loop())
vo = ValidationSubscriber(expected_values, self)
drop.supply_event_when_timeout(EventWatcher(), scheduler,
1.1).output().subscribe(vo)
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
def main(argv=sys.argv[1:]):
if len(argv) != 2:
print("%s threshold interval" % sys.argv[0])
return 1
threshold = float(argv[0])
interval = float(argv[1])
print("%f seconds interval and an led threshold of %f lux" %
(interval, threshold))
(lux, led) = setup(threshold)
scheduler = Scheduler(asyncio.get_event_loop())
stop = scheduler.schedule_periodic_on_separate_thread(lux, interval)
print("starting run...")
try:
scheduler.run_forever()
except KeyboardInterrupt:
led.on_completed()
stop()
return 0
def test_multiple_timeouts(self):
"""In this testcase, we pass two events, drop two events, etc.
We set the timeout to a bit longer than the event interval. The last
good value is supplied when the timeout expires. Thus, we should see
two good events, two repeats of the first event, two good events, etc.
"""
sensor = make_test_publisher_from_vallist(1, sensor_values)
drop = DropPeriodic(sensor, N=2)
scheduler = Scheduler(asyncio.get_event_loop())
vo = ValidationSubscriber(expected_values_multiple_timeouts, self)
drop.supply_event_when_timeout(EventWatcher(), scheduler,
1.1).output().subscribe(vo)
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
def run_example():
sensor = SensorPub(DummyTempSensor('temp-1', input_sequence))
sensor.output() # let us see the raw values
dispatcher = sensor.transduce(RunningAvg(4))\
.passthrough(lambda evt:
print("Running avg temp: %s" %
round(evt.val, 2))) \
.dispatch([(lambda v: v[2]>=T_high, 't_high'),
(lambda v: v[2]<=T_low, 't_low')])
controller = Controller()
dispatcher.subscribe(controller, topic_mapping=('t_high', 't_high'))
dispatcher.subscribe(controller, topic_mapping=('t_low', 't_low'))
dispatcher.subscribe(controller, topic_mapping=('default', 'between'))
controller.subscribe(BypassValveActuator())
sensor.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 0.5)
scheduler.run_forever()
print("got to the end")
class TestCase(unittest.TestCase):
def setUp(self):
self.scheduler = Scheduler(asyncio.get_event_loop())
self.sensor = ValueListSensor(1, value_stream)
def test_sensor_event_sliding_window(self):
vs = ValidationSubscriber(mean_stream, self)
self.scheduler.schedule_sensor(self.sensor, 0.1,
transduce(SensorSlidingMean(4)),
parallel(vs, output()))
self.scheduler.run_forever()
self.assertTrue(vs.completed)
def test_periodic_median_transducer(self):
vs = ValidationSubscriber(periodic_median_stream, self)
self.scheduler.schedule_sensor(self.sensor, 0.1,
transduce(PeriodicMedianTransducer(3)),
parallel(vs, output()))
self.scheduler.run_forever()
self.assertTrue(vs.completed)
def test_complex_workflow(self):
THRESHOLD = 300
lux = ValueListSensor('lux-1', lux_data)
scheduler = Scheduler(asyncio.get_event_loop())
vs1 = ValidationSubscriber(lux_data, self)
vs2 = ValidationSubscriber(
[False, False, False, True, False, False, True],
self,
extract_value_fn=lambda v: v)
scheduler.schedule_sensor(
lux,
0.5,
passthrough(output()),
passthrough(vs1),
map(lambda event: event.val > THRESHOLD),
passthrough(lambda v: print('ON' if v else 'OFF')),
vs2,
print_downstream=True)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
print("That's all folks")
def test_bokeh_manager():
loop = asyncio.get_event_loop()
s1 = ValueListSensor(1, value_stream)
p1 = SensorPub(s1)
s2 = ValueListSensor(1, value_stream2)
p2 = SensorPub(s2)
bm = BokehPlotManager()
bplot1 = BokehPlot('Sensor1', y_axis_label='value')
bplot2 = BokehPlot('Sensor2', y_axis_label='value')
bm.register(bplot1)
bm.register(bplot2)
p1.map(lambda v: ('Sensor1', v) ).subscribe(bm)
p2.map(lambda v: ('Sensor2', v) ).subscribe(bm)
bm.start()
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p1, 1.0) # sample every second
scheduler.schedule_periodic(p2, 0.5) # sample twice every second
scheduler.run_forever()
# self.assertTrue(vo.completed,
# "Schedule exited before validation observer completed")
print("That's all folks")
def main_kalman():
dim_x = 2
dim_u = 1
dim_z = 1
initial_state_mean = np.array([[1.0], [0.0]])
initial_state_covariance = 1000 * np.eye(dim_x)
F = np.array([[1., 1.], [0., 1.]])
B = np.zeros((2, 1))
Q = Q_discrete_white_noise(dim=2, dt=0.1, var=0.13)
H = np.array([[1., 0.]])
R = 5 * np.eye(1)
model = KalmanFilterModel(dim_x, dim_u, dim_z, initial_state_mean,
initial_state_covariance, F, B, Q, H, R)
measurement_stream = from_iterable(iter([[1.0], [0.0]]))
# measurement_stream = from_iterable(iter([ np.array([ [1.0, 1.0] ]) ]))
measurement_stream.subscribe(model, topic_mapping=('default', 'observe'))
model.subscribe(print, topic_mapping=('predict', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(measurement_stream, 1)
scheduler.run_forever()
if self.events_since_last == self.period:
val = median(self.samples)
event = SensorEvent(sensor_id=v.sensor_id, ts=v.ts, val=val)
self.events_since_last = 0
return event
else:
self.last_event = v # save in case we complete before completing a period
return None
def complete(self):
if self.events_since_last > 0:
# if we have some partial state, we emit one final event that
# averages whatever we saw since the last emission.
return SensorEvent(sensor_id=self.last_event.sensor_id,
ts=self.last_event.ts,
val=median(
self.samples[0:self.events_since_last]))
SENSOR_ID = 'sensor-1'
scheduler = Scheduler(asyncio.get_event_loop())
sensor = SensorPub(
RandomSensor(SENSOR_ID, mean=10, stddev=5, stop_after_events=12))
sensor.csv_writer('raw_data.csv').subscribe(
lambda x: print("raw data: %s" % repr(x)))
sensor.transduce(PeriodicMedianTransducer()).mqtt_async_send(
URL, SENSOR_ID, scheduler).output()
scheduler.schedule_periodic(sensor, 0.5)
scheduler.run_forever()
print("that's all folks")
print("LED Completed")
def __str__(self):
return 'LED'
# instantiate an LED
led = LED()
# Now, build a pipeline to sample events returned from the sensor,
# convert to a boolean based on whether the value is greater than
# the mean, and output to the LED.
import antevents.linq.select
sensor.select(lambda evt: evt.val > MEAN).subscribe(led)
# If you want to see the raw value of each sensor, just add the output() element
import antevents.linq.output
sensor.output()
# Call a debug method on the base publisher class to see the element tree rooted
# at sensor.
sensor.print_downstream()
# Now, we need to schedule the sensor to be sampled
import asyncio
from antevents.base import Scheduler
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 1.0) # sample once a second
scheduler.run_forever() # run until all sensors complete
print("That's all folks!")