def test_batching(self):
"""We write out a set of event from two simulated sensors using an odd batch size (3).
We then read them back and verify that we got all the events.
"""
sensor1 = make_test_output_thing_from_vallist(TEST_SENSOR1, VALUE_STREAM)
sensor2 = make_test_output_thing_from_vallist(TEST_SENSOR2, VALUE_STREAM)
writer = PredixWriter(PREDIX_INGEST_URL, PREDIX_ZONE_ID, PREDIX_TOKEN,
extractor=EventExtractor(attributes={'test':True}),
batch_size=3)
sensor1.connect(writer)
sensor2.connect(writer)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor1, 0.5)
scheduler.schedule_periodic(sensor2, 0.5)
start_time = time.time()
scheduler.run_forever()
# Now we read the events back
reader1 = PredixReader(PREDIX_QUERY_URL, PREDIX_ZONE_ID, PREDIX_TOKEN, TEST_SENSOR1,
start_time=start_time,
one_shot=False)
reader2 = PredixReader(PREDIX_QUERY_URL, PREDIX_ZONE_ID, PREDIX_TOKEN, TEST_SENSOR2,
start_time=start_time,
one_shot=False)
ti1 = TestInput(reader1, 'sensor-1')
ti2 = TestInput(reader2, 'sensor-2')
scheduler.schedule_periodic(reader1, 2)
scheduler.schedule_periodic(reader2, 2)
scheduler.run_forever()
self.assertListEqual(VALUE_STREAM, ti1.values)
self.assertListEqual(VALUE_STREAM, ti2.values)
def run(args, token):
sensor1 = TestSensor.output_thing(TEST_SENSOR1, 5)
writer = PredixWriter(args.ingest_url,
args.predix_zone_id,
token,
extractor=EventExtractor(attributes={'test': True}),
batch_size=3)
sensor1.connect(writer)
sensor1.connect(print) # also print the event
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor1, 0.5)
start_time = time.time()
scheduler.run_forever()
print("Reading back events")
reader1 = PredixReader(args.query_url,
args.predix_zone_id,
token,
TEST_SENSOR1,
start_time=start_time,
one_shot=True)
reader1.connect(print)
scheduler.schedule_recurring(reader1)
scheduler.run_forever()
def test_mqtt(self):
loop = asyncio.get_event_loop()
s = Scheduler(loop)
sensor = make_test_output_thing_from_vallist(1, sensor_data)
mqtt_writer = MQTTWriter('localhost', topics=[
('bogus/bogus', 0),
])
sensor.to_json().connect(mqtt_writer)
s.schedule_periodic(sensor, 0.5)
mqtt_reader = MQTTReader("localhost", topics=[
('bogus/bogus', 0),
])
vs = ValidationInputThing(sensor_data, self)
mqtt_reader.take(5).select(mqtt_msg_to_unicode).from_json(constructor=SensorEvent) \
.output().connect(vs)
c = s.schedule_on_private_event_loop(mqtt_reader)
stop = StopLoopAfter(5, c)
mqtt_reader.connect(stop)
mqtt_reader.print_downstream()
sensor.print_downstream()
s.run_forever()
loop.stop()
self.assertTrue(vs.completed)
print("that's it")
def test_function_filter(self):
"""Verify the function filter class
"""
s = ValueListSensor(1, value_stream)
st = SensorAsOutputThing(s)
captured_list_ref = [[]]
got_completed_ref = [
False,
]
def on_next(self, x):
captured_list_ref[0].append(x.val)
self._dispatch_next(x)
def on_completed(self):
got_completed_ref[0] = True
self._dispatch_completed()
ff = FunctionFilter(st, on_next=on_next, on_completed=on_completed)
vo = ValidationInputThing(value_stream, self.test_function_filter)
ff.connect(vo)
st.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(st, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertEqual(value_stream, captured_list_ref[0])
self.assertTrue(got_completed_ref[0])
print("That's all folks")
def test_influx_output(self):
loop = asyncio.get_event_loop()
s = ValueListSensor(1, value_stream)
p = SensorAsOutputThing(s)
b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor',
fields=['val', 'ts'],
tags=['sensor_id']),
generate_timestamp=False,
username=INFLUXDB_USER,
password=INFLUXDB_PASSWORD,
database=INFLUXDB_DATABASE)
p.connect(b)
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
# Now play back
rs = self.c.query('SELECT * FROM Sensor;').get_points()
for d in rs:
print(d)
# Play back using an output thing
p = InfluxDBReader('SELECT * FROM Sensor;',
database=INFLUXDB_DATABASE,
username=INFLUXDB_USER,
password=INFLUXDB_PASSWORD)
p.connect(CallableAsInputThing(print))
scheduler = Scheduler(loop)
scheduler.schedule_periodic(p, 0.2) # sample five times every second
scheduler.run_forever()
print("That's all folks")
def test_periodic(self):
s = SensorAsOutputThing(RandomSensor(1))
PrintAndDeschedule(s)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(s, 0.25)
scheduler.run_forever()
print("Exited successfully")
def test_function_filter_error_handling(self):
"""Verify the error handling functionality of the function filter. We do
this by connecting two downstream paths to the sensor. The first includes
a function filter that throws an error when it encouters a sensor reading
of 120. This should disconnect th stream at this point. The second is
a normal validation input thing. It is connected directly to the sensor,
and thus should not see any errors.
"""
s = ValueListSensor(1, value_stream)
st = SensorAsOutputThing(s)
captured_list_ref = [[]]
got_completed_ref = [
False,
]
got_on_error_ref = [
False,
]
def on_next_throw_exc(self, x):
if x.val == 120:
raise Exception("expected exc")
else:
captured_list_ref[0].append(x.val)
self._dispatch_next(x)
def on_completed(self):
got_completed_ref[0] = True
self._dispatch_completed()
def on_error(self, e):
got_on_error_ref[0] = True
self._dispatch_error(e)
ff = FunctionFilter(st,
on_next=on_next_throw_exc,
on_completed=on_completed,
on_error=on_error)
ct = CaptureInputThing(expecting_error=True)
ff.map(lambda x: x.val).connect(ct)
vo = ValidationInputThing(value_stream,
self.test_function_filter_error_handling)
st.connect(vo)
st.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(st, 0.5) # sample twice every second
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertFalse(ct.completed,
"Capture thing should not have completed")
self.assertTrue(ct.errored, "Capture thing should have seen an error")
self.assertFalse(got_completed_ref[0])
self.assertTrue(got_on_error_ref[0])
self.assertEqual([20, 30, 100], ct.events,
"Capture thing event mismatch")
self.assertEqual([20, 30, 100], captured_list_ref[0],
"captured_list_ref mismatch")
print("That's all folks")
def test(self):
scheduler = Scheduler(asyncio.get_event_loop())
sensor = make_test_output_thing_from_vallist(1, values)
scheduler.schedule_periodic(sensor, 1)
blocking_subscriber = TestInputThing(scheduler, values, self)
sensor.connect(blocking_subscriber)
scheduler.run_forever()
self.assertTrue(blocking_subscriber.completed)
def test_gpio(self):
import thingflow.adapters.rpi.gpio
o = thingflow.adapters.rpi.gpio.GpioPinOut()
sensor_thing = SensorAsOutputThing(ValueListSensor("sensor-1", values))
sensor_thing.map(lambda evt: evt.val > 0).passthrough(
output()).connect(o)
s = Scheduler(asyncio.get_event_loop())
s.schedule_periodic(sensor_thing, 1.0)
s.run_forever()
def test_error_handling(self):
"""This is a non-fatal error, so we should just print the error and
exit cleanly without propagating the exception.
"""
s = IterableAsOutputThing(ErrorIterator(expected_stream))
s.output()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(s, 0.5) # sample twice every second
s.print_downstream()
scheduler.run_forever()
class TestCase(unittest.TestCase):
def setUp(self):
# Creating a new event loop each test case does not seem to work.
# I think it is due to hbmqtt not cleaning up some state in the asyncio
# layer.
#self.loop = asyncio.new_event_loop()
self.loop = asyncio.get_event_loop()
self.sched = Scheduler(self.loop)
def tearDown(self):
pass
#self.loop.stop()
#self.loop.close()
def test_client_only(self):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
self.sched.schedule_periodic(sensor, 0.5)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
print("test_client_only completed")
def send_and_recv_body(self, sleep_timeout):
SENSOR_ID = 'sensor-1'
TOPIC = get_topic_name(self)
sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES))
td = sensor.transduce(PeriodicMedianTransducer(period=3))
qw = QueueWriter(td, URL, TOPIC, self.sched)
qw.output()
qr = QueueReader(URL, TOPIC, self.sched, timeout=sleep_timeout)
self.sched.schedule_periodic(sensor, 0.5)
stop_qr = self.sched.schedule_on_main_event_loop(qr)
vs = ValidateAndStopInputThing(EXPECTED, self, stop_qr)
qr.select(msg_to_event).connect(vs)
self.sched.run_forever()
self.assertFalse(
qw.has_pending_requests(),
"QueueWriter has pending requests: %s" % qw.dump_state())
self.assertEqual(qr.state, QueueReader.FINAL_STATE)
self.assertEqual(vs.next_idx, len(EXPECTED))
print("send_and_recv_bod(%s) completed" % sleep_timeout)
def test_short_timeout(self):
self.send_and_recv_body(0.1)
def test_long_timeout(self):
self.send_and_recv_body(3.0)
def test_passthrough_as_a_method(self):
"""Verify that, when passthrough is used as a method, it can still take
thunks.
"""
scheduler = Scheduler(asyncio.get_event_loop())
luxpub = SensorAsOutputThing(ValueListSensor('lux-2', lux_data))
vs1 = ValidationInputThing([450, 600], self)
vs2 = ValidationInputThing(lux_data, self)
luxpub.passthrough(compose(where(lambda evt: evt.val > 300),
vs1)).connect(vs2)
scheduler.schedule_periodic(luxpub, 0.5)
scheduler.run_forever()
self.assertTrue(vs1.completed)
self.assertTrue(vs2.completed)
def test_where(self):
s = make_test_output_thing_from_vallist(1, value_stream)
w = s.where(predicate)
w.output()
vo = ValidationInputThing(expected_stream, self)
w.connect(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(s, 0.5) # sample twice every second
s.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
self.assertTrue(vo.completed)
print("That's all folks")
def main_linear():
obs_stream = from_iterable(
iter([[[[1.0, 1.0], [2.0, 2.0]], [1.0, 2.0]],
[[[6.0, 6.0], [9.0, 9.0]], [6.0, 9.0]]]))
pred_stream = from_iterable(iter([[3.0, 3.0]]))
model = SGDLinearRegressionModel()
obs_stream.connect(model, port_mapping=('default', 'train'))
obs_stream.connect(print)
pred_stream.connect(model, port_mapping=('default', 'observe'))
model.connect(print, port_mapping=('predict', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(obs_stream, 1)
scheduler.schedule_periodic(pred_stream, 5)
scheduler.run_forever()
def test_tsl2591(self):
import thingflow.sensors.rpi.lux_sensor
sensor = SensorAsOutputThing(
thingflow.sensors.rpi.lux_sensor.LuxSensor())
s = Scheduler(asyncio.get_event_loop())
stop = s.schedule_periodic(sensor, 1.0)
StopAfterN(sensor, stop, N=4).output()
s.run_forever()
def test_where(self):
"""In this version, we create a publisher and use method chaining to
compose the filters"""
s = ValueListSensor(1, value_stream)
p = SensorAsOutputThing(s)
w = p.where(predicate)
w.output()
vo = ValidationInputThing(expected_stream, self.test_where)
w.connect(vo)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(p, 0.5) # sample twice every second
p.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
print("That's all folks")
def test_case(self):
sensor = make_test_output_thing(1, stop_after_events=10)
split = SplitOutputThing()
sensor.connect(split)
split.connect(lambda x: print("above:%s" % x),
port_mapping=('above', 'default'))
split.connect(lambda x: print("below:%s" % x),
port_mapping=('below', 'default'))
split.connect(lambda x: print("within:%s" % x),
port_mapping=('within', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
print("that's all")
def test_multiple_timeouts(self):
"""In this testcase, we pass two events, drop two events, etc.
We set the timeout to a bit longer than the event interval. The last
good value is supplied when the timeout expires. Thus, we should see
two good events, two repeats of the first event, two good events, etc.
"""
sensor = make_test_output_thing_from_vallist(1, sensor_values)
drop = DropPeriodic(sensor, N=2)
scheduler = Scheduler(asyncio.get_event_loop())
vo = ValidationInputThing(expected_values_multiple_timeouts, self)
drop.supply_event_when_timeout(EventWatcher(), scheduler,
1.1).output().connect(vo)
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
def test_supplying_event_on_timeout(self):
"""In this testcase, we drop every other event.
We set the timeout to a bit longer than the event interval of
one second. It then supplies the previous event. The resulting
output stream will show every other value repeated twice.
"""
sensor = make_test_output_thing_from_vallist(1, sensor_values)
drop = DropPeriodic(sensor)
scheduler = Scheduler(asyncio.get_event_loop())
vo = ValidationInputThing(expected_values, self)
drop.supply_event_when_timeout(EventWatcher(), scheduler,
1.1).output().connect(vo)
scheduler.schedule_periodic(sensor, 1)
sensor.print_downstream()
scheduler.run_forever()
self.assertTrue(
vo.completed,
"Schedule exited before validation observer completed")
def run_example():
sensor = SensorAsOutputThing(DummyTempSensor('temp-1', input_sequence))
sensor.output() # let us see the raw values
dispatcher = sensor.transduce(RunningAvg(4))\
.passthrough(lambda evt:
print("Running avg temp: %s" %
round(evt.val, 2))) \
.dispatch([(lambda v: v[2]>=T_high, 't_high'),
(lambda v: v[2]<=T_low, 't_low')])
controller = Controller()
dispatcher.connect(controller, port_mapping=('t_high', 't_high'))
dispatcher.connect(controller, port_mapping=('t_low', 't_low'))
dispatcher.connect(controller, port_mapping=('default', 'between'))
controller.connect(BypassValveActuator())
sensor.print_downstream()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 0.5)
scheduler.run_forever()
print("got to the end")
def test_daniels_bug(self):
"""Test bug reported by Daniel (issue #1). If you call the mqtt writer without
serializing the message, you should get a fatal error.
"""
import time
import asyncio
import thingflow.filters.output # This has output side-effect
from thingflow.base import Scheduler, from_list
from thingflow.adapters.mqtt import MQTTReader, MQTTWriter
from collections import namedtuple
StripEvent = namedtuple('StripEvent', ['strip_id', 'ts', 'val'])
strip_events = (StripEvent('strip-1', 1500000000,
50), StripEvent('strip-1', 1500000000, 5),
StripEvent('strip-1', 1500000000, 50))
mqtt = MQTTWriter('localhost', topics=[
('strip-data', 0),
])
strip = from_list(strip_events)
strip.connect(mqtt)
strip.output()
sched = Scheduler(asyncio.get_event_loop())
sched.schedule_periodic(strip, 1.0)
try:
sched.run_forever()
except ScheduleError as e:
# verify the cause of the error
dispatch_error = e.__cause__
self.assertTrue(
isinstance(dispatch_error, ExcInDispatch),
"expecting cause to be a dispatch error, instead got %s" %
repr(dispatch_error))
orig_error = dispatch_error.__cause__
self.assertTrue(
isinstance(orig_error, TypeError),
"expecting original exception to be a TypeError, intead got %s"
% repr(orig_error))
print("Got expected exception: '%s'" % e)
def main_kalman():
dim_x = 2
dim_u = 1
dim_z = 1
initial_state_mean = np.array([[1.0], [0.0]])
initial_state_covariance = 1000 * np.eye(dim_x)
F = np.array([[1., 1.], [0., 1.]])
B = np.zeros((2, 1))
Q = Q_discrete_white_noise(dim=2, dt=0.1, var=0.13)
H = np.array([[1., 0.]])
R = 5 * np.eye(1)
model = KalmanFilterModel(dim_x, dim_u, dim_z, initial_state_mean,
initial_state_covariance, F, B, Q, H, R)
measurement_stream = from_iterable(iter([[1.0], [0.0]]))
# measurement_stream = from_iterable(iter([ np.array([ [1.0, 1.0] ]) ]))
measurement_stream.connect(model, port_mapping=('default', 'observe'))
model.connect(print, port_mapping=('predict', 'default'))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(measurement_stream, 1)
scheduler.run_forever()
def main(argv=sys.argv[1:]):
if len(argv) != 1:
print("%s interval" % sys.argv[0])
return 1
interval = float(argv[0])
print("%f seconds interval" % interval)
lux1, lux2 = setup()
scheduler = Scheduler(asyncio.get_event_loop())
stop1 = scheduler.schedule_periodic(lux1, interval)
stop2 = scheduler.schedule_periodic(lux2, interval)
print("starting run...")
try:
scheduler.run_forever()
except KeyboardInterrupt:
stop1()
stop2()
return 0
print("LED Completed")
def __str__(self):
return 'LED'
# instantiate an LED
led = LED()
# Now, build a pipeline to sample events returned from the sensor,
# convert to a boolean based on whether the value is greater than
# the mean, and output to the LED.
import thingflow.filters.map
sensor.map(lambda evt: evt.val > MEAN).connect(led)
# If you want to see the raw value of each sensor, just add the output() element
import thingflow.filters.output
sensor.output()
# Call a debug method on the base output_thing class to see the element tree rooted
# at sensor.
sensor.print_downstream()
# Now, we need to schedule the sensor to be sampled
import asyncio
from thingflow.base import Scheduler
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor, 1.0) # sample once a second
scheduler.run_forever() # run until all sensors complete
print("That's all folks!")
class Agent:
def __init__(self):
self.logger = Logger()
self.config = Config()
self.id = self.config.get('id')
self.type = self.config.get('type')
self.sensors = self.config.get('sensors')
self.data = dict()
self.data['id'] = self.id
self.data['type'] = self.type
self.data['model'] = self.config.get('model')
self.data['implementation'] = dict()
self.data['implementation']['name'] = sys.implementation.name
self.data['implementation']['version'] = '.'.join(
str(x) for x in sys.implementation.version)
self.data['platform'] = sys.platform
self.data['sensors'] = dict()
self.data['os'] = self.getOsData()
self.logger.notice("MicroNet Agent started on %s (%s)" %
(self.id, self.type))
self.conn = MQTTConnector(self.id)
self.conn.set_last_will('micronet/devices/' + self.id + '/online',
'false')
self.scheduler = Scheduler(asyncio.get_event_loop())
def getOsData(self):
u = uname()
res = dict()
res['architecture'] = u.machine
res['kernel'] = u.sysname
res['version'] = u.release
return res
def start(self):
self.conn.connect()
self.conn.publish('micronet/devices/' + self.id + '/online',
'true',
retain=True,
qos=0)
self.conn.publish('micronet/devices/' + self.id + '/info',
json.dumps(self.data),
retain=True,
qos=0)
self.scheduler.run_forever()
def schedule(self):
self.logger.info("Scheduling sensors...")
informer = Informer(data=self.data, id=self.id, conn=self.conn)
informer_out = SensorAsOutputThing(informer)
informer_out.connect(StdoutConnector(informer.info))
informer_out.connect(
self.conn.writer('informer',
informer.info,
topic='micronet/devices/' + self.id + '/online'))
self.scheduler.schedule_periodic(informer_out, 5)
for k, v in self.sensors.items():
id = k.split(':')[0]
unit = id
if 'unit' in v:
unit = v['unit']
try:
module = __import__("sensor_" + unit)
SensorClass = getattr(module, to_pascal_case(id + "_sensor"))
v['id'] = k
sensor = SensorClass(**v)
sensor_output = SensorAsOutputThing(sensor)
self.data['sensors'][k] = sensor.info
sensor_output.connect(StdoutConnector(sensor.info))
sensor_output.connect(self.conn.writer(k, sensor.info))
self.scheduler.schedule_periodic(sensor_output, v['freq'])
self.logger.info("Sensor '{0}' sampling every {1}s".format(
k, v['freq']))
except Exception as e:
self.logger.warning(e)
if self.events_since_last == self.period:
val = median(self.samples)
event = SensorEvent(sensor_id=v.sensor_id, ts=v.ts, val=val)
self.events_since_last = 0
return event
else:
self.last_event = v # save in case we complete before completing a period
return None
def complete(self):
if self.events_since_last > 0:
# if we have some partial state, we emit one final event that
# averages whatever we saw since the last emission.
return SensorEvent(sensor_id=self.last_event.sensor_id,
ts=self.last_event.ts,
val=median(
self.samples[0:self.events_since_last]))
SENSOR_ID = 'sensor-1'
scheduler = Scheduler(asyncio.get_event_loop())
sensor = SensorAsOutputThing(
RandomSensor(SENSOR_ID, mean=10, stddev=5, stop_after_events=12))
sensor.csv_writer('raw_data.csv').connect(
lambda x: print("raw data: %s" % repr(x)))
sensor.transduce(PeriodicMedianTransducer()).mqtt_async_send(
URL, SENSOR_ID, scheduler).output()
scheduler.schedule_periodic(sensor, 0.5)
scheduler.run_forever()
print("that's all folks")