def test_file_write_read(self):
tf = NamedTemporaryFile(mode='w', delete=False)
tf.close()
try:
sensor = make_test_output_thing(1, stop_after_events=NUM_EVENTS)
capture = CaptureInputThing()
sensor.connect(capture)
sensor.csv_writer(tf.name)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
print("Writing sensor events to temp file")
scheduler.run_forever()
self.assertTrue(capture.completed,
"CaptureInputThing did not complete")
self.assertEqual(
len(capture.events), NUM_EVENTS,
"number of events captured did not match generated events")
reader = CsvReader(tf.name)
vs = SensorEventValidationInputThing(capture.events, self)
reader.connect(vs)
scheduler.schedule_recurring(reader)
print("reading sensor events back from temp file")
scheduler.run_forever()
self.assertTrue(vs.completed,
"ValidationInputThing did not complete")
finally:
os.remove(tf.name)
def test_recurring(self):
s = SensorAsOutputThing(RandomSensor(1))
PrintAndDeschedule(s)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(s)
scheduler.run_forever()
print("Exited successfully")
def test_dispatch(self):
"""Test a scenario where we dispatch to one of several writers
depending on the sensor id.
"""
def generator():
for e in EVENTS2:
yield e
sensor = IterableAsOutputThing(generator(), name='sensor')
dispatcher = sensor.dispatch(dispatch_rules)
for s in sensor_ids:
dispatcher.rolling_csv_writer('.', s, sub_port=s)
dispatcher.connect(
lambda x: self.assertTrue(False, "bad dispatch of %s" % x))
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
scheduler.run_forever()
for f in FILES2:
self.assertTrue(os.path.exists(f), 'did not find file %s' % f)
cnt = 0
with open(f, 'r') as fobj:
for line in fobj:
cnt += 1
self.assertEqual(2, cnt, "File %s did not have 2 lines" % f)
print("found log file %s" % f)
def run(args, token):
sensor1 = TestSensor.output_thing(TEST_SENSOR1, 5)
writer = PredixWriter(args.ingest_url,
args.predix_zone_id,
token,
extractor=EventExtractor(attributes={'test': True}),
batch_size=3)
sensor1.connect(writer)
sensor1.connect(print) # also print the event
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_periodic(sensor1, 0.5)
start_time = time.time()
scheduler.run_forever()
print("Reading back events")
reader1 = PredixReader(args.query_url,
args.predix_zone_id,
token,
TEST_SENSOR1,
start_time=start_time,
one_shot=True)
reader1.connect(print)
scheduler.schedule_recurring(reader1)
scheduler.run_forever()
def test_rollover(self):
def generator():
for e in EVENTS:
yield e
sensor = IterableAsOutputThing(generator(), name='sensor')
sensor.rolling_csv_writer('.', 'dining-room')
vs = SensorEventValidationInputThing(EVENTS, self)
sensor.connect(vs)
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(sensor)
scheduler.run_forever()
for f in FILES:
self.assertTrue(os.path.exists(f), 'did not find file %s' % f)
print("found log file %s" % f)
def test_publish_and_subscribe(self):
sensor = ValueListSensor(1, sensor_values)
scheduler = Scheduler(asyncio.get_event_loop())
pg = PostgresWriter(scheduler, self.connect_string, self.mapping)
capture = CaptureInputThing()
scheduler.schedule_sensor(sensor, 0.5,
parallel(pg, output, capture))
scheduler.run_forever()
print("finish writing to the database")
row_source = PostgresReader(self.connect_string, self.mapping)
row_source.output()
validate = SensorEventValidationInputThing(capture.seq, self)
row_source.connect(validate)
scheduler.schedule_recurring(row_source)
scheduler.run_forever()
self.assertTrue(validate.completed)
print("finished reading rows")
def test_pandas(self):
s = ValueListSensor(1, value_stream)
p = SensorAsOutputThing(s)
import thingflow.adapters.pandas
import numpy
w = thingflow.adapters.pandas.PandasSeriesWriter()
p.connect(w)
sch = Scheduler(asyncio.get_event_loop())
sch.schedule_recurring(p)
sch.run_forever()
self.assertTrue(w.result is not None, "Result of pandas never set")
# now we verify each element
for (i, v) in enumerate(value_stream):
pv = w.result[i]
self.assertTrue(
isinstance(pv, numpy.int64),
"Expecting pandas value '%s' to be numpy.int64, but instead was %s"
% (pv, repr(type(pv))))
self.assertEqual(
v, pv, "Pandas value '%s' not equal to original value '%s'" %
(repr(pv), repr(v)))
print("Validate pandas array")
val=float(row[2]))
self._dispatch_next(event)
except StopIteration:
self.file.close()
self._dispatch_completed()
except FatalError:
self._close()
raise
except Exception as e:
self.file.close()
self._dispatch_error(e)
# If we are running this as a script, read events from the specified
# file and print them via output().
if __name__ == '__main__':
# check command line arguments
if len(sys.argv) != 2:
# did not provide filename or provided too many arguments
sys.stderr.write("%s FILENAME\n" % sys.argv[0])
if len(sys.argv) == 1:
sys.stderr.write(" FILENAME is a required parameter\n")
sys.exit(1)
reader = SimpleCsvReader(sys.argv[1])
reader.output()
scheduler = Scheduler(asyncio.get_event_loop())
scheduler.schedule_recurring(reader)
scheduler.run_forever()
sys.exit(0)
while True:
yield random.gauss(mean, stddev)
self.generator = generator()
def sample(self):
return self.generator.__next__()
def __repr__(self):
if self.stop_after_events is None:
return 'RandomSensor(%s, mean=%s, stddev=%s)' % \
(self.sensor_id, self.mean, self.stddev)
else:
return 'RandomSensor(%s, mean=%s, stddev=%s, stop_after_events=%s)' % \
(self.sensor_id, self.mean, self.stddev, self.stop_after_events)
scheduler = Scheduler(asyncio.get_event_loop())
sensor = SensorAsOutputThing(
RandomSensor(1, mean=10, stddev=5, stop_after_events=10))
mtthing = MultiPortOutputThing(sensor)
mtthing.connect(lambda v: print("even: %s" % v),
port_mapping=('divisible_by_two', 'default'))
mtthing.connect(lambda v: print("divisible by three: %s" % v),
port_mapping=('divisible_by_three', 'default'))
mtthing.connect(lambda v: print("not divisible: %s" % v),
port_mapping=('other', 'default'))
mtthing.print_downstream()
scheduler.schedule_recurring(sensor)
scheduler.run_forever()