def test_counter_walker():
"""Make sure counter walkers work correctly."""
model = DeviceModel()
log = SensorLog(model=model)
walk = log.create_walker(DataStreamSelector.FromString('counter 1'))
stream = DataStream.FromString('counter 1')
reading = IOTileReading(0, 1, 1)
assert walk.count() == 0
log.push(stream, IOTileReading(0, 1, 1))
assert walk.count() == 1
assert walk.peek().value == 1
log.push(stream, IOTileReading(0, 1, 3))
assert walk.count() == 2
assert walk.peek().value == 3
val = walk.pop()
assert walk.count() == 1
assert val.value == 3
val = walk.pop()
assert walk.count() == 0
assert val.value == 3
with pytest.raises(StreamEmptyError):
walk.pop()
def __init__(self, address, args, device=None):
name = args.get('name', 'refcn1')
self._device = device
self._logger = logging.getLogger(__name__)
self._post_config_subsystems = []
model = DeviceModel()
EmulatedTile.__init__(self, address, name, device)
# Initialize all of the controller subsystems
ClockManagerMixin.__init__(
self, has_rtc=False
) #FIXME: Load the controller model info to get whether it has an RTC
ConfigDatabaseMixin.__init__(
self, 4096,
4096) #FIXME: Load the controller model info to get its memory map
TileManagerMixin.__init__(self)
RemoteBridgeMixin.__init__(self)
RawSensorLogMixin.__init__(self, model)
StreamingSubsystemMixin.__init__(self, basic=True)
SensorGraphMixin.__init__(self,
self.sensor_log,
self.stream_manager,
model=model)
# Establish required post-init linkages between subsystems
self.clock_manager.graph_input = self.sensor_graph.process_input
self.sensor_graph.get_timestamp = self.clock_manager.get_time
self.stream_manager.get_timestamp = self.clock_manager.get_time
self.stream_manager.get_uptime = lambda: self.clock_manager.get_time(
False)
self.app_info = (0, "0.0")
self.os_info = (0, "0.0")
def test_seek_walker():
"""Make sure we can seek a walker can count with an offset."""
model = DeviceModel()
log = SensorLog(model=model)
stream = DataStream.FromString('buffered 1')
reading = IOTileReading(stream.encode(), 0, 1)
log.push(stream, reading)
log.push(stream, reading)
log.push(DataStream.FromString('buffered 2'), reading)
log.push(stream, reading)
walk = log.create_walker(DataStreamSelector.FromString('buffered 1'),
skip_all=False)
assert walk.offset == 0
assert walk.count() == 3
walk.seek(1)
assert walk.offset == 1
assert walk.count() == 2
# Make sure we can seek to a position corresponding to another stream
walk.seek(2)
assert walk.offset == 2
assert walk.count() == 1
# Make sure we can find a reading by reading ID
walk = log.create_walker(DataStreamSelector.FromString('output 1'),
skip_all=False)
output1 = DataStream.FromString('output 1')
output2 = DataStream.FromString('output 2')
log.push(output1, IOTileReading(0, 0, 1, reading_id=1))
log.push(output1, IOTileReading(0, 0, 1, reading_id=2))
log.push(output2, IOTileReading(0, 0, 1, reading_id=3))
log.push(output1, IOTileReading(0, 0, 1, reading_id=4))
exact = walk.seek(2, target='id')
assert exact is True
assert walk.count() == 2
assert walk.offset == 1
exact = walk.seek(3, target='id')
assert exact is False
assert walk.count() == 1
assert walk.offset == 2
# Verify exceptions thrown by seek()
with pytest.raises(UnresolvedIdentifierError):
walk.seek(5, target='id')
with pytest.raises(UnresolvedIdentifierError):
walk.seek(5, target=u'id')
with pytest.raises(UnresolvedIdentifierError):
walk.seek(5, target='offset')
with pytest.raises(ArgumentError):
walk.seek(2, target="unsupported")
def test_storage_walker():
"""Make sure the storage walker works."""
model = DeviceModel()
log = SensorLog(model=model)
walk = log.create_walker(DataStreamSelector.FromString('buffered 1'))
stream = DataStream.FromString('buffered 1')
storage_size = model.get('max_storage_buffer')
erase_size = model.get('buffer_erase_size')
for i in range(0, storage_size):
reading = IOTileReading(0, stream.encode(), i)
log.push(stream, reading)
assert walk.count() == (i + 1)
# Make sure the overwrite happens correctly
old_count = walk.count()
reading = IOTileReading(0, stream.encode(), storage_size)
log.push(stream, reading)
assert walk.count() == (old_count - erase_size + 1)
def __init__(self, engine=None, model=None):
self._monitors = {}
self._last_values = {}
self._virtual_walkers = []
self._queue_walkers = []
if model is None:
model = DeviceModel()
if engine is None:
engine = InMemoryStorageEngine(model=model)
self._engine = engine
self._model = model
def test_basic_parsing():
"""Make sure we can parse a basic node description."""
node, inputs, processing = parse_node_descriptor(
'(input 1 always && input 2 when count >= 1) => buffered node 1 using copy_all_a',
DeviceModel())
assert processing == u'copy_all_a'
assert str(node.stream) == u'buffered 1'
assert str(inputs[0][0]) == u'input 1'
assert inputs[0][1] is None
assert str(inputs[1][0]) == u'input 2'
assert inputs[1][1] is not None
def test_walker_at_beginning():
"""Make sure we can start a walker at the beginning of a stream."""
model = DeviceModel()
log = SensorLog(model=model)
stream = DataStream.FromString('buffered 1')
reading = IOTileReading(stream.encode(), 0, 1)
log.push(stream, reading)
log.push(stream, reading)
log.push(DataStream.FromString('buffered 2'), reading)
walk = log.create_walker(DataStreamSelector.FromString('buffered 1'),
skip_all=False)
assert walk.offset == 0
assert walk.count() == 2
def test_storage_streaming_walkers():
"""Make sure the storage and streaming walkers work simultaneously."""
model = DeviceModel()
log = SensorLog(model=model)
storage_walk = log.create_walker(DataStreamSelector.FromString('buffered 1'))
output_walk = log.create_walker(DataStreamSelector.FromString('output 1'))
storage1 = DataStream.FromString('buffered 1')
storage2 = DataStream.FromString('buffered 2')
output1 = DataStream.FromString('output 1')
output2 = DataStream.FromString('output 2')
assert storage_walk.offset == 0
assert output_walk.offset == 0
for i in range(0, 5000):
reading = IOTileReading(0, 0, i)
log.push(storage1, reading)
log.push(storage2, reading)
log.push(output1, reading)
log.push(output2, reading)
assert storage_walk.count() == (i + 1)
assert output_walk.count() == (i + 1)
assert storage_walk.offset == 0
assert output_walk.offset == 0
for i in range(0, 5000):
reading1 = storage_walk.pop()
reading2 = output_walk.pop()
assert reading1.value == i
assert reading1.stream == storage1.encode()
assert reading2.value == i
assert reading2.stream == output1.encode()
assert storage_walk.offset == (2 * i) + 1
assert output_walk.offset == (2* i) + 1
log.clear()
assert storage_walk.count() == 0
assert storage_walk.offset == 0
assert output_walk.count() == 0
assert output_walk.offset == 0
def __init__(self, engine=None, model=None, id_assigner=None):
self._monitors = {}
self._last_values = {}
self._virtual_walkers = []
self._queue_walkers = []
if model is None:
model = DeviceModel()
if engine is None:
engine = InMemoryStorageEngine(model=model)
self._engine = engine
self._model = model
self._rollover_streaming = True
self._rollover_storage = True
self.id_assigner = id_assigner
def test_fill_stop():
"""Make sure we can configure SensorLog into fill-stop mode."""
model = DeviceModel()
log = SensorLog(model=model)
storage = DataStream.FromString('buffered 1')
output = DataStream.FromString('output 1')
reading = IOTileReading(0, 0, 1)
# Neither fill-stop
for _i in range(0, 50000):
log.push(storage, reading)
for _i in range(0, 50000):
log.push(output, reading)
log.clear()
log.set_rollover('storage', False)
with pytest.raises(StorageFullError):
for _i in range(0, 50000):
log.push(storage, reading)
for _i in range(0, 50000):
log.push(output, reading)
assert log.count() == (16128, 48720)
log.clear()
log.set_rollover('streaming', False)
with pytest.raises(StorageFullError):
for _i in range(0, 50000):
log.push(storage, reading)
with pytest.raises(StorageFullError):
for _i in range(0, 50000):
log.push(output, reading)
assert log.count() == (16128, 48896)
def test_string_generation():
"""Make sure we can print nodes."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node(
'(input 1 always && input 2 when count >= 1) => buffered node 1 using copy_all_a'
)
assert str(
sg.nodes[-1]
) == u'(input 1 always && input 2 when count >= 1) => buffered 1 using copy_all_a'
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node(
'(input 1 when value < 0x10) => buffered node 1 using copy_all_a')
assert str(sg.nodes[-1]
) == u'(input 1 when value < 16) => buffered 1 using copy_all_a'
def test_constant_walker():
"""Make sure constant walkers can be read any number of times."""
model = DeviceModel()
log = SensorLog(model=model)
walk = log.create_walker(DataStreamSelector.FromString('constant 1'))
stream = DataStream.FromString('constant 1')
reading = IOTileReading(0, 1, 1)
log.push(stream, reading)
assert walk.count() == 0xFFFFFFFF
log.push(stream, reading)
assert walk.count() == 0xFFFFFFFF
val = walk.pop()
assert walk.count() == 0xFFFFFFFF
val = walk.pop()
assert walk.count() == 0xFFFFFFFF
def test_unbuffered_walker():
"""Make sure unbuffered walkers hold only 1 reading."""
model = DeviceModel()
log = SensorLog(model=model)
walk = log.create_walker(DataStreamSelector.FromString('unbuffered 1'))
stream = DataStream.FromString('unbuffered 1')
assert walk.count() == 0
log.push(stream, IOTileReading(0, 1, 1))
assert walk.count() == 1
assert walk.peek().value == 1
log.push(stream, IOTileReading(0, 1, 3))
assert walk.count() == 1
assert walk.peek().value == 3
val = walk.pop()
assert walk.count() == 0
assert val.value == 3
with pytest.raises(StreamEmptyError):
walk.pop()
def load_sgf(self, sgf_data):
"""Load, persist a sensor_graph file.
The data passed in `sgf_data` can either be a path or the already
loaded sgf lines as a string. It is determined to be sgf lines if
there is a '\n' character in the data, otherwise it is interpreted as
a path.
Note that this scenario just loads the sensor_graph directly into the
persisted sensor_graph inside the device. You will still need to
reset the device for the sensor_graph to enabled and run.
Args:
sgf_data (str): Either the path to an sgf file or its contents
as a string.
"""
if '\n' not in sgf_data:
with open(sgf_data, "r") as infile:
sgf_data = infile.read()
model = DeviceModel()
parser = SensorGraphFileParser()
parser.parse_file(data=sgf_data)
parser.compile(model)
opt = SensorGraphOptimizer()
opt.optimize(parser.sensor_graph, model=model)
sensor_graph = parser.sensor_graph
self._logger.info(
"Loading sensor_graph with %d nodes, %d streamers and %d configs",
len(sensor_graph.nodes), len(sensor_graph.streamers),
len(sensor_graph.config_database))
# Directly load the sensor_graph into our persisted storage
self.sensor_graph.persisted_nodes = sensor_graph.dump_nodes()
self.sensor_graph.persisted_streamers = sensor_graph.dump_streamers()
self.sensor_graph.persisted_constants = []
for stream, value in sorted(sensor_graph.constant_database.items(),
key=lambda x: x[0].encode()):
reading = IOTileReading(stream.encode(), 0, value)
self.sensor_graph.persisted_constants.append((stream, reading))
self.sensor_graph.persisted_exists = True
# Clear all config variables and load in those from this sgf file
self.config_database.clear()
for slot in sorted(sensor_graph.config_database,
key=lambda x: x.encode()):
for conf_var, (conf_type, conf_val) in sorted(
sensor_graph.config_database[slot].items()):
self.config_database.add_direct(slot, conf_var, conf_type,
conf_val)
# If we have an app tag and version set program them in
app_tag = sensor_graph.metadata_database.get('app_tag')
app_version = sensor_graph.metadata_database.get('app_version')
if app_tag is not None:
if app_version is None:
app_version = "0.0"
self.app_info = (app_tag, app_version)
def test_default_values():
"""Make sure we can get properties with default values."""
model = DeviceModel()
assert model.get('max_nodes') == 32
assert model.get(u'max_nodes') == 32
model.set('max_nodes', 16)
assert model.get('max_nodes') == 16
assert model.get(u'max_nodes') == 16
model.set(u'max_nodes', 17)
assert model.get('max_nodes') == 17
assert model.get(u'max_nodes') == 17
with pytest.raises(ArgumentError):
model.get('unknown_parameter')
with pytest.raises(ArgumentError):
model.set('unknown_parameter', 15)
def test_dump_restore():
"""Make sure we can properly dump and restore a SensorLog."""
model = DeviceModel()
log = SensorLog(model=model)
storage = DataStream.FromString('buffered 1')
output = DataStream.FromString('output 1')
reading = IOTileReading(0, 0, 1)
for _i in range(0, 25):
log.push(storage, reading)
for _i in range(0, 20):
log.push(output, reading)
out1 = log.create_walker(DataStreamSelector.FromString('output 1'),
skip_all=False)
store1 = log.create_walker(DataStreamSelector.FromString('buffered 1'),
skip_all=False)
count1 = log.create_walker(DataStreamSelector.FromString('counter 1'))
const1 = log.create_walker(DataStreamSelector.FromString('constant 1'))
unbuf1 = log.create_walker(DataStreamSelector.FromString('unbuffered 1'))
log.push(DataStream.FromString('counter 1'), reading)
log.push(DataStream.FromString('counter 1'), reading)
log.push(DataStream.FromString('constant 1'), reading)
log.push(DataStream.FromString('unbuffered 1'), reading)
state = log.dump()
log.clear()
log.destroy_all_walkers()
out1 = log.create_walker(DataStreamSelector.FromString('output 1'),
skip_all=False)
store1 = log.create_walker(DataStreamSelector.FromString('buffered 1'),
skip_all=False)
count1 = log.create_walker(DataStreamSelector.FromString('counter 1'))
const1 = log.create_walker(DataStreamSelector.FromString('constant 1'))
unbuf1 = log.create_walker(DataStreamSelector.FromString('unbuffered 1'))
log.restore(state)
assert store1.count() == 25
assert out1.count() == 20
assert count1.count() == 2
assert const1.count() == 0xFFFFFFFF
assert unbuf1.count() == 1
# Test permissive and non-permissive restores
_unbuf2 = log.create_walker(DataStreamSelector.FromString('unbuffered 2'))
with pytest.raises(ArgumentError):
log.restore(state)
log.clear()
log.restore(state, permissive=True)
assert store1.count() == 25
assert out1.count() == 20
assert count1.count() == 2
assert const1.count() == 0xFFFFFFFF
assert unbuf1.count() == 1
# Test restoring a stream walker
log.clear()
log.destroy_all_walkers()
walk = log.create_walker(DataStreamSelector.FromString(str(storage)))
for _i in range(0, 25):
log.push(storage, reading)
assert walk.count() == 25
dump = walk.dump()
log.destroy_all_walkers()
walk2 = log.restore_walker(dump)
assert walk2.count() == 25
def test_storage_scan():
"""Make sure scan_storage works."""
model = DeviceModel()
engine = InMemoryStorageEngine(model)
log = SensorLog(engine, model=model)
storage1 = DataStream.FromString('buffered 1')
output1 = DataStream.FromString('output 1')
for i in range(0, 10):
reading = IOTileReading(0, 0, i)
log.push(storage1, reading)
for i in range(7, 14):
reading = IOTileReading(0, 0, i)
log.push(output1, reading)
shared = [None, 0]
def _max_counter(i, reading):
if reading.value > shared[1]:
shared[0] = i
shared[1] = reading.value
# Make sure scanning storage works
shared = [None, 0]
count = engine.scan_storage('storage', _max_counter)
assert count == 10
assert shared == [9, 9]
shared = [None, 0]
count = engine.scan_storage('storage', _max_counter, start=5, stop=9)
assert count == 5
assert shared == [9, 9]
shared = [None, 0]
count = engine.scan_storage('storage', _max_counter, start=5, stop=8)
assert count == 4
assert shared == [8, 8]
# Make sure scanning steaming works
shared = [None, 0]
count = engine.scan_storage('streaming', _max_counter)
assert count == 7
assert shared == [6, 13]
shared = [None, 0]
count = engine.scan_storage('streaming', _max_counter, start=2, stop=6)
assert count == 5
assert shared == [6, 13]
shared = [None, 0]
count = engine.scan_storage('streaming', _max_counter, start=2, stop=5)
assert count == 4
assert shared == [5, 12]
# Make sure errors are thrown correctly
with pytest.raises(ArgumentError):
engine.scan_storage('other_name', _max_counter)
with pytest.raises(ArgumentError):
engine.scan_storage('streaming', _max_counter, stop=7)