def __init__(self, emulator, model):
super(SensorLogSubsystem, self).__init__(emulator)
self.engine = InMemoryStorageEngine(model=model)
self.storage = SensorLog(self.engine, model=model, id_assigner=lambda x, y: self.allocate_id())
self.dump_walker = None
self.next_id = 1
self._logger = logging.getLogger(__name__)
def callrpc_sg():
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node(
'(system input 2 always && constant 1 always) => unbuffered 2 using call_rpc'
)
log.push(DataStream.FromString('constant 1'),
IOTileReading(0, 0, 0x000a8000))
return sg
def test_stream_allocation():
"""Make sure we can allocate DataStreams."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
# TODO Finish this function
alloc = StreamAllocator(sg, model=model)
stream1 = alloc.allocate_stream(DataStream.ConstantType)
assert len(sg.nodes) == 0
stream2 = alloc.attach_stream(stream1)
assert len(sg.nodes) == 0
stream3 = alloc.attach_stream(stream1)
assert len(sg.nodes) == 0
stream4 = alloc.attach_stream(stream1)
assert len(sg.nodes) == 0
stream5 = alloc.attach_stream(stream1)
assert len(sg.nodes) == 1
assert stream1 == stream2
assert stream2 == stream3
assert stream4 == stream1
assert stream5 != stream1
def compile(self, model):
"""Compile this file into a SensorGraph.
You must have preivously called parse_file to parse a
sensor graph file into statements that are then executed
by this command to build a sensor graph.
The results are stored in self.sensor_graph and can be
inspected before running optimization passes.
Args:
model (DeviceModel): The device model that we should compile
this sensor graph for.
"""
log = SensorLog(InMemoryStorageEngine(model), model)
self.sensor_graph = SensorGraph(log, model)
allocator = StreamAllocator(self.sensor_graph, model)
self._scope_stack = []
# Create a root scope
root = RootScope(self.sensor_graph, allocator)
self._scope_stack.append(root)
for statement in self.statements:
statement.execute(self.sensor_graph, self._scope_stack)
self.sensor_graph.initialize_remaining_constants()
self.sensor_graph.sort_nodes()
def basic_sg():
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node('(system input 2 always) => unbuffered 1 using copy_all_a')
return sg
def usertick_sg():
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node('(system input 3 always) => counter 1 using copy_latest_a')
sg.add_config(SlotIdentifier.FromString('controller'),
config_fast_tick_secs, 'uint32_t', 2)
return sg
def test_string_generation():
"""Make sure we can print nodes."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node(
'(input 1 always && input 2 when count >= 1) => buffered node 1 using copy_all_a'
)
assert str(
sg.nodes[-1]
) == u'(input 1 always && input 2 when count >= 1) => buffered 1 using copy_all_a'
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node(
'(input 1 when value < 0x10) => buffered node 1 using copy_all_a')
assert str(sg.nodes[-1]
) == u'(input 1 when value < 16) => buffered 1 using copy_all_a'
def test_usertick():
"""Make sure we properly can set the user tick input."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
assert sg.get_tick('fast') == 0
sg.add_config(SlotIdentifier.FromString('controller'), config_fast_tick_secs, 'uint32_t', 1)
assert sg.get_tick('fast') == 1
def tick2_sg():
"""A sensorgrah that listens to tick1."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node('(system input 6 always) => counter 1 using copy_latest_a')
sg.add_config(SlotIdentifier.FromString('controller'), config_tick2_secs,
'uint32_t', 2)
return sg
def test_basic_sensorgraph():
"""Make sure we can parse, load and run a basic sensor graph."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node('(input 1 always && input 2 when count >= 1) => unbuffered 1 using copy_all_a')
sg.process_input(DataStream.FromString('input 1'), IOTileReading(0, 1, 1), rpc_executor=None)
sg.process_input(DataStream.FromString('input 2'), IOTileReading(0, 1, 1), rpc_executor=None)
assert sg.sensor_log.inspect_last(DataStream.FromString('unbuffered 1')).value == 1
def test_iteration():
"""Make sure we can iterate over the graph."""
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node(
'(input 1 always && input 2 when count >= 1) => unbuffered 1 using copy_all_a'
)
sg.add_node(
'(input 1 always && input 3 when count >= 1) => unbuffered 2 using copy_all_a'
)
sg.add_node(
'(unbuffered 2 always && unbuffered 1 always) => unbuffered 3 using copy_all_a'
)
sg.add_node('(unbuffered 1 always) => unbuffered 3 using copy_all_a')
iterator = sg.iterate_bfs()
node1, in1, out1 = next(iterator)
assert str(node1.stream) == u'unbuffered 1'
assert len(in1) == 0
assert len(out1) == 2
assert str(out1[0].stream) == u'unbuffered 3'
assert str(out1[1].stream) == u'unbuffered 3'
node1, in1, out1 = next(iterator)
assert str(node1.stream) == u'unbuffered 2'
assert len(in1) == 0
assert len(out1) == 1
assert str(out1[0].stream) == u'unbuffered 3'
node1, in1, out1 = next(iterator)
assert str(node1.stream) == u'unbuffered 3'
assert len(in1) == 2
assert len(out1) == 0
assert str(in1[0].stream) == u'unbuffered 2'
assert str(in1[1].stream) == u'unbuffered 1'
node1, in1, out1 = next(iterator)
assert str(node1.stream) == u'unbuffered 3'
assert len(in1) == 1
assert len(out1) == 0
assert str(in1[0].stream) == u'unbuffered 1'
def test_triggering_streamers():
model = DeviceModel()
log = SensorLog(model=model)
sg = SensorGraph(log, model=model)
sg.add_node('(input 1 always) => output 1 using copy_all_a')
sg.add_node('(input 1 always) => output 2 using copy_all_a')
sg.add_streamer(parse_string_descriptor('streamer on output 1'))
sg.add_streamer(
parse_string_descriptor('manual streamer on output 2 with streamer 0'))
triggered = sg.check_streamers()
assert len(triggered) == 0
sg.process_input(DataStream.FromString('input 1'),
IOTileReading(0, 1, 1),
rpc_executor=None)
triggered = sg.check_streamers()
assert len(triggered) == 2
class SensorLogSubsystem(ControllerSubsystemBase):
"""Container for raw sensor log state."""
def __init__(self, emulator, model):
super(SensorLogSubsystem, self).__init__(emulator)
self.engine = InMemoryStorageEngine(model=model)
self.storage = SensorLog(self.engine,
model=model,
id_assigner=lambda x, y: self.allocate_id())
self.dump_walker = None
self.next_id = 1
self._logger = logging.getLogger(__name__)
def dump(self):
"""Serialize the state of this subsystem into a dict.
Returns:
dict: The serialized state
"""
walker = self.dump_walker
if walker is not None:
walker = walker.dump()
state = {
'storage': self.storage.dump(),
'dump_walker': walker,
'next_id': self.next_id
}
return state
def prepare_for_restore(self):
"""Prepare the SensorLog subsystem for a restore.
This must be called at the start of the restore to clear all of the
stream walkers in the SensorLog storage engine, which are then added
back by the SensorGraph and Streaming subsystems and then restored
to their previous positions when restore() is called on this subsystem.
"""
self.storage.destroy_all_walkers()
def restore(self, state):
"""Restore the state of this subsystem from a prior call to dump().
Calling restore must be properly sequenced with calls to other
subsystems that include stream walkers so that their walkers are
properly restored.
Args:
state (dict): The results of a prior call to dump().
"""
self.storage.restore(state.get('storage'))
dump_walker = state.get('dump_walker')
if dump_walker is not None:
dump_walker = self.storage.restore_walker(dump_walker)
self.dump_walker = dump_walker
self.next_id = state.get('next_id', 1)
def clear(self, timestamp):
"""Clear all data from the RSL.
This pushes a single reading once we clear everything so that
we keep track of the highest ID that we have allocated to date.
This needs the current timestamp to be able to properly timestamp
the cleared storage reading that it pushes.
Args:
timestamp (int): The current timestamp to store with the
reading.
"""
self.storage.clear()
self.push(streams.DATA_CLEARED, timestamp, 1)
def clear_to_reset(self, config_vars):
"""Clear all volatile information across a reset."""
self._logger.info("Config vars in sensor log reset: %s", config_vars)
super(SensorLogSubsystem, self).clear_to_reset(config_vars)
self.storage.destroy_all_walkers()
self.dump_walker = None
if config_vars.get('storage_fillstop', False):
self._logger.debug("Marking storage log fill/stop")
self.storage.set_rollover('storage', False)
if config_vars.get('streaming_fillstop', False):
self._logger.debug("Marking streaming log fill/stop")
self.storage.set_rollover('streaming', False)
def count(self):
"""Count many many readings are persistently stored.
Returns:
(int, int): The number of readings in storage and output areas.
"""
return self.storage.count()
def allocate_id(self):
"""Get the next unique ID.
Returns:
int: A unique reading ID.
"""
next_id = self.next_id
self.next_id += 1
return next_id
def push(self, stream_id, timestamp, value):
"""Push a value to a stream.
Args:
stream_id (int): The stream we want to push to.
timestamp (int): The raw timestamp of the value we want to
store.
value (int): The 32-bit integer value we want to push.
Returns:
int: Packed 32-bit error code.
"""
stream = DataStream.FromEncoded(stream_id)
reading = IOTileReading(stream_id, timestamp, value)
try:
self.storage.push(stream, reading)
return Error.NO_ERROR
except StorageFullError:
return pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.RING_BUFFER_FULL)
def inspect_virtual(self, stream_id):
"""Inspect the last value written into a virtual stream.
Args:
stream_id (int): The virtual stream was want to inspect.
Returns:
(int, int): An error code and the stream value.
"""
stream = DataStream.FromEncoded(stream_id)
if stream.buffered:
return [
pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.VIRTUAL_STREAM_NOT_FOUND), 0
]
try:
reading = self.storage.inspect_last(stream, only_allocated=True)
return [Error.NO_ERROR, reading.value]
except StreamEmptyError:
return [Error.NO_ERROR, 0]
except UnresolvedIdentifierError:
return [
pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.VIRTUAL_STREAM_NOT_FOUND), 0
]
def dump_begin(self, selector_id):
"""Start dumping a stream.
Args:
selector_id (int): The buffered stream we want to dump.
Returns:
(int, int, int): Error code, second error code, number of available readings
"""
if self.dump_walker is not None:
self.storage.destroy_walker(self.dump_walker)
selector = DataStreamSelector.FromEncoded(selector_id)
self.dump_walker = self.storage.create_walker(selector, skip_all=False)
return Error.NO_ERROR, Error.NO_ERROR, self.dump_walker.count()
def dump_seek(self, reading_id):
"""Seek the dump streamer to a given ID.
Returns:
(int, int, int): Two error codes and the count of remaining readings.
The first error code covers the seeking process.
The second error code covers the stream counting process (cannot fail)
The third item in the tuple is the number of readings left in the stream.
"""
if self.dump_walker is None:
return (pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.STREAM_WALKER_NOT_INITIALIZED),
Error.NO_ERROR, 0)
try:
exact = self.dump_walker.seek(reading_id, target='id')
except UnresolvedIdentifierError:
return (pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.NO_MORE_READINGS),
Error.NO_ERROR, 0)
error = Error.NO_ERROR
if not exact:
error = pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.ID_FOUND_FOR_ANOTHER_STREAM)
return (error, error.NO_ERROR, self.dump_walker.count())
def dump_next(self):
"""Dump the next reading from the stream.
Returns:
IOTileReading: The next reading or None if there isn't one
"""
if self.dump_walker is None:
return pack_error(ControllerSubsystem.SENSOR_LOG,
SensorLogError.STREAM_WALKER_NOT_INITIALIZED)
try:
return self.dump_walker.pop()
except StreamEmptyError:
return None
def highest_stored_id(self):
"""Scan through the stored readings and report the highest stored id.
Returns:
int: The highest stored id.
"""
shared = [0]
def _keep_max(_i, reading):
if reading.reading_id > shared[0]:
shared[0] = reading.reading_id
self.engine.scan_storage('storage', _keep_max)
self.engine.scan_storage('streaming', _keep_max)
return shared[0]
def __init__(self, model):
self.engine = InMemoryStorageEngine(model=model)
self.storage = SensorLog(self.engine, model=model, id_assigner=lambda x, y: self.allocate_id())
self.dump_walker = None
self.next_id = 1