def test_islice():
sl = SortedKeyList(key=modulo)
sl._reset(7)
assert [] == list(sl.islice())
values = sorted(range(100), key=modulo)
sl.update(values)
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop,
reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(sl.islice(start=start)) == values[start:]
assert list(sl.islice(start=start,
reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(sl.islice(stop=stop)) == values[:stop]
assert list(sl.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_islice():
return
slt = SortedKeyList(key=negate)
slt._reset(7)
assert [] == list(slt.islice())
values = sorted(range(53), key=negate)
slt.update(values)
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop,
reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(slt.islice(start=start)) == values[start:]
assert list(slt.islice(start=start,
reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(slt.islice(stop=stop)) == values[:stop]
assert list(slt.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_islice():
return
slt = SortedKeyList(key=negate)
slt._reset(7)
assert [] == list(slt.islice())
values = sorted(range(53), key=negate)
slt.update(values)
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop, reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(slt.islice(start=start)) == values[start:]
assert list(slt.islice(start=start, reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(slt.islice(stop=stop)) == values[:stop]
assert list(slt.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_islice():
sl = SortedKeyList(key=modulo)
sl._reset(7)
assert [] == list(sl.islice())
values = sorted(range(100), key=modulo)
sl.update(values)
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop, reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(sl.islice(start=start)) == values[start:]
assert list(sl.islice(start=start, reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(sl.islice(stop=stop)) == values[:stop]
assert list(sl.islice(stop=stop, reverse=True)) == values[:stop][::-1]
class UTCAssigner:
"""Helper class to assign UTC timestamps to device readings.
This class contains logic to infer UTC timestamps for readings that are
stamped in uptime only by looking for nearby readings for which the UTC
timestamp is known. The relative offset between these anchor points and
the reading in question is then used to infer the UTC timestamp from the
anchor point either exactly or approximately.
The underlying truth that this class relies on is the fact that all
readings produced by an IOTile device have a mononotically increasing
reading_id that can be used to absolutely order them. In contrast to the
readings timestamp, which may reset to 0 if the device reboots and does
not have a realtime clock, the reading_id is guaranteed to never decrease.
"""
_Y2KReference = datetime.datetime(2000, 1, 1)
_EpochReference = datetime.datetime(1970, 1, 1)
def __init__(self):
self._anchor_points = SortedKeyList(key=lambda x: x.reading_id)
self._prepared = False
self._anchor_streams = {}
self._break_streams = set()
self._logger = logging.getLogger(__name__)
self._known_converters = {
'rtc': UTCAssigner._convert_rtc_anchor,
'epoch': UTCAssigner._convert_epoch_anchor
}
def _load_known_breaks(self):
self._break_streams.add(0x5C00)
def anchor_stream(self, stream_id, converter="rtc"):
"""Mark a stream as containing anchor points."""
if isinstance(converter, str):
converter = self._known_converters.get(converter)
if converter is None:
raise ArgumentError(
"Unknown anchor converter string: %s" % converter,
known_converters=list(self._known_converters))
self._anchor_streams[stream_id] = converter
def id_range(self):
"""Get the range of archor reading_ids.
Returns:
(int, int): The lowest and highest reading ids.
If no reading ids have been loaded, (0, 0) is returned.
"""
if len(self._anchor_points) == 0:
return (0, 0)
return (self._anchor_points[0].reading_id,
self._anchor_points[-1].reading_id)
@classmethod
def convert_rtc(cls, timestamp):
"""Convert a number of seconds since 1/1/2000 to UTC time."""
if timestamp & (1 << 31):
timestamp &= ~(1 << 31)
delta = datetime.timedelta(seconds=timestamp)
return cls._Y2KReference + delta
@classmethod
def _convert_rtc_anchor(cls, reading):
"""Convert a reading containing an RTC timestamp to datetime."""
return cls.convert_rtc(reading.value)
@classmethod
def _convert_epoch_anchor(cls, reading):
"""Convert a reading containing an epoch timestamp to datetime."""
delta = datetime.timedelta(seconds=reading.value)
return cls._EpochReference + delta
def add_point(self, reading_id, uptime=None, utc=None, is_break=False):
"""Add a time point that could be used as a UTC reference."""
if reading_id == 0:
return
if uptime is None and utc is None:
return
if uptime is not None and uptime & (1 << 31):
if utc is not None:
return
uptime &= ~(1 << 31)
utc = self.convert_rtc(uptime)
uptime = None
anchor = _TimeAnchor(reading_id,
uptime,
utc,
is_break,
exact=utc is not None)
if anchor in self._anchor_points:
return
self._anchor_points.add(anchor)
self._prepared = False
def add_reading(self, reading):
"""Add an IOTileReading."""
is_break = False
utc = None
if reading.stream in self._break_streams:
is_break = True
if reading.stream in self._anchor_streams:
utc = self._anchor_streams[reading.stream](reading)
self.add_point(reading.reading_id,
reading.raw_time,
utc,
is_break=is_break)
def add_report(self, report, ignore_errors=False):
"""Add all anchors from a report."""
if not isinstance(report, SignedListReport):
if ignore_errors:
return
raise ArgumentError(
"You can only add SignedListReports to a UTCAssigner",
report=report)
for reading in report.visible_readings:
self.add_reading(reading)
self.add_point(report.report_id, report.sent_timestamp,
report.received_time)
def assign_utc(self, reading_id, uptime=None, prefer="before"):
"""Assign a utc datetime to a reading id.
This method will return an object with assignment information or None
if a utc value cannot be assigned. The assignment object returned
contains a utc property that has the asssigned UTC as well as other
properties describing how reliable the assignment is.
Args:
reading_id (int): The monotonic reading id that we wish to assign
a utc timestamp to.
uptime (int): Optional uptime that should be associated with the
reading id. If this is not specified and the reading_id is
found in the anchor points passed to this class then the
uptime from the corresponding anchor point will be used.
prefer (str): There are two possible directions that can be used
to assign a UTC timestamp (the nearest anchor before or after the
reading). If both directions are of similar quality, the choice
is arbitrary. Passing prefer="before" will use the anchor point
before the reading. Passing prefer="after" will use the anchor
point after the reading. Default: before.
Returns:
UTCAssignment: The assigned UTC time or None if assignment is impossible.
"""
if prefer not in ("before", "after"):
raise ArgumentError(
"Invalid prefer parameter: {}, must be 'before' or 'after'".
format(prefer))
if len(self._anchor_points) == 0:
return None
if reading_id > self._anchor_points[-1].reading_id:
return None
i = self._anchor_points.bisect_key_left(reading_id)
found_id = False
crossed_break = False
exact = True
last = self._anchor_points[i].copy()
if uptime is not None:
last.uptime = uptime
if last.reading_id == reading_id:
found_id = True
if last.utc is not None:
return UTCAssignment(reading_id, last.utc, found_id, exact,
crossed_break)
left_assign = self._fix_left(reading_id, last, i, found_id)
if left_assign is not None and left_assign.exact:
return left_assign
right_assign = self._fix_right(reading_id, last, i, found_id)
if right_assign is not None and right_assign.exact:
return right_assign
return self._pick_best_fix(left_assign, right_assign, prefer)
def ensure_prepared(self):
"""Calculate and cache UTC values for all exactly known anchor points."""
if self._prepared:
return
exact_count = 0
fixed_count = 0
inexact_count = 0
self._logger.debug("Preparing UTCAssigner (%d total anchors)",
len(self._anchor_points))
for curr in self._anchor_points:
if not curr.exact:
assignment = self.assign_utc(curr.reading_id, curr.uptime)
if assignment is not None and assignment.exact:
curr.utc = assignment.utc
curr.exact = True
fixed_count += 1
else:
inexact_count += 1
else:
exact_count += 1
self._logger.debug(
"Prepared UTCAssigner with %d reference points, "
"%d exact anchors and %d inexact anchors", exact_count,
fixed_count, inexact_count)
self._prepared = True
def fix_report(self, report, errors="drop", prefer="before"):
"""Perform utc assignment on all readings in a report.
The returned report will have all reading timestamps in UTC. This only
works on SignedListReport objects. Note that the report should
typically have previously been added to the UTC assigner using
add_report or no reference points from the report will be used.
Args:
report (SignedListReport): The report that we should fix.
errors (str): The behavior that we should have when we can't
fix a given reading. The only currently support behavior is
drop, which means that the reading will be dropped and not
included in the new report.
prefer (str): Whether to prefer fixing readings by looking for
reference points after the reading or before, all other things
being equal. See the description of ``assign_utc``.
Returns:
SignedListReport: The report with UTC timestamps.
"""
if not isinstance(report, SignedListReport):
raise ArgumentError("Report must be a SignedListReport",
report=report)
if errors not in ('drop', ):
raise ArgumentError(
"Unknown errors handler: {}, supported=['drop']".format(
errors))
self.ensure_prepared()
fixed_readings = []
dropped_readings = 0
for reading in report.visible_readings:
assignment = self.assign_utc(reading.reading_id,
reading.raw_time,
prefer=prefer)
if assignment is None:
dropped_readings += 1
continue
fixed_reading = IOTileReading(assignment.rtc_value,
reading.stream,
reading.value,
reading_time=assignment.utc,
reading_id=reading.reading_id)
fixed_readings.append(fixed_reading)
fixed_report = SignedListReport.FromReadings(
report.origin,
fixed_readings,
report_id=report.report_id,
selector=report.streamer_selector,
streamer=report.origin_streamer,
sent_timestamp=report.sent_timestamp)
fixed_report.received_time = report.received_time
if dropped_readings > 0:
self._logger.warning(
"Dropped %d readings of %d when fixing UTC timestamps in report 0x%08X for device 0x%08X",
dropped_readings, len(report.visible_readings),
report.report_id, report.origin)
return fixed_report
def _pick_best_fix(self, before, after, prefer):
if before is None and after is None:
return None
if after is None:
return before
if before is None:
return after
if after.crossed_break and not before.crossed_break:
return before
if before.crossed_break and not after.crossed_break:
return after
if before.exact and not after.exact:
return before
if after.exact and not before.exact:
return after
if prefer == 'before':
return before
return after
def _fix_right(self, reading_id, last, start, found_id):
"""Fix a reading by looking for the nearest anchor point after it."""
accum_delta = 0
exact = True
crossed_break = False
if start == len(self._anchor_points) - 1:
return None
for curr in self._anchor_points.islice(start + 1):
if curr.uptime is None or last.uptime is None:
exact = False
elif curr.is_break or curr.uptime < last.uptime:
exact = False
crossed_break = True
else:
accum_delta += curr.uptime - last.uptime
if curr.utc is not None:
time_delta = datetime.timedelta(seconds=accum_delta)
return UTCAssignment(reading_id, curr.utc - time_delta,
found_id, exact, crossed_break)
last = curr
return None
def _fix_left(self, reading_id, last, start, found_id):
"""Fix a reading by looking for the nearest anchor point before it."""
accum_delta = 0
exact = True
crossed_break = False
if start == 0:
return None
for curr in self._anchor_points.islice(None, start, reverse=True):
if curr.uptime is None or last.uptime is None:
exact = False
elif curr.is_break or last.uptime < curr.uptime:
exact = False
crossed_break = True
else:
accum_delta += last.uptime - curr.uptime
if curr.utc is not None:
time_delta = datetime.timedelta(seconds=accum_delta)
return UTCAssignment(reading_id, curr.utc + time_delta,
found_id, exact, crossed_break)
last = curr
return None
class UTCAssigner(object):
"""Helper class to assign UTC timestamps to device readings."""
_Y2KReference = datetime.datetime(2000, 1, 1)
_EpochReference = datetime.datetime(1970, 1, 1)
def __init__(self):
self._anchor_points = SortedKeyList(key=lambda x: x.reading_id)
self._anchor_streams = {}
self._break_streams = set()
self._known_converters = {
'rtc': UTCAssigner._convert_rtc_anchor,
'epoch': UTCAssigner._convert_epoch_anchor
}
def _load_known_breaks(self):
self._break_streams.add(0x5C00)
def anchor_stream(self, stream_id, converter="rtc"):
"""Mark a stream as containing anchor points."""
if isinstance(converter, str):
converter = self._known_converters.get(converter)
if converter is None:
raise ArgumentError(
"Unknown anchor converter string: %s" % converter,
known_converters=list(self._known_converters))
self._anchor_streams[stream_id] = converter
@classmethod
def convert_rtc(cls, timestamp):
"""Convert a number of seconds since 1/1/2000 to UTC time."""
if timestamp & (1 << 31):
timestamp &= ~(1 << 31)
delta = datetime.timedelta(seconds=timestamp)
return cls._Y2KReference + delta
@classmethod
def _convert_rtc_anchor(cls, reading):
"""Convert a reading containing an RTC timestamp to datetime."""
return cls.convert_rtc(reading.value)
@classmethod
def _convert_epoch_anchor(cls, reading):
"""Convert a reading containing an epoch timestamp to datetime."""
delta = datetime.timedelta(seconds=reading.value)
return cls._EpochReference + delta
def add_point(self, reading_id, uptime=None, utc=None, is_break=False):
"""Add a time point that could be used as a UTC reference."""
if reading_id == 0:
return
if uptime is None and utc is None:
return
if uptime is not None and uptime & (1 << 31):
if utc is not None:
return
uptime &= ~(1 << 31)
utc = self.convert_rtc(uptime)
uptime = None
anchor = _TimeAnchor(reading_id, uptime, utc, is_break)
if anchor in self._anchor_points:
return
self._anchor_points.add(anchor)
def add_reading(self, reading):
"""Add an IOTileReading."""
is_break = False
utc = None
if reading.stream in self._break_streams:
is_break = True
if reading.stream in self._anchor_streams:
utc = self._anchor_streams[reading.stream](reading)
self.add_point(reading.reading_id,
reading.raw_time,
utc,
is_break=is_break)
def add_report(self, report, ignore_errors=False):
"""Add all anchors from a report."""
if not isinstance(report, SignedListReport):
if ignore_errors:
return
raise ArgumentError(
"You can only add SignedListReports to a UTCAssigner",
report=report)
for reading in report.visible_readings:
self.add_reading(reading)
self.add_point(report.report_id, report.sent_timestamp,
report.received_time)
def assign_utc(self, reading_id, uptime=None):
"""Assign a utc datetime to a reading id.
This method will return an object with assignment information or None
if a utc value cannot be assigned. The assignment object returned
contains a utc property that has the asssigned UTC as well as other
properties describing how reliable the assignment is.
"""
if len(self._anchor_points) == 0:
return None
if reading_id > self._anchor_points[-1].reading_id:
return None
i = self._anchor_points.bisect_key_left(reading_id)
found_id = False
crossed_break = False
exact = True
last = self._anchor_points[i].copy()
if uptime is not None:
last.uptime = uptime
if last.reading_id == reading_id:
found_id = True
if last.utc is not None:
return last.utc
accum_delta = 0
for curr in self._anchor_points.islice(i + 1):
if curr.uptime is None or last.uptime is None:
exact = False
elif curr.is_break or curr.uptime < last.uptime:
exact = False
crossed_break = True
else:
accum_delta += curr.uptime - last.uptime
if curr.utc is not None:
time_delta = datetime.timedelta(seconds=accum_delta)
return UTCAssignment(reading_id, curr.utc - time_delta,
found_id, exact, crossed_break)
last = curr
return None
