def setUp(self): # make a canned event self.msec = 1458768183949 self.data = {'a': 3, 'b': 6, 'c': 9} self.aware_ts = aware_utcnow() self.canned_event = self._create_event(self.msec, self.data)
def setUp(self): self.test_end_ts = aware_utcnow() self.test_begin_ts = self.test_end_ts - datetime.timedelta(hours=12) self.test_end_ms = ms_from_dt(self.test_end_ts) self.test_begin_ms = ms_from_dt(self.test_begin_ts) self.canned_range = self._create_time_range(self.test_begin_ts, self.test_end_ts)
def test_is_duplicate(self): """Test Event.is_duplicate()""" # events # pylint: disable=invalid-name e_ts = aware_utcnow() e1 = Event(e_ts, 23) e2 = Event(e_ts, 23) self.assertTrue(Event.is_duplicate(e1, e2)) self.assertTrue(Event.is_duplicate(e1, e2, ignore_values=False)) e3 = Event(e_ts, 25) self.assertTrue(Event.is_duplicate(e1, e3)) self.assertFalse(Event.is_duplicate(e1, e3, ignore_values=False)) # indexed events ie1 = IndexedEvent('1d-12355', {'value': 42}) ie2 = IndexedEvent('1d-12355', {'value': 42}) self.assertTrue(Event.is_duplicate(ie1, ie2)) self.assertTrue(Event.is_duplicate(ie1, ie2, ignore_values=False)) ie3 = IndexedEvent('1d-12355', {'value': 44}) self.assertTrue(Event.is_duplicate(ie1, ie3)) self.assertFalse(Event.is_duplicate(ie1, ie3, ignore_values=False)) # time range events test_end_ts = aware_utcnow() test_begin_ts = test_end_ts - datetime.timedelta(hours=12) test_end_ms = ms_from_dt(test_end_ts) test_begin_ms = ms_from_dt(test_begin_ts) tre1 = TimeRangeEvent((test_begin_ms, test_end_ms), 11) tre2 = TimeRangeEvent((test_begin_ms, test_end_ms), 11) self.assertTrue(Event.is_duplicate(tre1, tre2)) self.assertTrue(Event.is_duplicate(tre1, tre2, ignore_values=False)) tre3 = TimeRangeEvent((test_begin_ms, test_end_ms), 22) self.assertTrue(Event.is_duplicate(tre1, tre3)) self.assertFalse(Event.is_duplicate(tre1, tre3, ignore_values=False))
def setUp(self): super(TestTimeRangeEvent, self).setUp() self.test_end_ts = aware_utcnow() self.test_begin_ts = self.test_end_ts - datetime.timedelta(hours=12) self.test_end_ms = ms_from_dt(self.test_end_ts) self.test_begin_ms = ms_from_dt(self.test_begin_ts) self.canned_time_range = TimeRangeEvent((self.test_begin_ms, self.test_end_ms), 11)
def setUp(self): super(TestTimeRangeEvent, self).setUp() self.test_end_ts = aware_utcnow() self.test_begin_ts = self.test_end_ts - datetime.timedelta(hours=12) self.test_end_ms = ms_from_dt(self.test_end_ts) self.test_begin_ms = ms_from_dt(self.test_begin_ts) self.canned_time_range = TimeRangeEvent( (self.test_begin_ms, self.test_end_ms), 11)
def test_nested_merge(self): """trigger merging nested data.""" # pylint: disable=invalid-name e_ts = aware_utcnow() e1 = Event(e_ts, dict(payload=dict(a=1))) e2 = Event(e_ts, dict(payload=dict(b=2))) emerge = Event.merge([e1, e2]) self.assertEqual(emerge[0].get('payload.a'), 1) self.assertEqual(emerge[0].get('payload.b'), 2)
def test_at_key_and_dedup(self): """test Collection.at_key() and dedup()""" # events coll = Collection(EVENT_LIST_DUP) key_time = dt_from_ms(1429673460000) find = coll.at_key(key_time) self.assertEqual(len(find), 2) self.assertEqual(find[0].get('in'), 3) self.assertEqual(find[1].get('in'), 4) ddcoll = coll.dedup() self.assertEqual(ddcoll.size(), 3) self.assertEqual(ddcoll.at(1).get('in'), 4) # the second dup event # indexed events coll = Collection(IDX_EVENT_DUP) find = coll.at_key('1d-12355') self.assertEqual(len(find), 2) self.assertEqual(find[0].get('value'), 43) self.assertEqual(find[1].get('value'), 44) ddcoll = coll.dedup() self.assertEqual(ddcoll.size(), 3) self.assertEqual(ddcoll.at(1).get('value'), 44) # the second dup event # time range events test_end_ts = aware_utcnow() test_begin_ts = test_end_ts - datetime.timedelta(hours=12) test_end_ms = ms_from_dt(test_end_ts) test_begin_ms = ms_from_dt(test_begin_ts) dup_tre = [ TimeRangeEvent((test_begin_ms, test_end_ms), 11), TimeRangeEvent((test_begin_ms + 60000, test_end_ms + 60000), 12), TimeRangeEvent((test_begin_ms + 60000, test_end_ms + 60000), 13), TimeRangeEvent((test_begin_ms + 120000, test_end_ms + 120000), 14), ] coll = Collection(dup_tre) search = TimeRange(test_begin_ms + 60000, test_end_ms + 60000) find = coll.at_key(search) self.assertEqual(len(find), 2) self.assertEqual(find[0].get('value'), 12) self.assertEqual(find[1].get('value'), 13) ddcoll = coll.dedup() self.assertEqual(ddcoll.size(), 3) self.assertEqual(ddcoll.at(1).get('value'), 13) # the second dup event
def test_create_collection(self): """test collection creation and methods related to internal payload.""" # event list col_1 = Collection(EVENT_LIST) self.assertEqual(col_1.size(), 3) self.assertEqual(col_1.type(), Event) self.assertEqual(col_1.size_valid('in'), 3) # copy ctor col_2 = Collection(col_1) self.assertEqual(col_2.size(), 3) self.assertEqual(col_2.type(), Event) self.assertEqual(col_2.size_valid('in'), 3) # copy ctor - no event copy col_3 = Collection(col_2, copy_events=False) self.assertEqual(col_3.size(), 0) self.assertEqual(col_3.size_valid('in'), 0) # pass in an immutable - use a pre- _check()'ed one col_4 = Collection(col_1._event_list) # pylint: disable=protected-access self.assertEqual(col_4.size(), 3) self.assertEqual(col_4.size_valid('in'), 3) # other event types for coverage ie1 = IndexedEvent('1d-12355', {'value': 42}) ie2 = IndexedEvent('1d-12356', {'value': 4242}) col_5 = Collection([ie1, ie2]) self.assertEqual(col_5.size(), 2) tre = TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(hours=24)), {'in': 100}) col_6 = Collection([tre]) self.assertEqual(col_6.size(), 1)
def test_invalid_constructor_args(self): """test invalid constructor args""" # test both (two, args) and ([list, arg]) inputs to work different logic # unaware datetime input u_begin = datetime.datetime.utcnow() - datetime.timedelta(hours=12) u_end = datetime.datetime.utcnow() with self.assertRaises(TimeRangeException): self._create_time_range(u_begin, u_end) with self.assertRaises(TimeRangeException): self._create_time_range([u_begin, u_end]) # invalid types - pass in floats end = time.time() * 1000 begin = end - 10000 with self.assertRaises(TimeRangeException): self._create_time_range(begin, end) with self.assertRaises(TimeRangeException): self._create_time_range([begin, end]) # type mismatch with self.assertRaises(TimeRangeException): self._create_time_range((int(begin), aware_utcnow())) with self.assertRaises(TimeRangeException): self._create_time_range(int(begin), aware_utcnow()) with self.assertRaises(TimeRangeException): self._create_time_range( [aware_utcnow() - datetime.timedelta(hours=12), int(end)]) with self.assertRaises(TimeRangeException): self._create_time_range( aware_utcnow() - datetime.timedelta(hours=12), int(end)) # end time before begin time bad_begin = aware_utcnow() bad_end = aware_utcnow() - datetime.timedelta(hours=12) with self.assertRaises(TimeRangeException): self._create_time_range(bad_begin, bad_end) with self.assertRaises(TimeRangeException): self._create_time_range([bad_begin, bad_end]) with self.assertRaises(TimeRangeException): self._create_time_range(ms_from_dt(bad_begin), ms_from_dt(bad_end)) with self.assertRaises(TimeRangeException): self._create_time_range( (ms_from_dt(bad_begin), ms_from_dt(bad_end))) # wrong number of args with self.assertRaises(TimeRangeException): TimeRange((begin, end, end))
def test_invalid_constructor_args(self): """test invalid constructor args""" # test both (two, args) and ([list, arg]) inputs to work different logic # unaware datetime input u_begin = datetime.datetime.utcnow() - datetime.timedelta(hours=12) u_end = datetime.datetime.utcnow() with self.assertRaises(TimeRangeException): self._create_time_range(u_begin, u_end) with self.assertRaises(TimeRangeException): self._create_time_range([u_begin, u_end]) # invalid types - pass in floats end = time.time() * 1000 begin = end - 10000 with self.assertRaises(TimeRangeException): self._create_time_range(begin, end) with self.assertRaises(TimeRangeException): self._create_time_range([begin, end]) # type mismatch with self.assertRaises(TimeRangeException): self._create_time_range((int(begin), aware_utcnow())) with self.assertRaises(TimeRangeException): self._create_time_range(int(begin), aware_utcnow()) with self.assertRaises(TimeRangeException): self._create_time_range([aware_utcnow() - datetime.timedelta(hours=12), int(end)]) with self.assertRaises(TimeRangeException): self._create_time_range(aware_utcnow() - datetime.timedelta(hours=12), int(end)) # end time before begin time bad_begin = aware_utcnow() bad_end = aware_utcnow() - datetime.timedelta(hours=12) with self.assertRaises(TimeRangeException): self._create_time_range(bad_begin, bad_end) with self.assertRaises(TimeRangeException): self._create_time_range([bad_begin, bad_end]) with self.assertRaises(TimeRangeException): self._create_time_range(ms_from_dt(bad_begin), ms_from_dt(bad_end)) with self.assertRaises(TimeRangeException): self._create_time_range((ms_from_dt(bad_begin), ms_from_dt(bad_end))) # wrong number of args with self.assertRaises(TimeRangeException): TimeRange((begin, end, end))
def test_round_trip(self): """Test ms -> dt -> ms and dt -> ms -> dt""" # ms -> dt -> ms to_dt = dt_from_ms(self.ms_reference) from_dt = ms_from_dt(to_dt) self.assertEqual(from_dt, self.ms_reference) # dt -> ms -> dt to test rounding in aware_utcnow() now = aware_utcnow() to_ms = ms_from_dt(now) back_to_dt = dt_from_ms(to_ms) self.assertEqual(now, back_to_dt) # dt from unixtime -> ms -> dt utc = datetime.datetime.utcfromtimestamp(1459442035).replace(tzinfo=pytz.UTC) utcms = ms_from_dt(utc) back_to_utc = dt_from_ms(utcms) self.assertEqual(utc, back_to_utc)
def test_round_trip(self): """Test ms -> dt -> ms and dt -> ms -> dt""" # ms -> dt -> ms to_dt = dt_from_ms(self.ms_reference) from_dt = ms_from_dt(to_dt) self.assertEqual(from_dt, self.ms_reference) # dt -> ms -> dt to test rounding in aware_utcnow() now = aware_utcnow() to_ms = ms_from_dt(now) back_to_dt = dt_from_ms(to_ms) self.assertEqual(now, back_to_dt) # dt from unixtime -> ms -> dt utc = datetime.datetime.utcfromtimestamp(1459442035).replace( tzinfo=pytz.UTC) utcms = ms_from_dt(utc) back_to_utc = dt_from_ms(utcms) self.assertEqual(utc, back_to_utc)
def test_sanitize_dt(self): """Test datetime timezone conversion to UTC. """ # aware utc should just go in and out. utc = aware_utcnow() sanitized_utc = sanitize_dt(utc) self.assertTrue(dt_is_aware(sanitized_utc)) self.assertEqual(utc, sanitized_utc) # Sanitize a time zone aware localtime to UTC. The # sanitized object # Use .localize and not datetime.replace to generate # the local date because that doesn't handle DST correctly. def get_unrounded_local(): """get an unrounded local time deal - a datetime object with real microsecond precision.""" pacific = pytz.timezone('US/Pacific') local = pacific.localize(datetime.datetime.now()) if len(str(local.microsecond)) < 6: # Prevent a microsecond value that starts with # a zero - ie: .023493. This will interefere # with the janky string-based rounding that this # test is doing for the purposes of this test by # becoming 23493. time.sleep(.1) return get_unrounded_local() elif local.microsecond % 1000 != 0: # The "good" case - we have a six decimal place # microseconds value that has greater than millisecond # precision. This is to work sanitizing incoming # microsecond values down to milliseconds to have # parity with the javascript precision. return local else: # Unlikely - this will happen if the microsecond # value is spontaneously exactly at millisecond # precision. return get_unrounded_local() local = get_unrounded_local() # testing mode to suppress warnings local_utc = sanitize_dt(local, testing=True) # objects will not be equal since sanitize is rounding to milliseconds self.assertNotEqual(local_utc, local) # this is a terrible way to round to milliseconds, but in this # case, we are trying to leave the time zone difference intact # whereas pypond.util wants to force everything to UTC. # But in general DO NOT DO THIS. -MMG msec = '{ms}000'.format(ms=str(local.microsecond)[0:3]) local = local.replace(microsecond=int(msec)) self.assertEqual(local_utc, local) # double check that delta is zero local_utc_delta = local_utc - local self.assertEqual(int(local_utc_delta.total_seconds()), 0) # test sanity check stopping naive datetime objects with self.assertRaises(UtilityException): sanitize_dt(self.naive)
def test_aware(self): """Verify test_aware function.""" self.assertFalse(dt_is_aware(self.naive)) aware = aware_utcnow() self.assertTrue(dt_is_aware(aware))
def test_local_info(self): """test the code that extracts localized values from a UTC datetime.""" utcnow = aware_utcnow() l_info = localtime_info_from_utc(utcnow) self.assertTrue(isinstance(l_info, dict))
def test_fill_event_variants(self): """fill time range and indexed events.""" range_list = [ TimeRangeEvent((aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=1)), {'in': 100}), TimeRangeEvent((aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=2)), {'in': None}), TimeRangeEvent((aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=3)), {'in': None}), TimeRangeEvent((aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=4)), {'in': 90}), TimeRangeEvent((aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=5)), {'in': 80}), TimeRangeEvent((aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=6)), {'in': 70}), ] coll = Collection(range_list) # canned series objects rts = TimeSeries(dict(name='collection', collection=coll)) new_rts = rts.fill(field_spec='in') self.assertEqual(new_rts.at(1).get('in'), 0) self.assertEqual(new_rts.at(2).get('in'), 0) # indexed events index_list = [ IndexedEvent('1d-12355', {'value': 42}), IndexedEvent('1d-12356', {'value': None}), IndexedEvent('1d-12357', {'value': None}), IndexedEvent('1d-12358', {'value': 52}), IndexedEvent('1d-12359', {'value': 55}), IndexedEvent('1d-12360', {'value': 58}), ] coll = Collection(index_list) its = TimeSeries(dict(name='collection', collection=coll)) new_its = its.fill() self.assertEqual(new_its.at(1).get(), 0) self.assertEqual(new_its.at(2).get(), 0)
def test_fill_event_variants(self): """fill time range and indexed events.""" range_list = [ TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=1)), {'in': 100} ), TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=2)), {'in': None} ), TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=3)), {'in': None} ), TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=4)), {'in': 90} ), TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=5)), {'in': 80} ), TimeRangeEvent( (aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=6)), {'in': 70} ), ] coll = Collection(range_list) # canned series objects rts = TimeSeries( dict(name='collection', collection=coll)) new_rts = rts.fill(field_spec='in') self.assertEqual(new_rts.at(1).get('in'), 0) self.assertEqual(new_rts.at(2).get('in'), 0) # indexed events index_list = [ IndexedEvent('1d-12355', {'value': 42}), IndexedEvent('1d-12356', {'value': None}), IndexedEvent('1d-12357', {'value': None}), IndexedEvent('1d-12358', {'value': 52}), IndexedEvent('1d-12359', {'value': 55}), IndexedEvent('1d-12360', {'value': 58}), ] coll = Collection(index_list) its = TimeSeries( dict(name='collection', collection=coll)) new_its = its.fill() self.assertEqual(new_its.at(1).get(), 0) self.assertEqual(new_its.at(2).get(), 0)