def test_fill_event_variants(self):
"""fill time range and indexed events."""
range_list = [
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=1)),
{'in': 100}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=2)),
{'in': None}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=3)),
{'in': None}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=4)),
{'in': 90}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=5)),
{'in': 80}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=6)),
{'in': 70}),
]
coll = Collection(range_list)
# canned series objects
rts = TimeSeries(dict(name='collection', collection=coll))
new_rts = rts.fill(field_spec='in')
self.assertEqual(new_rts.at(1).get('in'), 0)
self.assertEqual(new_rts.at(2).get('in'), 0)
# indexed events
index_list = [
IndexedEvent('1d-12355', {'value': 42}),
IndexedEvent('1d-12356', {'value': None}),
IndexedEvent('1d-12357', {'value': None}),
IndexedEvent('1d-12358', {'value': 52}),
IndexedEvent('1d-12359', {'value': 55}),
IndexedEvent('1d-12360', {'value': 58}),
]
coll = Collection(index_list)
its = TimeSeries(dict(name='collection', collection=coll))
new_its = its.fill()
self.assertEqual(new_its.at(1).get(), 0)
self.assertEqual(new_its.at(2).get(), 0)
def test_at_key_and_dedup(self):
"""test Collection.at_key() and dedup()"""
# events
coll = Collection(EVENT_LIST_DUP)
key_time = dt_from_ms(1429673460000)
find = coll.at_key(key_time)
self.assertEqual(len(find), 2)
self.assertEqual(find[0].get('in'), 3)
self.assertEqual(find[1].get('in'), 4)
ddcoll = coll.dedup()
self.assertEqual(ddcoll.size(), 3)
self.assertEqual(ddcoll.at(1).get('in'), 4) # the second dup event
# indexed events
coll = Collection(IDX_EVENT_DUP)
find = coll.at_key('1d-12355')
self.assertEqual(len(find), 2)
self.assertEqual(find[0].get('value'), 43)
self.assertEqual(find[1].get('value'), 44)
ddcoll = coll.dedup()
self.assertEqual(ddcoll.size(), 3)
self.assertEqual(ddcoll.at(1).get('value'), 44) # the second dup event
# time range events
test_end_ts = aware_utcnow()
test_begin_ts = test_end_ts - datetime.timedelta(hours=12)
test_end_ms = ms_from_dt(test_end_ts)
test_begin_ms = ms_from_dt(test_begin_ts)
dup_tre = [
TimeRangeEvent((test_begin_ms, test_end_ms), 11),
TimeRangeEvent((test_begin_ms + 60000, test_end_ms + 60000), 12),
TimeRangeEvent((test_begin_ms + 60000, test_end_ms + 60000), 13),
TimeRangeEvent((test_begin_ms + 120000, test_end_ms + 120000), 14),
]
coll = Collection(dup_tre)
search = TimeRange(test_begin_ms + 60000, test_end_ms + 60000)
find = coll.at_key(search)
self.assertEqual(len(find), 2)
self.assertEqual(find[0].get('value'), 12)
self.assertEqual(find[1].get('value'), 13)
ddcoll = coll.dedup()
self.assertEqual(ddcoll.size(), 3)
self.assertEqual(ddcoll.at(1).get('value'), 13) # the second dup event
def test_equality_methods(self):
"""test equal/same static methods."""
self.assertTrue(
Collection.equal(self._canned_collection, self._canned_collection))
self.assertTrue(
Collection.same(self._canned_collection, self._canned_collection))
self.assertFalse(
Collection.equal(self._canned_collection, Collection(EVENT_LIST)))
self.assertTrue(
Collection.same(self._canned_collection, Collection(EVENT_LIST)))
copy_ctor = Collection(self._canned_collection)
self.assertTrue(Collection.equal(self._canned_collection, copy_ctor))
self.assertFalse(copy_ctor is self._canned_collection)
def test_list_as_map(self):
"""test collection.list_as_map()"""
coll = Collection(EVENT_LIST_DUP)
cmap = coll.event_list_as_map()
self.assertEqual(len(cmap), 3)
self.assertEqual(len(cmap.get(1429673400000)), 1)
self.assertEqual(len(cmap.get(1429673460000)), 2) # dups
def test_bad_args(self):
"""pass in bad values"""
with warnings.catch_warnings(record=True) as wrn:
bad_col = Collection(dict())
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, CollectionWarning))
self.assertEqual(bad_col.size(), 0)
with self.assertRaises(CollectionException):
self._canned_collection.set_events(dict())
def setUp(self):
"""setup."""
# canned collection
self._canned_collection = Collection(EVENT_LIST)
# canned series objects
self._canned_event_series = TimeSeries(
dict(name='collection', collection=self._canned_collection))
self._canned_wire_series = TimeSeries(DATA)
# canned index
self._canned_index_series = TimeSeries(INDEXED_DATA)
def setUp(self):
"""set up for all tests."""
# canned collection
self._canned_collection = Collection(EVENT_LIST)
# canned series objects
self._canned_event_series = TimeSeries(
dict(name='collection', collection=self._canned_collection))
global RESULTS
RESULTS = None
def test_sort_by_time(self):
"""test Collection.sort_by_time()"""
reversed_events = [EVENT_LIST[2], EVENT_LIST[1], EVENT_LIST[0]]
bad_order = Collection(reversed_events)
self.assertNotEqual(bad_order.event_list_as_list(), EVENT_LIST)
self.assertFalse(bad_order.is_chronological())
good_order = bad_order.sort_by_time()
self.assertEqual(good_order.event_list_as_list(), EVENT_LIST)
self.assertTrue(good_order.is_chronological())
def test_equality_methods(self):
"""test equal/same static methods."""
self.assertTrue(
Collection.equal(self._canned_collection, self._canned_collection))
self.assertTrue(
Collection.same(self._canned_collection, self._canned_collection))
self.assertFalse(
Collection.equal(self._canned_collection, Collection(EVENT_LIST)))
self.assertTrue(
Collection.same(self._canned_collection, Collection(EVENT_LIST)))
copy_ctor = Collection(self._canned_collection)
self.assertTrue(Collection.equal(self._canned_collection, copy_ctor))
self.assertFalse(copy_ctor is self._canned_collection)
def test_accessor_methods(self):
"""test various access methods. Mostly for coverage."""
col = self._canned_collection
# basic accessors
self.assertEqual(col.to_json(), EVENT_LIST)
self.assertEqual(len(json.loads(col.to_string())), 3)
# test at() - corollary to array index
self.assertTrue(Event.same(col.at(2), EVENT_LIST[2]))
# overshoot
with self.assertRaises(CollectionException):
col.at(5)
# test at_time()
# get timestamp of second event and add some time to it
ref_dtime = EVENT_LIST[1].timestamp() + datetime.timedelta(seconds=3)
self.assertTrue(Event.same(col.at_time(ref_dtime), EVENT_LIST[1]))
# overshoot the end of the list for coverage
ref_dtime = EVENT_LIST[2].timestamp() + datetime.timedelta(seconds=3)
self.assertTrue(Event.same(col.at_time(ref_dtime), EVENT_LIST[2]))
# make sure returning index 0 works
ref_dtime = EVENT_LIST[0].timestamp() + datetime.timedelta(seconds=3)
self.assertTrue(Event.same(col.at_time(ref_dtime), EVENT_LIST[0]))
# hit dead on for coverage
self.assertEqual(col.at_time(EVENT_LIST[1].timestamp()).get('in'), 3)
# empty collection for coverage
empty_coll = Collection(col, copy_events=False)
self.assertIsNone(empty_coll.at_time(ref_dtime))
# at_first() and at_last()
self.assertTrue(Event.same(col.at_first(), EVENT_LIST[0]))
self.assertTrue(Event.same(col.at_last(), EVENT_LIST[2]))
# get the raw event list
self.assertTrue(is_pvector(col.event_list()))
self.assertTrue(isinstance(col.event_list_as_list(), list))
def test_aggregation_filtering(self):
"""Test the new filtering methods for cleaning stuff."""
elist = [
Event(1429673400000, {
'in': 1,
'out': 1
}),
Event(1429673460000, {
'in': 2,
'out': 5
}),
Event(1429673520000, {
'in': 3,
'out': None
}),
]
coll = Collection(elist)
self.assertEqual(coll.aggregate(Functions.sum(), 'in'), 6)
self.assertEqual(
coll.aggregate(Functions.sum(Filters.propagate_missing), 'in'), 6)
self.assertEqual(
coll.aggregate(Functions.sum(Filters.propagate_missing), 'out'),
None)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.ignore_missing), 'in'), 2)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.ignore_missing), 'out'), 3)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.zero_missing), 'in'), 2)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.zero_missing), 'out'), 2)
def test_create_collection(self):
"""test collection creation and methods related to internal payload."""
# event list
col_1 = Collection(EVENT_LIST)
self.assertEqual(col_1.size(), 3)
self.assertEqual(col_1.type(), Event)
self.assertEqual(col_1.size_valid('in'), 3)
# copy ctor
col_2 = Collection(col_1)
self.assertEqual(col_2.size(), 3)
self.assertEqual(col_2.type(), Event)
self.assertEqual(col_2.size_valid('in'), 3)
# copy ctor - no event copy
col_3 = Collection(col_2, copy_events=False)
self.assertEqual(col_3.size(), 0)
self.assertEqual(col_3.size_valid('in'), 0)
# pass in an immutable - use a pre- _check()'ed one
col_4 = Collection(col_1._event_list) # pylint: disable=protected-access
self.assertEqual(col_4.size(), 3)
self.assertEqual(col_4.size_valid('in'), 3)
# other event types for coverage
ie1 = IndexedEvent('1d-12355', {'value': 42})
ie2 = IndexedEvent('1d-12356', {'value': 4242})
col_5 = Collection([ie1, ie2])
self.assertEqual(col_5.size(), 2)
tre = TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(hours=24)),
{'in': 100})
col_6 = Collection([tre])
self.assertEqual(col_6.size(), 1)