def test_sum_and_find_max(self):
"""sum elements, find max get result out."""
def cback(event):
"""catch the return"""
self.assertEqual(event.get('max_total'), 117)
timeseries = TimeSeries(IN_OUT_DATA)
(
Pipeline()
.from_source(timeseries)
.emit_on('flush')
.collapse(['in', 'out'], 'total', Functions.sum())
.aggregate(dict(max_total=dict(total=Functions.max())))
.to(EventOut, cback)
)
# Same test but as an event list
elist = (
Pipeline()
.from_source(timeseries)
.emit_on('flush')
.collapse(['in', 'out'], 'total', Functions.sum())
.aggregate(dict(max_total=dict(total=Functions.max())))
.to_event_list()
)
self.assertEqual(len(elist), 1)
self.assertEqual(elist[0].get('max_total'), 117)
def test_event_collapse(self):
"""test collapse()"""
ev1 = self._create_event(self.aware_ts, {'a': 5, 'b': 6, 'c': 7})
ev2 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=True)
self.assertEqual(len(list(ev2.data().keys())), 4)
self.assertEqual(ev2.get('a_to_c'), 12)
ev3 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=False)
self.assertEqual(len(list(ev3.data().keys())), 1)
self.assertEqual(ev3.get('a_to_c'), 12)
def test_event_collapse(self):
"""test collapse()"""
ev1 = self._create_event(self.aware_ts, {'a': 5, 'b': 6, 'c': 7})
ev2 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=True)
self.assertEqual(len(list(ev2.data().keys())), 4)
self.assertEqual(ev2.get('a_to_c'), 12)
ev3 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=False)
self.assertEqual(len(list(ev3.data().keys())), 1)
self.assertEqual(ev3.get('a_to_c'), 12)
def test_various_bad_args(self):
"""ensure proper exceptions are being raised."""
ser1 = TimeSeries(DATA)
with self.assertRaises(CollectionException):
ser1.aggregate(dict())
with self.assertRaises(CollectionException):
ser1.aggregate(Functions.sum(), dict())
def test_collection_collapse(self):
"""test Collection.collaps()"""
col = self._canned_collection
collapsed_col = col.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
self.assertEqual(collapsed_col.size(), 3)
for i in collapsed_col.events():
self.assertEqual(len(list(i.data().keys())), 3)
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_ts_collapse(self):
"""
Test TimeSeries.collapse()
"""
ces = self._canned_event_series
collapsed_ces = ces.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
for i in collapsed_ces.events():
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_various_bad_args(self):
"""ensure proper exceptions are being raised."""
ser1 = TimeSeries(DATA)
with self.assertRaises(CollectionException):
ser1.aggregate(dict())
with self.assertRaises(CollectionException):
ser1.aggregate(Functions.sum(), dict())
def test_underlying_methods(self):
"""basically aliases for underlying collection methods."""
self.assertEqual(self._canned_event_series.count(), len(EVENT_LIST))
tser = self._canned_event_series
self.assertEqual(tser.sum('in'), 9)
self.assertEqual(tser.avg('out'), 4)
self.assertEqual(tser.mean('out'), 4)
self.assertEqual(tser.min('in'), 1)
self.assertEqual(tser.max('in'), 5)
self.assertEqual(tser.median('out'), 4)
self.assertEqual(tser.stdev('out'), 1.632993161855452)
# redundant, but for coverage
self.assertEqual(tser.aggregate(Functions.sum(), 'in'), 9)
self.assertEqual(tser.aggregate(Functions.sum(), ('in',)), 9)
ser1 = TimeSeries(DATA)
self.assertEqual(ser1.aggregate(Functions.sum()), 189)
def test_underlying_methods(self):
"""basically aliases for underlying collection methods."""
self.assertEqual(self._canned_event_series.count(), len(EVENT_LIST))
tser = self._canned_event_series
self.assertEqual(tser.sum('in'), 9)
self.assertEqual(tser.avg('out'), 4)
self.assertEqual(tser.mean('out'), 4)
self.assertEqual(tser.min('in'), 1)
self.assertEqual(tser.max('in'), 5)
self.assertEqual(tser.median('out'), 4)
self.assertEqual(tser.stdev('out'), 1.632993161855452)
# redundant, but for coverage
self.assertEqual(tser.aggregate(Functions.sum(), 'in'), 9)
self.assertEqual(tser.aggregate(Functions.sum(), ('in', )), 9)
ser1 = TimeSeries(DATA)
self.assertEqual(ser1.aggregate(Functions.sum()), 189)
def test_ts_collapse(self):
"""
Test TimeSeries.collapse()
"""
ces = self._canned_event_series
collapsed_ces = ces.collapse(['in', 'out'], 'in_out_sum',
Functions.sum())
for i in collapsed_ces.events():
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_collection_collapse(self):
"""test Collection.collaps()"""
col = self._canned_collection
collapsed_col = col.collapse(['in', 'out'], 'in_out_sum',
Functions.sum())
self.assertEqual(collapsed_col.size(), 3)
for i in collapsed_col.events():
self.assertEqual(len(list(i.data().keys())), 3)
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_simple_collapse(self):
"""collapse a subset of columns."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (Pipeline().from_source(timeseries).collapse(
['in', 'out'], 'in_out_sum',
Functions.sum()).emit_on('flush').to_keyed_collections())
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
def test_sum_and_find_max(self):
"""sum elements, find max get result out."""
def cback(event):
"""catch the return"""
self.assertEqual(event.get('max_total'), 117)
timeseries = TimeSeries(IN_OUT_DATA)
(Pipeline().from_source(timeseries).emit_on('flush').collapse(
['in', 'out'], 'total', Functions.sum()).aggregate(
dict(max_total=dict(total=Functions.max()))).to(
EventOut, cback))
# Same test but as an event list
elist = (Pipeline().from_source(timeseries).emit_on('flush').collapse(
['in', 'out'], 'total', Functions.sum()).aggregate(
dict(max_total=dict(total=Functions.max()))).to_event_list())
self.assertEqual(len(elist), 1)
self.assertEqual(elist[0].get('max_total'), 117)
def test_aggregation_filtering(self):
"""Test the new filtering methods for cleaning stuff."""
elist = [
Event(1429673400000, {'in': 1, 'out': 1}),
Event(1429673460000, {'in': 2, 'out': 5}),
Event(1429673520000, {'in': 3, 'out': None}),
]
coll = Collection(elist)
self.assertEqual(coll.aggregate(Functions.sum(), 'in'), 6)
self.assertEqual(coll.aggregate(Functions.sum(Filters.propagate_missing), 'in'), 6)
self.assertEqual(coll.aggregate(Functions.sum(Filters.propagate_missing), 'out'), None)
self.assertEqual(coll.aggregate(Functions.avg(Filters.ignore_missing), 'in'), 2)
self.assertEqual(coll.aggregate(Functions.avg(Filters.ignore_missing), 'out'), 3)
self.assertEqual(coll.aggregate(Functions.avg(Filters.zero_missing), 'in'), 2)
self.assertEqual(coll.aggregate(Functions.avg(Filters.zero_missing), 'out'), 2)
def test_aggregation_filtering(self):
"""Test the new filtering methods for cleaning stuff."""
elist = [
Event(1429673400000, {
'in': 1,
'out': 1
}),
Event(1429673460000, {
'in': 2,
'out': 5
}),
Event(1429673520000, {
'in': 3,
'out': None
}),
]
coll = Collection(elist)
self.assertEqual(coll.aggregate(Functions.sum(), 'in'), 6)
self.assertEqual(
coll.aggregate(Functions.sum(Filters.propagate_missing), 'in'), 6)
self.assertEqual(
coll.aggregate(Functions.sum(Filters.propagate_missing), 'out'),
None)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.ignore_missing), 'in'), 2)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.ignore_missing), 'out'), 3)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.zero_missing), 'in'), 2)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.zero_missing), 'out'), 2)
def test_simple_collapse(self):
"""collapse a subset of columns."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
.emit_on('flush')
.to_keyed_collections()
)
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
def test_multiple_collapse_chains(self):
"""multiple collapsers."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (Pipeline().from_source(timeseries).collapse(
['in', 'out'], 'in_out_sum', Functions.sum()).collapse(
['in', 'out'], 'in_out_max',
Functions.max()).emit_on('flush').to_keyed_collections())
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
self.assertEqual(kcol.get('all').at(0).get('in_out_max'), 80)
self.assertEqual(kcol.get('all').at(1).get('in_out_max'), 88)
self.assertEqual(kcol.get('all').at(2).get('in_out_max'), 56)
def test_multiple_collapse_chains(self):
"""multiple collapsers."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
.collapse(['in', 'out'], 'in_out_max', Functions.max())
.emit_on('flush')
.to_keyed_collections()
)
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
self.assertEqual(kcol.get('all').at(0).get('in_out_max'), 80)
self.assertEqual(kcol.get('all').at(1).get('in_out_max'), 88)
self.assertEqual(kcol.get('all').at(2).get('in_out_max'), 56)