def test_aggregate_deep_path(self):
"""Make sure that the aggregator will work on a deep path."""
elist = (Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('flush').aggregate(
dict(out_max={'direction.out': Functions.max()
})).to_event_list())
self.assertEqual(elist[0].get('out_max'), 4)
# Make sure it works with the the non-string version to aggregate
# multiple columns
elist = (Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('flush').aggregate({
'in_max': {
'direction.in': Functions.max()
},
'out_max': {
'direction.out': Functions.max()
},
}).to_event_list())
self.assertEqual(elist[0].get('out_max'), 4)
self.assertEqual(elist[0].get('in_max'), 8)
def test_sum_and_find_max(self):
"""sum elements, find max get result out."""
def cback(event):
"""catch the return"""
self.assertEqual(event.get('max_total'), 117)
timeseries = TimeSeries(IN_OUT_DATA)
(
Pipeline()
.from_source(timeseries)
.emit_on('flush')
.collapse(['in', 'out'], 'total', Functions.sum())
.aggregate(dict(max_total=dict(total=Functions.max())))
.to(EventOut, cback)
)
# Same test but as an event list
elist = (
Pipeline()
.from_source(timeseries)
.emit_on('flush')
.collapse(['in', 'out'], 'total', Functions.sum())
.aggregate(dict(max_total=dict(total=Functions.max())))
.to_event_list()
)
self.assertEqual(len(elist), 1)
self.assertEqual(elist[0].get('max_total'), 117)
def test_fixed_window(self):
"""Test fixed window rollup"""
timeseries = TimeSeries(SEPT_2014_DATA)
daily_avg = timeseries.fixed_window_rollup(
'1d', dict(value=dict(value=Functions.avg())))
self.assertEqual(daily_avg.size(), 5)
self.assertEqual(daily_avg.at(0).value(), 46.875)
self.assertEqual(daily_avg.at(2).value(), 54.083333333333336)
self.assertEqual(daily_avg.at(4).value(), 51.85)
# not really a rollup, each data point will create one
# aggregation index.
timeseries = TimeSeries(SEPT_2014_DATA)
hourly_avg = timeseries.hourly_rollup(
dict(value=dict(value=Functions.avg())))
self.assertEqual(hourly_avg.size(), len(SEPT_2014_DATA.get('points')))
self.assertEqual(hourly_avg.at(0).value(), 80.0)
self.assertEqual(hourly_avg.at(2).value(), 52.0)
self.assertEqual(hourly_avg.at(4).value(), 26.0)
def test_aggregate_deep_path(self):
"""Make sure that the aggregator will work on a deep path."""
elist = (
Pipeline()
.from_source(TimeSeries(dict(name='events', events=DEEP_EVENT_LIST)))
.emit_on('flush')
.aggregate(dict(out_max={'direction.out': Functions.max()}))
.to_event_list()
)
self.assertEqual(elist[0].get('out_max'), 4)
# Make sure it works with the the non-string version to aggregate
# multiple columns
elist = (
Pipeline()
.from_source(TimeSeries(dict(name='events', events=DEEP_EVENT_LIST)))
.emit_on('flush')
.aggregate(
{
'in_max': {'direction.in': Functions.max()},
'out_max': {'direction.out': Functions.max()},
}
)
.to_event_list()
)
self.assertEqual(elist[0].get('out_max'), 4)
self.assertEqual(elist[0].get('in_max'), 8)
def test_non_fixed_rollups(self):
"""Work the calendar rollup logic / utc / etc."""
timeseries = TimeSeries(SEPT_2014_DATA)
# just silence the warnings, not do anything with them.
with warnings.catch_warnings(record=True):
daily_avg = timeseries.daily_rollup(dict(value=dict(value=Functions.avg())))
ts_1 = SEPT_2014_DATA.get('points')[0][0]
self.assertEqual(
Index.get_daily_index_string(dt_from_ms(ts_1), utc=False),
daily_avg.at(0).index().to_string()
)
monthly_avg = timeseries.monthly_rollup(dict(value=dict(value=Functions.avg())))
self.assertEqual(
Index.get_monthly_index_string(dt_from_ms(ts_1), utc=False),
monthly_avg.at(0).index().to_string()
)
yearly_avg = timeseries.yearly_rollup(dict(value=dict(value=Functions.avg())))
self.assertEqual(
Index.get_yearly_index_string(dt_from_ms(ts_1), utc=False),
yearly_avg.at(0).index().to_string()
)
def test_non_fixed_rollups(self):
"""Work the calendar rollup logic / utc / etc."""
timeseries = TimeSeries(SEPT_2014_DATA)
# just silence the warnings, not do anything with them.
with warnings.catch_warnings(record=True):
daily_avg = timeseries.daily_rollup(
dict(value=dict(value=Functions.avg())))
ts_1 = SEPT_2014_DATA.get('points')[0][0]
self.assertEqual(
Index.get_daily_index_string(dt_from_ms(ts_1), utc=False),
daily_avg.at(0).index().to_string())
monthly_avg = timeseries.monthly_rollup(
dict(value=dict(value=Functions.avg())))
self.assertEqual(
Index.get_monthly_index_string(dt_from_ms(ts_1), utc=False),
monthly_avg.at(0).index().to_string())
yearly_avg = timeseries.yearly_rollup(
dict(value=dict(value=Functions.avg())))
self.assertEqual(
Index.get_yearly_index_string(dt_from_ms(ts_1), utc=False),
yearly_avg.at(0).index().to_string())
def test_fixed_window(self):
"""Test fixed window rollup"""
timeseries = TimeSeries(SEPT_2014_DATA)
daily_avg = timeseries.fixed_window_rollup(
'1d',
dict(value=dict(value=Functions.avg())))
self.assertEqual(daily_avg.size(), 5)
self.assertEqual(daily_avg.at(0).value(), 46.875)
self.assertEqual(daily_avg.at(2).value(), 54.083333333333336)
self.assertEqual(daily_avg.at(4).value(), 51.85)
# not really a rollup, each data point will create one
# aggregation index.
timeseries = TimeSeries(SEPT_2014_DATA)
hourly_avg = timeseries.hourly_rollup(dict(value=dict(value=Functions.avg())))
self.assertEqual(hourly_avg.size(), len(SEPT_2014_DATA.get('points')))
self.assertEqual(hourly_avg.at(0).value(), 80.0)
self.assertEqual(hourly_avg.at(2).value(), 52.0)
self.assertEqual(hourly_avg.at(4).value(), 26.0)
def test_aggregate_and_conversion(self):
"""Aggregate/average and convert to TimeRangeEvent."""
events_in = [
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=1, minute=57)),
{'in': 3, 'out': 1}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=1, minute=58)),
{'in': 9, 'out': 2}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=1, minute=59)),
{'in': 6, 'out': 6}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=2, minute=0)),
{'in': 4, 'out': 7}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=2, minute=1)),
{'in': 5, 'out': 9}
),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(ms_from_dt(event.timestamp()))] = event
uin = Stream()
(
Pipeline()
.from_source(uin)
.window_by('1h')
.emit_on('eachEvent')
.aggregate(
{
'in_avg': {'in': Functions.avg()},
'out_avg': {'out': Functions.avg()},
}
)
.as_time_range_events(dict(alignment='lag'))
.to(EventOut, cback)
)
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1426294800000').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1426294800000').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1426298400000').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1426298400000').get('out_avg'), 8)
def test_windowed_average(self):
"""aggregate events into by windowed avg."""
events_in = [
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=57)),
{'in': 3, 'out': 1}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=58)),
{'in': 9, 'out': 2}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=59)),
{'in': 6, 'out': 6}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=0)),
{'in': 4, 'out': 7}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=1)),
{'in': 5, 'out': 9}
),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(event.index())] = event
uin = Stream()
(
Pipeline()
.from_source(uin)
.window_by('1h')
.emit_on('eachEvent')
.aggregate(
{
'in_avg': {'in': Functions.avg()},
'out_avg': {'out': Functions.avg()}
}
)
.to(EventOut, cback)
)
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1h-396200').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1h-396200').get('out_avg'), 8)
def test_event_collapse(self):
"""test collapse()"""
ev1 = self._create_event(self.aware_ts, {'a': 5, 'b': 6, 'c': 7})
ev2 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=True)
self.assertEqual(len(list(ev2.data().keys())), 4)
self.assertEqual(ev2.get('a_to_c'), 12)
ev3 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=False)
self.assertEqual(len(list(ev3.data().keys())), 1)
self.assertEqual(ev3.get('a_to_c'), 12)
def test_event_collapse(self):
"""test collapse()"""
ev1 = self._create_event(self.aware_ts, {'a': 5, 'b': 6, 'c': 7})
ev2 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=True)
self.assertEqual(len(list(ev2.data().keys())), 4)
self.assertEqual(ev2.get('a_to_c'), 12)
ev3 = ev1.collapse(['a', 'c'], 'a_to_c', Functions.sum(), append=False)
self.assertEqual(len(list(ev3.data().keys())), 1)
self.assertEqual(ev3.get('a_to_c'), 12)
def test_sum_events_with_combine(self):
"""test summing multiple events together via combine on the back end."""
# combine them all
events = [
self._create_event(self.aware_ts, {'a': 5, 'b': 6, 'c': 7}),
self._create_event(self.aware_ts, {'a': 2, 'b': 3, 'c': 4}),
self._create_event(self.aware_ts, {'a': 1, 'b': 2, 'c': 3}),
]
result = Event.sum(events)
self.assertEqual(result[0].get('a'), 8)
self.assertEqual(result[0].get('b'), 11)
self.assertEqual(result[0].get('c'), 14)
# combine single field
result = Event.sum(events, 'a')
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertIsNone(result[0].get('c'))
# grab multiple fields
result = Event.sum(events, ['a', 'c'])
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertEqual(result[0].get('c'), 14)
# average
result = Event.avg(
events + [self._create_event(self.aware_ts, {'a': 1, 'b': 1, 'c': 2})],
'c')
self.assertEqual(result[0].get('c'), 4)
# bad arg
self.assertEqual(Event.sum([]), [])
self.assertEqual(Event.avg([]), [])
# work the extra reducer functions in Functions module
result = Event.combine(events, 'c', Functions.max())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.min())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.count())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.first())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.last())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.difference())
self.assertEqual(result[0].get('c'), 4)
self.assertIsNone(Functions.first()([]))
self.assertIsNone(Functions.last()([]))
def test_bad_args(self):
"""Trigger exceptions and warnings, etc."""
uin = Stream()
with warnings.catch_warnings(record=True) as wrn:
Pipeline().from_source(uin).window_by('1h', utc=False)
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, PipelineWarning))
# bad arg
with self.assertRaises(PipelineException):
Pipeline().from_source(dict())
# no source
with self.assertRaises(PipelineException):
Pipeline().to_keyed_collections()
# can't iterate on unbounded source
with self.assertRaises(PipelineIOException):
list(uin.events())
# bad emit on type
with self.assertRaises(PipelineIOException):
(
Pipeline()
.from_source(TimeSeries(dict(name='events', events=DEEP_EVENT_LIST)))
.emit_on('BOGUS')
.aggregate(
{'max_in': {'direction.in': Functions.max()}}
)
.to_event_list()
)
def test_multiple_collapse_chains(self):
"""multiple collapsers."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (Pipeline().from_source(timeseries).collapse(
['in', 'out'], 'in_out_sum', Functions.sum()).collapse(
['in', 'out'], 'in_out_max',
Functions.max()).emit_on('flush').to_keyed_collections())
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
self.assertEqual(kcol.get('all').at(0).get('in_out_max'), 80)
self.assertEqual(kcol.get('all').at(1).get('in_out_max'), 88)
self.assertEqual(kcol.get('all').at(2).get('in_out_max'), 56)
def test_bad_args(self):
"""Trigger exceptions and warnings, etc."""
uin = Stream()
with warnings.catch_warnings(record=True) as wrn:
Pipeline().from_source(uin).window_by('1h', utc=False)
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, PipelineWarning))
# bad arg
with self.assertRaises(PipelineException):
Pipeline().from_source(dict())
# no source
with self.assertRaises(PipelineException):
Pipeline().to_keyed_collections()
# can't iterate on unbounded source
with self.assertRaises(PipelineIOException):
list(uin.events())
# bad emit on type
with self.assertRaises(PipelineIOException):
(Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('BOGUS').aggregate({
'max_in': {
'direction.in': Functions.max()
}
}).to_event_list())
def test_aggregation_filtering(self):
"""test the filtering modifers to the agg functions."""
event_objects = [
Event(1429673400000, {'in': 1, 'out': 2}),
Event(1429673460000, {'in': 3, 'out': None}),
Event(1429673520000, {'in': 5, 'out': 6}),
]
series = TimeSeries(dict(name='events', events=event_objects))
self.assertEqual(series.sum('out', Filters.ignore_missing), 8)
self.assertEqual(series.avg('out', Filters.ignore_missing), 4)
self.assertEqual(series.min('out', Filters.zero_missing), 0)
self.assertEqual(series.max('out', Filters.propagate_missing), None)
self.assertEqual(series.mean('out', Filters.ignore_missing), 4)
self.assertEqual(series.median('out', Filters.zero_missing), 2)
self.assertEqual(series.stdev('out', Filters.zero_missing), 2.494438257849294)
avg_f = Functions.avg(Filters.none_if_empty)
self.assertIsNone(avg_f([]))
def bad_filtering_function(): # pylint: disable=missing-docstring
pass
with self.assertRaises(FilterException):
series.sum('out', bad_filtering_function)
def test_various_bad_args(self):
"""ensure proper exceptions are being raised."""
ser1 = TimeSeries(DATA)
with self.assertRaises(CollectionException):
ser1.aggregate(dict())
with self.assertRaises(CollectionException):
ser1.aggregate(Functions.sum(), dict())
def test_ts_collapse(self):
"""
Test TimeSeries.collapse()
"""
ces = self._canned_event_series
collapsed_ces = ces.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
for i in collapsed_ces.events():
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_underlying_methods(self):
"""basically aliases for underlying collection methods."""
self.assertEqual(self._canned_event_series.count(), len(EVENT_LIST))
tser = self._canned_event_series
self.assertEqual(tser.sum('in'), 9)
self.assertEqual(tser.avg('out'), 4)
self.assertEqual(tser.mean('out'), 4)
self.assertEqual(tser.min('in'), 1)
self.assertEqual(tser.max('in'), 5)
self.assertEqual(tser.median('out'), 4)
self.assertEqual(tser.stdev('out'), 1.632993161855452)
# redundant, but for coverage
self.assertEqual(tser.aggregate(Functions.sum(), 'in'), 9)
self.assertEqual(tser.aggregate(Functions.sum(), ('in',)), 9)
ser1 = TimeSeries(DATA)
self.assertEqual(ser1.aggregate(Functions.sum()), 189)
def test_various_bad_args(self):
"""ensure proper exceptions are being raised."""
ser1 = TimeSeries(DATA)
with self.assertRaises(CollectionException):
ser1.aggregate(dict())
with self.assertRaises(CollectionException):
ser1.aggregate(Functions.sum(), dict())
def test_event_map_function_arg_and_reduce(self): # pylint: disable=invalid-name
"""Test Event.map() with a custom function and Event.reduce()"""
def map_sum(event): # pylint: disable=missing-docstring
# return 'sum', event.get('in') + event.get('out')
return dict(sum=event.get('in') + event.get('out'))
result = Event.map(self._get_event_series(), map_sum)
self.assertEqual(set(result), set({'sum': [13, 17, 21, 26]}))
res = Event.reduce(result, Functions.avg())
self.assertEqual(set(res), set({'sum': 19.25}))
def test_collection_collapse(self):
"""test Collection.collaps()"""
col = self._canned_collection
collapsed_col = col.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
self.assertEqual(collapsed_col.size(), 3)
for i in collapsed_col.events():
self.assertEqual(len(list(i.data().keys())), 3)
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_underlying_methods(self):
"""basically aliases for underlying collection methods."""
self.assertEqual(self._canned_event_series.count(), len(EVENT_LIST))
tser = self._canned_event_series
self.assertEqual(tser.sum('in'), 9)
self.assertEqual(tser.avg('out'), 4)
self.assertEqual(tser.mean('out'), 4)
self.assertEqual(tser.min('in'), 1)
self.assertEqual(tser.max('in'), 5)
self.assertEqual(tser.median('out'), 4)
self.assertEqual(tser.stdev('out'), 1.632993161855452)
# redundant, but for coverage
self.assertEqual(tser.aggregate(Functions.sum(), 'in'), 9)
self.assertEqual(tser.aggregate(Functions.sum(), ('in', )), 9)
ser1 = TimeSeries(DATA)
self.assertEqual(ser1.aggregate(Functions.sum()), 189)
def test_simple_collapse(self):
"""collapse a subset of columns."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (Pipeline().from_source(timeseries).collapse(
['in', 'out'], 'in_out_sum',
Functions.sum()).emit_on('flush').to_keyed_collections())
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
def test_event_map_function_arg_and_reduce(self): # pylint: disable=invalid-name
"""Test Event.map() with a custom function and Event.reduce()"""
def map_sum(event): # pylint: disable=missing-docstring
# return 'sum', event.get('in') + event.get('out')
return dict(sum=event.get('in') + event.get('out'))
result = Event.map(self._get_event_series(), map_sum)
self.assertEqual(set(result), set({'sum': [13, 17, 21, 26]}))
res = Event.reduce(result, Functions.avg())
self.assertEqual(set(res), set({'sum': 19.25}))
def test_ts_collapse(self):
"""
Test TimeSeries.collapse()
"""
ces = self._canned_event_series
collapsed_ces = ces.collapse(['in', 'out'], 'in_out_sum',
Functions.sum())
for i in collapsed_ces.events():
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_collection_collapse(self):
"""test Collection.collaps()"""
col = self._canned_collection
collapsed_col = col.collapse(['in', 'out'], 'in_out_sum',
Functions.sum())
self.assertEqual(collapsed_col.size(), 3)
for i in collapsed_col.events():
self.assertEqual(len(list(i.data().keys())), 3)
self.assertEqual(i.get('in') + i.get('out'), i.get('in_out_sum'))
def test_multiple_collapse_chains(self):
"""multiple collapsers."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
.collapse(['in', 'out'], 'in_out_max', Functions.max())
.emit_on('flush')
.to_keyed_collections()
)
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
self.assertEqual(kcol.get('all').at(0).get('in_out_max'), 80)
self.assertEqual(kcol.get('all').at(1).get('in_out_max'), 88)
self.assertEqual(kcol.get('all').at(2).get('in_out_max'), 56)
def test_sum_and_find_max(self):
"""sum elements, find max get result out."""
def cback(event):
"""catch the return"""
self.assertEqual(event.get('max_total'), 117)
timeseries = TimeSeries(IN_OUT_DATA)
(Pipeline().from_source(timeseries).emit_on('flush').collapse(
['in', 'out'], 'total', Functions.sum()).aggregate(
dict(max_total=dict(total=Functions.max()))).to(
EventOut, cback))
# Same test but as an event list
elist = (Pipeline().from_source(timeseries).emit_on('flush').collapse(
['in', 'out'], 'total', Functions.sum()).aggregate(
dict(max_total=dict(total=Functions.max()))).to_event_list())
self.assertEqual(len(elist), 1)
self.assertEqual(elist[0].get('max_total'), 117)
def test_simple_collapse(self):
"""collapse a subset of columns."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
.emit_on('flush')
.to_keyed_collections()
)
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
def test_aggregation_filtering(self):
"""Test the new filtering methods for cleaning stuff."""
elist = [
Event(1429673400000, {'in': 1, 'out': 1}),
Event(1429673460000, {'in': 2, 'out': 5}),
Event(1429673520000, {'in': 3, 'out': None}),
]
coll = Collection(elist)
self.assertEqual(coll.aggregate(Functions.sum(), 'in'), 6)
self.assertEqual(coll.aggregate(Functions.sum(Filters.propagate_missing), 'in'), 6)
self.assertEqual(coll.aggregate(Functions.sum(Filters.propagate_missing), 'out'), None)
self.assertEqual(coll.aggregate(Functions.avg(Filters.ignore_missing), 'in'), 2)
self.assertEqual(coll.aggregate(Functions.avg(Filters.ignore_missing), 'out'), 3)
self.assertEqual(coll.aggregate(Functions.avg(Filters.zero_missing), 'in'), 2)
self.assertEqual(coll.aggregate(Functions.avg(Filters.zero_missing), 'out'), 2)
def test_aggregation_filtering(self):
"""test the filtering modifers to the agg functions."""
event_objects = [
Event(1429673400000, {
'in': 1,
'out': 2
}),
Event(1429673460000, {
'in': 3,
'out': None
}),
Event(1429673520000, {
'in': 5,
'out': 6
}),
]
series = TimeSeries(dict(name='events', events=event_objects))
self.assertEqual(series.sum('out', Filters.ignore_missing), 8)
self.assertEqual(series.avg('out', Filters.ignore_missing), 4)
self.assertEqual(series.min('out', Filters.zero_missing), 0)
self.assertEqual(series.max('out', Filters.propagate_missing), None)
self.assertEqual(series.mean('out', Filters.ignore_missing), 4)
self.assertEqual(series.median('out', Filters.zero_missing), 2)
self.assertEqual(series.stdev('out', Filters.zero_missing),
2.494438257849294)
avg_f = Functions.avg(Filters.none_if_empty)
self.assertIsNone(avg_f([]))
def bad_filtering_function(): # pylint: disable=missing-docstring
pass
with self.assertRaises(FilterException):
series.sum('out', bad_filtering_function)
def test_aggregation_filtering(self):
"""Test the new filtering methods for cleaning stuff."""
elist = [
Event(1429673400000, {
'in': 1,
'out': 1
}),
Event(1429673460000, {
'in': 2,
'out': 5
}),
Event(1429673520000, {
'in': 3,
'out': None
}),
]
coll = Collection(elist)
self.assertEqual(coll.aggregate(Functions.sum(), 'in'), 6)
self.assertEqual(
coll.aggregate(Functions.sum(Filters.propagate_missing), 'in'), 6)
self.assertEqual(
coll.aggregate(Functions.sum(Filters.propagate_missing), 'out'),
None)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.ignore_missing), 'in'), 2)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.ignore_missing), 'out'), 3)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.zero_missing), 'in'), 2)
self.assertEqual(
coll.aggregate(Functions.avg(Filters.zero_missing), 'out'), 2)
def test_simple_map_reduce(self):
"""test simple map/reduce."""
result = Event.map_reduce(self._get_event_series(), ['in', 'out'],
Functions.avg())
self.assertEqual(set(result), set({'in': 5.0, 'out': 14.25}))
def test_bad_processor_args(self):
"""Feed the Processors bad args."""
# neither Pipeline or copy ctor
with self.assertRaises(ProcessorException):
Aggregator(dict())
with self.assertRaises(ProcessorException):
Collapser(dict())
with self.assertRaises(ProcessorException):
Converter(dict())
with self.assertRaises(ProcessorException):
Filter(dict())
with self.assertRaises(ProcessorException):
Mapper(dict())
with self.assertRaises(ProcessorException):
Offset(dict())
with self.assertRaises(ProcessorException):
Selector(dict())
with self.assertRaises(ProcessorException):
Taker(dict())
pip = Pipeline()
# not passed a callable function
with self.assertRaises(ProcessorException):
Filter(pip)
# bad agg args
# no opts
with self.assertRaises(ProcessorException):
Aggregator(pip)
# wrong opt type
with self.assertRaises(ProcessorException):
Aggregator(
pip,
Options(
fields=list()
)
)
# bad opt keys
with self.assertRaises(ProcessorException):
Aggregator(
pip,
Options(
fields={1: 'foo'}
)
)
# bad opt value
with self.assertRaises(ProcessorException):
Aggregator(
pip,
Options(
fields={'in': 'foo'}
)
)
# stream w/no window strat
with self.assertRaises(ProcessorException):
pip2 = Pipeline(pip._d.update(dict(mode='stream'))) # pylint: disable=protected-access
Aggregator(
pip2,
Options(
fields={'in': Functions.avg()}
)
)
# bad Converter args
# no type in opts
with self.assertRaises(ProcessorException):
Converter(pip)
# bad opt type
with self.assertRaises(ProcessorException):
Converter(
pip,
Options(
type=Pipeline
)
)
# bad Mapper Args
with self.assertRaises(ProcessorException):
Mapper(dict())
with self.assertRaises(ProcessorException):
Mapper(pip)
def test_simple_map_reduce(self):
"""test simple map/reduce."""
result = Event.map_reduce(self._get_event_series(), ['in', 'out'], Functions.avg())
self.assertEqual(set(result), set({'in': 5.0, 'out': 14.25}))
def test_sum_events_with_combine(self):
"""test summing multiple events together via combine on the back end."""
# combine them all
events = [
self._create_event(self.aware_ts, {
'a': 5,
'b': 6,
'c': 7
}),
self._create_event(self.aware_ts, {
'a': 2,
'b': 3,
'c': 4
}),
self._create_event(self.aware_ts, {
'a': 1,
'b': 2,
'c': 3
}),
]
result = Event.sum(events)
self.assertEqual(result[0].get('a'), 8)
self.assertEqual(result[0].get('b'), 11)
self.assertEqual(result[0].get('c'), 14)
# combine single field
result = Event.sum(events, 'a')
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertIsNone(result[0].get('c'))
# grab multiple fields
result = Event.sum(events, ['a', 'c'])
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertEqual(result[0].get('c'), 14)
# average
result = Event.avg(
events +
[self._create_event(self.aware_ts, {
'a': 1,
'b': 1,
'c': 2
})], 'c')
self.assertEqual(result[0].get('c'), 4)
# bad arg
self.assertEqual(Event.sum([]), [])
self.assertEqual(Event.avg([]), [])
# work the extra reducer functions in Functions module
result = Event.combine(events, 'c', Functions.max())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.min())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.count())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.first())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.last())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.difference())
self.assertEqual(result[0].get('c'), 4)
self.assertIsNone(Functions.first()([]))
self.assertIsNone(Functions.last()([]))
def test_bad_processor_args(self):
"""Feed the Processors bad args."""
# neither Pipeline or copy ctor
with self.assertRaises(ProcessorException):
Aggregator(dict())
with self.assertRaises(ProcessorException):
Collapser(dict())
with self.assertRaises(ProcessorException):
Converter(dict())
with self.assertRaises(ProcessorException):
Filter(dict())
with self.assertRaises(ProcessorException):
Mapper(dict())
with self.assertRaises(ProcessorException):
Offset(dict())
with self.assertRaises(ProcessorException):
Selector(dict())
with self.assertRaises(ProcessorException):
Taker(dict())
pip = Pipeline()
# not passed a callable function
with self.assertRaises(ProcessorException):
Filter(pip)
# bad agg args
# no opts
with self.assertRaises(ProcessorException):
Aggregator(pip)
# wrong opt type
with self.assertRaises(ProcessorException):
Aggregator(pip, Options(fields=list()))
# bad opt keys
with self.assertRaises(ProcessorException):
Aggregator(pip, Options(fields={1: 'foo'}))
# bad opt value
with self.assertRaises(ProcessorException):
Aggregator(pip, Options(fields={'in': 'foo'}))
# stream w/no window strat
with self.assertRaises(ProcessorException):
pip2 = Pipeline(pip._d.update(dict(mode='stream'))) # pylint: disable=protected-access
Aggregator(pip2, Options(fields={'in': Functions.avg()}))
# bad Converter args
# no type in opts
with self.assertRaises(ProcessorException):
Converter(pip)
# bad opt type
with self.assertRaises(ProcessorException):
Converter(pip, Options(type=Pipeline))
# bad Mapper Args
with self.assertRaises(ProcessorException):
Mapper(dict())
with self.assertRaises(ProcessorException):
Mapper(pip)
def test_collect_and_aggregate(self):
"""collect events together and aggregate."""
events_in = [
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=57)),
{'type': 'a', 'in': 3, 'out': 1}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=58)),
{'type': 'a', 'in': 9, 'out': 2}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=59)),
{'type': 'b', 'in': 6, 'out': 6}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=0)),
{'type': 'a', 'in': 4, 'out': 7}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=1)),
{'type': 'b', 'in': 5, 'out': 9}
),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}:{1}'.format(event.index(), event.get('type'))] = event
uin = Stream()
(
Pipeline()
.from_source(uin)
.group_by('type')
.window_by(
Capsule(
duration='1h',
type='fixed'
)
)
.emit_on('eachEvent')
.aggregate(
{
'type': {'type': Functions.keep()},
'in_avg': {'in': Functions.avg()},
'out_avg': {'out': Functions.avg()}
}
)
.to(EventOut, cback)
)
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199:a').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:a').get('out_avg'), 1.5)
self.assertEqual(RESULTS.get('1h-396199:b').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:b').get('out_avg'), 6)
self.assertEqual(RESULTS.get('1h-396200:a').get('in_avg'), 4)
self.assertEqual(RESULTS.get('1h-396200:a').get('out_avg'), 7)
self.assertEqual(RESULTS.get('1h-396200:b').get('in_avg'), 5)
self.assertEqual(RESULTS.get('1h-396200:b').get('out_avg'), 9)
def test_aggregate_and_conversion(self):
"""Aggregate/average and convert to TimeRangeEvent."""
events_in = [
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=1, minute=57)), {
'in': 3,
'out': 1
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=1, minute=58)), {
'in': 9,
'out': 2
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=1, minute=59)), {
'in': 6,
'out': 6
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=2, minute=0)), {
'in': 4,
'out': 7
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=2, minute=1)), {
'in': 5,
'out': 9
}),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(ms_from_dt(event.timestamp()))] = event
uin = Stream()
(Pipeline().from_source(uin).window_by('1h').emit_on(
'eachEvent').aggregate({
'in_avg': {
'in': Functions.avg()
},
'out_avg': {
'out': Functions.avg()
},
}).as_time_range_events(dict(alignment='lag')).to(EventOut, cback))
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1426294800000').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1426294800000').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1426298400000').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1426298400000').get('out_avg'), 8)
def test_windowed_average(self):
"""aggregate events into by windowed avg."""
events_in = [
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=57)), {
'in': 3,
'out': 1
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=58)), {
'in': 9,
'out': 2
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=59)), {
'in': 6,
'out': 6
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=0)), {
'in': 4,
'out': 7
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=1)), {
'in': 5,
'out': 9
}),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(event.index())] = event
uin = Stream()
(Pipeline().from_source(uin).window_by('1h').emit_on(
'eachEvent').aggregate({
'in_avg': {
'in': Functions.avg()
},
'out_avg': {
'out': Functions.avg()
}
}).to(EventOut, cback))
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1h-396200').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1h-396200').get('out_avg'), 8)
def test_collect_and_aggregate(self):
"""collect events together and aggregate."""
events_in = [
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=57)), {
'type': 'a',
'in': 3,
'out': 1
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=58)), {
'type': 'a',
'in': 9,
'out': 2
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=59)), {
'type': 'b',
'in': 6,
'out': 6
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=0)), {
'type': 'a',
'in': 4,
'out': 7
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=1)), {
'type': 'b',
'in': 5,
'out': 9
}),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}:{1}'.format(event.index(), event.get('type'))] = event
uin = Stream()
(Pipeline().from_source(uin).group_by('type').window_by(
Capsule(duration='1h',
type='fixed')).emit_on('eachEvent').aggregate({
'type': {
'type': Functions.keep()
},
'in_avg': {
'in': Functions.avg()
},
'out_avg': {
'out': Functions.avg()
}
}).to(EventOut, cback))
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199:a').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:a').get('out_avg'), 1.5)
self.assertEqual(RESULTS.get('1h-396199:b').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:b').get('out_avg'), 6)
self.assertEqual(RESULTS.get('1h-396200:a').get('in_avg'), 4)
self.assertEqual(RESULTS.get('1h-396200:a').get('out_avg'), 7)
self.assertEqual(RESULTS.get('1h-396200:b').get('in_avg'), 5)
self.assertEqual(RESULTS.get('1h-396200:b').get('out_avg'), 9)