def test_sum_and_find_max(self):
"""sum elements, find max get result out."""
def cback(event):
"""catch the return"""
self.assertEqual(event.get('max_total'), 117)
timeseries = TimeSeries(IN_OUT_DATA)
(
Pipeline()
.from_source(timeseries)
.emit_on('flush')
.collapse(['in', 'out'], 'total', Functions.sum())
.aggregate(dict(max_total=dict(total=Functions.max())))
.to(EventOut, cback)
)
# Same test but as an event list
elist = (
Pipeline()
.from_source(timeseries)
.emit_on('flush')
.collapse(['in', 'out'], 'total', Functions.sum())
.aggregate(dict(max_total=dict(total=Functions.max())))
.to_event_list()
)
self.assertEqual(len(elist), 1)
self.assertEqual(elist[0].get('max_total'), 117)
def test_aggregate_deep_path(self):
"""Make sure that the aggregator will work on a deep path."""
elist = (
Pipeline()
.from_source(TimeSeries(dict(name='events', events=DEEP_EVENT_LIST)))
.emit_on('flush')
.aggregate(dict(out_max={'direction.out': Functions.max()}))
.to_event_list()
)
self.assertEqual(elist[0].get('out_max'), 4)
# Make sure it works with the the non-string version to aggregate
# multiple columns
elist = (
Pipeline()
.from_source(TimeSeries(dict(name='events', events=DEEP_EVENT_LIST)))
.emit_on('flush')
.aggregate(
{
'in_max': {'direction.in': Functions.max()},
'out_max': {'direction.out': Functions.max()},
}
)
.to_event_list()
)
self.assertEqual(elist[0].get('out_max'), 4)
self.assertEqual(elist[0].get('in_max'), 8)
def test_aggregate_deep_path(self):
"""Make sure that the aggregator will work on a deep path."""
elist = (Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('flush').aggregate(
dict(out_max={'direction.out': Functions.max()
})).to_event_list())
self.assertEqual(elist[0].get('out_max'), 4)
# Make sure it works with the the non-string version to aggregate
# multiple columns
elist = (Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('flush').aggregate({
'in_max': {
'direction.in': Functions.max()
},
'out_max': {
'direction.out': Functions.max()
},
}).to_event_list())
self.assertEqual(elist[0].get('out_max'), 4)
self.assertEqual(elist[0].get('in_max'), 8)
def test_bad_args(self):
"""Trigger exceptions and warnings, etc."""
uin = Stream()
with warnings.catch_warnings(record=True) as wrn:
Pipeline().from_source(uin).window_by('1h', utc=False)
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, PipelineWarning))
# bad arg
with self.assertRaises(PipelineException):
Pipeline().from_source(dict())
# no source
with self.assertRaises(PipelineException):
Pipeline().to_keyed_collections()
# can't iterate on unbounded source
with self.assertRaises(PipelineIOException):
list(uin.events())
# bad emit on type
with self.assertRaises(PipelineIOException):
(
Pipeline()
.from_source(TimeSeries(dict(name='events', events=DEEP_EVENT_LIST)))
.emit_on('BOGUS')
.aggregate(
{'max_in': {'direction.in': Functions.max()}}
)
.to_event_list()
)
def test_bad_args(self):
"""Trigger exceptions and warnings, etc."""
uin = Stream()
with warnings.catch_warnings(record=True) as wrn:
Pipeline().from_source(uin).window_by('1h', utc=False)
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, PipelineWarning))
# bad arg
with self.assertRaises(PipelineException):
Pipeline().from_source(dict())
# no source
with self.assertRaises(PipelineException):
Pipeline().to_keyed_collections()
# can't iterate on unbounded source
with self.assertRaises(PipelineIOException):
list(uin.events())
# bad emit on type
with self.assertRaises(PipelineIOException):
(Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('BOGUS').aggregate({
'max_in': {
'direction.in': Functions.max()
}
}).to_event_list())
def test_sum_events_with_combine(self):
"""test summing multiple events together via combine on the back end."""
# combine them all
events = [
self._create_event(self.aware_ts, {'a': 5, 'b': 6, 'c': 7}),
self._create_event(self.aware_ts, {'a': 2, 'b': 3, 'c': 4}),
self._create_event(self.aware_ts, {'a': 1, 'b': 2, 'c': 3}),
]
result = Event.sum(events)
self.assertEqual(result[0].get('a'), 8)
self.assertEqual(result[0].get('b'), 11)
self.assertEqual(result[0].get('c'), 14)
# combine single field
result = Event.sum(events, 'a')
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertIsNone(result[0].get('c'))
# grab multiple fields
result = Event.sum(events, ['a', 'c'])
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertEqual(result[0].get('c'), 14)
# average
result = Event.avg(
events + [self._create_event(self.aware_ts, {'a': 1, 'b': 1, 'c': 2})],
'c')
self.assertEqual(result[0].get('c'), 4)
# bad arg
self.assertEqual(Event.sum([]), [])
self.assertEqual(Event.avg([]), [])
# work the extra reducer functions in Functions module
result = Event.combine(events, 'c', Functions.max())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.min())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.count())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.first())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.last())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.difference())
self.assertEqual(result[0].get('c'), 4)
self.assertIsNone(Functions.first()([]))
self.assertIsNone(Functions.last()([]))
def test_sum_and_find_max(self):
"""sum elements, find max get result out."""
def cback(event):
"""catch the return"""
self.assertEqual(event.get('max_total'), 117)
timeseries = TimeSeries(IN_OUT_DATA)
(Pipeline().from_source(timeseries).emit_on('flush').collapse(
['in', 'out'], 'total', Functions.sum()).aggregate(
dict(max_total=dict(total=Functions.max()))).to(
EventOut, cback))
# Same test but as an event list
elist = (Pipeline().from_source(timeseries).emit_on('flush').collapse(
['in', 'out'], 'total', Functions.sum()).aggregate(
dict(max_total=dict(total=Functions.max()))).to_event_list())
self.assertEqual(len(elist), 1)
self.assertEqual(elist[0].get('max_total'), 117)
def test_multiple_collapse_chains(self):
"""multiple collapsers."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (Pipeline().from_source(timeseries).collapse(
['in', 'out'], 'in_out_sum', Functions.sum()).collapse(
['in', 'out'], 'in_out_max',
Functions.max()).emit_on('flush').to_keyed_collections())
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
self.assertEqual(kcol.get('all').at(0).get('in_out_max'), 80)
self.assertEqual(kcol.get('all').at(1).get('in_out_max'), 88)
self.assertEqual(kcol.get('all').at(2).get('in_out_max'), 56)
def test_multiple_collapse_chains(self):
"""multiple collapsers."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.collapse(['in', 'out'], 'in_out_sum', Functions.sum())
.collapse(['in', 'out'], 'in_out_max', Functions.max())
.emit_on('flush')
.to_keyed_collections()
)
self.assertEqual(kcol.get('all').at(0).get('in_out_sum'), 117)
self.assertEqual(kcol.get('all').at(1).get('in_out_sum'), 110)
self.assertEqual(kcol.get('all').at(2).get('in_out_sum'), 108)
self.assertEqual(kcol.get('all').at(0).get('in_out_max'), 80)
self.assertEqual(kcol.get('all').at(1).get('in_out_max'), 88)
self.assertEqual(kcol.get('all').at(2).get('in_out_max'), 56)
def test_sum_events_with_combine(self):
"""test summing multiple events together via combine on the back end."""
# combine them all
events = [
self._create_event(self.aware_ts, {
'a': 5,
'b': 6,
'c': 7
}),
self._create_event(self.aware_ts, {
'a': 2,
'b': 3,
'c': 4
}),
self._create_event(self.aware_ts, {
'a': 1,
'b': 2,
'c': 3
}),
]
result = Event.sum(events)
self.assertEqual(result[0].get('a'), 8)
self.assertEqual(result[0].get('b'), 11)
self.assertEqual(result[0].get('c'), 14)
# combine single field
result = Event.sum(events, 'a')
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertIsNone(result[0].get('c'))
# grab multiple fields
result = Event.sum(events, ['a', 'c'])
self.assertEqual(result[0].get('a'), 8)
self.assertIsNone(result[0].get('b'))
self.assertEqual(result[0].get('c'), 14)
# average
result = Event.avg(
events +
[self._create_event(self.aware_ts, {
'a': 1,
'b': 1,
'c': 2
})], 'c')
self.assertEqual(result[0].get('c'), 4)
# bad arg
self.assertEqual(Event.sum([]), [])
self.assertEqual(Event.avg([]), [])
# work the extra reducer functions in Functions module
result = Event.combine(events, 'c', Functions.max())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.min())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.count())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.first())
self.assertEqual(result[0].get('c'), 7)
result = Event.combine(events, 'c', Functions.last())
self.assertEqual(result[0].get('c'), 3)
result = Event.combine(events, 'c', Functions.difference())
self.assertEqual(result[0].get('c'), 4)
self.assertIsNone(Functions.first()([]))
self.assertIsNone(Functions.last()([]))