def test_bad_args(self):
"""Trigger errors for coverage."""
with self.assertRaises(UtilityException):
aware_dt_from_args(('year', 2015))
with self.assertRaises(UtilityException):
ms_from_dt(localtime_from_ms(time.time() * 1000))
def test_dt_generation_from_args(self):
"""check aware_dt_from_args()
This generates a datetime object from the kwargs that are passed to
datetime.datetime. So all of the attrs will be the same between the
UTC and local version.
"""
dtime = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=32, second=22))
dtime_loc = aware_dt_from_args(dict(year=2015,
month=3,
day=14,
hour=7,
minute=32,
second=22),
localize=True)
self.assertEqual(dtime.year, dtime_loc.year)
self.assertEqual(dtime.month, dtime_loc.month)
self.assertEqual(dtime.day, dtime_loc.day)
self.assertEqual(dtime.hour, dtime_loc.hour)
self.assertEqual(dtime.minute, dtime_loc.minute)
self.assertEqual(dtime.second, dtime_loc.second)
self.assertNotEqual(dtime.tzinfo, dtime_loc.tzinfo)
def test_bad_args(self):
"""Trigger errors for coverage."""
with self.assertRaises(UtilityException):
aware_dt_from_args(('year', 2015))
with self.assertRaises(UtilityException):
ms_from_dt(localtime_from_ms(time.time() * 1000))
def test_get_index_string_list(self):
"""
test get_index_string_list - 2 dt-> timerange -> idx_list
Used to be:
const d1 = Date.UTC(2015, 2, 14, 7, 30, 0);
const d2 = Date.UTC(2015, 2, 14, 8, 29, 59);
it("should have the correct index list for a date range", done => {
const bucketList = generator.bucketList(d1, d2);
const expectedBegin = "5m-4754394";
const expectedEnd = "5m-4754405";
// _.each(bucketList, (b) => {
// console.log(" -", b.index().asString(), b.index().asTimerange().humanize())
// })
expect(bucketList.length).to.equal(12);
Zero based month in play again.
"""
dtime_1 = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=30, second=0))
dtime_2 = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=29, second=59))
idx_list = Index.get_index_string_list('5m',
TimeRange(dtime_1, dtime_2))
self.assertEqual(len(idx_list), 12)
self.assertEqual(idx_list[0], '5m-4754394')
self.assertEqual(idx_list[-1], '5m-4754405')
def test_get_index_string_list(self):
"""
test get_index_string_list - 2 dt-> timerange -> idx_list
Used to be:
const d1 = Date.UTC(2015, 2, 14, 7, 30, 0);
const d2 = Date.UTC(2015, 2, 14, 8, 29, 59);
it("should have the correct index list for a date range", done => {
const bucketList = generator.bucketList(d1, d2);
const expectedBegin = "5m-4754394";
const expectedEnd = "5m-4754405";
// _.each(bucketList, (b) => {
// console.log(" -", b.index().asString(), b.index().asTimerange().humanize())
// })
expect(bucketList.length).to.equal(12);
Zero based month in play again.
"""
dtime_1 = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=30, second=0))
dtime_2 = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=29, second=59))
idx_list = Index.get_index_string_list('5m', TimeRange(dtime_1, dtime_2))
self.assertEqual(len(idx_list), 12)
self.assertEqual(idx_list[0], '5m-4754394')
self.assertEqual(idx_list[-1], '5m-4754405')
def test_dt_generation_from_args(self):
"""check aware_dt_from_args()
This generates a datetime object from the kwargs that are passed to
datetime.datetime. So all of the attrs will be the same between the
UTC and local version.
"""
local_offset = datetime.datetime.utcnow() - datetime.datetime.now()
if local_offset.total_seconds() < 1:
# if our local clock is UTC, this test will fail. skip it.
return
dtime = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=32, second=22))
dtime_loc = aware_dt_from_args(dict(year=2015,
month=3,
day=14,
hour=7,
minute=32,
second=22),
localize=True)
self.assertEqual(dtime.year, dtime_loc.year)
self.assertEqual(dtime.month, dtime_loc.month)
self.assertEqual(dtime.day, dtime_loc.day)
self.assertEqual(dtime.hour, dtime_loc.hour)
self.assertEqual(dtime.minute, dtime_loc.minute)
self.assertEqual(dtime.second, dtime_loc.second)
self.assertNotEqual(dtime.tzinfo, dtime_loc.tzinfo)
def test_aggregate_and_conversion(self):
"""Aggregate/average and convert to TimeRangeEvent."""
events_in = [
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=1, minute=57)),
{'in': 3, 'out': 1}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=1, minute=58)),
{'in': 9, 'out': 2}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=1, minute=59)),
{'in': 6, 'out': 6}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=2, minute=0)),
{'in': 4, 'out': 7}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=2, minute=1)),
{'in': 5, 'out': 9}
),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(ms_from_dt(event.timestamp()))] = event
uin = Stream()
(
Pipeline()
.from_source(uin)
.window_by('1h')
.emit_on('eachEvent')
.aggregate(
{
'in_avg': {'in': Functions.avg()},
'out_avg': {'out': Functions.avg()},
}
)
.as_time_range_events(dict(alignment='lag'))
.to(EventOut, cback)
)
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1426294800000').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1426294800000').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1426298400000').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1426298400000').get('out_avg'), 8)
def test_windowed_average(self):
"""aggregate events into by windowed avg."""
events_in = [
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=57)),
{'in': 3, 'out': 1}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=58)),
{'in': 9, 'out': 2}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=59)),
{'in': 6, 'out': 6}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=0)),
{'in': 4, 'out': 7}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=1)),
{'in': 5, 'out': 9}
),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(event.index())] = event
uin = Stream()
(
Pipeline()
.from_source(uin)
.window_by('1h')
.emit_on('eachEvent')
.aggregate(
{
'in_avg': {'in': Functions.avg()},
'out_avg': {'out': Functions.avg()}
}
)
.to(EventOut, cback)
)
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1h-396200').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1h-396200').get('out_avg'), 8)
def test_bad_args(self):
"""Trigger errors for coverage."""
local_offset = datetime.datetime.utcnow() - datetime.datetime.now()
if local_offset.total_seconds() < 1:
# if our local clock is UTC, this test will fail. skip it.
return
with self.assertRaises(UtilityException):
aware_dt_from_args(('year', 2015))
with self.assertRaises(UtilityException):
ms_from_dt(localtime_from_ms(time.time() * 1000))
def test_formatted_index_strings(self):
"""test the staic methods to return date strings."""
dtime = aware_dt_from_args(
dict(year=2016, month=1, day=1, hour=1, minute=0))
self.assertEqual(Index.get_yearly_index_string(dtime), '2016')
self.assertEqual(Index.get_monthly_index_string(dtime), '2016-01')
self.assertEqual(Index.get_daily_index_string(dtime), '2016-01-01')
def test_get_index_string(self):
"""
test get_index_string - datetime -> index
Used to be:
const d = Date.UTC(2015, 2, 14, 7, 32, 22);
const generator = new Generator("5m");
it("should have the correct index", done => {
const b = generator.bucket(d);
const expected = "5m-4754394";
expect(b.index().asString()).to.equal(expected);
done();
});
REMEMBER: JS Date.UTC month (arg2) is ZERO INDEXED
get_index_string() calls window_position_from_date() which
in turn calls window_duration()
"""
dtime = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=32, second=22))
self.assertEqual(dtime, dt_from_ms(1426318342000))
idx_str = Index.get_index_string('5m', dtime)
self.assertEqual(idx_str, '5m-4754394')
def test_formatted_index_strings(self):
"""test the staic methods to return date strings."""
dtime = aware_dt_from_args(
dict(year=2016, month=1, day=1, hour=1, minute=0))
self.assertEqual(Index.get_yearly_index_string(dtime), '2016')
self.assertEqual(Index.get_monthly_index_string(dtime), '2016-01')
self.assertEqual(Index.get_daily_index_string(dtime), '2016-01-01')
def test_get_index_string(self):
"""
test get_index_string - datetime -> index
Used to be:
const d = Date.UTC(2015, 2, 14, 7, 32, 22);
const generator = new Generator("5m");
it("should have the correct index", done => {
const b = generator.bucket(d);
const expected = "5m-4754394";
expect(b.index().asString()).to.equal(expected);
done();
});
REMEMBER: JS Date.UTC month (arg2) is ZERO INDEXED
get_index_string() calls window_position_from_date() which
in turn calls window_duration()
"""
dtime = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=32, second=22))
self.assertEqual(dtime, dt_from_ms(1426318342000))
idx_str = Index.get_index_string('5m', dtime)
self.assertEqual(idx_str, '5m-4754394')
def test_index_accessors(self):
"""test the various accessor methods - mostly for coverage."""
# various accessors
self.assertEqual(self._canned_index.to_json(), self._day_index)
self.assertEqual(self._canned_index.to_string(), self._day_index)
self.assertEqual(str(self._canned_index), self._day_index)
# make sure end is what it should be
beg = aware_dt_from_args(dict(year=2014, month=9, day=17))
end = beg + datetime.timedelta(hours=23, minutes=59, seconds=59)
self.assertEqual(self._canned_index.end(), end)
def test_dt_generation_from_args(self):
"""check aware_dt_from_args()
This generates a datetime object from the kwargs that are passed to
datetime.datetime. So all of the attrs will be the same between the
UTC and local version.
"""
dtime = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=32, second=22))
dtime_loc = aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=32, second=22),
localize=True)
self.assertEqual(dtime.year, dtime_loc.year)
self.assertEqual(dtime.month, dtime_loc.month)
self.assertEqual(dtime.day, dtime_loc.day)
self.assertEqual(dtime.hour, dtime_loc.hour)
self.assertEqual(dtime.minute, dtime_loc.minute)
self.assertEqual(dtime.second, dtime_loc.second)
self.assertNotEqual(dtime.tzinfo, dtime_loc.tzinfo)
def test_index_accessors(self):
"""test the various accessor methods - mostly for coverage."""
# various accessors
self.assertEqual(self._canned_index.to_json(), self._day_index)
self.assertEqual(self._canned_index.to_string(), self._day_index)
self.assertEqual(str(self._canned_index), self._day_index)
# make sure end is what it should be
beg = aware_dt_from_args(dict(year=2014, month=9, day=17))
end = beg + datetime.timedelta(hours=23, minutes=59, seconds=59)
self.assertEqual(self._canned_index.end(), end)
def test_collect_and_aggregate(self):
"""collect events together and aggregate."""
events_in = [
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=57)),
{'type': 'a', 'in': 3, 'out': 1}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=58)),
{'type': 'a', 'in': 9, 'out': 2}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=7, minute=59)),
{'type': 'b', 'in': 6, 'out': 6}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=0)),
{'type': 'a', 'in': 4, 'out': 7}
),
Event(
aware_dt_from_args(dict(year=2015, month=3, day=14, hour=8, minute=1)),
{'type': 'b', 'in': 5, 'out': 9}
),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}:{1}'.format(event.index(), event.get('type'))] = event
uin = Stream()
(
Pipeline()
.from_source(uin)
.group_by('type')
.window_by(
Capsule(
duration='1h',
type='fixed'
)
)
.emit_on('eachEvent')
.aggregate(
{
'type': {'type': Functions.keep()},
'in_avg': {'in': Functions.avg()},
'out_avg': {'out': Functions.avg()}
}
)
.to(EventOut, cback)
)
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199:a').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:a').get('out_avg'), 1.5)
self.assertEqual(RESULTS.get('1h-396199:b').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:b').get('out_avg'), 6)
self.assertEqual(RESULTS.get('1h-396200:a').get('in_avg'), 4)
self.assertEqual(RESULTS.get('1h-396200:a').get('out_avg'), 7)
self.assertEqual(RESULTS.get('1h-396200:b').get('in_avg'), 5)
self.assertEqual(RESULTS.get('1h-396200:b').get('out_avg'), 9)
def test_aggregate_and_conversion(self):
"""Aggregate/average and convert to TimeRangeEvent."""
events_in = [
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=1, minute=57)), {
'in': 3,
'out': 1
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=1, minute=58)), {
'in': 9,
'out': 2
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=1, minute=59)), {
'in': 6,
'out': 6
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=2, minute=0)), {
'in': 4,
'out': 7
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=2, minute=1)), {
'in': 5,
'out': 9
}),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(ms_from_dt(event.timestamp()))] = event
uin = Stream()
(Pipeline().from_source(uin).window_by('1h').emit_on(
'eachEvent').aggregate({
'in_avg': {
'in': Functions.avg()
},
'out_avg': {
'out': Functions.avg()
},
}).as_time_range_events(dict(alignment='lag')).to(EventOut, cback))
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1426294800000').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1426294800000').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1426298400000').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1426298400000').get('out_avg'), 8)
def test_collect_and_aggregate(self):
"""collect events together and aggregate."""
events_in = [
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=57)), {
'type': 'a',
'in': 3,
'out': 1
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=58)), {
'type': 'a',
'in': 9,
'out': 2
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=59)), {
'type': 'b',
'in': 6,
'out': 6
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=0)), {
'type': 'a',
'in': 4,
'out': 7
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=1)), {
'type': 'b',
'in': 5,
'out': 9
}),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}:{1}'.format(event.index(), event.get('type'))] = event
uin = Stream()
(Pipeline().from_source(uin).group_by('type').window_by(
Capsule(duration='1h',
type='fixed')).emit_on('eachEvent').aggregate({
'type': {
'type': Functions.keep()
},
'in_avg': {
'in': Functions.avg()
},
'out_avg': {
'out': Functions.avg()
}
}).to(EventOut, cback))
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199:a').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:a').get('out_avg'), 1.5)
self.assertEqual(RESULTS.get('1h-396199:b').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199:b').get('out_avg'), 6)
self.assertEqual(RESULTS.get('1h-396200:a').get('in_avg'), 4)
self.assertEqual(RESULTS.get('1h-396200:a').get('out_avg'), 7)
self.assertEqual(RESULTS.get('1h-396200:b').get('in_avg'), 5)
self.assertEqual(RESULTS.get('1h-396200:b').get('out_avg'), 9)
def test_windowed_average(self):
"""aggregate events into by windowed avg."""
events_in = [
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=57)), {
'in': 3,
'out': 1
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=58)), {
'in': 9,
'out': 2
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=7, minute=59)), {
'in': 6,
'out': 6
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=0)), {
'in': 4,
'out': 7
}),
Event(
aware_dt_from_args(
dict(year=2015, month=3, day=14, hour=8, minute=1)), {
'in': 5,
'out': 9
}),
]
def cback(event):
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
if RESULTS is None:
RESULTS = dict()
RESULTS['{0}'.format(event.index())] = event
uin = Stream()
(Pipeline().from_source(uin).window_by('1h').emit_on(
'eachEvent').aggregate({
'in_avg': {
'in': Functions.avg()
},
'out_avg': {
'out': Functions.avg()
}
}).to(EventOut, cback))
for i in events_in:
uin.add_event(i)
self.assertEqual(RESULTS.get('1h-396199').get('in_avg'), 6)
self.assertEqual(RESULTS.get('1h-396199').get('out_avg'), 3)
self.assertEqual(RESULTS.get('1h-396200').get('in_avg'), 4.5)
self.assertEqual(RESULTS.get('1h-396200').get('out_avg'), 8)