def test_fill_event_variants(self):
"""fill time range and indexed events."""
range_list = [
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=1)),
{'in': 100}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=2)),
{'in': None}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=3)),
{'in': None}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=4)),
{'in': 90}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=5)),
{'in': 80}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=6)),
{'in': 70}
),
]
coll = Collection(range_list)
# canned series objects
rts = TimeSeries(
dict(name='collection', collection=coll))
new_rts = rts.fill(field_spec='in')
self.assertEqual(new_rts.at(1).get('in'), 0)
self.assertEqual(new_rts.at(2).get('in'), 0)
# indexed events
index_list = [
IndexedEvent('1d-12355', {'value': 42}),
IndexedEvent('1d-12356', {'value': None}),
IndexedEvent('1d-12357', {'value': None}),
IndexedEvent('1d-12358', {'value': 52}),
IndexedEvent('1d-12359', {'value': 55}),
IndexedEvent('1d-12360', {'value': 58}),
]
coll = Collection(index_list)
its = TimeSeries(
dict(name='collection', collection=coll))
new_its = its.fill()
self.assertEqual(new_its.at(1).get(), 0)
self.assertEqual(new_its.at(2).get(), 0)
def test_complex_zero_fill(self):
"""make sure more complex nested paths work OK"""
complex_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000,
{'in': {'tcp': 1, 'udp': 3}, 'out': {'tcp': 2, 'udp': 3}}],
[1400425948000,
{'in': {'tcp': 3, 'udp': None}, 'out': {'tcp': 4, 'udp': 3}}],
[1400425949000,
{'in': {'tcp': 5, 'udp': None}, 'out': {'tcp': None, 'udp': 3}}],
[1400425950000,
{'in': {'tcp': 7, 'udp': None}, 'out': {'tcp': None, 'udp': 3}}],
[1400425960000,
{'in': {'tcp': 9, 'udp': 4}, 'out': {'tcp': 6, 'udp': 3}}],
[1400425970000,
{'in': {'tcp': 11, 'udp': 5}, 'out': {'tcp': 8, 'udp': 3}}],
]
)
ts = TimeSeries(complex_missing_data)
# zero fill everything
new_ts = ts.fill(field_spec=['direction.out.tcp', 'direction.in.udp'])
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
# do it again, but only fill the out.tcp
new_ts = ts.fill(field_spec=['direction.out.tcp'])
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
def test_zero_fill(self):
"""test using the filler to fill missing values with zero."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': None,
'out': 4
}],
[1400425949000, {
'in': 5,
'out': None
}],
[1400425950000, {
'in': None,
'out': 8
}],
[1400425960000, {
'in': 9,
'out': None
}],
[1400425970000, {
'in': None,
'out': 12
}],
])
ts = TimeSeries(simple_missing_data)
# fill all invalid values
new_ts = ts.fill(field_spec=['direction.in', 'direction.out'])
self.assertEqual(new_ts.size(), 6)
self.assertEqual(new_ts.at(0).get('direction.out'), 0)
self.assertEqual(new_ts.at(2).get('direction.out'), 0)
self.assertEqual(new_ts.at(1).get('direction.in'), 0)
# fill one column
new_ts = ts.fill(field_spec='direction.in')
self.assertEqual(new_ts.size(), 6)
self.assertEqual(new_ts.at(1).get('direction.in'), 0)
self.assertEqual(new_ts.at(3).get('direction.in'), 0)
self.assertIsNone(new_ts.at(0).get('direction.out'))
self.assertIsNone(new_ts.at(2).get('direction.out'))
def test_fill_event_variants(self):
"""fill time range and indexed events."""
range_list = [
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=1)),
{'in': 100}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=2)),
{'in': None}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=3)),
{'in': None}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=4)),
{'in': 90}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=5)),
{'in': 80}),
TimeRangeEvent((aware_utcnow(),
aware_utcnow() + datetime.timedelta(minutes=6)),
{'in': 70}),
]
coll = Collection(range_list)
# canned series objects
rts = TimeSeries(dict(name='collection', collection=coll))
new_rts = rts.fill(field_spec='in')
self.assertEqual(new_rts.at(1).get('in'), 0)
self.assertEqual(new_rts.at(2).get('in'), 0)
# indexed events
index_list = [
IndexedEvent('1d-12355', {'value': 42}),
IndexedEvent('1d-12356', {'value': None}),
IndexedEvent('1d-12357', {'value': None}),
IndexedEvent('1d-12358', {'value': 52}),
IndexedEvent('1d-12359', {'value': 55}),
IndexedEvent('1d-12360', {'value': 58}),
]
coll = Collection(index_list)
its = TimeSeries(dict(name='collection', collection=coll))
new_its = its.fill()
self.assertEqual(new_its.at(1).get(), 0)
self.assertEqual(new_its.at(2).get(), 0)
def test_assymetric_linear_fill(self):
"""Test new chained/assymetric linear default fill in TimeSeries."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': None, 'out': None}],
[1400425949000, {'in': None, 'out': None}],
[1400425950000, {'in': 3, 'out': 8}],
[1400425960000, {'in': None, 'out': None}],
[1400425970000, {'in': 5, 'out': 12}],
[1400425980000, {'in': 6, 'out': 13}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(method='linear', field_spec=['direction.in', 'direction.out'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1.6666666666666665) # filled
self.assertEqual(new_ts.at(2).get('direction.in'), 2.333333333333333) # filled
self.assertEqual(new_ts.at(3).get('direction.in'), 3)
self.assertEqual(new_ts.at(4).get('direction.in'), 4.0) # filled
self.assertEqual(new_ts.at(5).get('direction.in'), 5)
self.assertEqual(new_ts.at(0).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(1).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(2).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(3).get('direction.out'), 8)
self.assertEqual(new_ts.at(4).get('direction.out'), 10.0) # filled
self.assertEqual(new_ts.at(5).get('direction.out'), 12)
def test_scan_stop(self):
"""stop seeking good values if there are none - for coverage."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': 3, 'out': None}],
[1400425949000, {'in': None, 'out': None}],
[1400425950000, {'in': None, 'out': 8}],
[1400425960000, {'in': None, 'out': None}],
[1400425970000, {'in': None, 'out': 12}],
[1400425980000, {'in': None, 'out': 13}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(field_spec='direction.out', method='linear')
self.assertEqual(new_ts.at(2).get('direction.in'), None)
self.assertEqual(new_ts.at(3).get('direction.in'), None)
self.assertEqual(new_ts.at(4).get('direction.in'), None)
self.assertEqual(new_ts.at(5).get('direction.in'), None)
self.assertEqual(new_ts.at(6).get('direction.in'), None)
def test_assymetric_linear_fill(self):
"""Test new chained/assymetric linear default fill in TimeSeries."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': None,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': 3,
'out': 8
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': 5,
'out': 12
}],
[1400425980000, {
'in': 6,
'out': 13
}],
])
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(method='linear',
field_spec=['direction.in', 'direction.out'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'),
1.6666666666666665) # filled
self.assertEqual(new_ts.at(2).get('direction.in'),
2.333333333333333) # filled
self.assertEqual(new_ts.at(3).get('direction.in'), 3)
self.assertEqual(new_ts.at(4).get('direction.in'), 4.0) # filled
self.assertEqual(new_ts.at(5).get('direction.in'), 5)
self.assertEqual(new_ts.at(0).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(1).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(2).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(3).get('direction.out'), 8)
self.assertEqual(new_ts.at(4).get('direction.out'), 10.0) # filled
self.assertEqual(new_ts.at(5).get('direction.out'), 12)
def test_zero_fill(self):
"""test using the filler to fill missing values with zero."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': None, 'out': 4}],
[1400425949000, {'in': 5, 'out': None}],
[1400425950000, {'in': None, 'out': 8}],
[1400425960000, {'in': 9, 'out': None}],
[1400425970000, {'in': None, 'out': 12}],
]
)
ts = TimeSeries(simple_missing_data)
# fill all invalid values
new_ts = ts.fill(field_spec=['direction.in', 'direction.out'])
self.assertEqual(new_ts.size(), 6)
self.assertEqual(new_ts.at(0).get('direction.out'), 0)
self.assertEqual(new_ts.at(2).get('direction.out'), 0)
self.assertEqual(new_ts.at(1).get('direction.in'), 0)
# fill one column
new_ts = ts.fill(field_spec='direction.in')
self.assertEqual(new_ts.size(), 6)
self.assertEqual(new_ts.at(1).get('direction.in'), 0)
self.assertEqual(new_ts.at(3).get('direction.in'), 0)
self.assertIsNone(new_ts.at(0).get('direction.out'))
self.assertIsNone(new_ts.at(2).get('direction.out'))
def test_scan_stop(self):
"""stop seeking good values if there are none - for coverage."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': 3,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': None,
'out': 8
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': None,
'out': 12
}],
[1400425980000, {
'in': None,
'out': 13
}],
])
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(field_spec='direction.out', method='linear')
self.assertEqual(new_ts.at(2).get('direction.in'), None)
self.assertEqual(new_ts.at(3).get('direction.in'), None)
self.assertEqual(new_ts.at(4).get('direction.in'), None)
self.assertEqual(new_ts.at(5).get('direction.in'), None)
self.assertEqual(new_ts.at(6).get('direction.in'), None)
def test_pad(self):
"""Test the pad style fill."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None, 'drop': None}],
[1400425948000, {'in': None, 'out': 4, 'drop': None}],
[1400425949000, {'in': None, 'out': None, 'drop': 13}],
[1400425950000, {'in': None, 'out': None, 'drop': 14}],
[1400425960000, {'in': 9, 'out': 8, 'drop': None}],
[1400425970000, {'in': 11, 'out': 10, 'drop': 16}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(method='pad',
field_spec=['direction.in', 'direction.out', 'direction.drop'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(2).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(3).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(4).get('direction.in'), 9)
self.assertEqual(new_ts.at(5).get('direction.in'), 11)
self.assertEqual(new_ts.at(0).get('direction.out'), None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.out'), 4)
self.assertEqual(new_ts.at(2).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(3).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(4).get('direction.out'), 8)
self.assertEqual(new_ts.at(5).get('direction.out'), 10)
self.assertEqual(new_ts.at(0).get('direction.drop'), None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.drop'), None) # bad prev can't pad
self.assertEqual(new_ts.at(2).get('direction.drop'), 13)
self.assertEqual(new_ts.at(3).get('direction.drop'), 14)
self.assertEqual(new_ts.at(4).get('direction.drop'), 14) # padded
self.assertEqual(new_ts.at(5).get('direction.drop'), 16)
def test_linear(self):
"""Test linear interpolation filling returned by to_keyed_collections()."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': 2}],
[1400425948000, {'in': None, 'out': None}],
[1400425949000, {'in': None, 'out': None}],
[1400425950000, {'in': 3, 'out': None}],
[1400425960000, {'in': None, 'out': None}],
[1400425970000, {'in': 5, 'out': 12}],
[1400425980000, {'in': 6, 'out': 13}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(field_spec=['direction.in', 'direction.out'],
method='linear')
self.assertEqual(new_ts.size(), 7)
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1.6666666666666665) # filled
self.assertEqual(new_ts.at(2).get('direction.in'), 2.333333333333333) # filled
self.assertEqual(new_ts.at(3).get('direction.in'), 3)
self.assertEqual(new_ts.at(4).get('direction.in'), 4.0) # filled
self.assertEqual(new_ts.at(5).get('direction.in'), 5)
self.assertEqual(new_ts.at(0).get('direction.out'), 2)
self.assertEqual(new_ts.at(1).get('direction.out'), 2.4347826086956523) # filled
self.assertEqual(new_ts.at(2).get('direction.out'), 2.8695652173913047) # filled
self.assertEqual(new_ts.at(3).get('direction.out'), 3.304347826086957) # filled
self.assertEqual(new_ts.at(4).get('direction.out'), 7.6521739130434785) # filled
self.assertEqual(new_ts.at(5).get('direction.out'), 12)
def test_pad_and_zero_limiting(self):
"""test the limiting on pad and zero options."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': None, 'out': None}],
[1400425949000, {'in': None, 'out': None}],
[1400425950000, {'in': 3, 'out': 8}],
[1400425960000, {'in': None, 'out': None}],
[1400425970000, {'in': None, 'out': 12}],
[1400425980000, {'in': None, 'out': 13}],
[1400425990000, {'in': 7, 'out': None}],
[1400426000000, {'in': 8, 'out': None}],
[1400426010000, {'in': 9, 'out': None}],
[1400426020000, {'in': 10, 'out': None}],
]
)
ts = TimeSeries(simple_missing_data)
# verify fill limit for zero fill
zero_ts = ts.fill(method='zero', fill_limit=2,
field_spec=['direction.in', 'direction.out'])
self.assertEqual(zero_ts.at(0).get('direction.in'), 1)
self.assertEqual(zero_ts.at(1).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(2).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(3).get('direction.in'), 3)
self.assertEqual(zero_ts.at(4).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(5).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(6).get('direction.in'), None) # over limit skip
self.assertEqual(zero_ts.at(7).get('direction.in'), 7)
self.assertEqual(zero_ts.at(8).get('direction.in'), 8)
self.assertEqual(zero_ts.at(9).get('direction.in'), 9)
self.assertEqual(zero_ts.at(10).get('direction.in'), 10)
self.assertEqual(zero_ts.at(0).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(1).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(2).get('direction.out'), None) # over limit skip
self.assertEqual(zero_ts.at(3).get('direction.out'), 8)
self.assertEqual(zero_ts.at(4).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(5).get('direction.out'), 12)
self.assertEqual(zero_ts.at(6).get('direction.out'), 13)
self.assertEqual(zero_ts.at(7).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(8).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(9).get('direction.out'), None) # over limit skip
self.assertEqual(zero_ts.at(10).get('direction.out'), None) # over limit skip
# verify fill limit for pad fill
pad_ts = ts.fill(method='pad', fill_limit=2,
field_spec=['direction.in', 'direction.out'])
self.assertEqual(pad_ts.at(0).get('direction.in'), 1)
self.assertEqual(pad_ts.at(1).get('direction.in'), 1) # fill
self.assertEqual(pad_ts.at(2).get('direction.in'), 1) # fill
self.assertEqual(pad_ts.at(3).get('direction.in'), 3)
self.assertEqual(pad_ts.at(4).get('direction.in'), 3) # fill
self.assertEqual(pad_ts.at(5).get('direction.in'), 3) # fill
self.assertEqual(pad_ts.at(6).get('direction.in'), None) # over limit skip
self.assertEqual(pad_ts.at(7).get('direction.in'), 7)
self.assertEqual(pad_ts.at(8).get('direction.in'), 8)
self.assertEqual(pad_ts.at(9).get('direction.in'), 9)
self.assertEqual(pad_ts.at(10).get('direction.in'), 10)
self.assertEqual(pad_ts.at(0).get('direction.out'), None) # no fill start
self.assertEqual(pad_ts.at(1).get('direction.out'), None) # no fill start
self.assertEqual(pad_ts.at(2).get('direction.out'), None) # no fill start
self.assertEqual(pad_ts.at(3).get('direction.out'), 8)
self.assertEqual(pad_ts.at(4).get('direction.out'), 8) # fill
self.assertEqual(pad_ts.at(5).get('direction.out'), 12)
self.assertEqual(pad_ts.at(6).get('direction.out'), 13)
self.assertEqual(pad_ts.at(7).get('direction.out'), 13) # fill
self.assertEqual(pad_ts.at(8).get('direction.out'), 13) # fill
self.assertEqual(pad_ts.at(9).get('direction.out'), None) # over limit skip
self.assertEqual(pad_ts.at(10).get('direction.out'), None) # over limit skip
def test_pad(self):
"""Test the pad style fill."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None,
'drop': None
}],
[1400425948000, {
'in': None,
'out': 4,
'drop': None
}],
[1400425949000, {
'in': None,
'out': None,
'drop': 13
}],
[1400425950000, {
'in': None,
'out': None,
'drop': 14
}],
[1400425960000, {
'in': 9,
'out': 8,
'drop': None
}],
[1400425970000, {
'in': 11,
'out': 10,
'drop': 16
}],
])
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(
method='pad',
field_spec=['direction.in', 'direction.out', 'direction.drop'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(2).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(3).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(4).get('direction.in'), 9)
self.assertEqual(new_ts.at(5).get('direction.in'), 11)
self.assertEqual(new_ts.at(0).get('direction.out'),
None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.out'), 4)
self.assertEqual(new_ts.at(2).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(3).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(4).get('direction.out'), 8)
self.assertEqual(new_ts.at(5).get('direction.out'), 10)
self.assertEqual(new_ts.at(0).get('direction.drop'),
None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.drop'),
None) # bad prev can't pad
self.assertEqual(new_ts.at(2).get('direction.drop'), 13)
self.assertEqual(new_ts.at(3).get('direction.drop'), 14)
self.assertEqual(new_ts.at(4).get('direction.drop'), 14) # padded
self.assertEqual(new_ts.at(5).get('direction.drop'), 16)
def test_pad_and_zero_limiting(self):
"""test the limiting on pad and zero options."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': None,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': 3,
'out': 8
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': None,
'out': 12
}],
[1400425980000, {
'in': None,
'out': 13
}],
[1400425990000, {
'in': 7,
'out': None
}],
[1400426000000, {
'in': 8,
'out': None
}],
[1400426010000, {
'in': 9,
'out': None
}],
[1400426020000, {
'in': 10,
'out': None
}],
])
ts = TimeSeries(simple_missing_data)
# verify fill limit for zero fill
zero_ts = ts.fill(method='zero',
fill_limit=2,
field_spec=['direction.in', 'direction.out'])
self.assertEqual(zero_ts.at(0).get('direction.in'), 1)
self.assertEqual(zero_ts.at(1).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(2).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(3).get('direction.in'), 3)
self.assertEqual(zero_ts.at(4).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(5).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(6).get('direction.in'),
None) # over limit skip
self.assertEqual(zero_ts.at(7).get('direction.in'), 7)
self.assertEqual(zero_ts.at(8).get('direction.in'), 8)
self.assertEqual(zero_ts.at(9).get('direction.in'), 9)
self.assertEqual(zero_ts.at(10).get('direction.in'), 10)
self.assertEqual(zero_ts.at(0).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(1).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(2).get('direction.out'),
None) # over limit skip
self.assertEqual(zero_ts.at(3).get('direction.out'), 8)
self.assertEqual(zero_ts.at(4).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(5).get('direction.out'), 12)
self.assertEqual(zero_ts.at(6).get('direction.out'), 13)
self.assertEqual(zero_ts.at(7).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(8).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(9).get('direction.out'),
None) # over limit skip
self.assertEqual(zero_ts.at(10).get('direction.out'),
None) # over limit skip
# verify fill limit for pad fill
pad_ts = ts.fill(method='pad',
fill_limit=2,
field_spec=['direction.in', 'direction.out'])
self.assertEqual(pad_ts.at(0).get('direction.in'), 1)
self.assertEqual(pad_ts.at(1).get('direction.in'), 1) # fill
self.assertEqual(pad_ts.at(2).get('direction.in'), 1) # fill
self.assertEqual(pad_ts.at(3).get('direction.in'), 3)
self.assertEqual(pad_ts.at(4).get('direction.in'), 3) # fill
self.assertEqual(pad_ts.at(5).get('direction.in'), 3) # fill
self.assertEqual(pad_ts.at(6).get('direction.in'),
None) # over limit skip
self.assertEqual(pad_ts.at(7).get('direction.in'), 7)
self.assertEqual(pad_ts.at(8).get('direction.in'), 8)
self.assertEqual(pad_ts.at(9).get('direction.in'), 9)
self.assertEqual(pad_ts.at(10).get('direction.in'), 10)
self.assertEqual(pad_ts.at(0).get('direction.out'),
None) # no fill start
self.assertEqual(pad_ts.at(1).get('direction.out'),
None) # no fill start
self.assertEqual(pad_ts.at(2).get('direction.out'),
None) # no fill start
self.assertEqual(pad_ts.at(3).get('direction.out'), 8)
self.assertEqual(pad_ts.at(4).get('direction.out'), 8) # fill
self.assertEqual(pad_ts.at(5).get('direction.out'), 12)
self.assertEqual(pad_ts.at(6).get('direction.out'), 13)
self.assertEqual(pad_ts.at(7).get('direction.out'), 13) # fill
self.assertEqual(pad_ts.at(8).get('direction.out'), 13) # fill
self.assertEqual(pad_ts.at(9).get('direction.out'),
None) # over limit skip
self.assertEqual(pad_ts.at(10).get('direction.out'),
None) # over limit skip
def test_linear(self):
"""Test linear interpolation filling returned by to_keyed_collections()."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': 2
}],
[1400425948000, {
'in': None,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': 3,
'out': None
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': 5,
'out': 12
}],
[1400425980000, {
'in': 6,
'out': 13
}],
])
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(field_spec=['direction.in', 'direction.out'],
method='linear')
self.assertEqual(new_ts.size(), 7)
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'),
1.6666666666666665) # filled
self.assertEqual(new_ts.at(2).get('direction.in'),
2.333333333333333) # filled
self.assertEqual(new_ts.at(3).get('direction.in'), 3)
self.assertEqual(new_ts.at(4).get('direction.in'), 4.0) # filled
self.assertEqual(new_ts.at(5).get('direction.in'), 5)
self.assertEqual(new_ts.at(0).get('direction.out'), 2)
self.assertEqual(
new_ts.at(1).get('direction.out'), 2.4347826086956523) # filled
self.assertEqual(
new_ts.at(2).get('direction.out'), 2.8695652173913047) # filled
self.assertEqual(new_ts.at(3).get('direction.out'),
3.304347826086957) # filled
self.assertEqual(
new_ts.at(4).get('direction.out'), 7.6521739130434785) # filled
self.assertEqual(new_ts.at(5).get('direction.out'), 12)
def test_complex_zero_fill(self):
"""make sure more complex nested paths work OK"""
complex_missing_data = dict(name="traffic",
columns=["time", "direction"],
points=[
[
1400425947000, {
'in': {
'tcp': 1,
'udp': 3
},
'out': {
'tcp': 2,
'udp': 3
}
}
],
[
1400425948000, {
'in': {
'tcp': 3,
'udp': None
},
'out': {
'tcp': 4,
'udp': 3
}
}
],
[
1400425949000, {
'in': {
'tcp': 5,
'udp': None
},
'out': {
'tcp': None,
'udp': 3
}
}
],
[
1400425950000, {
'in': {
'tcp': 7,
'udp': None
},
'out': {
'tcp': None,
'udp': 3
}
}
],
[
1400425960000, {
'in': {
'tcp': 9,
'udp': 4
},
'out': {
'tcp': 6,
'udp': 3
}
}
],
[
1400425970000, {
'in': {
'tcp': 11,
'udp': 5
},
'out': {
'tcp': 8,
'udp': 3
}
}
],
])
ts = TimeSeries(complex_missing_data)
# zero fill everything
new_ts = ts.fill(field_spec=['direction.out.tcp', 'direction.in.udp'])
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
# do it again, but only fill the out.tcp
new_ts = ts.fill(field_spec=['direction.out.tcp'])
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
def test_bad_args(self):
"""Trigger error states for coverage."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None, 'drop': None}],
[1400425948000, {'in': None, 'out': 4, 'drop': None}],
[1400425949000, {'in': None, 'out': None, 'drop': 13}],
[1400425950000, {'in': None, 'out': None, 'drop': 14}],
[1400425960000, {'in': 9, 'out': 8, 'drop': None}],
[1400425970000, {'in': 11, 'out': 10, 'drop': 16}],
]
)
ts = TimeSeries(simple_missing_data)
# bad ctor arg
with self.assertRaises(ProcessorException):
f = Filler(dict())
# invalid method
with self.assertRaises(TimeSeriesException):
ts.fill(method='bogus')
# limit not int
with self.assertRaises(ProcessorException):
ts.fill(fill_limit='z')
# direct access to filler via pipeline needs to take a single path
with self.assertRaises(ProcessorException):
pip = Pipeline()
pip.fill(method='linear', field_spec=['direction.in', 'direction.out'])
# invalid method
with self.assertRaises(ProcessorException):
pip = Pipeline()
pip.fill(method='bogus')
# catch bad path at various points
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='bad.path')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='bad.path', method='linear')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='direction.bogus')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
# trigger warnings about non-numeric values in linear.
with warnings.catch_warnings(record=True) as wrn:
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': 'non_numeric', 'out': 4}],
[1400425949000, {'in': 5, 'out': None}],
]
)
ts = TimeSeries(simple_missing_data)
ts.fill(field_spec='direction.in', method='linear')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
# empty series for coverage caught a bug
empty = TimeSeries(dict(
name="Sensor values",
columns=["time", "temperature"],
points=[
]
))
self.assertEqual(empty.fill(field_spec='temperature').size(), 0)
def test_bad_args(self):
"""Trigger error states for coverage."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None,
'drop': None
}],
[1400425948000, {
'in': None,
'out': 4,
'drop': None
}],
[1400425949000, {
'in': None,
'out': None,
'drop': 13
}],
[1400425950000, {
'in': None,
'out': None,
'drop': 14
}],
[1400425960000, {
'in': 9,
'out': 8,
'drop': None
}],
[1400425970000, {
'in': 11,
'out': 10,
'drop': 16
}],
])
ts = TimeSeries(simple_missing_data)
# bad ctor arg
with self.assertRaises(ProcessorException):
f = Filler(dict())
# invalid method
with self.assertRaises(TimeSeriesException):
ts.fill(method='bogus')
# limit not int
with self.assertRaises(ProcessorException):
ts.fill(fill_limit='z')
# direct access to filler via pipeline needs to take a single path
with self.assertRaises(ProcessorException):
pip = Pipeline()
pip.fill(method='linear',
field_spec=['direction.in', 'direction.out'])
# invalid method
with self.assertRaises(ProcessorException):
pip = Pipeline()
pip.fill(method='bogus')
# catch bad path at various points
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='bad.path')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='bad.path', method='linear')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='direction.bogus')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
# trigger warnings about non-numeric values in linear.
with warnings.catch_warnings(record=True) as wrn:
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': 'non_numeric',
'out': 4
}],
[1400425949000, {
'in': 5,
'out': None
}],
])
ts = TimeSeries(simple_missing_data)
ts.fill(field_spec='direction.in', method='linear')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
# empty series for coverage caught a bug
empty = TimeSeries(
dict(name="Sensor values",
columns=["time", "temperature"],
points=[]))
self.assertEqual(empty.fill(field_spec='temperature').size(), 0)