def test_invalid_point(self):
"""make sure non-numeric values are handled properly."""
bad_point = copy.deepcopy(SIMPLE_GAP_DATA)
bad_point.get('points')[-2][1] = 'non_numeric_value'
ts = TimeSeries(bad_point)
with warnings.catch_warnings(record=True) as wrn:
aligned = ts.align(window='1m')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), 1.0)
self.assertEqual(aligned.at(4).get(), 1.0)
self.assertEqual(aligned.at(5).get(), 1.0)
self.assertEqual(aligned.at(6).get(), None) # bad value
self.assertEqual(aligned.at(7).get(), None) # bad value
with warnings.catch_warnings(record=True) as wrn:
a_diff = aligned.rate()
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(a_diff.at(5).get(), None) # bad value
self.assertEqual(a_diff.at(6).get(), None) # bad value
def test_callback_offset_chain(self):
"""pass a callback in rather than retrieving a keyed collection."""
def cback(collection, window_key, group_by): # pylint: disable=unused-argument
"""callback to pass in."""
global RESULTS # pylint: disable=global-statement
RESULTS = collection
timeseries = TimeSeries(DATA)
(
Pipeline()
.from_source(timeseries.collection())
.offset_by(1, 'value')
.offset_by(2)
.to(CollectionOut, cback)
)
# Spurious lint error due to upstream tinkering
# with the global variable
# pylint: disable=no-member
self.assertEqual(RESULTS.at(0).get(), 55)
self.assertEqual(RESULTS.at(1).get(), 21)
self.assertEqual(RESULTS.at(2).get(), 29)
self.assertEqual(RESULTS.at(3).get(), 96)
def test_assymetric_linear_fill(self):
"""Test new chained/assymetric linear default fill in TimeSeries."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': None, 'out': None}],
[1400425949000, {'in': None, 'out': None}],
[1400425950000, {'in': 3, 'out': 8}],
[1400425960000, {'in': None, 'out': None}],
[1400425970000, {'in': 5, 'out': 12}],
[1400425980000, {'in': 6, 'out': 13}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(method='linear', field_spec=['direction.in', 'direction.out'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1.6666666666666665) # filled
self.assertEqual(new_ts.at(2).get('direction.in'), 2.333333333333333) # filled
self.assertEqual(new_ts.at(3).get('direction.in'), 3)
self.assertEqual(new_ts.at(4).get('direction.in'), 4.0) # filled
self.assertEqual(new_ts.at(5).get('direction.in'), 5)
self.assertEqual(new_ts.at(0).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(1).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(2).get('direction.out'), None) # can't fill
self.assertEqual(new_ts.at(3).get('direction.out'), 8)
self.assertEqual(new_ts.at(4).get('direction.out'), 10.0) # filled
self.assertEqual(new_ts.at(5).get('direction.out'), 12)
def test_scan_stop(self):
"""stop seeking good values if there are none - for coverage."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None}],
[1400425948000, {'in': 3, 'out': None}],
[1400425949000, {'in': None, 'out': None}],
[1400425950000, {'in': None, 'out': 8}],
[1400425960000, {'in': None, 'out': None}],
[1400425970000, {'in': None, 'out': 12}],
[1400425980000, {'in': None, 'out': 13}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(field_spec='direction.out', method='linear')
self.assertEqual(new_ts.at(2).get('direction.in'), None)
self.assertEqual(new_ts.at(3).get('direction.in'), None)
self.assertEqual(new_ts.at(4).get('direction.in'), None)
self.assertEqual(new_ts.at(5).get('direction.in'), None)
self.assertEqual(new_ts.at(6).get('direction.in'), None)
def test_aggregation_filtering(self):
"""test the filtering modifers to the agg functions."""
event_objects = [
Event(1429673400000, {'in': 1, 'out': 2}),
Event(1429673460000, {'in': 3, 'out': None}),
Event(1429673520000, {'in': 5, 'out': 6}),
]
series = TimeSeries(dict(name='events', events=event_objects))
self.assertEqual(series.sum('out', Filters.ignore_missing), 8)
self.assertEqual(series.avg('out', Filters.ignore_missing), 4)
self.assertEqual(series.min('out', Filters.zero_missing), 0)
self.assertEqual(series.max('out', Filters.propagate_missing), None)
self.assertEqual(series.mean('out', Filters.ignore_missing), 4)
self.assertEqual(series.median('out', Filters.zero_missing), 2)
self.assertEqual(series.stdev('out', Filters.zero_missing), 2.494438257849294)
avg_f = Functions.avg(Filters.none_if_empty)
self.assertIsNone(avg_f([]))
def bad_filtering_function(): # pylint: disable=missing-docstring
pass
with self.assertRaises(FilterException):
series.sum('out', bad_filtering_function)
def test_fill_event_variants(self):
"""fill time range and indexed events."""
range_list = [
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=1)),
{'in': 100}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=2)),
{'in': None}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=3)),
{'in': None}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=4)),
{'in': 90}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=5)),
{'in': 80}
),
TimeRangeEvent(
(aware_utcnow(), aware_utcnow() + datetime.timedelta(minutes=6)),
{'in': 70}
),
]
coll = Collection(range_list)
# canned series objects
rts = TimeSeries(
dict(name='collection', collection=coll))
new_rts = rts.fill(field_spec='in')
self.assertEqual(new_rts.at(1).get('in'), 0)
self.assertEqual(new_rts.at(2).get('in'), 0)
# indexed events
index_list = [
IndexedEvent('1d-12355', {'value': 42}),
IndexedEvent('1d-12356', {'value': None}),
IndexedEvent('1d-12357', {'value': None}),
IndexedEvent('1d-12358', {'value': 52}),
IndexedEvent('1d-12359', {'value': 55}),
IndexedEvent('1d-12360', {'value': 58}),
]
coll = Collection(index_list)
its = TimeSeries(
dict(name='collection', collection=coll))
new_its = its.fill()
self.assertEqual(new_its.at(1).get(), 0)
self.assertEqual(new_its.at(2).get(), 0)
def test_bad_args(self):
"""error states for coverage."""
# various bad values
with self.assertRaises(ProcessorException):
Align(dict())
with self.assertRaises(ProcessorException):
Rate(dict())
with self.assertRaises(ProcessorException):
self._simple_ts.align(method='bogus')
with self.assertRaises(ProcessorException):
self._simple_ts.align(limit='bogus')
# non event types
ticket_range = dict(
name="outages",
columns=["timerange", "title", "esnet_ticket"],
points=[
[[1429673400000, 1429707600000], "BOOM", "ESNET-20080101-001"],
[[1429673400000, 1429707600000], "BAM!", "ESNET-20080101-002"],
],
)
ts = TimeSeries(ticket_range)
with self.assertRaises(ProcessorException):
ts.align()
with self.assertRaises(ProcessorException):
ts.rate()
def test_fixed_window(self):
"""Test fixed window rollup"""
timeseries = TimeSeries(SEPT_2014_DATA)
daily_avg = timeseries.fixed_window_rollup(
'1d',
dict(value=dict(value=Functions.avg())))
self.assertEqual(daily_avg.size(), 5)
self.assertEqual(daily_avg.at(0).value(), 46.875)
self.assertEqual(daily_avg.at(2).value(), 54.083333333333336)
self.assertEqual(daily_avg.at(4).value(), 51.85)
# not really a rollup, each data point will create one
# aggregation index.
timeseries = TimeSeries(SEPT_2014_DATA)
hourly_avg = timeseries.hourly_rollup(dict(value=dict(value=Functions.avg())))
self.assertEqual(hourly_avg.size(), len(SEPT_2014_DATA.get('points')))
self.assertEqual(hourly_avg.at(0).value(), 80.0)
self.assertEqual(hourly_avg.at(2).value(), 52.0)
self.assertEqual(hourly_avg.at(4).value(), 26.0)
def test_non_fixed_rollups(self):
"""Work the calendar rollup logic / utc / etc."""
timeseries = TimeSeries(SEPT_2014_DATA)
# just silence the warnings, not do anything with them.
with warnings.catch_warnings(record=True):
daily_avg = timeseries.daily_rollup(dict(value=dict(value=Functions.avg())))
ts_1 = SEPT_2014_DATA.get('points')[0][0]
self.assertEqual(
Index.get_daily_index_string(dt_from_ms(ts_1), utc=False),
daily_avg.at(0).index().to_string()
)
monthly_avg = timeseries.monthly_rollup(dict(value=dict(value=Functions.avg())))
self.assertEqual(
Index.get_monthly_index_string(dt_from_ms(ts_1), utc=False),
monthly_avg.at(0).index().to_string()
)
yearly_avg = timeseries.yearly_rollup(dict(value=dict(value=Functions.avg())))
self.assertEqual(
Index.get_yearly_index_string(dt_from_ms(ts_1), utc=False),
yearly_avg.at(0).index().to_string()
)
def test_negative_derivatives(self):
"""Test behavior on counter resets."""
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 50]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
# lower counter will produce negative derivatives
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), -0.5434782608695656)
self.assertEqual(rates.at(1).get('value_rate'), -0.5434782608695646)
self.assertEqual(rates.at(2).get('value_rate'), -0.5434782608695653)
rates = ts.align(window='30s').rate(allow_negative=False)
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), None)
self.assertEqual(rates.at(1).get('value_rate'), None)
self.assertEqual(rates.at(2).get('value_rate'), None)
def test_rate_bins(self):
"""replicate basic esmond rates."""
# | 100 | | | | 200 | v
# | | | | | | | |
# 60 89 90 120 150 180 181 210 t ->
# | | | | | |
# |<- ? --------->|<- 1.08/s --->|<- 1.08/s --->|<- 1.08/s --->|<- ? ------->| result
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 200]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), 1.0869565217391313)
self.assertEqual(rates.at(1).get('value_rate'), 1.0869565217391293)
self.assertEqual(rates.at(2).get('value_rate'), 1.0869565217391313)
def test_merge_sum_and_map(self):
"""test the time series merging/map static methods."""
t_in = TimeSeries(TRAFFIC_DATA_IN)
t_out = TimeSeries(TRAFFIC_DATA_OUT)
t_merged = TimeSeries.timeseries_list_merge(dict(name='traffic'), [t_in, t_out])
self.assertEqual(t_merged.at(2).get('in'), 26)
self.assertEqual(t_merged.at(2).get('out'), 67)
self.assertEqual(t_merged.name(), 'traffic')
t_summed = TimeSeries.timeseries_list_sum(
dict(name='traffic'), [t_in, t_in], 'in')
self.assertEqual(t_summed.at(0).get('in'), 104)
self.assertEqual(t_summed.at(1).get('in'), 36)
# more variations for coverage
test_idx_data = dict(
name="availability",
columns=["index", "uptime"],
points=[
["2014-07", 100],
["2014-08", 88],
["2014-09", 95],
["2014-10", 99],
["2014-11", 91],
["2014-12", 99],
["2015-01", 100],
["2015-02", 92],
["2015-03", 99],
["2015-04", 87],
["2015-05", 92],
["2015-06", 100],
]
)
t_idx = TimeSeries(test_idx_data)
idx_sum = TimeSeries.timeseries_list_sum(
dict(name='available'), [t_idx, t_idx], 'uptime')
self.assertEqual(idx_sum.at(0).get('uptime'), 200)
self.assertEqual(idx_sum.at(1).get('uptime'), 176)
self.assertEqual(idx_sum.at(2).get('uptime'), 190)
test_outage = dict(
name="outages",
columns=["timerange", "length", "esnet_ticket"],
points=[
[[1429673400000, 1429707600000], 23, "ESNET-20080101-001"],
[[1429673500000, 1429707700000], 54, "ESNET-20080101-002"],
],
)
t_tr = TimeSeries(test_outage)
tr_sum = TimeSeries.timeseries_list_sum(
dict(name='outage length'), [t_tr, t_tr], 'length')
self.assertEqual(tr_sum.at(0).get('length'), 46)
self.assertEqual(tr_sum.at(1).get('length'), 108)
def test_complex_zero_fill(self):
"""make sure more complex nested paths work OK"""
complex_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000,
{'in': {'tcp': 1, 'udp': 3}, 'out': {'tcp': 2, 'udp': 3}}],
[1400425948000,
{'in': {'tcp': 3, 'udp': None}, 'out': {'tcp': 4, 'udp': 3}}],
[1400425949000,
{'in': {'tcp': 5, 'udp': None}, 'out': {'tcp': None, 'udp': 3}}],
[1400425950000,
{'in': {'tcp': 7, 'udp': None}, 'out': {'tcp': None, 'udp': 3}}],
[1400425960000,
{'in': {'tcp': 9, 'udp': 4}, 'out': {'tcp': 6, 'udp': 3}}],
[1400425970000,
{'in': {'tcp': 11, 'udp': 5}, 'out': {'tcp': 8, 'udp': 3}}],
]
)
ts = TimeSeries(complex_missing_data)
# zero fill everything
new_ts = ts.fill(field_spec=['direction.out.tcp', 'direction.in.udp'])
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
# do it again, but only fill the out.tcp
new_ts = ts.fill(field_spec=['direction.out.tcp'])
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
def test_fixed_window_collect(self):
"""Make collections for each day in the timeseries."""
timeseries = TimeSeries(SEPT_2014_DATA)
colls = timeseries.collect_by_fixed_window('1d')
self.assertEqual(colls.get('1d-16314').size(), 24)
self.assertEqual(colls.get('1d-16318').size(), 20)
def test_various_bad_args(self):
"""ensure proper exceptions are being raised."""
ser1 = TimeSeries(DATA)
with self.assertRaises(CollectionException):
ser1.aggregate(dict())
with self.assertRaises(CollectionException):
ser1.aggregate(Functions.sum(), dict())
def test_percentile(self):
"""Test percentile of a series."""
series = TimeSeries(dict(
name="Sensor values",
columns=["time", "temperature"],
points=[
[1400425951000, 22.3],
[1400425952000, 32.4],
[1400425953000, 12.1],
[1400425955000, 76.8],
[1400425956000, 87.3],
[1400425957000, 54.6],
[1400425958000, 45.5],
[1400425959000, 87.9]
]
))
self.assertEqual(series.percentile(50, 'temperature'), 50.05)
self.assertEqual(series.percentile(95, 'temperature'), 87.69)
self.assertEqual(series.percentile(99, 'temperature'), 87.858)
self.assertEqual(series.percentile(99, 'temperature', 'lower'), 87.3)
self.assertEqual(series.percentile(99, 'temperature', 'higher'), 87.9)
self.assertEqual(series.percentile(99, 'temperature', 'nearest'), 87.9)
self.assertEqual(series.percentile(99, 'temperature', 'midpoint'), 87.6)
self.assertEqual(series.percentile(0, 'temperature'), 12.1)
self.assertEqual(series.percentile(100, 'temperature'), 87.9)
def test_percentile_empty(self):
"""percentile of an empty timeseries."""
series = TimeSeries(dict(
name="Sensor values",
columns=["time", "temperature"],
points=[
]
))
self.assertIsNone(series.percentile(0, 'temperature'))
self.assertIsNone(series.percentile(100, 'temperature'))
def test_rate_mag(self):
"""test the rate processor order of mag."""
ts = TimeSeries(RATE)
rate = ts.rate(field_spec='in')
# one less than source
self.assertEqual(rate.size(), len(RATE.get('points')) - 1)
self.assertEqual(rate.at(2).get('in_rate'), 1)
self.assertEqual(rate.at(3).get('in_rate'), 1)
self.assertEqual(rate.at(4).get('in_rate'), 2)
self.assertEqual(rate.at(8).get('in_rate'), 3)
self.assertEqual(rate.at(9).get('in_rate'), 4)
def test_simple_take(self):
"""take 10 events in batch."""
timeseries = TimeSeries(SEPT_2014_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.take(10)
.to_keyed_collections()
)
new_ts = TimeSeries(dict(name='result', collection=kcol.get('all')))
self.assertEqual(new_ts.size(), 10)
def test_single_select(self):
"""select a single column."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.select('in')
.to_keyed_collections()
)
new_ts = TimeSeries(dict(name='new_timeseries', collection=kcol.get('all')))
self.assertEqual(new_ts.columns(), ['in'])
def test_bad_args(self):
"""bad args for new TimeSeries functions/coverage."""
# def timeseries_list_reduce(data, series_list, reducer, field_spec=None):
def test_func():
"""test function."""
i = 1
return i
with self.assertRaises(TimeSeriesException):
TimeSeries.timeseries_list_reduce({}, {}, test_func)
with self.assertRaises(TimeSeriesException):
TimeSeries.timeseries_list_reduce({}, [], {})
def test_indexed_event_series(self):
"""test a series of IndexedEvent objects."""
indexed_event_series = dict(
name="availability",
columns=["index", "uptime"],
points=[
["2014-07", "100%"],
["2014-08", "88%"],
["2014-09", "95%"],
["2014-10", "99%"],
["2014-11", "91%"],
["2014-12", "99%"],
["2015-01", "100%"],
["2015-02", "92%"],
["2015-03", "99%"],
["2015-04", "87%"],
["2015-05", "92%"],
["2015-06", "100%"],
]
)
series = TimeSeries(indexed_event_series)
wire = self._call_interop_script('indexed_event', series.to_string())
new_series = TimeSeries(wire)
new_json = new_series.to_json()
self._validate_wire_points(indexed_event_series, new_json)
self.assertTrue(new_json.get('utc'))
# again with more involved data
availability_series = dict(
name="availability",
columns=["index", "uptime", "notes", "outages"],
points=[
["2014-08", 88, "", 17],
["2014-09", 100, "", 2],
["2014-09", 95, "", 6],
["2014-10", 99, "", 3],
["2014-11", 91, "", 14],
["2014-12", 99, "", 3],
["2015-01", 100, "", 0],
["2015-02", 92, "", 12],
["2015-03", 99, "Minor outage March 2", 4],
["2015-04", 87, "Planned downtime in April", 82],
["2015-05", 92, "Router failure June 12", 26],
["2015-06", 100, "", 0],
]
)
series = TimeSeries(availability_series)
wire = self._call_interop_script('indexed_event', series.to_string())
new_series = TimeSeries(wire)
new_json = new_series.to_json()
self._validate_wire_points(availability_series, new_json)
def test_subset_select(self):
"""select multiple columns."""
timeseries = TimeSeries(IN_OUT_DATA)
kcol = (
Pipeline()
.from_source(timeseries)
.select(['out', 'perpendicular'])
.to_keyed_collections()
)
new_ts = TimeSeries(dict(name='new_timeseries', collection=kcol.get('all')))
self.assertEqual(set(new_ts.columns()), set(['out', 'perpendicular']))
def test_ts_offset_chain(self):
"""test running the offset chain directly from the TimeSeries."""
timeseries = TimeSeries(DATA)
kcol = (
timeseries.pipeline()
.offset_by(1, 'value')
.offset_by(2)
.to_keyed_collections()
)
self.assertEqual(kcol['all'].at(0).get(), 55)
self.assertEqual(kcol['all'].at(1).get(), 21)
self.assertEqual(kcol['all'].at(2).get(), 29)
self.assertEqual(kcol['all'].at(3).get(), 96)
def test_percentile_single(self):
"""percentile of an timeseries with one point."""
series = TimeSeries(dict(
name="Sensor values",
columns=["time", "temperature"],
points=[
[1400425951000, 22.3]
]
))
self.assertEqual(series.percentile(0, 'temperature'), 22.3)
self.assertEqual(series.percentile(50, 'temperature'), 22.3)
self.assertEqual(series.percentile(100, 'temperature'), 22.3)
def test_equality_methods(self):
"""test equal/same static methods."""
ser1 = TimeSeries(DATA)
ser2 = TimeSeries(DATA)
self.assertTrue(TimeSeries.equal(ser1, ser1))
self.assertTrue(TimeSeries.same(ser1, ser1))
self.assertFalse(TimeSeries.equal(ser1, ser2))
self.assertTrue(TimeSeries.same(ser1, ser2))
copy_ctor = TimeSeries(ser1)
self.assertTrue(TimeSeries.equal(copy_ctor, ser1))
self.assertFalse(copy_ctor is ser1)
def test_simple_offset_chain(self):
"""test a simple offset chain."""
timeseries = TimeSeries(DATA)
kcol = (
Pipeline()
.from_source(timeseries.collection())
.offset_by(1, 'value')
.offset_by(2)
.to_keyed_collections()
)
self.assertEqual(kcol['all'].at(0).get(), 55)
self.assertEqual(kcol['all'].at(1).get(), 21)
self.assertEqual(kcol['all'].at(2).get(), 29)
self.assertEqual(kcol['all'].at(3).get(), 96)
def test_rate_bins_long(self):
"""replicate counter to rate conversion with more data."""
raw_ts = TimeSeries(RAW)
base_rates = raw_ts.align(window='30s').rate()
for i in enumerate(base_rates.collection().events()):
# there are going to be decimal rounding variations but
# for the purposes of this sanity check, if they are
# equal to one decimal place is close enough.
self.assertAlmostEqual(
i[1].get('value_rate'),
BASE.get('points')[i[0]][1],
places=1
)
def test_underlying_methods(self):
"""basically aliases for underlying collection methods."""
self.assertEqual(self._canned_event_series.count(), len(EVENT_LIST))
tser = self._canned_event_series
self.assertEqual(tser.sum('in'), 9)
self.assertEqual(tser.avg('out'), 4)
self.assertEqual(tser.mean('out'), 4)
self.assertEqual(tser.min('in'), 1)
self.assertEqual(tser.max('in'), 5)
self.assertEqual(tser.median('out'), 4)
self.assertEqual(tser.stdev('out'), 1.632993161855452)
# redundant, but for coverage
self.assertEqual(tser.aggregate(Functions.sum(), 'in'), 9)
self.assertEqual(tser.aggregate(Functions.sum(), ('in',)), 9)
ser1 = TimeSeries(DATA)
self.assertEqual(ser1.aggregate(Functions.sum()), 189)
def test_pad(self):
"""Test the pad style fill."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {'in': 1, 'out': None, 'drop': None}],
[1400425948000, {'in': None, 'out': 4, 'drop': None}],
[1400425949000, {'in': None, 'out': None, 'drop': 13}],
[1400425950000, {'in': None, 'out': None, 'drop': 14}],
[1400425960000, {'in': 9, 'out': 8, 'drop': None}],
[1400425970000, {'in': 11, 'out': 10, 'drop': 16}],
]
)
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(method='pad',
field_spec=['direction.in', 'direction.out', 'direction.drop'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(2).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(3).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(4).get('direction.in'), 9)
self.assertEqual(new_ts.at(5).get('direction.in'), 11)
self.assertEqual(new_ts.at(0).get('direction.out'), None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.out'), 4)
self.assertEqual(new_ts.at(2).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(3).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(4).get('direction.out'), 8)
self.assertEqual(new_ts.at(5).get('direction.out'), 10)
self.assertEqual(new_ts.at(0).get('direction.drop'), None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.drop'), None) # bad prev can't pad
self.assertEqual(new_ts.at(2).get('direction.drop'), 13)
self.assertEqual(new_ts.at(3).get('direction.drop'), 14)
self.assertEqual(new_ts.at(4).get('direction.drop'), 14) # padded
self.assertEqual(new_ts.at(5).get('direction.drop'), 16)
def test_event_series(self):
"""test a series that contains basic event objects."""
event_series = dict(name="traffic",
columns=["time", "value", "status"],
points=[[1400425947000, 52, "ok"],
[1400425948000, 18, "ok"],
[1400425949000, 26, "fail"],
[1400425950000, 93, "offline"]])
series = TimeSeries(event_series)
wire = self._call_interop_script('event', series.to_string())
new_series = TimeSeries(wire)
new_json = new_series.to_json()
self._validate_wire_points(event_series, new_json)
self.assertTrue(new_json.get('utc'))
# try something a bit fancier with different types
interface_series = dict(
name="star-cr5:to_anl_ip-a_v4",
description="star-cr5->anl(as683):100ge:site-ex:show:intercloud",
device="star-cr5",
id=169,
interface="to_anl_ip-a_v4",
is_ipv6=False,
is_oscars=False,
oscars_id=None,
resource_uri="",
site="anl",
site_device="noni",
site_interface="et-1/0/0",
stats_type="Standard",
title=None,
columns=["time", "in", "out"],
points=[[1400425947000, 52, 34], [1400425948000, 18, 13],
[1400425949000, 26, 67], [1400425950000, 93, 91]])
series = TimeSeries(interface_series)
wire = self._call_interop_script('event', series.to_string())
new_series = TimeSeries(wire)
new_json = new_series.to_json()
self._validate_wire_points(interface_series, new_json)
# Now with a list of events
event_objects = [
Event(1429673400000, {
'in': 1,
'out': 2
}),
Event(1429673460000, {
'in': 3,
'out': 4
}),
Event(1429673520000, {
'in': 5,
'out': 6
}),
]
series = TimeSeries(dict(name='events', events=event_objects))
wire = self._call_interop_script('event', series.to_string())
new_series = TimeSeries(wire)
for i in enumerate(event_objects):
self.assertTrue(Event.same(i[1], new_series.at(i[0])))
def test_bad_args(self):
"""Trigger error states for coverage."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None,
'drop': None
}],
[1400425948000, {
'in': None,
'out': 4,
'drop': None
}],
[1400425949000, {
'in': None,
'out': None,
'drop': 13
}],
[1400425950000, {
'in': None,
'out': None,
'drop': 14
}],
[1400425960000, {
'in': 9,
'out': 8,
'drop': None
}],
[1400425970000, {
'in': 11,
'out': 10,
'drop': 16
}],
])
ts = TimeSeries(simple_missing_data)
# bad ctor arg
with self.assertRaises(ProcessorException):
f = Filler(dict())
# invalid method
with self.assertRaises(TimeSeriesException):
ts.fill(method='bogus')
# limit not int
with self.assertRaises(ProcessorException):
ts.fill(fill_limit='z')
# direct access to filler via pipeline needs to take a single path
with self.assertRaises(ProcessorException):
pip = Pipeline()
pip.fill(method='linear',
field_spec=['direction.in', 'direction.out'])
# invalid method
with self.assertRaises(ProcessorException):
pip = Pipeline()
pip.fill(method='bogus')
# catch bad path at various points
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='bad.path')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='bad.path', method='linear')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
with warnings.catch_warnings(record=True) as wrn:
ts.fill(field_spec='direction.bogus')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
# trigger warnings about non-numeric values in linear.
with warnings.catch_warnings(record=True) as wrn:
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': 'non_numeric',
'out': 4
}],
[1400425949000, {
'in': 5,
'out': None
}],
])
ts = TimeSeries(simple_missing_data)
ts.fill(field_spec='direction.in', method='linear')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
# empty series for coverage caught a bug
empty = TimeSeries(
dict(name="Sensor values",
columns=["time", "temperature"],
points=[]))
self.assertEqual(empty.fill(field_spec='temperature').size(), 0)
def test_complex_zero_fill(self):
"""make sure more complex nested paths work OK"""
complex_missing_data = dict(name="traffic",
columns=["time", "direction"],
points=[
[
1400425947000, {
'in': {
'tcp': 1,
'udp': 3
},
'out': {
'tcp': 2,
'udp': 3
}
}
],
[
1400425948000, {
'in': {
'tcp': 3,
'udp': None
},
'out': {
'tcp': 4,
'udp': 3
}
}
],
[
1400425949000, {
'in': {
'tcp': 5,
'udp': None
},
'out': {
'tcp': None,
'udp': 3
}
}
],
[
1400425950000, {
'in': {
'tcp': 7,
'udp': None
},
'out': {
'tcp': None,
'udp': 3
}
}
],
[
1400425960000, {
'in': {
'tcp': 9,
'udp': 4
},
'out': {
'tcp': 6,
'udp': 3
}
}
],
[
1400425970000, {
'in': {
'tcp': 11,
'udp': 5
},
'out': {
'tcp': 8,
'udp': 3
}
}
],
])
ts = TimeSeries(complex_missing_data)
# zero fill everything
new_ts = ts.fill(field_spec=['direction.out.tcp', 'direction.in.udp'])
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
# do it again, but only fill the out.tcp
new_ts = ts.fill(field_spec=['direction.out.tcp'])
self.assertEqual(new_ts.at(0).get('direction.out.tcp'), 2)
self.assertEqual(new_ts.at(1).get('direction.out.tcp'), 4)
self.assertEqual(new_ts.at(2).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(3).get('direction.out.tcp'), 0) # fill
self.assertEqual(new_ts.at(4).get('direction.out.tcp'), 6)
self.assertEqual(new_ts.at(5).get('direction.out.tcp'), 8)
self.assertEqual(new_ts.at(0).get('direction.in.udp'), 3)
self.assertEqual(new_ts.at(1).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(2).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(3).get('direction.in.udp'), None) # no fill
self.assertEqual(new_ts.at(4).get('direction.in.udp'), 4)
self.assertEqual(new_ts.at(5).get('direction.in.udp'), 5)
def test_linear(self):
"""Test linear interpolation filling returned by to_keyed_collections()."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': 2
}],
[1400425948000, {
'in': None,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': 3,
'out': None
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': 5,
'out': 12
}],
[1400425980000, {
'in': 6,
'out': 13
}],
])
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(field_spec=['direction.in', 'direction.out'],
method='linear')
self.assertEqual(new_ts.size(), 7)
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'),
1.6666666666666665) # filled
self.assertEqual(new_ts.at(2).get('direction.in'),
2.333333333333333) # filled
self.assertEqual(new_ts.at(3).get('direction.in'), 3)
self.assertEqual(new_ts.at(4).get('direction.in'), 4.0) # filled
self.assertEqual(new_ts.at(5).get('direction.in'), 5)
self.assertEqual(new_ts.at(0).get('direction.out'), 2)
self.assertEqual(
new_ts.at(1).get('direction.out'), 2.4347826086956523) # filled
self.assertEqual(
new_ts.at(2).get('direction.out'), 2.8695652173913047) # filled
self.assertEqual(new_ts.at(3).get('direction.out'),
3.304347826086957) # filled
self.assertEqual(
new_ts.at(4).get('direction.out'), 7.6521739130434785) # filled
self.assertEqual(new_ts.at(5).get('direction.out'), 12)
def setUp(self):
"""setup for all tests."""
self._simple_ts = TimeSeries(SIMPLE_GAP_DATA)
def test_rename(self):
"""Test the renamer facility."""
# rename an Event series
ts = copy.deepcopy(self._canned_event_series)
renamed = ts.rename_columns({'in': 'new_in', 'out': 'new_out'})
self.assertEqual(
renamed.at(0).get('new_in'),
self._canned_event_series.at(0).get('in'))
self.assertEqual(
renamed.at(0).get('new_out'),
self._canned_event_series.at(0).get('out'))
self.assertEqual(
renamed.at(1).get('new_in'),
self._canned_event_series.at(1).get('in'))
self.assertEqual(
renamed.at(1).get('new_out'),
self._canned_event_series.at(1).get('out'))
self.assertEqual(
renamed.at(2).get('new_in'),
self._canned_event_series.at(2).get('in'))
self.assertEqual(
renamed.at(2).get('new_out'),
self._canned_event_series.at(2).get('out'))
# rename a TimeRangeEvent series
ts = TimeSeries(TICKET_RANGE)
renamed = ts.rename_columns({
'title': 'event',
'esnet_ticket': 'ticket'
})
self.assertEqual(renamed.at(0).get('event'), ts.at(0).get('title'))
self.assertEqual(
renamed.at(0).get('ticket'),
ts.at(0).get('esnet_ticket'))
self.assertEqual(renamed.at(1).get('event'), ts.at(1).get('title'))
self.assertEqual(
renamed.at(1).get('ticket'),
ts.at(1).get('esnet_ticket'))
self.assertEqual(renamed.at(0).timestamp(), ts.at(0).timestamp())
self.assertEqual(renamed.at(1).timestamp(), ts.at(1).timestamp())
# rename and IndexedEvent series
ts = TimeSeries(AVAILABILITY_DATA)
renamed = ts.rename_columns(dict(uptime='available'))
self.assertEqual(
renamed.at(0).get('available'),
ts.at(0).get('uptime'))
self.assertEqual(
renamed.at(2).get('available'),
ts.at(2).get('uptime'))
self.assertEqual(
renamed.at(4).get('available'),
ts.at(4).get('uptime'))
self.assertEqual(
renamed.at(6).get('available'),
ts.at(6).get('uptime'))
self.assertEqual(renamed.at(0).timestamp(), ts.at(0).timestamp())
self.assertEqual(renamed.at(1).timestamp(), ts.at(1).timestamp())
self.assertEqual(renamed.at(2).timestamp(), ts.at(2).timestamp())
def test_pad(self):
"""Test the pad style fill."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None,
'drop': None
}],
[1400425948000, {
'in': None,
'out': 4,
'drop': None
}],
[1400425949000, {
'in': None,
'out': None,
'drop': 13
}],
[1400425950000, {
'in': None,
'out': None,
'drop': 14
}],
[1400425960000, {
'in': 9,
'out': 8,
'drop': None
}],
[1400425970000, {
'in': 11,
'out': 10,
'drop': 16
}],
])
ts = TimeSeries(simple_missing_data)
new_ts = ts.fill(
method='pad',
field_spec=['direction.in', 'direction.out', 'direction.drop'])
self.assertEqual(new_ts.at(0).get('direction.in'), 1)
self.assertEqual(new_ts.at(1).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(2).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(3).get('direction.in'), 1) # padded
self.assertEqual(new_ts.at(4).get('direction.in'), 9)
self.assertEqual(new_ts.at(5).get('direction.in'), 11)
self.assertEqual(new_ts.at(0).get('direction.out'),
None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.out'), 4)
self.assertEqual(new_ts.at(2).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(3).get('direction.out'), 4) # padded
self.assertEqual(new_ts.at(4).get('direction.out'), 8)
self.assertEqual(new_ts.at(5).get('direction.out'), 10)
self.assertEqual(new_ts.at(0).get('direction.drop'),
None) # 1st can't pad
self.assertEqual(new_ts.at(1).get('direction.drop'),
None) # bad prev can't pad
self.assertEqual(new_ts.at(2).get('direction.drop'), 13)
self.assertEqual(new_ts.at(3).get('direction.drop'), 14)
self.assertEqual(new_ts.at(4).get('direction.drop'), 14) # padded
self.assertEqual(new_ts.at(5).get('direction.drop'), 16)
def test_group_by_variants(self):
"""test group by with strings and arrays."""
data = dict(name="traffic",
columns=["time", "value", "status"],
points=[[1400425947000, 52, "ok"],
[1400425948000, 18, "ok"],
[1400425949000, 26, "fail"],
[1400425950000, 93, "offline"]])
# group on a single column with string input to group_by
kcol = (Pipeline().from_source(TimeSeries(data)).emit_on(
'flush').group_by('status').to_keyed_collections())
self.assertEqual(kcol.get('ok').size(), 2)
self.assertEqual(kcol.get('fail').size(), 1)
self.assertEqual(kcol.get('offline').size(), 1)
# group on a deep/nested column with an array arg
kcol = (Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('flush').group_by(
['direction', 'status']).to_keyed_collections())
self.assertEqual(kcol.get('OK').size(), 3)
self.assertEqual(kcol.get('FAIL').size(), 1)
self.assertEqual(kcol.get('OK').at(0).value('direction').get('out'), 2)
self.assertEqual(kcol.get('OK').at(1).value('direction').get('in'), 3)
self.assertEqual(
kcol.get('FAIL').at(0).value('direction').get('out'), 0)
# same thing but with the old.school.style
kcol = (Pipeline().from_source(
TimeSeries(dict(name='events', events=DEEP_EVENT_LIST))).emit_on(
'flush').group_by('direction.status').to_keyed_collections())
self.assertEqual(kcol.get('OK').size(), 3)
self.assertEqual(kcol.get('FAIL').size(), 1)
self.assertEqual(kcol.get('OK').at(0).value('direction').get('out'), 2)
self.assertEqual(kcol.get('OK').at(1).value('direction').get('in'), 3)
self.assertEqual(
kcol.get('FAIL').at(0).value('direction').get('out'), 0)
# and with a tuple
kcol = (Pipeline().from_source(
TimeSeries(
dict(name='events',
events=DEEP_EVENT_LIST))).emit_on('flush').group_by((
'direction',
'status',
)).to_keyed_collections())
self.assertEqual(kcol.get('OK').size(), 3)
self.assertEqual(kcol.get('FAIL').size(), 1)
self.assertEqual(kcol.get('OK').at(0).value('direction').get('out'), 2)
self.assertEqual(kcol.get('OK').at(1).value('direction').get('in'), 3)
self.assertEqual(
kcol.get('FAIL').at(0).value('direction').get('out'), 0)
def test_pad_and_zero_limiting(self):
"""test the limiting on pad and zero options."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': None,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': 3,
'out': 8
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': None,
'out': 12
}],
[1400425980000, {
'in': None,
'out': 13
}],
[1400425990000, {
'in': 7,
'out': None
}],
[1400426000000, {
'in': 8,
'out': None
}],
[1400426010000, {
'in': 9,
'out': None
}],
[1400426020000, {
'in': 10,
'out': None
}],
])
ts = TimeSeries(simple_missing_data)
# verify fill limit for zero fill
zero_ts = ts.fill(method='zero',
fill_limit=2,
field_spec=['direction.in', 'direction.out'])
self.assertEqual(zero_ts.at(0).get('direction.in'), 1)
self.assertEqual(zero_ts.at(1).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(2).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(3).get('direction.in'), 3)
self.assertEqual(zero_ts.at(4).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(5).get('direction.in'), 0) # fill
self.assertEqual(zero_ts.at(6).get('direction.in'),
None) # over limit skip
self.assertEqual(zero_ts.at(7).get('direction.in'), 7)
self.assertEqual(zero_ts.at(8).get('direction.in'), 8)
self.assertEqual(zero_ts.at(9).get('direction.in'), 9)
self.assertEqual(zero_ts.at(10).get('direction.in'), 10)
self.assertEqual(zero_ts.at(0).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(1).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(2).get('direction.out'),
None) # over limit skip
self.assertEqual(zero_ts.at(3).get('direction.out'), 8)
self.assertEqual(zero_ts.at(4).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(5).get('direction.out'), 12)
self.assertEqual(zero_ts.at(6).get('direction.out'), 13)
self.assertEqual(zero_ts.at(7).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(8).get('direction.out'), 0) # fill
self.assertEqual(zero_ts.at(9).get('direction.out'),
None) # over limit skip
self.assertEqual(zero_ts.at(10).get('direction.out'),
None) # over limit skip
# verify fill limit for pad fill
pad_ts = ts.fill(method='pad',
fill_limit=2,
field_spec=['direction.in', 'direction.out'])
self.assertEqual(pad_ts.at(0).get('direction.in'), 1)
self.assertEqual(pad_ts.at(1).get('direction.in'), 1) # fill
self.assertEqual(pad_ts.at(2).get('direction.in'), 1) # fill
self.assertEqual(pad_ts.at(3).get('direction.in'), 3)
self.assertEqual(pad_ts.at(4).get('direction.in'), 3) # fill
self.assertEqual(pad_ts.at(5).get('direction.in'), 3) # fill
self.assertEqual(pad_ts.at(6).get('direction.in'),
None) # over limit skip
self.assertEqual(pad_ts.at(7).get('direction.in'), 7)
self.assertEqual(pad_ts.at(8).get('direction.in'), 8)
self.assertEqual(pad_ts.at(9).get('direction.in'), 9)
self.assertEqual(pad_ts.at(10).get('direction.in'), 10)
self.assertEqual(pad_ts.at(0).get('direction.out'),
None) # no fill start
self.assertEqual(pad_ts.at(1).get('direction.out'),
None) # no fill start
self.assertEqual(pad_ts.at(2).get('direction.out'),
None) # no fill start
self.assertEqual(pad_ts.at(3).get('direction.out'), 8)
self.assertEqual(pad_ts.at(4).get('direction.out'), 8) # fill
self.assertEqual(pad_ts.at(5).get('direction.out'), 12)
self.assertEqual(pad_ts.at(6).get('direction.out'), 13)
self.assertEqual(pad_ts.at(7).get('direction.out'), 13) # fill
self.assertEqual(pad_ts.at(8).get('direction.out'), 13) # fill
self.assertEqual(pad_ts.at(9).get('direction.out'),
None) # over limit skip
self.assertEqual(pad_ts.at(10).get('direction.out'),
None) # over limit skip
class AlignTest(unittest.TestCase):
"""
tests for the align processor
"""
def setUp(self):
"""setup for all tests."""
self._simple_ts = TimeSeries(SIMPLE_GAP_DATA)
def test_basic_linear_align(self):
"""test basic align"""
aligned = self._simple_ts.align(window='1m')
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), 1.0)
self.assertEqual(aligned.at(4).get(), 1.0)
self.assertEqual(aligned.at(5).get(), 1.0)
self.assertEqual(aligned.at(6).get(), 1.5)
self.assertEqual(aligned.at(7).get(), 2.5)
def test_basic_hold_align(self):
"""test basic hold align."""
aligned = self._simple_ts.align(window='1m', method='hold')
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), .75)
self.assertEqual(aligned.at(1).get(), 2)
self.assertEqual(aligned.at(2).get(), 2)
self.assertEqual(aligned.at(3).get(), 1)
self.assertEqual(aligned.at(4).get(), 1)
self.assertEqual(aligned.at(5).get(), 1)
self.assertEqual(aligned.at(6).get(), 1)
self.assertEqual(aligned.at(7).get(), 1)
def test_align_limit(self):
"""test basic hold align."""
aligned = self._simple_ts.align(window='1m', method='hold', limit=2)
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), .75)
self.assertEqual(aligned.at(1).get(), 2)
self.assertEqual(aligned.at(2).get(), 2)
self.assertEqual(aligned.at(3).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(4).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(5).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(6).get(), 1)
self.assertEqual(aligned.at(7).get(), 1)
aligned = self._simple_ts.align(field_spec='value', window='1m', method='linear', limit=2)
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(4).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(5).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(6).get(), 1.5)
self.assertEqual(aligned.at(7).get(), 2.5)
def test_invalid_point(self):
"""make sure non-numeric values are handled properly."""
bad_point = copy.deepcopy(SIMPLE_GAP_DATA)
bad_point.get('points')[-2][1] = 'non_numeric_value'
ts = TimeSeries(bad_point)
with warnings.catch_warnings(record=True) as wrn:
aligned = ts.align(window='1m')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), 1.0)
self.assertEqual(aligned.at(4).get(), 1.0)
self.assertEqual(aligned.at(5).get(), 1.0)
self.assertEqual(aligned.at(6).get(), None) # bad value
self.assertEqual(aligned.at(7).get(), None) # bad value
with warnings.catch_warnings(record=True) as wrn:
a_diff = aligned.rate()
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(a_diff.at(5).get(), None) # bad value
self.assertEqual(a_diff.at(6).get(), None) # bad value
def test_rate_mag(self):
"""test the rate processor order of mag."""
ts = TimeSeries(RATE)
rate = ts.rate(field_spec='in')
# one less than source
self.assertEqual(rate.size(), len(RATE.get('points')) - 1)
self.assertEqual(rate.at(2).get('in_rate'), 1)
self.assertEqual(rate.at(3).get('in_rate'), 1)
self.assertEqual(rate.at(4).get('in_rate'), 2)
self.assertEqual(rate.at(8).get('in_rate'), 3)
self.assertEqual(rate.at(9).get('in_rate'), 4)
def test_rate_bins(self):
"""replicate basic esmond rates."""
# | 100 | | | | 200 | v
# | | | | | | | |
# 60 89 90 120 150 180 181 210 t ->
# | | | | | |
# |<- ? --------->|<- 1.08/s --->|<- 1.08/s --->|<- 1.08/s --->|<- ? ------->| result
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 200]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), 1.0869565217391313)
self.assertEqual(rates.at(1).get('value_rate'), 1.0869565217391293)
self.assertEqual(rates.at(2).get('value_rate'), 1.0869565217391313)
def test_rate_bins_long(self):
"""replicate counter to rate conversion with more data."""
raw_ts = TimeSeries(RAW)
base_rates = raw_ts.align(window='30s').rate()
for i in enumerate(base_rates.collection().events()):
# there are going to be decimal rounding variations but
# for the purposes of this sanity check, if they are
# equal to one decimal place is close enough.
self.assertAlmostEqual(
i[1].get('value_rate'),
BASE.get('points')[i[0]][1],
places=1
)
def test_negative_derivatives(self):
"""Test behavior on counter resets."""
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 50]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
# lower counter will produce negative derivatives
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), -0.5434782608695656)
self.assertEqual(rates.at(1).get('value_rate'), -0.5434782608695646)
self.assertEqual(rates.at(2).get('value_rate'), -0.5434782608695653)
rates = ts.align(window='30s').rate(allow_negative=False)
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), None)
self.assertEqual(rates.at(1).get('value_rate'), None)
self.assertEqual(rates.at(2).get('value_rate'), None)
def test_bad_args(self):
"""error states for coverage."""
# various bad values
with self.assertRaises(ProcessorException):
Align(dict())
with self.assertRaises(ProcessorException):
Rate(dict())
with self.assertRaises(ProcessorException):
self._simple_ts.align(method='bogus')
with self.assertRaises(ProcessorException):
self._simple_ts.align(limit='bogus')
# non event types
ticket_range = dict(
name="outages",
columns=["timerange", "title", "esnet_ticket"],
points=[
[[1429673400000, 1429707600000], "BOOM", "ESNET-20080101-001"],
[[1429673400000, 1429707600000], "BAM!", "ESNET-20080101-002"],
],
)
ts = TimeSeries(ticket_range)
with self.assertRaises(ProcessorException):
ts.align()
with self.assertRaises(ProcessorException):
ts.rate()
def test_first_point(self):
"""Make sure the first point is handled right when it is perfectly aligned."""
data = dict(
name="traffic",
columns=["time", "value"],
points=[
[1473490770000, 10],
[1473490800000, 20],
[1473490830000, 30],
[1473490860000, 40]
]
)
base_30_sec = (
Pipeline()
.from_source(TimeSeries(data))
.align(window='30s', method='linear', limit=10)
.to_keyed_collections()
)
self.assertEqual(base_30_sec.get('all').size(), 4)
def test_linear_list(self):
"""Test linear interpolation returned as an event list."""
simple_missing_data = dict(
name="traffic",
columns=["time", "direction"],
points=[
[1400425947000, {
'in': 1,
'out': None
}],
[1400425948000, {
'in': None,
'out': None
}],
[1400425949000, {
'in': None,
'out': None
}],
[1400425950000, {
'in': 3,
'out': 8
}],
[1400425960000, {
'in': None,
'out': None
}],
[1400425970000, {
'in': 5,
'out': 12
}],
[1400425980000, {
'in': 6,
'out': 13
}],
])
ts = TimeSeries(simple_missing_data)
# also test chaining multiple fillers together. in this series,
# field_spec=['direction.in', 'direction.out'] would not start
# filling until the 4th point so points 2 and 3 of direction.in
# would not be filled. A chain like this will ensure both
# columns will be fully filled.
elist = (Pipeline().from_source(ts).fill(
field_spec='direction.in',
method='linear').fill(field_spec='direction.out',
method='linear').to_event_list())
self.assertEqual(len(elist), len(simple_missing_data.get('points')))
self.assertEqual(elist[0].get('direction.in'), 1)
self.assertEqual(elist[1].get('direction.in'),
1.6666666666666665) # filled
self.assertEqual(elist[2].get('direction.in'),
2.333333333333333) # filled
self.assertEqual(elist[3].get('direction.in'), 3)
self.assertEqual(elist[4].get('direction.in'), 4.0) # filled
self.assertEqual(elist[5].get('direction.in'), 5)
self.assertEqual(elist[0].get('direction.out'), None) # can't fill
self.assertEqual(elist[1].get('direction.out'), None) # can't fill
self.assertEqual(elist[2].get('direction.out'), None) # can't fill
self.assertEqual(elist[3].get('direction.out'), 8)
self.assertEqual(elist[4].get('direction.out'), 10.0) # filled
self.assertEqual(elist[5].get('direction.out'), 12)
def test_indexed_event_series(self):
"""test a series of IndexedEvent objects."""
indexed_event_series = dict(name="availability",
columns=["index", "uptime"],
points=[
["2014-07", "100%"],
["2014-08", "88%"],
["2014-09", "95%"],
["2014-10", "99%"],
["2014-11", "91%"],
["2014-12", "99%"],
["2015-01", "100%"],
["2015-02", "92%"],
["2015-03", "99%"],
["2015-04", "87%"],
["2015-05", "92%"],
["2015-06", "100%"],
])
series = TimeSeries(indexed_event_series)
wire = self._call_interop_script('indexed_event', series.to_string())
new_series = TimeSeries(wire)
new_json = new_series.to_json()
self._validate_wire_points(indexed_event_series, new_json)
self.assertTrue(new_json.get('utc'))
# again with more involved data
availability_series = dict(
name="availability",
columns=["index", "uptime", "notes", "outages"],
points=[
["2014-08", 88, "", 17],
["2014-09", 100, "", 2],
["2014-09", 95, "", 6],
["2014-10", 99, "", 3],
["2014-11", 91, "", 14],
["2014-12", 99, "", 3],
["2015-01", 100, "", 0],
["2015-02", 92, "", 12],
["2015-03", 99, "Minor outage March 2", 4],
["2015-04", 87, "Planned downtime in April", 82],
["2015-05", 92, "Router failure June 12", 26],
["2015-06", 100, "", 0],
])
series = TimeSeries(availability_series)
wire = self._call_interop_script('indexed_event', series.to_string())
new_series = TimeSeries(wire)
new_json = new_series.to_json()
self._validate_wire_points(availability_series, new_json)