def test_bad_args(self):
"""error states for coverage."""
# various bad values
with self.assertRaises(ProcessorException):
Align(dict())
with self.assertRaises(ProcessorException):
Rate(dict())
with self.assertRaises(ProcessorException):
self._simple_ts.align(method='bogus')
with self.assertRaises(ProcessorException):
self._simple_ts.align(limit='bogus')
# non event types
ticket_range = dict(
name="outages",
columns=["timerange", "title", "esnet_ticket"],
points=[
[[1429673400000, 1429707600000], "BOOM", "ESNET-20080101-001"],
[[1429673400000, 1429707600000], "BAM!", "ESNET-20080101-002"],
],
)
ts = TimeSeries(ticket_range)
with self.assertRaises(ProcessorException):
ts.align()
with self.assertRaises(ProcessorException):
ts.rate()
def test_negative_derivatives(self):
"""Test behavior on counter resets."""
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 50]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
# lower counter will produce negative derivatives
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), -0.5434782608695656)
self.assertEqual(rates.at(1).get('value_rate'), -0.5434782608695646)
self.assertEqual(rates.at(2).get('value_rate'), -0.5434782608695653)
rates = ts.align(window='30s').rate(allow_negative=False)
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), None)
self.assertEqual(rates.at(1).get('value_rate'), None)
self.assertEqual(rates.at(2).get('value_rate'), None)
def test_rate_bins(self):
"""replicate basic esmond rates."""
# | 100 | | | | 200 | v
# | | | | | | | |
# 60 89 90 120 150 180 181 210 t ->
# | | | | | |
# |<- ? --------->|<- 1.08/s --->|<- 1.08/s --->|<- 1.08/s --->|<- ? ------->| result
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 200]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), 1.0869565217391313)
self.assertEqual(rates.at(1).get('value_rate'), 1.0869565217391293)
self.assertEqual(rates.at(2).get('value_rate'), 1.0869565217391313)
def test_invalid_point(self):
"""make sure non-numeric values are handled properly."""
bad_point = copy.deepcopy(SIMPLE_GAP_DATA)
bad_point.get('points')[-2][1] = 'non_numeric_value'
ts = TimeSeries(bad_point)
with warnings.catch_warnings(record=True) as wrn:
aligned = ts.align(window='1m')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), 1.0)
self.assertEqual(aligned.at(4).get(), 1.0)
self.assertEqual(aligned.at(5).get(), 1.0)
self.assertEqual(aligned.at(6).get(), None) # bad value
self.assertEqual(aligned.at(7).get(), None) # bad value
with warnings.catch_warnings(record=True) as wrn:
a_diff = aligned.rate()
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(a_diff.at(5).get(), None) # bad value
self.assertEqual(a_diff.at(6).get(), None) # bad value
def test_rate_bins_long(self):
"""replicate counter to rate conversion with more data."""
raw_ts = TimeSeries(RAW)
base_rates = raw_ts.align(window='30s').rate()
for i in enumerate(base_rates.collection().events()):
# there are going to be decimal rounding variations but
# for the purposes of this sanity check, if they are
# equal to one decimal place is close enough.
self.assertAlmostEqual(
i[1].get('value_rate'),
BASE.get('points')[i[0]][1],
places=1
)
class AlignTest(unittest.TestCase):
"""
tests for the align processor
"""
def setUp(self):
"""setup for all tests."""
self._simple_ts = TimeSeries(SIMPLE_GAP_DATA)
def test_basic_linear_align(self):
"""test basic align"""
aligned = self._simple_ts.align(window='1m')
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), 1.0)
self.assertEqual(aligned.at(4).get(), 1.0)
self.assertEqual(aligned.at(5).get(), 1.0)
self.assertEqual(aligned.at(6).get(), 1.5)
self.assertEqual(aligned.at(7).get(), 2.5)
def test_basic_hold_align(self):
"""test basic hold align."""
aligned = self._simple_ts.align(window='1m', method='hold')
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), .75)
self.assertEqual(aligned.at(1).get(), 2)
self.assertEqual(aligned.at(2).get(), 2)
self.assertEqual(aligned.at(3).get(), 1)
self.assertEqual(aligned.at(4).get(), 1)
self.assertEqual(aligned.at(5).get(), 1)
self.assertEqual(aligned.at(6).get(), 1)
self.assertEqual(aligned.at(7).get(), 1)
def test_align_limit(self):
"""test basic hold align."""
aligned = self._simple_ts.align(window='1m', method='hold', limit=2)
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), .75)
self.assertEqual(aligned.at(1).get(), 2)
self.assertEqual(aligned.at(2).get(), 2)
self.assertEqual(aligned.at(3).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(4).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(5).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(6).get(), 1)
self.assertEqual(aligned.at(7).get(), 1)
aligned = self._simple_ts.align(field_spec='value', window='1m', method='linear', limit=2)
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(4).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(5).get(), None) # over limit, fill with None
self.assertEqual(aligned.at(6).get(), 1.5)
self.assertEqual(aligned.at(7).get(), 2.5)
def test_invalid_point(self):
"""make sure non-numeric values are handled properly."""
bad_point = copy.deepcopy(SIMPLE_GAP_DATA)
bad_point.get('points')[-2][1] = 'non_numeric_value'
ts = TimeSeries(bad_point)
with warnings.catch_warnings(record=True) as wrn:
aligned = ts.align(window='1m')
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(aligned.size(), 8)
self.assertEqual(aligned.at(0).get(), 1.25)
self.assertEqual(aligned.at(1).get(), 1.8571428571428572)
self.assertEqual(aligned.at(2).get(), 1.2857142857142856)
self.assertEqual(aligned.at(3).get(), 1.0)
self.assertEqual(aligned.at(4).get(), 1.0)
self.assertEqual(aligned.at(5).get(), 1.0)
self.assertEqual(aligned.at(6).get(), None) # bad value
self.assertEqual(aligned.at(7).get(), None) # bad value
with warnings.catch_warnings(record=True) as wrn:
a_diff = aligned.rate()
self.assertEqual(len(wrn), 1)
self.assertTrue(issubclass(wrn[0].category, ProcessorWarning))
self.assertEqual(a_diff.at(5).get(), None) # bad value
self.assertEqual(a_diff.at(6).get(), None) # bad value
def test_rate_mag(self):
"""test the rate processor order of mag."""
ts = TimeSeries(RATE)
rate = ts.rate(field_spec='in')
# one less than source
self.assertEqual(rate.size(), len(RATE.get('points')) - 1)
self.assertEqual(rate.at(2).get('in_rate'), 1)
self.assertEqual(rate.at(3).get('in_rate'), 1)
self.assertEqual(rate.at(4).get('in_rate'), 2)
self.assertEqual(rate.at(8).get('in_rate'), 3)
self.assertEqual(rate.at(9).get('in_rate'), 4)
def test_rate_bins(self):
"""replicate basic esmond rates."""
# | 100 | | | | 200 | v
# | | | | | | | |
# 60 89 90 120 150 180 181 210 t ->
# | | | | | |
# |<- ? --------->|<- 1.08/s --->|<- 1.08/s --->|<- 1.08/s --->|<- ? ------->| result
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 200]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), 1.0869565217391313)
self.assertEqual(rates.at(1).get('value_rate'), 1.0869565217391293)
self.assertEqual(rates.at(2).get('value_rate'), 1.0869565217391313)
def test_rate_bins_long(self):
"""replicate counter to rate conversion with more data."""
raw_ts = TimeSeries(RAW)
base_rates = raw_ts.align(window='30s').rate()
for i in enumerate(base_rates.collection().events()):
# there are going to be decimal rounding variations but
# for the purposes of this sanity check, if they are
# equal to one decimal place is close enough.
self.assertAlmostEqual(
i[1].get('value_rate'),
BASE.get('points')[i[0]][1],
places=1
)
def test_negative_derivatives(self):
"""Test behavior on counter resets."""
raw_rates = dict(
name="traffic",
columns=["time", "value"],
points=[
[89000, 100],
[181000, 50]
]
)
ts = TimeSeries(raw_rates)
rates = ts.align(window='30s').rate()
# lower counter will produce negative derivatives
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), -0.5434782608695656)
self.assertEqual(rates.at(1).get('value_rate'), -0.5434782608695646)
self.assertEqual(rates.at(2).get('value_rate'), -0.5434782608695653)
rates = ts.align(window='30s').rate(allow_negative=False)
self.assertEqual(rates.size(), 3)
self.assertEqual(rates.at(0).get('value_rate'), None)
self.assertEqual(rates.at(1).get('value_rate'), None)
self.assertEqual(rates.at(2).get('value_rate'), None)
def test_bad_args(self):
"""error states for coverage."""
# various bad values
with self.assertRaises(ProcessorException):
Align(dict())
with self.assertRaises(ProcessorException):
Rate(dict())
with self.assertRaises(ProcessorException):
self._simple_ts.align(method='bogus')
with self.assertRaises(ProcessorException):
self._simple_ts.align(limit='bogus')
# non event types
ticket_range = dict(
name="outages",
columns=["timerange", "title", "esnet_ticket"],
points=[
[[1429673400000, 1429707600000], "BOOM", "ESNET-20080101-001"],
[[1429673400000, 1429707600000], "BAM!", "ESNET-20080101-002"],
],
)
ts = TimeSeries(ticket_range)
with self.assertRaises(ProcessorException):
ts.align()
with self.assertRaises(ProcessorException):
ts.rate()
def test_first_point(self):
"""Make sure the first point is handled right when it is perfectly aligned."""
data = dict(
name="traffic",
columns=["time", "value"],
points=[
[1473490770000, 10],
[1473490800000, 20],
[1473490830000, 30],
[1473490860000, 40]
]
)
base_30_sec = (
Pipeline()
.from_source(TimeSeries(data))
.align(window='30s', method='linear', limit=10)
.to_keyed_collections()
)
self.assertEqual(base_30_sec.get('all').size(), 4)
