def test_TimeSeries_consolidate(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value",
0, len(values)/2, 1, values)
self.assertEqual(series.valuesPerPoint, 1)
series.consolidate(2)
self.assertEqual(series.valuesPerPoint, 2)
def test_holt_winters(self):
timespan = 3600 * 24 * 8 # 8 days
stop = int(time.time())
step = 100
series = TimeSeries('foo.bar',
stop - timespan,
stop,
step,
[x**1.5 for x in range(0, timespan, step)])
series[10] = None
series.pathExpression = 'foo.bar'
self.write_series(series, [(100, timespan)])
ctx = {
'startTime': parseATTime('-1d'),
}
analysis = functions.holtWintersForecast(ctx, [series])
self.assertEqual(len(analysis), 1)
analysis = functions.holtWintersConfidenceBands(ctx, [series])
self.assertEqual(len(analysis), 2)
analysis = functions.holtWintersConfidenceArea(ctx, [series])
self.assertEqual(len(analysis), 2)
analysis = functions.holtWintersAberration(ctx, [series])
self.assertEqual(len(analysis), 1)
def test_TimeSeries_equal_list_color_bad2(self):
values = range(0, 100)
series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
values)
series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
values)
series1.color = 'white'
with self.assertRaises(AssertionError):
self.assertEqual(series1, series2)
def test_TimeSeries_iterate_valuesPerPoint_2_invalid(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value",
0, 5, 1, values, consolidate='bogus')
self.assertEqual(series.valuesPerPoint, 1)
series.consolidate(2)
self.assertEqual(series.valuesPerPoint, 2)
with self.assertRaises(Exception):
list(series)
def test_TimeSeries_equal_list_color(self):
values = range(0, 100)
series1 = TimeSeries("collectd.test-db.load.value",
0, len(values), 1, values)
series1.color = 'white'
series2 = TimeSeries("collectd.test-db.load.value",
0, len(values), 1, values)
series2.color = 'white'
self.assertEqual(series1, series2)
def test_TimeSeries_equal_list_color_bad2(self):
values = range(0, 100)
series1 = TimeSeries("collectd.test-db.load.value",
0, len(values), 1, values)
series2 = TimeSeries("collectd.test-db.load.value",
0, len(values), 1, values)
series1.color = 'white'
with self.assertRaises(AssertionError):
self.assertEqual(series1, series2)
def _generate_series_list(self):
seriesList = []
config = [range(101), range(101), [1] + [None] * 100]
for i, c in enumerate(config):
name = "collectd.test-db{0}.load.value".format(i + 1)
series = TimeSeries(name, 0, 101, 1, c)
series.pathExpression = name
seriesList.append(series)
return seriesList
def test_TimeSeries_iterate_valuesPerPoint_2_min(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value",
0, 5, 1, values, consolidate='min')
self.assertEqual(series.valuesPerPoint, 1)
series.consolidate(2)
self.assertEqual(series.valuesPerPoint, 2)
expected = TimeSeries("collectd.test-db.load.value",
0, 5, 1, list(range(0, 100, 2)) + [None])
self.assertEqual(list(series), list(expected))
def test_TimeSeries_iterate_valuesPerPoint_2_none_values(self):
values = [None, None, None, None, None]
series = TimeSeries("collectd.test-db.load.value",
0, len(values)/2, 1, values)
self.assertEqual(series.valuesPerPoint, 1)
series.consolidate(2)
self.assertEqual(series.valuesPerPoint, 2)
expected = TimeSeries("collectd.test-db.load.value",
0, 5, 1, [None, None, None])
self.assertEqual(list(series), list(expected))
def _generate_mr_series(self):
seriesList = [
TimeSeries('group.server1.metric1', 0, 1, 1, [None]),
TimeSeries('group.server1.metric2', 0, 1, 1, [None]),
TimeSeries('group.server2.metric1', 0, 1, 1, [None]),
TimeSeries('group.server2.metric2', 0, 1, 1, [None]),
]
mappedResult = [[seriesList[0], seriesList[1]],
[seriesList[2], seriesList[3]]]
return seriesList, mappedResult
def test_TimeSeries_iterate_valuesPerPoint_2_avg(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value",
0, len(values)/2, 1, values)
self.assertEqual(series.valuesPerPoint, 1)
series.consolidate(2)
self.assertEqual(series.valuesPerPoint, 2)
expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1,
list(map(lambda x: x+0.5, range(0, 100, 2))) +
[None])
self.assertEqual(list(series), list(expected))
def _generate_series_list(self, config=(range(101), range(2, 103), [1] * 2 + [None] * 90 + [1] * 2 + [None] * 7)):
seriesList = []
now = int(time.time())
for i, c in enumerate(config):
name = "collectd.test-db{0}.load.value".format(i + 1)
series = TimeSeries(name, now - 101, now, 1, c)
series.pathExpression = name
seriesList.append(series)
return seriesList
def test_remove_emtpy(self):
series = [
TimeSeries('foo.bar', 0, 100, 10,
[None, None, None, 0, 0, 0, 1, 1, 1, None]),
TimeSeries('foo.baz', 0, 100, 10, [None] * 10),
TimeSeries('foo.blah', 0, 100, 10,
[None, None, None, 0, 0, 0, 0, 0, 0, None]),
]
results = functions.removeEmptySeries({}, series)
self.assertEqual(results, [series[0], series[2]])
def _generate_series_list(self,
config=(range(101), range(2, 103), [1] * 2 +
[None] * 90 + [1] * 2 + [None] * 7)):
seriesList = []
now = int(time.time())
for i, c in enumerate(config):
name = "collectd.test-db{0}.load.value".format(i + 1)
series = TimeSeries(name, now - 101, now, 1, c)
series.pathExpression = name
seriesList.append(series)
return seriesList
def test_time_stack(self):
timespan = 3600 * 24 * 8 # 8 days
stop = int(time.time())
step = 100
series = TimeSeries("foo.bar", stop - timespan, stop, step, [x ** 1.5 for x in range(0, timespan, step)])
series[10] = None
series.pathExpression = "foo.bar"
self.write_series(series, [(100, timespan)])
ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")}
stack = functions.timeStack(ctx, [series], "1d", 0, 7)
self.assertEqual(len(stack), 7)
stack = functions.timeStack(ctx, [series], "-1d", 0, 7)
self.assertEqual(len(stack), 7)
def test_reduceSeries(self):
sl, inputList = self._generate_mr_series()
expectedResult = [
TimeSeries('group.server1.reduce.mock', 0, 1, 1, [None]),
TimeSeries('group.server2.reduce.mock', 0, 1, 1, [None])
]
resultSeriesList = [TimeSeries('mock(series)', 0, 1, 1, [None])]
mock = MagicMock(return_value=resultSeriesList)
with patch.dict(app.config['GRAPHITE']['functions'], {'mock': mock}):
results = functions.reduceSeries({}, copy.deepcopy(inputList),
"mock", 2, "metric1", "metric2")
self.assertEqual(results, expectedResult)
self.assertEqual(mock.mock_calls,
[call({}, inputList[0]),
call({}, inputList[1])])
def test_null_zero_sum(self):
s = TimeSeries("s", 0, 1, 1, [None])
s.pathExpression = 's'
[series] = functions.sumSeries({}, [s])
self.assertEqual(list(series), [None])
s = TimeSeries("s", 0, 1, 1, [None, 1])
s.pathExpression = 's'
t = TimeSeries("s", 0, 1, 1, [None, None])
t.pathExpression = 't'
[series] = functions.sumSeries({}, [s, t])
self.assertEqual(list(series), [None, 1])
def test_time_stack(self):
timespan = 3600 * 24 * 8 # 8 days
stop = int(time.time())
step = 100
series = TimeSeries('foo.bar', stop - timespan, stop, step,
[x**1.5 for x in range(0, timespan, step)])
series[10] = None
series.pathExpression = 'foo.bar'
self.write_series(series, [(100, timespan)])
ctx = {'startTime': parseATTime('-1d'), 'endTime': parseATTime('now')}
stack = functions.timeStack(ctx, [series], '1d', 0, 7)
self.assertEqual(len(stack), 7)
stack = functions.timeStack(ctx, [series], '-1d', 0, 7)
self.assertEqual(len(stack), 7)
def test_timeslice(self):
series = [
TimeSeries('test.value', 0, 600, 60,
[None, 1, 2, 3, None, 5, 6, None, 7, 8, 9]),
]
expected = [
TimeSeries('timeSlice(test.value, 180, 480)', 0, 600, 60,
[None, None, None, 3, None, 5, 6, None, 7, None, None]),
]
results = functions.timeSlice(
{
'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc),
'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc),
'data': [],
}, series, '00:03 19700101', '00:08 19700101')
self.assertEqual(results, expected)
def _generate_series_list(self, config=None):
seriesList = []
if not config:
config = [range(101), range(101), [1, None, None, None, None]]
for i, c in enumerate(config):
name = "collectd.test-db{0}.load.value".format(i + 1)
seriesList.append(TimeSeries(name, 0, 1, 1, c))
return seriesList
def mostChange(requestContext, seriesList):
"""
Takes one metric or a wildcard seriesList. For each series, determine the delta (last
value minus first value) and create a new series with the delta as its only (first) value.
Really only useful to create a bar graph showing metrics with the most change over a time period.
"""
results = []
for series in seriesList:
newValues = []
delta = seriesdelta(series)
newValues.append(delta)
newName = "mostChange(%s)" % series.name
newSeries = TimeSeries(newName, series.start, series.end, series.step,
newValues)
newSeries.pathExpression = newName
results.append(newSeries)
return results
def test_time_stack(self):
timespan = 3600 * 24 * 8 # 8 days
stop = int(time.time())
step = 100
series = TimeSeries('foo.bar',
stop - timespan,
stop,
step,
[x**1.5 for x in range(0, timespan, step)])
series[10] = None
series.pathExpression = 'foo.bar'
self.write_series(series, [(100, timespan)])
ctx = {'startTime': parseATTime('-1d'),
'endTime': parseATTime('now')}
stack = functions.timeStack(ctx, [series], '1d', 0, 7)
self.assertEqual(len(stack), 7)
stack = functions.timeStack(ctx, [series], '-1d', 0, 7)
self.assertEqual(len(stack), 7)
def test_TimeSeries_equal_list_color(self):
values = range(0, 100)
series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
values)
series1.color = 'white'
series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
values)
series2.color = 'white'
self.assertEqual(series1, series2)
def test_holt_winters(self):
timespan = 3600 * 24 * 8 # 8 days
stop = int(time.time())
step = 100
series = TimeSeries('foo.bar', stop - timespan, stop, step,
[x**1.5 for x in range(0, timespan, step)])
series[10] = None
series.pathExpression = 'foo.bar'
self.write_series(series, [(100, timespan)])
ctx = {
'startTime': parseATTime('-1d'),
}
analysis = functions.holtWintersForecast(ctx, [series])
self.assertEqual(len(analysis), 1)
analysis = functions.holtWintersConfidenceBands(ctx, [series])
self.assertEqual(len(analysis), 2)
analysis = functions.holtWintersConfidenceArea(ctx, [series])
self.assertEqual(len(analysis), 2)
analysis = functions.holtWintersAberration(ctx, [series])
self.assertEqual(len(analysis), 1)
def test_null_zero_sum(self):
s = TimeSeries("s", 0, 1, 1, [None])
s.pathExpression = 's'
[series] = functions.sumSeries({}, [s])
self.assertEqual(list(series), [None])
s = TimeSeries("s", 0, 1, 1, [None, 1])
s.pathExpression = 's'
t = TimeSeries("s", 0, 1, 1, [None, None])
t.pathExpression = 't'
[series] = functions.sumSeries({}, [s, t])
self.assertEqual(list(series), [None, 1])
def test_n_percentile(self):
seriesList = []
config = [
[15, 35, 20, 40, 50],
range(1, 101),
range(1, 201),
range(1, 301),
range(0, 100),
range(0, 200),
range(0, 300),
# Ensure None values in list has no effect.
[None, None, None] + list(range(0, 300)),
]
for i, c in enumerate(config):
seriesList.append(TimeSeries('Test(%d)' % i, 0, 1, 1, c))
def n_percentile(perc, expected):
result = functions.nPercentile({}, seriesList, perc)
self.assertEqual(expected, result)
n_percentile(30, [[20], [31], [61], [91], [30], [60], [90], [90]])
n_percentile(90, [[50], [91], [181], [271], [90], [180], [270], [270]])
n_percentile(95, [[50], [96], [191], [286], [95], [190], [285], [285]])
def test_reduceSeries_asPercent(self):
seriesList = [
TimeSeries('group.server1.bytes_used', 0, 1, 1, [1]),
TimeSeries('group.server1.total_bytes', 0, 1, 1, [2]),
TimeSeries('group.server2.bytes_used', 0, 1, 1, [3]),
TimeSeries('group.server2.total_bytes', 0, 1, 1, [4]),
]
for series in seriesList:
series.pathExpression = "tempPath"
expectedResult = [
# 50 == 100 * 1 / 2
TimeSeries('group.server1.reduce.asPercent', 0, 1, 1, [50]),
# 100 * 3 / 4
TimeSeries('group.server2.reduce.asPercent', 0, 1, 1, [75]),
]
mappedResult = ([seriesList[0]], [seriesList[1]], [seriesList[2]],
[seriesList[3]])
results = functions.reduceSeries({}, copy.deepcopy(mappedResult),
"asPercent", 2, "bytes_used",
"total_bytes")
self.assertEqual(results, expectedResult)
def test_multiply_with_wildcards(self):
s1 = [
TimeSeries('web.host-1.avg-response.value', 0, 1, 1, [1, 10, 11]),
TimeSeries('web.host-2.avg-response.value', 0, 1, 1, [2, 20, 21]),
TimeSeries('web.host-3.avg-response.value', 0, 1, 1, [3, 30, 31]),
TimeSeries('web.host-4.avg-response.value', 0, 1, 1, [4, 40, 41]),
]
s2 = [
TimeSeries('web.host-4.total-request.value', 0, 1, 1, [4, 8, 12]),
TimeSeries('web.host-3.total-request.value', 0, 1, 1, [3, 7, 11]),
TimeSeries('web.host-1.total-request.value', 0, 1, 1, [1, 5, 9]),
TimeSeries('web.host-2.total-request.value', 0, 1, 1, [2, 6, 10]),
]
expected = [
TimeSeries('web.host-1', 0, 1, 1, [1, 50, 99]),
TimeSeries('web.host-2', 0, 1, 1, [4, 120, 210]),
TimeSeries('web.host-3', 0, 1, 1, [9, 210, 341]),
TimeSeries('web.host-4', 0, 1, 1, [16, 320, 492]),
]
results = functions.multiplySeriesWithWildcards({}, s1 + s2, 2, 3)
self.assertEqual(results, expected)
def test_nonempty_true(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value",
0, len(values), 1, values)
self.assertTrue(nonempty(series))
def test_legend_value_with_system_preserves_sign(self):
series = [TimeSeries("foo", 0, 1, 1, [-10000, -20000, -30000, -40000])]
[result] = functions.legendValue({}, series, "avg", "si")
self.assertEqual(result.name, "foo avg -25.00K ")
def test_TimeSeries_init_no_args(self):
with self.assertRaises(TypeError):
TimeSeries()
def ASAP(requestContext, seriesList, resolution=1000):
'''
use the ASAP smoothing on a series
https://arxiv.org/pdf/1703.00983.pdf
https://raw.githubusercontent.com/stanford-futuredata/ASAP/master/ASAP.py
:param requestContext:
:param seriesList:
:param resolution: either number of points to keep or a time resolution
:return: smoothed(seriesList)
'''
if not seriesList:
return []
windowInterval = None
if isinstance(resolution, six.string_types):
delta = parseTimeOffset(resolution)
windowInterval = to_seconds(delta)
if windowInterval:
previewSeconds = windowInterval
else:
previewSeconds = max([s.step for s in seriesList]) * int(resolution)
# ignore original data and pull new, including our preview
# data from earlier is needed to calculate the early results
newContext = requestContext.copy()
newContext['startTime'] = (requestContext['startTime'] -
timedelta(seconds=previewSeconds))
previewList = evaluateTokens(newContext, requestContext['args'][0])
result = []
for series in previewList:
if windowInterval:
# the resolution here is really the number of points to maintain
# so we need to convert the "seconds" to num points
windowPoints = round((series.end - series.start) / windowInterval)
else:
use_res = int(resolution)
if len(series) < use_res:
use_res = len(series)
windowPoints = use_res
if isinstance(resolution, six.string_types):
newName = 'asap(%s,"%s")' % (series.name, resolution)
else:
newName = "asap(%s,%s)" % (series.name, resolution)
step_guess = (series.end - series.start) // windowPoints
newSeries = TimeSeries(newName, series.start, series.end, step_guess,
[])
newSeries.pathExpression = newName
# detect "none" lists
if len([v for v in series if v is not None]) <= 1:
newSeries.extend(series)
else:
# the "resolution" is a suggestion,
# the algo will alter it some inorder
# to get the best view for things
new_s = smooth(series, windowPoints)
# steps need to be ints, so we must force the issue
new_step = round((series.end - series.start) / len(new_s))
newSeries.step = new_step
newSeries.extend(new_s)
result.append(newSeries)
return result
def test_TimeSeries_equal_list(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
values)
with self.assertRaises(AssertionError):
self.assertEqual(values, series)
def test_TimeSeries_iterate(self):
values = range(0, 100)
series = TimeSeries("collectd.test-db.load.value",
0, len(values), 1, values)
for i, val in enumerate(series):
self.assertEqual(val, values[i])
def test_nonempty_false_nones(self):
series = TimeSeries("collectd.test-db.load.value",
0, 4, 1, [None, None, None, None])
self.assertFalse(nonempty(series))
def test_TimeSeries_init_string_values(self):
series = TimeSeries("collectd.test-db.load.value", 0, 2, 1, "ab")
expected = TimeSeries("collectd.test-db.load.value", 0, 2, 1,
["a", "b"])
self.assertEqual(series, expected)