def test_TimeSeries_consolidate(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value",
                         0, len(values)/2, 1, values)
     self.assertEqual(series.valuesPerPoint, 1)
     series.consolidate(2)
     self.assertEqual(series.valuesPerPoint, 2)
Example #2
0
    def test_holt_winters(self):
        timespan = 3600 * 24 * 8  # 8 days
        stop = int(time.time())
        step = 100
        series = TimeSeries('foo.bar',
                            stop - timespan,
                            stop,
                            step,
                            [x**1.5 for x in range(0, timespan, step)])
        series[10] = None
        series.pathExpression = 'foo.bar'
        self.write_series(series, [(100, timespan)])

        ctx = {
            'startTime': parseATTime('-1d'),
        }
        analysis = functions.holtWintersForecast(ctx, [series])
        self.assertEqual(len(analysis), 1)

        analysis = functions.holtWintersConfidenceBands(ctx, [series])
        self.assertEqual(len(analysis), 2)

        analysis = functions.holtWintersConfidenceArea(ctx, [series])
        self.assertEqual(len(analysis), 2)

        analysis = functions.holtWintersAberration(ctx, [series])
        self.assertEqual(len(analysis), 1)
Example #3
0
 def test_TimeSeries_equal_list_color_bad2(self):
     values = range(0, 100)
     series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                          values)
     series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                          values)
     series1.color = 'white'
     with self.assertRaises(AssertionError):
         self.assertEqual(series1, series2)
 def test_TimeSeries_iterate_valuesPerPoint_2_invalid(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value",
                         0, 5, 1, values, consolidate='bogus')
     self.assertEqual(series.valuesPerPoint, 1)
     series.consolidate(2)
     self.assertEqual(series.valuesPerPoint, 2)
     with self.assertRaises(Exception):
         list(series)
 def test_TimeSeries_equal_list_color(self):
     values = range(0, 100)
     series1 = TimeSeries("collectd.test-db.load.value",
                          0, len(values), 1, values)
     series1.color = 'white'
     series2 = TimeSeries("collectd.test-db.load.value",
                          0, len(values), 1, values)
     series2.color = 'white'
     self.assertEqual(series1, series2)
 def test_TimeSeries_equal_list_color_bad2(self):
     values = range(0, 100)
     series1 = TimeSeries("collectd.test-db.load.value",
                          0, len(values), 1, values)
     series2 = TimeSeries("collectd.test-db.load.value",
                          0, len(values), 1, values)
     series1.color = 'white'
     with self.assertRaises(AssertionError):
         self.assertEqual(series1, series2)
Example #7
0
    def _generate_series_list(self):
        seriesList = []
        config = [range(101), range(101), [1] + [None] * 100]

        for i, c in enumerate(config):
            name = "collectd.test-db{0}.load.value".format(i + 1)
            series = TimeSeries(name, 0, 101, 1, c)
            series.pathExpression = name
            seriesList.append(series)
        return seriesList
 def test_TimeSeries_iterate_valuesPerPoint_2_min(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value",
                         0, 5, 1, values, consolidate='min')
     self.assertEqual(series.valuesPerPoint, 1)
     series.consolidate(2)
     self.assertEqual(series.valuesPerPoint, 2)
     expected = TimeSeries("collectd.test-db.load.value",
                           0, 5, 1, list(range(0, 100, 2)) + [None])
     self.assertEqual(list(series), list(expected))
 def test_TimeSeries_iterate_valuesPerPoint_2_none_values(self):
     values = [None, None, None, None, None]
     series = TimeSeries("collectd.test-db.load.value",
                         0, len(values)/2, 1, values)
     self.assertEqual(series.valuesPerPoint, 1)
     series.consolidate(2)
     self.assertEqual(series.valuesPerPoint, 2)
     expected = TimeSeries("collectd.test-db.load.value",
                           0, 5, 1, [None, None, None])
     self.assertEqual(list(series), list(expected))
 def _generate_mr_series(self):
     seriesList = [
         TimeSeries('group.server1.metric1', 0, 1, 1, [None]),
         TimeSeries('group.server1.metric2', 0, 1, 1, [None]),
         TimeSeries('group.server2.metric1', 0, 1, 1, [None]),
         TimeSeries('group.server2.metric2', 0, 1, 1, [None]),
     ]
     mappedResult = [[seriesList[0], seriesList[1]],
                     [seriesList[2], seriesList[3]]]
     return seriesList, mappedResult
Example #11
0
 def test_TimeSeries_iterate_valuesPerPoint_2_avg(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value",
                         0, len(values)/2, 1, values)
     self.assertEqual(series.valuesPerPoint, 1)
     series.consolidate(2)
     self.assertEqual(series.valuesPerPoint, 2)
     expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1,
                           list(map(lambda x: x+0.5, range(0, 100, 2))) +
                           [None])
     self.assertEqual(list(series), list(expected))
Example #12
0
    def _generate_series_list(self, config=(range(101), range(2, 103), [1] * 2 + [None] * 90 + [1] * 2 + [None] * 7)):
        seriesList = []

        now = int(time.time())

        for i, c in enumerate(config):
            name = "collectd.test-db{0}.load.value".format(i + 1)
            series = TimeSeries(name, now - 101, now, 1, c)
            series.pathExpression = name
            seriesList.append(series)
        return seriesList
Example #13
0
    def test_remove_emtpy(self):
        series = [
            TimeSeries('foo.bar', 0, 100, 10,
                       [None, None, None, 0, 0, 0, 1, 1, 1, None]),
            TimeSeries('foo.baz', 0, 100, 10, [None] * 10),
            TimeSeries('foo.blah', 0, 100, 10,
                       [None, None, None, 0, 0, 0, 0, 0, 0, None]),
        ]

        results = functions.removeEmptySeries({}, series)
        self.assertEqual(results, [series[0], series[2]])
    def _generate_series_list(self,
                              config=(range(101), range(2, 103), [1] * 2 +
                                      [None] * 90 + [1] * 2 + [None] * 7)):
        seriesList = []

        now = int(time.time())

        for i, c in enumerate(config):
            name = "collectd.test-db{0}.load.value".format(i + 1)
            series = TimeSeries(name, now - 101, now, 1, c)
            series.pathExpression = name
            seriesList.append(series)
        return seriesList
Example #15
0
    def test_time_stack(self):
        timespan = 3600 * 24 * 8  # 8 days
        stop = int(time.time())
        step = 100
        series = TimeSeries("foo.bar", stop - timespan, stop, step, [x ** 1.5 for x in range(0, timespan, step)])
        series[10] = None
        series.pathExpression = "foo.bar"
        self.write_series(series, [(100, timespan)])

        ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")}
        stack = functions.timeStack(ctx, [series], "1d", 0, 7)
        self.assertEqual(len(stack), 7)

        stack = functions.timeStack(ctx, [series], "-1d", 0, 7)
        self.assertEqual(len(stack), 7)
 def test_reduceSeries(self):
     sl, inputList = self._generate_mr_series()
     expectedResult = [
         TimeSeries('group.server1.reduce.mock', 0, 1, 1, [None]),
         TimeSeries('group.server2.reduce.mock', 0, 1, 1, [None])
     ]
     resultSeriesList = [TimeSeries('mock(series)', 0, 1, 1, [None])]
     mock = MagicMock(return_value=resultSeriesList)
     with patch.dict(app.config['GRAPHITE']['functions'], {'mock': mock}):
         results = functions.reduceSeries({}, copy.deepcopy(inputList),
                                          "mock", 2, "metric1", "metric2")
         self.assertEqual(results, expectedResult)
     self.assertEqual(mock.mock_calls,
                      [call({}, inputList[0]),
                       call({}, inputList[1])])
Example #17
0
    def test_null_zero_sum(self):
        s = TimeSeries("s", 0, 1, 1, [None])
        s.pathExpression = 's'
        [series] = functions.sumSeries({}, [s])
        self.assertEqual(list(series), [None])

        s = TimeSeries("s", 0, 1, 1, [None, 1])
        s.pathExpression = 's'
        t = TimeSeries("s", 0, 1, 1, [None, None])
        t.pathExpression = 't'
        [series] = functions.sumSeries({}, [s, t])
        self.assertEqual(list(series), [None, 1])
    def test_time_stack(self):
        timespan = 3600 * 24 * 8  # 8 days
        stop = int(time.time())
        step = 100
        series = TimeSeries('foo.bar', stop - timespan, stop, step,
                            [x**1.5 for x in range(0, timespan, step)])
        series[10] = None
        series.pathExpression = 'foo.bar'
        self.write_series(series, [(100, timespan)])

        ctx = {'startTime': parseATTime('-1d'), 'endTime': parseATTime('now')}
        stack = functions.timeStack(ctx, [series], '1d', 0, 7)
        self.assertEqual(len(stack), 7)

        stack = functions.timeStack(ctx, [series], '-1d', 0, 7)
        self.assertEqual(len(stack), 7)
Example #19
0
    def test_timeslice(self):
        series = [
            TimeSeries('test.value', 0, 600, 60,
                       [None, 1, 2, 3, None, 5, 6, None, 7, 8, 9]),
        ]

        expected = [
            TimeSeries('timeSlice(test.value, 180, 480)', 0, 600, 60,
                       [None, None, None, 3, None, 5, 6, None, 7, None, None]),
        ]

        results = functions.timeSlice(
            {
                'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc),
                'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc),
                'data': [],
            }, series, '00:03 19700101', '00:08 19700101')
        self.assertEqual(results, expected)
Example #20
0
    def _generate_series_list(self, config=None):
        seriesList = []
        if not config:
            config = [range(101), range(101), [1, None, None, None, None]]

        for i, c in enumerate(config):
            name = "collectd.test-db{0}.load.value".format(i + 1)
            seriesList.append(TimeSeries(name, 0, 1, 1, c))
        return seriesList
def mostChange(requestContext, seriesList):
    """
    Takes one metric or a wildcard seriesList.  For each series, determine the delta (last
    value minus first value) and create a new series with the delta as its only (first) value.
    Really only useful to create a bar graph showing metrics with the most change over a time period.

    """

    results = []
    for series in seriesList:
        newValues = []
        delta = seriesdelta(series)
        newValues.append(delta)
        newName = "mostChange(%s)" % series.name
        newSeries = TimeSeries(newName, series.start, series.end, series.step,
                               newValues)
        newSeries.pathExpression = newName
        results.append(newSeries)
    return results
Example #22
0
    def test_time_stack(self):
        timespan = 3600 * 24 * 8  # 8 days
        stop = int(time.time())
        step = 100
        series = TimeSeries('foo.bar',
                            stop - timespan,
                            stop,
                            step,
                            [x**1.5 for x in range(0, timespan, step)])
        series[10] = None
        series.pathExpression = 'foo.bar'
        self.write_series(series, [(100, timespan)])

        ctx = {'startTime': parseATTime('-1d'),
               'endTime': parseATTime('now')}
        stack = functions.timeStack(ctx, [series], '1d', 0, 7)
        self.assertEqual(len(stack), 7)

        stack = functions.timeStack(ctx, [series], '-1d', 0, 7)
        self.assertEqual(len(stack), 7)
Example #23
0
 def test_TimeSeries_equal_list_color(self):
     values = range(0, 100)
     series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                          values)
     series1.color = 'white'
     series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                          values)
     series2.color = 'white'
     self.assertEqual(series1, series2)
    def test_holt_winters(self):
        timespan = 3600 * 24 * 8  # 8 days
        stop = int(time.time())
        step = 100
        series = TimeSeries('foo.bar', stop - timespan, stop, step,
                            [x**1.5 for x in range(0, timespan, step)])
        series[10] = None
        series.pathExpression = 'foo.bar'
        self.write_series(series, [(100, timespan)])

        ctx = {
            'startTime': parseATTime('-1d'),
        }
        analysis = functions.holtWintersForecast(ctx, [series])
        self.assertEqual(len(analysis), 1)

        analysis = functions.holtWintersConfidenceBands(ctx, [series])
        self.assertEqual(len(analysis), 2)

        analysis = functions.holtWintersConfidenceArea(ctx, [series])
        self.assertEqual(len(analysis), 2)

        analysis = functions.holtWintersAberration(ctx, [series])
        self.assertEqual(len(analysis), 1)
Example #25
0
    def test_null_zero_sum(self):
        s = TimeSeries("s", 0, 1, 1, [None])
        s.pathExpression = 's'
        [series] = functions.sumSeries({}, [s])
        self.assertEqual(list(series), [None])

        s = TimeSeries("s", 0, 1, 1, [None, 1])
        s.pathExpression = 's'
        t = TimeSeries("s", 0, 1, 1, [None, None])
        t.pathExpression = 't'
        [series] = functions.sumSeries({}, [s, t])
        self.assertEqual(list(series), [None, 1])
    def test_n_percentile(self):
        seriesList = []
        config = [
            [15, 35, 20, 40, 50],
            range(1, 101),
            range(1, 201),
            range(1, 301),
            range(0, 100),
            range(0, 200),
            range(0, 300),
            # Ensure None values in list has no effect.
            [None, None, None] + list(range(0, 300)),
        ]

        for i, c in enumerate(config):
            seriesList.append(TimeSeries('Test(%d)' % i, 0, 1, 1, c))

        def n_percentile(perc, expected):
            result = functions.nPercentile({}, seriesList, perc)
            self.assertEqual(expected, result)

        n_percentile(30, [[20], [31], [61], [91], [30], [60], [90], [90]])
        n_percentile(90, [[50], [91], [181], [271], [90], [180], [270], [270]])
        n_percentile(95, [[50], [96], [191], [286], [95], [190], [285], [285]])
Example #27
0
 def test_reduceSeries_asPercent(self):
     seriesList = [
         TimeSeries('group.server1.bytes_used', 0, 1, 1, [1]),
         TimeSeries('group.server1.total_bytes', 0, 1, 1, [2]),
         TimeSeries('group.server2.bytes_used', 0, 1, 1, [3]),
         TimeSeries('group.server2.total_bytes', 0, 1, 1, [4]),
     ]
     for series in seriesList:
         series.pathExpression = "tempPath"
     expectedResult = [
         # 50 == 100 * 1 / 2
         TimeSeries('group.server1.reduce.asPercent', 0, 1, 1, [50]),
         # 100 * 3 / 4
         TimeSeries('group.server2.reduce.asPercent', 0, 1, 1, [75]),
     ]
     mappedResult = ([seriesList[0]], [seriesList[1]], [seriesList[2]],
                     [seriesList[3]])
     results = functions.reduceSeries({}, copy.deepcopy(mappedResult),
                                      "asPercent", 2, "bytes_used",
                                      "total_bytes")
     self.assertEqual(results, expectedResult)
Example #28
0
 def test_multiply_with_wildcards(self):
     s1 = [
         TimeSeries('web.host-1.avg-response.value', 0, 1, 1, [1, 10, 11]),
         TimeSeries('web.host-2.avg-response.value', 0, 1, 1, [2, 20, 21]),
         TimeSeries('web.host-3.avg-response.value', 0, 1, 1, [3, 30, 31]),
         TimeSeries('web.host-4.avg-response.value', 0, 1, 1, [4, 40, 41]),
     ]
     s2 = [
         TimeSeries('web.host-4.total-request.value', 0, 1, 1, [4, 8, 12]),
         TimeSeries('web.host-3.total-request.value', 0, 1, 1, [3, 7, 11]),
         TimeSeries('web.host-1.total-request.value', 0, 1, 1, [1, 5, 9]),
         TimeSeries('web.host-2.total-request.value', 0, 1, 1, [2, 6, 10]),
     ]
     expected = [
         TimeSeries('web.host-1', 0, 1, 1, [1, 50, 99]),
         TimeSeries('web.host-2', 0, 1, 1, [4, 120, 210]),
         TimeSeries('web.host-3', 0, 1, 1, [9, 210, 341]),
         TimeSeries('web.host-4', 0, 1, 1, [16, 320, 492]),
     ]
     results = functions.multiplySeriesWithWildcards({}, s1 + s2, 2, 3)
     self.assertEqual(results, expected)
Example #29
0
 def test_nonempty_true(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value",
                         0, len(values), 1, values)
     self.assertTrue(nonempty(series))
 def test_legend_value_with_system_preserves_sign(self):
     series = [TimeSeries("foo", 0, 1, 1, [-10000, -20000, -30000, -40000])]
     [result] = functions.legendValue({}, series, "avg", "si")
     self.assertEqual(result.name, "foo                 avg  -25.00K   ")
Example #31
0
 def test_TimeSeries_init_no_args(self):
     with self.assertRaises(TypeError):
         TimeSeries()
Example #32
0
def ASAP(requestContext, seriesList, resolution=1000):
    '''
    use the ASAP smoothing on a series

    https://arxiv.org/pdf/1703.00983.pdf
    https://raw.githubusercontent.com/stanford-futuredata/ASAP/master/ASAP.py


    :param requestContext:
    :param seriesList:
    :param resolution: either number of points to keep or a time resolution
    :return: smoothed(seriesList)
    '''

    if not seriesList:
        return []

    windowInterval = None
    if isinstance(resolution, six.string_types):
        delta = parseTimeOffset(resolution)
        windowInterval = to_seconds(delta)

    if windowInterval:
        previewSeconds = windowInterval
    else:
        previewSeconds = max([s.step for s in seriesList]) * int(resolution)

    # ignore original data and pull new, including our preview
    # data from earlier is needed to calculate the early results
    newContext = requestContext.copy()
    newContext['startTime'] = (requestContext['startTime'] -
                               timedelta(seconds=previewSeconds))
    previewList = evaluateTokens(newContext, requestContext['args'][0])
    result = []
    for series in previewList:

        if windowInterval:
            # the resolution here is really the number of points to maintain
            # so we need to convert the "seconds" to num points
            windowPoints = round((series.end - series.start) / windowInterval)
        else:
            use_res = int(resolution)
            if len(series) < use_res:
                use_res = len(series)
            windowPoints = use_res

        if isinstance(resolution, six.string_types):
            newName = 'asap(%s,"%s")' % (series.name, resolution)
        else:
            newName = "asap(%s,%s)" % (series.name, resolution)

        step_guess = (series.end - series.start) // windowPoints

        newSeries = TimeSeries(newName, series.start, series.end, step_guess,
                               [])
        newSeries.pathExpression = newName

        # detect "none" lists
        if len([v for v in series if v is not None]) <= 1:
            newSeries.extend(series)
        else:
            # the "resolution" is a suggestion,
            # the algo will alter it some inorder
            # to get the best view for things
            new_s = smooth(series, windowPoints)
            # steps need to be ints, so we must force the issue
            new_step = round((series.end - series.start) / len(new_s))
            newSeries.step = new_step
            newSeries.extend(new_s)
        result.append(newSeries)

    return result
Example #33
0
 def test_TimeSeries_equal_list(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                         values)
     with self.assertRaises(AssertionError):
         self.assertEqual(values, series)
Example #34
0
 def test_TimeSeries_iterate(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value",
                         0, len(values), 1, values)
     for i, val in enumerate(series):
         self.assertEqual(val, values[i])
Example #35
0
 def test_nonempty_false_nones(self):
     series = TimeSeries("collectd.test-db.load.value",
                         0, 4, 1, [None, None, None, None])
     self.assertFalse(nonempty(series))
Example #36
0
 def test_TimeSeries_init_string_values(self):
     series = TimeSeries("collectd.test-db.load.value", 0, 2, 1, "ab")
     expected = TimeSeries("collectd.test-db.load.value", 0, 2, 1,
                           ["a", "b"])
     self.assertEqual(series, expected)