Exemplo n.º 1
0
def integral(requestContext, seriesList):
  """

  This will show the sum over time, sort of like a continuous addition function.  
  Useful for finding totals or trends in metrics that are collected per minute. 

  Example: 

  .. code-block:: none

    &target=integral(company.sales.perMinute)

  This would start at zero on the left side of the graph, adding the sales each
  minute, and show the total sales for the time period selected at the right 
  side, (time now, or the time specified by '&until=').
  """
  results = []
  for series in seriesList:
    newValues = []
    current = 0.0
    for val in series:
      if val is None:
        newValues.append(None)
      else:
        current += val
        newValues.append(current)
    newName = "integral(%s)" % series.name
    newSeries = TimeSeries(newName, series.start, series.end, series.step, newValues)
    newSeries.pathExpression = newName
    results.append(newSeries)
  return results
Exemplo n.º 2
0
def derivative(requestContext, seriesList):
  """

  This is the opposite of the integral function.  This is useful for taking a
  running total metric and showing how many requests per minute were handled. 

  Example:

  .. code-block:: none

    &target=derivative(company.server.application01.ifconfig.TXPackets)

  Each time you run ifconfig, the RX and TXPackets are higher (assuming there
  is network traffic.) By applying the derivative function, you can get an 
  idea of the packets per minute sent or received, even though you're only 
  recording the total.
  """
  results = []
  for series in seriesList:
    newValues = []
    prev = None
    for val in series:
      if None in (prev,val):
        newValues.append(None)
        prev = val
        continue
      newValues.append(val - prev)
      prev = val
    newName = "derivative(%s)" % series.name
    newSeries = TimeSeries(newName, series.start, series.end, series.step, newValues)
    newSeries.pathExpression = newName
    results.append(newSeries)
  return results
Exemplo n.º 3
0
def log(requestContext, seriesList, base=10):
  """

  Takes one metric or a wildcard seriesList, a base, and draws the y-axis in logarithmic 
  format.  If base is omitted, the function defaults to base 10.

  Example:

  .. code-block:: none

    &target=log(carbon.agents.hostname.avgUpdateTime,2)

  """
  results = []
  for series in seriesList:
    newValues = []
    for val in series:
      if val is None:
        newValues.append(None)
      elif val <= 0:
        newValues.append(None)
      else:
        newValues.append(math.log(val, base))
    newName = "log(%s, %s)" % (series.name, base)
    newSeries = TimeSeries(newName, series.start, series.end, series.step, newValues)
    newSeries.pathExpression = newName
    results.append(newSeries)
  return results
Exemplo n.º 4
0
 def test_TimeSeries_equal_list_color(self):
   values = range(0,100)
   series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   series1.color = 'white'
   series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   series2.color = 'white'
   self.assertEqual(series1, series2)
Exemplo n.º 5
0
def percentileOfSeries(requestContext, *args):
    levels = []
    seriesList = []
    for arg in args:
        logging.info("Arg: %s", arg)
        if isinstance(arg, (int, long, float)):
            levels.append(arg)
        elif isinstance(arg, basestring):
            levels += [float(x) for x in arg.split(";")]
        else:
            seriesList += arg

    logging.info("Levels: %s", levels)
    logging.info("Series: %s", seriesList)

    result = []
    for level in levels:
        if levels <= 0:
            raise ValueError('The requested percent is required to be greater than 0')

        name = 'percentilesOfSeries(%s,%g)' % (seriesList[0].pathExpression, level)
        (start, end, step) = functions.normalize([seriesList])[1:]
        values = [functions._getPercentile(row, level, False) for row in functions.izip(*seriesList)]
        resultSeries = TimeSeries(name, start, end, step, values)
        resultSeries.pathExpression = name
        result.append(resultSeries)

    return result
Exemplo n.º 6
0
 def test_TimeSeries_equal_list_color_bad2(self):
   values = range(0,100)
   series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   series1.color = 'white'
   with self.assertRaises(AssertionError):
     self.assertEqual(series1, series2)
Exemplo n.º 7
0
def sumSeries(requestContext, *seriesLists):
  """
  Short form: sum()

  This will add metrics together and return the sum at each datapoint. (See 
  integral for a sum over time)

  Example:

  .. code-block:: none

    &target=sum(company.server.application*.requestsHandled)

  This would show the sum of all requests handled per minute (provided 
  requestsHandled are collected once a minute).   If metrics with different 
  retention rates are combined, the coarsest metric is graphed, and the sum
  of the other metrics is averaged for the metrics with finer retention rates.

  """

  try:
    (seriesList,start,end,step) = normalize(seriesLists)
  except:
    return []
  #name = "sumSeries(%s)" % ','.join((s.name for s in seriesList))
  name = "sumSeries(%s)" % ','.join(set([s.pathExpression for s in seriesList]))
  values = ( safeSum(row) for row in izip(*seriesList) )
  series = TimeSeries(name,start,end,step,values)
  series.pathExpression = name
  return [series]
Exemplo n.º 8
0
    def test_linearRegression(self):
        original = functions.evaluateTarget
        try:
            # series starts at 60 seconds past the epoch and continues for 600 seconds (ten minutes)
            # steps are every 60 seconds
            savedSeries = TimeSeries('test.value',180,480,60,[3,None,5,6,None,8]),
            functions.evaluateTarget = lambda x, y: savedSeries

            # input values will be ignored and replaced by regression function
            inputSeries = TimeSeries('test.value',1200,1500,60,[123,None,None,456,None,None,None])
            inputSeries.pathExpression = 'test.value'
            results = functions.linearRegression({
                'startTime': datetime(1970, 1, 1, 0, 20, 0, 0, pytz.timezone(settings.TIME_ZONE)),
                'endTime': datetime(1970, 1, 1, 0, 25, 0, 0, pytz.timezone(settings.TIME_ZONE)),
                'localOnly': False,
                'data': [],
            }, [ inputSeries ], '00:03 19700101', '00:08 19700101')

            # regression function calculated from datapoints on minutes 3 to 8
            expectedResult = [
                TimeSeries('linearRegression(test.value, 180, 480)',1200,1500,60,[20.0,21.0,22.0,23.0,24.0,25.0,26.0])
            ]

            self.assertEqual(results, expectedResult)
        finally:
            functions.evaluateTarget = original
Exemplo n.º 9
0
    def test_TimeSeries_consolidate(self):
      values = list(range(0,100))

      series = TimeSeries("collectd.test-db.load.value", 0, len(values)/2, 1, values)
      self.assertEqual(series.valuesPerPoint, 1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)
Exemplo n.º 10
0
 def test_TimeSeries_iterate_valuesPerPoint_2_none_values(self):
   values = [None, None, None, None, None]
   series = TimeSeries("collectd.test-db.load.value", 0, len(values)/2, 1, values)
   self.assertEqual(series.valuesPerPoint, 1)
   series.consolidate(2)
   self.assertEqual(series.valuesPerPoint, 2)
   expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, [None, None, None])
   self.assertEqual(list(series), list(expected))
Exemplo n.º 11
0
 def test_TimeSeries_iterate_valuesPerPoint_2_avg(self):
   values = range(0,100)
   series = TimeSeries("collectd.test-db.load.value", 0, len(values)/2, 1, values)
   self.assertEqual(series.valuesPerPoint, 1)
   series.consolidate(2)
   self.assertEqual(series.valuesPerPoint, 2)
   expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, [0.5, 2.5, 4.5, 6.5, 8.5, 10.5, 12.5, 14.5, 16.5, 18.5, 20.5, 22.5, 24.5, 26.5, 28.5, 30.5, 32.5, 34.5, 36.5, 38.5, 40.5, 42.5, 44.5, 46.5, 48.5, 50.5, 52.5, 54.5, 56.5, 58.5, 60.5, 62.5, 64.5, 66.5, 68.5, 70.5, 72.5, 74.5, 76.5, 78.5, 80.5, 82.5, 84.5, 86.5, 88.5, 90.5, 92.5, 94.5, 96.5, 98.5, None])
   self.assertEqual(list(series), list(expected))
Exemplo n.º 12
0
 def test_TimeSeries_iterate_valuesPerPoint_2_min(self):
   values = range(0,100)
   series = TimeSeries("collectd.test-db.load.value", 0, 5, 1, values, consolidate='min')
   self.assertEqual(series.valuesPerPoint, 1)
   series.consolidate(2)
   self.assertEqual(series.valuesPerPoint, 2)
   expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, range(0,100,2)+[None])
   self.assertEqual(list(series), list(expected))
Exemplo n.º 13
0
 def test_TimeSeries_iterate_valuesPerPoint_2_invalid(self):
   values = range(0,100)
   series = TimeSeries("collectd.test-db.load.value", 0, 5, 1, values, consolidate='bogus')
   self.assertEqual(series.valuesPerPoint, 1)
   series.consolidate(2)
   self.assertEqual(series.valuesPerPoint, 2)
   expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, range(0,100,2)+[None])
   with self.assertRaisesRegexp(Exception, "Invalid consolidation function: 'bogus'"):
     result = list(series)
Exemplo n.º 14
0
    def test_TimeSeries_iterate_valuesPerPoint_2_none_values(self):
      values = [None, None, None, None, None]

      series = TimeSeries("collectd.test-db.load.value", 0, len(values)/2, 1, values)
      self.assertEqual(series.valuesPerPoint, 1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, [None, None, None])

      values = [None, None, None, None, None, 1, 2, 3, 4]

      series = TimeSeries("collectd.test-db.load.value", 0, len(values)/2, 1, values, xFilesFactor=0.1)
      self.assertEqual(series.valuesPerPoint, 1)
      self.assertEqual(series.xFilesFactor, 0.1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)

      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, [None, None, 1, 2.5, 4])
      self.assertEqual(list(series), list(expected))

      series.xFilesFactor = 0.5
      self.assertEqual(list(series), list(expected))

      series.xFilesFactor = 0.500001
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, [None, None, None, 2.5, None])
      self.assertEqual(list(series), list(expected))

      series.xFilesFactor = 1
      self.assertEqual(list(series), list(expected))
Exemplo n.º 15
0
def movingAverage(requestContext, seriesList, windowSize):
  """

  Takes one metric or a wildcard seriesList followed by a number N of datapoints and graphs 
  the average of N previous datapoints.  N-1 datapoints are set to None at the
  beginning of the graph.

  .. code-block:: none

    &target=movingAverage(Server.instance01.threads.busy,10)

  """
  for seriesIndex, series in enumerate(seriesList):
    newName = "movingAverage(%s,%.1f)" % (series.name, float(windowSize))
    newSeries = TimeSeries(newName, series.start, series.end, series.step, [])
    newSeries.pathExpression = newName

    windowIndex = windowSize - 1

    for i in range( len(series) ):
      if i < windowIndex: # Pad the beginning with None's since we don't have enough data
        newSeries.append( None )

      else:
        window = series[i - windowIndex : i + 1]
        nonNull = [ v for v in window if v is not None ]
        if nonNull:
          newSeries.append( sum(nonNull) / len(nonNull) )
        else:
          newSeries.append(None)

    seriesList[ seriesIndex ] = newSeries

  return seriesList
Exemplo n.º 16
0
 def test_TimeSeries_getInfo(self):
   values = list(range(0,100))
   series = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   self.assertEqual(series.getInfo(), {
     'name': 'collectd.test-db.load.value',
     'values': values,
     'start': 0,
     'step': 1,
     'end': len(values),
     'pathExpression': 'collectd.test-db.load.value',
     'valuesPerPoint': 1,
     'consolidationFunc': 'average',
     'xFilesFactor': 0,
   })
Exemplo n.º 17
0
    def test_TimeSeries_iterate_valuesPerPoint_2_last(self):
      values = list(range(0,100))

      series = TimeSeries("collectd.test-db.load.value", 0, 5, 1, values, consolidate='last')
      self.assertEqual(series.valuesPerPoint, 1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(1,100,2)))
      self.assertEqual(list(series), list(expected))

      series.consolidate(3)
      self.assertEqual(series.valuesPerPoint, 3)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(2,100,3)) + [99])
      self.assertEqual(list(series), list(expected))
Exemplo n.º 18
0
def minSeries(requestContext, *seriesLists):
  """
  Takes one metric or a wildcard seriesList.
  For each datapoint from each metric passed in, pick the minimum value and graph it.

  Example:

  .. code-block:: none
      
    &target=minSeries(Server*.connections.total)
  """
  (seriesList, start, end, step) = normalize(seriesLists)
  pathExprs = list( set([s.pathExpression for s in seriesList]) )
  name = "minSeries(%s)" % ','.join(pathExprs)
  values = ( safeMin(row) for row in izip(*seriesList) )
  series = TimeSeries(name, start, end, step, values)
  series.pathExpression = name
  return [series]
Exemplo n.º 19
0
def diffSeries(requestContext, *seriesLists):
  """
  Can take two or more metrics, or a single metric and a constant.
  Subtracts parameters 2 through n from parameter 1.

  Example:

  .. code-block:: none
    
    &target=diffSeries(service.connections.total,service.connections.failed)
    &target=diffSeries(service.connections.total,5)

  """
  (seriesList,start,end,step) = normalize(seriesLists)
  name = "diffSeries(%s)" % ','.join(set([s.pathExpression for s in seriesList]))
  values = ( safeDiff(row) for row in izip(*seriesList) )
  series = TimeSeries(name,start,end,step,values)
  series.pathExpression = name
  return [series]
Exemplo n.º 20
0
def nonNegativeDerivative(requestContext, seriesList, maxValue=None):
  """

  Same as the derivative function above, but ignores datapoints that trend 
  down.  Useful for counters that increase for a long time, then wrap or 
  reset. (Such as if a network interface is destroyed and recreated by unloading 
  and re-loading a kernel module, common with USB / WiFi cards.

  Example:

  .. code-block:: none
        
    &target=derivative(company.server.application01.ifconfig.TXPackets)

  """
  results = []

  for series in seriesList:
    newValues = []
    prev = None

    for val in series:
      if None in (prev, val):
        newValues.append(None)
        prev = val
        continue

      diff = val - prev
      if diff >= 0:
        newValues.append(diff)
      elif maxValue is not None and maxValue >= val:
        newValues.append( (maxValue - prev) + val  + 1 )
      else:
        newValues.append(None)

      prev = val

    newName = "nonNegativeDerivative(%s)" % series.name
    newSeries = TimeSeries(newName, series.start, series.end, series.step, newValues)
    newSeries.pathExpression = newName
    results.append(newSeries)

  return results
Exemplo n.º 21
0
def divideSeries(requestContext, dividendSeriesList, divisorSeriesList):
  """

  Takes a dividend metric and a divisor metric and draws the division result.
  A constant may *not* be passed. To divide by a constant, use the scale() 
  function (which is essentially a multiplication operation) and use the inverse
  of the dividend. (Division by 8 = multiplication by 1/8 or 0.125)

  Example:

  .. code-block:: none

    &target=asPercent(Series.dividends,Series.divisors)


  """
  if len(divisorSeriesList) != 1:
    raise ValueError("divideSeries second argument must reference exactly 1 series")

  divisorSeries = divisorSeriesList[0]
  results = []

  for dividendSeries in dividendSeriesList:
    name = "divideSeries(%s,%s)" % (dividendSeries.name, divisorSeries.name)
    bothSeries = (dividendSeries, divisorSeries)
    step = reduce(lcm,[s.step for s in bothSeries])

    for s in bothSeries:
      s.consolidate( step / s.step )

    start = min([s.start for s in bothSeries])
    end = max([s.end for s in bothSeries])
    end -= (end - start) % step

    values = ( safeDiv(v1,v2) for v1,v2 in izip(*bothSeries) )

    quotientSeries = TimeSeries(name, start, end, step, values)
    quotientSeries.pathExpression = name
    results.append(quotientSeries)

  return results
Exemplo n.º 22
0
def averageSeries(requestContext, *seriesLists):
  """
  Short Alias: avg()

  Takes one metric or a wildcard seriesList.
  Draws the average value of all metrics passed at each time.

  Example:

  .. code-block:: none

    &target=averageSeries(company.server.*.threads.busy)
  
  """
  (seriesList,start,end,step) = normalize(seriesLists)
  #name = "averageSeries(%s)" % ','.join((s.name for s in seriesList))
  name = "averageSeries(%s)" % ','.join(set([s.pathExpression for s in seriesList]))
  values = ( safeDiv(safeSum(row),safeLen(row)) for row in izip(*seriesList) )
  series = TimeSeries(name,start,end,step,values)
  series.pathExpression = name
  return [series]
Exemplo n.º 23
0
    def _generate_series_list(self, config=None):
        seriesList = []
        if not config:
            config = [
                list(range(101)),
                list(range(101)), [1, None, None, None, None]
            ]

        for i, c in enumerate(config):
            name = "collectd.test-db{0}.load.value".format(i + 1)
            seriesList.append(TimeSeries(name, 0, 1, 1, c))
        return seriesList
Exemplo n.º 24
0
    def gen_series_list_partial_none(self, start=0):

        data = list(range(start, start + 15))
        data[2] = None
        data[8] = None

        seriesList = [
            TimeSeries('stuff.things.more.things', start, start + 15, 1, data)
        ]
        for series in seriesList:
            series.pathExpression = series.name
        return seriesList
Exemplo n.º 25
0
    def test_check_empty_lists(self):
        seriesList = []
        config = [[1000, 100, 10, 0], []]
        for i, c in enumerate(config):
            seriesList.append(TimeSeries('Test(%d)' % i, 0, 0, 0, c))

        self.assertTrue(functions.safeIsNotEmpty(seriesList[0]))
        self.assertFalse(functions.safeIsNotEmpty(seriesList[1]))

        result = functions.removeEmptySeries({}, seriesList)

        self.assertEqual(1, len(result))
Exemplo n.º 26
0
    def test_sorting_by_total(self):
        seriesList = []
        config = [[1000, 100, 10, 0], [1000, 100, 10, 1]]
        for i, c in enumerate(config):
            seriesList.append(TimeSeries('Test(%d)' % i, 0, 0, 0, c))

        self.assertEqual(1110, functions.safeSum(seriesList[0]))

        result = functions.sortByTotal({}, seriesList)

        self.assertEqual(1111, functions.safeSum(result[0]))
        self.assertEqual(1110, functions.safeSum(result[1]))
Exemplo n.º 27
0
 def test_TimeSeries_equal_list_color(self):
   values = list(range(0,100))
   series1 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   series1.color = 'white'
   series2 = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   series2.color = 'white'
   self.assertEqual(series1, series2)
Exemplo n.º 28
0
    def test__merge_results_multiple_series(self):
        pathExpr = 'collectd.test-db.load.value'
        startTime = datetime(1970, 1, 1, 0, 10, 0, 0,
                             pytz.timezone(settings.TIME_ZONE)),
        endTime = datetime(1970, 1, 1, 0, 20, 0, 0,
                           pytz.timezone(settings.TIME_ZONE))
        timeInfo = [startTime, endTime, 60]
        result_queue = [
            [
                pathExpr,
                [timeInfo, [0, 1, 2, 3, 4, None, None, None, None, None]]
            ],
            [
                pathExpr,
                [timeInfo, [None, None, None, None, None, 5, 6, 7, 8, 9]]
            ],
            [
                pathExpr,
                [
                    timeInfo,
                    [None, None, None, None, None, None, None, 7, 8, 9]
                ]
            ], [pathExpr, [timeInfo, [0, 1, 2, 3, 4, None, None, 7, 8, 9]]]
        ]

        seriesList = {
            'collectd.test-db.cpu.value':
            TimeSeries("collectd.test-db.cpu.value", startTime, endTime, 60,
                       [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        }
        requestContext = self._build_requestContext(startTime, endTime)
        results = _merge_results(pathExpr, startTime, endTime, result_queue,
                                 seriesList, requestContext)
        expectedResults = [
            TimeSeries("collectd.test-db.cpu.value", startTime, endTime, 60,
                       [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
            TimeSeries("collectd.test-db.load.value", startTime, endTime, 60,
                       [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
        ]
        self.assertEqual(results, expectedResults)
Exemplo n.º 29
0
def centered_mov_avg(requestContext, seriesList, windowSize):
    windowInterval = None
    if isinstance(windowSize, basestring):
        delta = functions.parseTimeOffset(windowSize)
        windowInterval = abs(delta.seconds + (delta.days * 86400))

    if windowInterval:
        bootstrapSeconds = windowInterval
    else:
        bootstrapSeconds = max([s.step for s in seriesList]) * int(windowSize)

    bootstrapList = functions._fetchWithBootstrap(requestContext, seriesList, seconds=bootstrapSeconds)
    result = []

    for bootstrap, series in zip(bootstrapList, seriesList):
        if windowInterval:
            windowPoints = windowInterval / series.step
        else:
            windowPoints = int(windowSize)

        if isinstance(windowSize, basestring):
            newName = 'centeredMovingAverage(%s,"%s")' % (series.name, windowSize)
        else:
            newName = "centeredMovingAverage(%s,%s)" % (series.name, windowSize)
        newSeries = TimeSeries(newName, series.start, series.end, series.step, [])
        newSeries.pathExpression = newName

        offset = len(bootstrap) - len(series)
        logging.info("Offset: %s", offset)
        logging.info("windowPoints: %s", windowPoints)

        for i in range(len(series)):
            window = bootstrap[i + offset - windowPoints + windowPoints / 2:i + offset + windowPoints / 2]
            logging.info("window: %s", len(window))
            newSeries.append(functions.safeAvg(window))

        result.append(newSeries)

    return result
Exemplo n.º 30
0
    def gen_series_list(self, start=0, use_none=False):

        data = range(start, start + 15)
        if use_none:
            n = [None for d in data]
            data = n

        seriesList = [
            TimeSeries('stuff.things.more.things', start, start + 15, 1, data)
        ]
        for series in seriesList:
            series.pathExpression = series.name
        return seriesList
Exemplo n.º 31
0
    def test_TimeSeries_iterate_valuesPerPoint_2_avg_alias(self):
      values = list(range(0,100))

      series = TimeSeries("collectd.test-db.load.value", 0, len(values)/2, 1, values, consolidate='avg')
      self.assertEqual(series.valuesPerPoint, 1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, [0.5, 2.5, 4.5, 6.5, 8.5, 10.5, 12.5, 14.5, 16.5, 18.5, 20.5, 22.5, 24.5, 26.5, 28.5, 30.5, 32.5, 34.5, 36.5, 38.5, 40.5, 42.5, 44.5, 46.5, 48.5, 50.5, 52.5, 54.5, 56.5, 58.5, 60.5, 62.5, 64.5, 66.5, 68.5, 70.5, 72.5, 74.5, 76.5, 78.5, 80.5, 82.5, 84.5, 86.5, 88.5, 90.5, 92.5, 94.5, 96.5, 98.5])
      self.assertEqual(list(series), list(expected))

      series.consolidate(3)
      self.assertEqual(series.valuesPerPoint, 3)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, map(float, list(range(1, 100, 3)) + [99]))
      self.assertEqual(list(series), list(expected))
Exemplo n.º 32
0
    def test_TimeSeries_iterate_valuesPerPoint_2_last(self):
      values = list(range(0,100))

      series = TimeSeries("collectd.test-db.load.value", 0, 5, 1, values, consolidate='last')
      self.assertEqual(series.valuesPerPoint, 1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(1,100,2)))
      self.assertEqual(list(series), list(expected))

      series.consolidate(3)
      self.assertEqual(series.valuesPerPoint, 3)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(2,100,3)) + [99])
      self.assertEqual(list(series), list(expected))
Exemplo n.º 33
0
def applyHotSax(requestContext, seriesLists):
	matrix = []
	time = {}

	f = open("/var/log/debug.txt", "w")
	f.write(str(len(seriesLists)))
	f.close()

	for i in range(len(seriesLists)):
		datapoints = seriesLists[i].datapoints()
		current_measurement = []
		index = 0
		for j in range(len(datapoints)):
			if datapoints[j][0] == None:
				continue
			time[index] = datapoints[j][1]
			index = index + 1
			current_measurement.append(datapoints[j][0])
		current_measurement = np.array(current_measurement)
		matrix.append(current_measurement)
	matrix = np.array(matrix).T

	#Apply Anomaly algorithm using SAX
	discord = hotsax.hotsax( matrix )

	r_time = []
	r_index = []
	r_value = []
	for i in range(len(discord)):
		r_time.append(time[discord[i][1]])
		r_index.append(discord[i][0])
		r_value.append(discord[i][2])
	start = seriesLists[0].start
	step = seriesLists[0].step
	end = start + step * len(discord)
	return [ TimeSeries(name = 'recov.result.timestamp', start = start, end = end, step = step, values = r_time),
		 TimeSeries(name = 'recov.result.column', start = start, end = end, step = step, values = r_index),
		 TimeSeries(name = 'recov.result.value', start = start, end = end, step = step, values = r_value) ]
Exemplo n.º 34
0
def summarize(requestContext, seriesList, intervalString):
    results = []
    delta = parseTimeOffset(intervalString)
    interval = delta.seconds + (delta.days * 86400)

    for series in seriesList:
        buckets = {}

        timestamps = range(int(series.start), int(series.end),
                           int(series.step))
        datapoints = zip(timestamps, series)

        for (timestamp, value) in datapoints:
            bucketInterval = timestamp - (timestamp % interval)

            if bucketInterval not in buckets:
                buckets[bucketInterval] = []

            if value is not None:
                buckets[bucketInterval].append(value)

        newStart = series.start - (series.start % interval)
        newEnd = series.end - (series.end % interval) + interval
        newValues = []
        for timestamp in range(newStart, newEnd, interval):
            bucket = buckets.get(timestamp, [])

            if bucket:
                newValues.append(sum(bucket))
            else:
                newValues.append(None)

        newName = "summarize(%s, \"%s\")" % (series.name, intervalString)
        newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues)
        newSeries.pathExpression = newName
        results.append(newSeries)

    return results
Exemplo n.º 35
0
def asPercent(requestContext, seriesList1, seriesList2orNumber):
  """

  Takes exactly two metrics, or a metric and a constant.
  Draws the first metric as a percent of the second.

  Example:

  .. code-block:: none

    &target=asPercent(Server01.connections.failed,Server01.connections,total)
    &target=asPercent(apache01.threads.busy,1500)

  """
  assert len(seriesList1) == 1, "asPercent series arguments must reference *exactly* 1 series"
  series1 = seriesList1[0]
  if type(seriesList2orNumber) is list:
    assert len(seriesList2orNumber) == 1, "asPercent series arguments must reference *exactly* 1 series"
    series2 = seriesList2orNumber[0]
    name = "asPercent(%s,%s)" % (series1.name,series2.name)
    series = (series1,series2)
    step = reduce(lcm,[s.step for s in series])
    for s in series:
      s.consolidate( step / s.step )
    start = min([s.start for s in series])
    end = max([s.end for s in series])
    end -= (end - start) % step
    values = ( safeMul( safeDiv(v1,v2), 100.0 ) for v1,v2 in izip(*series) )
  else:
    number = float(seriesList2orNumber)
    name = "asPercent(%s,%.1f)" % (series1.name,number)
    step = series1.step
    start = series1.start
    end = series1.end
    values = ( safeMul( safeDiv(v,number), 100.0 ) for v in series1 )
  series = TimeSeries(name,start,end,step,values)
  series.pathExpression = name
  return [series]
Exemplo n.º 36
0
def summarize(requestContext, seriesList, intervalString):
  results = []
  delta = parseTimeOffset(intervalString)
  interval = delta.seconds + (delta.days * 86400)

  for series in seriesList:
    buckets = {}

    timestamps = range( int(series.start), int(series.end), int(series.step) )
    datapoints = zip(timestamps, series)

    for (timestamp, value) in datapoints:
      bucketInterval = timestamp - (timestamp % interval)

      if bucketInterval not in buckets:
        buckets[bucketInterval] = []

      if value is not None:
        buckets[bucketInterval].append(value)

    newStart = series.start - (series.start % interval)
    newEnd = series.end - (series.end % interval) + interval
    newValues = []
    for timestamp in range(newStart, newEnd, interval):
      bucket = buckets.get(timestamp, [])

      if bucket:
        newValues.append( sum(bucket) )
      else:
        newValues.append( None )

    newName = "summarize(%s, \"%s\")" % (series.name, intervalString)
    newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues)
    newSeries.pathExpression = newName
    results.append(newSeries)

  return results
Exemplo n.º 37
0
def stacked(requestContext,seriesLists):
  """
  Takes one metric or a wildcard seriesList and change them so they are
  stacked. This is a way of stacking just a couple of metrics without having
  to use the stacked area mode (that stacks everything). By means of this a mixed
  stacked and non stacked graph can be made

  Example:

  .. code-block:: none

    &target=stacked(company.server.application01.ifconfig.TXPackets)

  """
  if 'totalStack' in requestContext:
    totalStack = requestContext['totalStack']
  else:
    totalStack = [];
  results = []
  for series in seriesLists:
    newValues = []
    for i in range(len(series)):
      if len(totalStack) <= i: totalStack.append(0)

      if series[i] is not None:
        totalStack[i] += series[i]
        newValues.append(totalStack[i])
      else:
        newValues.append(None)

    newName = "stacked(%s)" % series.name
    newSeries = TimeSeries(newName, series.start, series.end, series.step, newValues)
    newSeries.options['stacked'] = True
    newSeries.pathExpression = newName
    results.append(newSeries)
  requestContext['totalStack'] = totalStack
  return results
Exemplo n.º 38
0
    def test_linearRegression(self):
        original = functions.evaluateTarget
        try:
            # series starts at 60 seconds past the epoch and continues for 600 seconds (ten minutes)
            # steps are every 60 seconds
            savedSeries = TimeSeries('test.value', 180, 480, 60,
                                     [3, None, 5, 6, None, 8]),
            functions.evaluateTarget = lambda x, y: savedSeries

            # input values will be ignored and replaced by regression function
            inputSeries = TimeSeries('test.value', 1200, 1500, 60,
                                     [123, None, None, 456, None, None, None])
            inputSeries.pathExpression = 'test.value'
            results = functions.linearRegression(
                {
                    'startTime':
                    datetime(1970, 1, 1, 0, 20, 0, 0,
                             pytz.timezone(settings.TIME_ZONE)),
                    'endTime':
                    datetime(1970, 1, 1, 0, 25, 0, 0,
                             pytz.timezone(settings.TIME_ZONE)),
                    'localOnly':
                    False,
                    'data': [],
                }, [inputSeries], '00:03 19700101', '00:08 19700101')

            # regression function calculated from datapoints on minutes 3 to 8
            expectedResult = [
                TimeSeries('linearRegression(test.value, 180, 480)', 1200,
                           1500, 60,
                           [20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0])
            ]

            self.assertEqual(results, expectedResult)
        finally:
            functions.evaluateTarget = original
Exemplo n.º 39
0
def applyKNN(requestContext, labelSeries, labelColumn, unlabelSeries):
    labelMatrix = []
    for i in range(len(labelSeries)):
        datapoints = labelSeries[i].datapoints()
        current_measurements = []
        for j in range(len(datapoints)):
            if datapoints[j][0] == None:
                continue
            current_measurements.append(datapoints[j][0])
        current_measurements = np.array(current_measurements)
        labelMatrix.append(current_measurements)
    labelMatrix = np.array(labelMatrix).T

    unlabelMatrix = []
    for i in range(len(unlabelSeries)):
        datapoints = unlabelSeries[i].datapoints()
        current_measurements = []
        for j in range(len(datapoints)):
            if datapoints[j][0] == None:
                continue
            current_measurements.append(datapoints[j][0])
        current_measurements = np.array(current_measurements)
        unlabelMatrix.append(current_measurements)
    unlabelMatrix = np.array(unlabelMatrix).T

    labels = []
    datapoints = labelColumn[0].datapoints()
    for j in range(len(datapoints)):
        if datapoints[j][0] == None:
            continue
        labels.append(datapoints[j][0])

    result = knn.knn(labelMatrix, labels, unlabelMatrix, 3)

    startTime = labelSeries[0].start
    stepTime = labelSeries[0].step
    endTime = startTime + len(result) * stepTime
    return [
        TimeSeries(name="master.knn.dist",
                   start=startTime,
                   end=endTime,
                   step=stepTime,
                   values=result)
    ]
Exemplo n.º 40
0
def movingAverage(requestContext, seriesList, windowSize):
    for seriesIndex, series in enumerate(seriesList):
        newName = "movingAverage(%s,%.1f)" % (series.name, float(windowSize))
        newSeries = TimeSeries(newName, series.start, series.end, series.step,
                               [])
        newSeries.pathExpression = newName

        windowIndex = windowSize - 1

        for i in range(len(series)):
            if i < windowIndex:  # Pad the beginning with None's since we don't have enough data
                newSeries.append(None)

            else:
                window = series[i - windowIndex:i + 1]
                nonNull = [v for v in window if v is not None]
                if nonNull:
                    newSeries.append(sum(nonNull) / len(nonNull))
                else:
                    newSeries.append(None)

        seriesList[seriesIndex] = newSeries

    return seriesList
Exemplo n.º 41
0
    def test__merge_results_no_remote_store_merge_results(self):
      pathExpr = 'collectd.test-db.load.value'
      startTime=datetime(1970, 1, 1, 0, 10, 0, 0, pytz.timezone(settings.TIME_ZONE))
      endTime=datetime(1970, 1, 1, 0, 20, 0, 0, pytz.timezone(settings.TIME_ZONE))
      timeInfo = [startTime, endTime, 60]
      result_queue = [
                      [pathExpr, [timeInfo, [0,1,2,3,4,None,None,None,None,None]]],
                      [pathExpr, [timeInfo, [None,None,None,3,4,5,6,7,8,9]]],
                      [pathExpr, [timeInfo, [None,None,None,None,None,None,None,7,8,9]]]
                     ]

      seriesList = {}
      requestContext = self._build_requestContext(startTime, endTime)
      with self.settings(REMOTE_STORE_MERGE_RESULTS=False):
          results = _merge_results(pathExpr, startTime, endTime, result_queue, seriesList, requestContext)
      expectedResults = [
          TimeSeries("collectd.test-db.load.value", startTime, endTime, 60, [None,None,None,3,4,5,6,7,8,9]),
      ]
      self.assertEqual(results, expectedResults)
Exemplo n.º 42
0
def applyDstreeSearch(requestContext, seriesLists, columns):
	matrix = []
	for i in range(len(seriesLists)):
		datapoints = seriesLists[i].datapoints()
		current_measurements = []
		for j in range(len(datapoints)):
			if datapoints[j][0] == None:
				continue
			current_measurements.append( datapoints[j][0] )
		current_measurements = np.array(current_measurements)
		matrix.append(current_measurements)
	matrix = np.array(matrix)

	distances = IndexExactSearcher.search(matrix, "/var/log/out/index.idx_dyn_100_1_" + str(columns))

	startTime = seriesLists[0].start
	stepTime = seriesLists[0].step
	endTime = startTime + len(distances) * stepTime
	return [ TimeSeries(name = "master.dstree.dist", start = startTime, end = endTime, step = stepTime, values = distances) ]
Exemplo n.º 43
0
 def test_reduceSeries_asPercent(self):
     seriesList = [
         TimeSeries('group.server1.bytes_used',0,1,1,[1]),
         TimeSeries('group.server1.total_bytes',0,1,1,[2]),
         TimeSeries('group.server2.bytes_used',0,1,1,[3]),
         TimeSeries('group.server2.total_bytes',0,1,1,[4]),
     ]
     for series in seriesList:
         series.pathExpression = "tempPath"
     expectedResult   = [
         TimeSeries('group.server1.reduce.asPercent',0,1,1,[50]), #100*1/2
         TimeSeries('group.server2.reduce.asPercent',0,1,1,[75])  #100*3/4
     ]
     mappedResult = [seriesList[0]],[seriesList[1]], [seriesList[2]],[seriesList[3]]
     results = functions.reduceSeries({}, copy.deepcopy(mappedResult), "asPercent", 2, "bytes_used", "total_bytes")
     self.assertEqual(results,expectedResult)
Exemplo n.º 44
0
def applySaxRepresentation(requestContext, seriesLists):
	matrix = []
	for i in range(len(seriesLists)):
		datapoints = seriesLists[i].datapoints()
		current_measurements = []
		for j in range(len(datapoints)):
			if datapoints[j][0] == None:
				continue
			current_measurements.append( datapoints[j][0] )
		current_measurements = np.array(current_measurements)
		matrix.append(current_measurements)
	matrix = np.array(matrix).T

	saxRepresentation = saxtransformation.saxrepresentation(matrix)
	result = []
	timeSeriesCount = matrix.shape[1]
	startTime = seriesLists[0].start
	stepTime = seriesLists[0].step
	endTime = startTime + len(seriesLists[0]) * stepTime
	for j in range(timeSeriesCount):
		result.append( TimeSeries(name = "master.saxrepresentation.dim" + str(j), start = startTime, end = endTime, step = stepTime, values = list(saxRepresentation[j])) )
	return result
Exemplo n.º 45
0
def applyRecov(requestContext, seriesLists):
    matrix = []
    timestamps = []
    for i in range(len(seriesLists)):
        datapoints = seriesLists[i].datapoints()
        current_measurement = []
        for j in range(len(datapoints)):
            if datapoints[j][0] == None:
                continue
            current_measurement.append(datapoints[j][0])
        current_measurement = np.array(current_measurement)
        matrix.append(current_measurement)
    matrix = np.array(matrix).T

    n, m = matrix.shape
    rec_time, iter, rmse, rec_mat = recovery.recovery(matrix, n, m, 3, 0.2, 10)

    result = []
    for i in range(m):
        startTime = seriesLists[i].start
        stepTime = seriesLists[i].step

        datapoints = seriesLists[i].datapoints()
        current_measurement = []
        index_result = 0
        for j in range(len(datapoints)):
            if datapoints[j][0] == None:
                current_measurement.append(None)
            else:
                current_measurement.append(rec_mat[index_result][i])
                index_result = index_result + 1
        endTime = startTime + len(current_measurement) * stepTime
        result.append(
            TimeSeries(name="recov.result.dim" + str(i),
                       start=startTime,
                       end=endTime,
                       step=stepTime,
                       values=current_measurement))
    return result
Exemplo n.º 46
0
def applyCD(requestContext, seriesLists):
    """Custom function that runs cd"""

    # Extract the data into 2D numpy.array
    matrix = []
    for i in range(len(seriesLists)):
        datapoints = seriesLists[i].datapoints()
        current_measurements = []
        for j in range(len(datapoints)):
            if datapoints[j][0] == None:
                continue
            current_measurements.append(datapoints[j][0])
        current_measurements = np.array(current_measurements)
        matrix.append(current_measurements)
    matrix = np.array(matrix).T
    f = open("/var/log/debug.txt", "w")
    f.write(str(matrix.shape) + "\n")
    f.write(str(matrix))
    f.close()

    # Apply KMeans
    matrix_l, matrix_r, z = cd_ssv.CD(matrix, matrix.shape[0], matrix.shape[1])

    # Format the data into TimeSeries
    result = []
    timeSeriesCount = matrix_r.shape[1]
    startTime = seriesLists[0].start
    stepTime = seriesLists[0].step
    endTime = startTime + timeSeriesCount * stepTime
    for j in range(timeSeriesCount):
        result.append(
            TimeSeries(name="master.cd.dim" + str(j),
                       start=startTime,
                       end=endTime,
                       step=stepTime,
                       values=matrix_r.T[j].tolist()))

    return result
Exemplo n.º 47
0
    def testNPercentile(self):
        seriesList = []
        config = [
            [15, 35, 20, 40, 50],
            range(1, 101),
            range(1, 201),
            range(1, 301),
            range(0, 100),
            range(0, 200),
            range(0, 300),
            [None, None, None] + range(0, 300),  # Ensure None values in list has no affect.
        ]

        for i, c in enumerate(config):
          seriesList.append( TimeSeries('Test(%d)' % i, 0, 0, 0, c) )

        def TestNPercentile(perc, expected):
          result =  functions.nPercentile({}, seriesList, perc)
          self.assertEquals(expected, result)

        TestNPercentile(30, [ [20], [30], [60], [90], [29], [59], [89], [89] ])
        TestNPercentile(90, [ [50], [90], [180], [270], [89], [179], [269], [269] ])
        TestNPercentile(95, [ [50], [95], [190], [285], [94], [189], [284], [284] ])
Exemplo n.º 48
0
    def test_n_percentile(self):
        seriesList = []
        config = [
            [15, 35, 20, 40, 50],
            range(1, 101),
            range(1, 201),
            range(1, 301),
            range(0, 100),
            range(0, 200),
            range(0, 300),
            # Ensure None values in list has no effect.
            [None, None, None] + range(0, 300),
        ]

        for i, c in enumerate(config):
            seriesList.append(TimeSeries('Test(%d)' % i, 0, 1, 1, c))

        def n_percentile(perc, expected):
            result = functions.nPercentile({}, seriesList, perc)
            self.assertEqual(expected, result)

        n_percentile(30, [[20], [31], [61], [91], [30], [60], [90], [90]])
        n_percentile(90, [[50], [91], [181], [271], [90], [180], [270], [270]])
        n_percentile(95, [[50], [96], [191], [286], [95], [190], [285], [285]])
Exemplo n.º 49
0
    def test__fetchData_remote_fetch_data(self):
        pathExpr = 'collectd.test-db.load.value'
        startDateTime = datetime(1970, 1, 1, 0, 10, 0, 0,
                                 pytz.timezone(settings.TIME_ZONE))
        endDateTime = datetime(1970, 1, 1, 0, 20, 0, 0,
                               pytz.timezone(settings.TIME_ZONE))
        requestContext = self._build_requestContext(startDateTime, endDateTime)
        requestContext['now'] = endDateTime
        requestContext['forwardHeaders'] = None

        # Use this form of the start/end times
        (startTime, endTime, now) = timebounds(requestContext)

        # First item in list is a proper fetched response
        # Second item is None, which is what happens if there is no data back from wait_for_results
        prefetched_results = [[{
            'pathExpression': 'collectd.test-db.load.value',
            'name': 'collectd.test-db.load.value',
            'time_info': (startTime, endTime, now),
            'step': 60,
            'values': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
        }], None]

        # Get the remote data
        requestContext['prefetched'] = {}
        requestContext['prefetched'][(
            startTime, endTime, now)] = PrefetchedData(prefetched_results)

        with self.settings(REMOTE_PREFETCH_DATA=True):
            results = _fetchData(pathExpr, startTime, endTime, now,
                                 requestContext, {})
        expectedResults = [
            TimeSeries("collectd.test-db.load.value", startTime, endTime, 1200,
                       [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
        ]
        self.assertEqual(results, expectedResults)
Exemplo n.º 50
0
    def test_TimeSeries_iterate_valuesPerPoint_2_sum(self):
      values = list(range(0,100))

      series = TimeSeries("collectd.test-db.load.value", 0, 5, 1, values, consolidate='sum')
      self.assertEqual(series.valuesPerPoint, 1)

      series.consolidate(2)
      self.assertEqual(series.valuesPerPoint, 2)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(1,200,4)))
      self.assertEqual(list(series), list(expected))

      series.consolidate(3)
      self.assertEqual(series.valuesPerPoint, 3)
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(3,300,9)) + [99])
      self.assertEqual(list(series), list(expected))

      series.xFilesFactor = 0.4
      expected = TimeSeries("collectd.test-db.load.value", 0, 5, 1, list(range(3,300,9)) + [None])
      self.assertEqual(list(series), list(expected))
Exemplo n.º 51
0
    def test_applyByNode(self):
        seriesList = [
            TimeSeries('servers.s1.disk.bytes_used', 0, 3, 1, [10, 20, 30]),
            TimeSeries('servers.s1.disk.bytes_free', 0, 3, 1, [90, 80, 70]),
            TimeSeries('servers.s2.disk.bytes_used', 0, 3, 1, [1, 2, 3]),
            TimeSeries('servers.s2.disk.bytes_free', 0, 3, 1, [99, 98, 97])
        ]
        for series in seriesList:
            series.pathExpression = series.name

        def mock_data_fetcher(reqCtx, path_expression):
            rv = []
            for s in seriesList:
                if s.name == path_expression or fnmatch(
                        s.name, path_expression):
                    rv.append(s)
            if rv:
                return rv
            raise KeyError('{} not found!'.format(path_expression))

        expectedResults = [
            TimeSeries('servers.s1.disk.pct_used', 0, 3, 1,
                       [0.10, 0.20, 0.30]),
            TimeSeries('servers.s2.disk.pct_used', 0, 3, 1, [0.01, 0.02, 0.03])
        ]

        with patch('graphite.render.evaluator.fetchData', mock_data_fetcher):
            result = functions.applyByNode(
                {
                    'startTime':
                    datetime(1970, 1, 1, 0, 0, 0, 0,
                             pytz.timezone(settings.TIME_ZONE)),
                    'endTime':
                    datetime(1970, 1, 1, 0, 9, 0, 0,
                             pytz.timezone(settings.TIME_ZONE)),
                    'localOnly':
                    False,
                }, seriesList, 1,
                'divideSeries(%.disk.bytes_used, sumSeries(%.disk.bytes_*))',
                '%.disk.pct_used')
        self.assertEqual(result, expectedResults)
Exemplo n.º 52
0
    def test_fetch_no_tag_support(self):
        class TestFinderNoTags(BaseFinder):
            tags = False

            def find_nodes(self, query):
                pass

            def fetch(self,
                      patterns,
                      start_time,
                      end_time,
                      now=None,
                      requestContext=None):
                if patterns != ['notags;hello=tiger']:
                    raise Exception('Unexpected patterns %s' % str(patterns))

                return [{
                    'pathExpression': 'notags;hello=tiger',
                    'name': 'notags;hello=tiger',
                    'time_info': (0, 60, 1),
                    'values': [],
                }]

        tagdb = Mock()

        def mockFindSeries(exprs, requestContext=None):
            self.assertEqual(requestContext, request_context)
            if exprs == ('hello=tiger', ) or exprs == ('name=notags', ):
                return ['notags;hello=tiger']
            if exprs == ('name=testtags', ):
                return []
            raise Exception('Unexpected exprs %s' % str(exprs))

        tagdb.find_series.side_effect = mockFindSeries

        store = Store(finders=[TestFinderNoTags()], tagdb=tagdb)

        with patch('graphite.render.datalib.STORE', store):
            request_context = {
                'startTime': epoch_to_dt(0),
                'endTime': epoch_to_dt(60),
                'now': epoch_to_dt(60),
            }

            results = evaluateTarget(request_context, [
                'notags;hello=tiger', 'seriesByTag("hello=tiger")',
                'seriesByTag("name=testtags")', 'seriesByTag("name=notags")'
            ])
            self.assertEqual(tagdb.find_series.call_count, 3)
            self.assertEqual(results, [
                TimeSeries('notags;hello=tiger', 0, 60, 1, []),
                TimeSeries('notags;hello=tiger',
                           0,
                           60,
                           1, [],
                           pathExpression='seriesByTag("hello=tiger")'),
                TimeSeries('notags;hello=tiger',
                           0,
                           60,
                           1, [],
                           pathExpression='seriesByTag("name=notags")'),
            ])
Exemplo n.º 53
0
    def test_fetch_tag_support(self):
        class TestFinderTags(BaseFinder):
            tags = True

            def find_nodes(self, query):
                pass

            def fetch(self,
                      patterns,
                      start_time,
                      end_time,
                      now=None,
                      requestContext=None):
                if patterns != [
                        'seriesByTag("hello=tiger")',
                        'seriesByTag("name=notags")',
                        'seriesByTag("name=testtags")', 'testtags;hello=tiger'
                ]:
                    raise Exception('Unexpected patterns %s' % str(patterns))

                return [
                    {
                        'pathExpression': 'testtags;hello=tiger',
                        'name': 'testtags;hello=tiger',
                        'time_info': (0, 60, 1),
                        'values': [],
                    },
                    {
                        'pathExpression': 'seriesByTag("hello=tiger")',
                        'name': 'testtags;hello=tiger',
                        'time_info': (0, 60, 1),
                        'values': [],
                    },
                    {
                        'pathExpression': 'seriesByTag("name=testtags")',
                        'name': 'testtags;hello=tiger',
                        'time_info': (0, 60, 1),
                        'values': [],
                    },
                ]

        tagdb = Mock()

        store = Store(finders=[TestFinderTags()], tagdb=tagdb)

        request_context = {
            'startTime': epoch_to_dt(0),
            'endTime': epoch_to_dt(60),
            'now': epoch_to_dt(60),
        }

        with patch('graphite.render.datalib.STORE', store):
            results = evaluateTarget(request_context, [
                'testtags;hello=tiger', 'seriesByTag("hello=tiger")',
                'seriesByTag("name=testtags")', 'seriesByTag("name=notags")'
            ])
            self.assertEqual(results, [
                TimeSeries('testtags;hello=tiger', 0, 60, 1, []),
                TimeSeries('testtags;hello=tiger',
                           0,
                           60,
                           1, [],
                           pathExpression='seriesByTag("hello=tiger")'),
                TimeSeries('testtags;hello=tiger',
                           0,
                           60,
                           1, [],
                           pathExpression='seriesByTag("name=testtags")'),
            ])
Exemplo n.º 54
0
def stdev(requestContext, seriesList, time):
  """

  Takes one metric or a wildcard seriesList followed by an integer N.
  Draw the Standard Deviation of all metrics passed for the past N datapoints. 
  
  Example:
  
  .. code-block:: none

    &target=stddev(server*.instance*.threads.busy,30)

  """
  
  count = 0
  for series in seriesList:
    stddevs = TimeSeries("stddev(%s,%.1f)" % (series.name, float(time)), series.start, series.end, series.step, [])
    stddevs.pathExpression = "stddev(%s,%.1f)" % (series.name, float(time))
    avg = safeDiv(safeSum(series[:time]), time)

    if avg is not None:
      sumOfSquares = sum(map(lambda(x): x * x, [v for v in series[:time] if v is not None]))
      (sd, sumOfSquares) = doStdDev(sumOfSquares, 0, 0, time, avg)
      stddevs.append(sd)
    else:
      stddevs.append(None)

    for (index, el) in enumerate(series[time:]):
      if el is None:
        continue

      toDrop = series[index]
      if toDrop is None:
        toDrop = 0

      s = safeSum([safeMul(time, avg), el, -toDrop])
      avg = safeDiv(s, time)

      if avg is not None:
        (sd, sumOfSquares) = doStdDev(sumOfSquares, toDrop, series[index+time], time, avg)
        stddevs.append(sd)
      else:
        stddevs.append(None)

    for i in range(0, time-1):
      stddevs.insert(0, None)

    seriesList[count] = stddevs
    count = count + 1

  return seriesList
Exemplo n.º 55
0
 def test_nonempty_false_nones(self):
     series = TimeSeries("collectd.test-db.load.value", 0, 4, 1,
                         [None, None, None, None])
     self.assertFalse(nonempty(series))
Exemplo n.º 56
0
 def test_nonempty_true(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                         values)
     self.assertTrue(nonempty(series))
Exemplo n.º 57
0
 def test_TimeSeries_equal_list(self):
     values = range(0, 100)
     series = TimeSeries("collectd.test-db.load.value", 0, len(values), 1,
                         values)
     with self.assertRaises(AssertionError):
         self.assertEqual(values, series)
Exemplo n.º 58
0
 def test_TimeSeries_init_tag_parse_fail(self):
     series = TimeSeries("collectd.test-db.load.value;", 0, 2, 1, [1, 2])
     self.assertEqual(series.tags, {'name': 'collectd.test-db.load.value;'})
Exemplo n.º 59
0
def hitcount(requestContext, seriesList, intervalString):
  """
  Estimate hit counts from a list of time series.

  This function assumes the values in each time series represent
  hits per second.  It calculates hits per some larger interval
  such as per day or per hour.  This function is like summarize(),
  except that it compensates automatically for different time scales
  (so that a similar graph results from using either fine-grained
  or coarse-grained records) and handles rarely-occurring events
  gracefully.
  """
  results = []
  delta = parseTimeOffset(intervalString)
  interval = int(delta.seconds + (delta.days * 86400))

  for series in seriesList:
    length = len(series)
    step = int(series.step)
    bucket_count = int(math.ceil(float(series.end - series.start) / interval))
    buckets = [[] for _ in range(bucket_count)]
    newStart = int(series.end - bucket_count * interval)

    for i, value in enumerate(series):
      if value is None:
        continue

      start_time = int(series.start + i * step)
      start_bucket, start_mod = divmod(start_time - newStart, interval)
      end_time = start_time + step
      end_bucket, end_mod = divmod(end_time - newStart, interval)

      if end_bucket >= bucket_count:
        end_bucket = bucket_count - 1
        end_mod = interval

      if start_bucket == end_bucket:
        # All of the hits go to a single bucket.
        if start_bucket >= 0:
          buckets[start_bucket].append(value * (end_mod - start_mod))

      else:
        # Spread the hits among 2 or more buckets.
        if start_bucket >= 0:
          buckets[start_bucket].append(value * (interval - start_mod))
        hits_per_bucket = value * interval
        for j in range(start_bucket + 1, end_bucket):
          buckets[j].append(hits_per_bucket)
        if end_mod > 0:
          buckets[end_bucket].append(value * end_mod)

    newValues = []
    for bucket in buckets:
      if bucket:
        newValues.append( sum(bucket) )
      else:
        newValues.append(None)

    newName = 'hitcount(%s, "%s")' % (series.name, intervalString)
    newSeries = TimeSeries(newName, newStart, series.end, interval, newValues)
    newSeries.pathExpression = newName
    results.append(newSeries)

  return results
Exemplo n.º 60
0
 def test_TimeSeries_getInfo(self):
   values = range(0,100)
   series = TimeSeries("collectd.test-db.load.value", 0, len(values), 1, values)
   self.assertEqual(series.getInfo(), {'name': 'collectd.test-db.load.value', 'values': values, 'start': 0, 'step': 1, 'end': len(values), 'pathExpression': 'collectd.test-db.load.value'} )