示例#1
0
def fetchDataFromHyperTable(requestContext, pathExpr):
  #TODO: use django settings
  MIN_INTERVAL_SECS = 10
  COL_INTERVAL_SECS = 60 * 60


  log.info('fetching %s' % pathExpr)
  pathExpr = addPrefix(pathExpr)
  metricData = hypertable_index.findMetric(pathExpr)
  metrics = [addPrefix(m[0]) for m in metricData]
  metricRate = {}
  for m in metricData:
    if not m[1]:
      log.info("metric %s doesn't specify a rate! Not rendering..." % m[0])
      metrics.remove(m[0])
    else:
      metricRate[m[0]] = m[1]

  if not metrics:
    return []

  startDateTime = requestContext['startTime']
  endDateTime = requestContext['endTime']

  start, end = int(timestamp(requestContext['startTime'])), int(timestamp(requestContext['endTime']))

  startColString = startDateTime.strftime('metric:%Y-%m-%d %H')
  endColString = endDateTime.strftime('metric:%Y-%m-%d %H')
  cellIntervals = [ CellInterval(m, startColString, True, m, endColString, True) for m in metrics ]

  if cellIntervals == None:
    return []

  nanosStart = start * 10**9L
  nanosEnd = end * 10**9L

  scan_spec = ScanSpec(None, None, None, 1)
  scan_spec.start_time = nanosStart
  scan_spec.end_time = nanosEnd
  scan_spec.cell_intervals = cellIntervals
  scan_spec.versions = COL_INTERVAL_SECS / MIN_INTERVAL_SECS

  log.info(startDateTime)
  log.info(endDateTime)
  log.info(scan_spec)

  valuesMap = defaultdict(list)

  def processResult(key, family, column, val, ts):
    its = long(ts) / 10**9L  #nanoseconds -> seconds
    valuesMap[key].append((its, val))



  HyperTablePool.doScan(scan_spec, "metrics", processResult)

  elapsed = end - start

  for m in valuesMap.keys():
    # resample everything to 'best' granularity
    steps = int(end - start) / metricRate[m]

    # push final values
    finalValues = [None] * steps
    for x in valuesMap[m]:
      bucket = int(min(round(float(x[0] - start) / float(metricRate[m])), steps - 1))
      finalValues[bucket] = float(x[1])
    valuesMap[m] = finalValues

  seriesList = []
  for m in sorted(valuesMap.keys()):
    series = TimeSeries(removePrefix(m), start, end, metricRate[m], valuesMap[m])
    series.pathExpression = pathExpr # hack to pass expressions through to render functions
    seriesList.append(series)

  return seriesList
示例#2
0
def fetchDataFromHyperTable(requestContext, pathExpr):
    # TODO: use django settings
    MIN_INTERVAL_SECS = 10
    COL_INTERVAL_SECS = 60 * 60

    log.info("fetching %s" % pathExpr)
    pathExpr = addPrefix(pathExpr)
    metrics = [addPrefix(m) for m in hypertable_index.findMetric(pathExpr)]

    if not metrics:
        return []

    startDateTime = requestContext["startTime"]
    endDateTime = requestContext["endTime"]

    start, end = int(timestamp(requestContext["startTime"])), int(timestamp(requestContext["endTime"]))

    startColString = startDateTime.strftime("metric:%Y-%m-%d %H")
    endColString = endDateTime.strftime("metric:%Y-%m-%d %H")
    cellIntervals = [CellInterval(m, startColString, True, m, endColString, True) for m in metrics]

    if cellIntervals == None:
        return []

    nanosStart = start * 10 ** 9L
    nanosEnd = end * 10 ** 9L

    scan_spec = ScanSpec(None, None, None, 1)
    scan_spec.start_time = nanosStart
    scan_spec.end_time = nanosEnd
    scan_spec.cell_intervals = cellIntervals
    scan_spec.versions = COL_INTERVAL_SECS / MIN_INTERVAL_SECS

    log.info(startDateTime)
    log.info(endDateTime)
    log.info(scan_spec)

    valuesMap = defaultdict(list)
    sortedVals = {}
    metricStep = {}

    def processResult(key, family, column, val, ts):
        its = long(ts) / 10 ** 9L  # nanoseconds -> seconds
        valuesMap[key].append((its, val))

    HyperTablePool.doScan(scan_spec, "metrics", processResult)

    elapsed = end - start

    stepsSeen = defaultdict(int)

    for m in valuesMap.keys():
        # determine step size (the minimum evenly divisible step found)
        minStep = elapsed
        sortedVals[m] = sorted(valuesMap[m], key=lambda x: x[0])
        for i in range(1, len(sortedVals[m])):
            step = sortedVals[m][i][0] - sortedVals[m][i - 1][0]
            if elapsed % step == 0:
                stepsSeen[step] += 1
    mostCommonStep = -1
    mostCommonCount = 0
    for k, v in stepsSeen.iteritems():
        if v > mostCommonCount:
            mostCommonCount = v
            mostCommonStep = k
    # hack for no data
    if mostCommonStep == -1:
        mostCommonStep = 60

    for m in valuesMap.keys():
        # resample everything to 'best' granularity
        steps = int(end - start) / mostCommonStep

        # push final values
        finalValues = [None] * steps
        for x in sortedVals[m]:
            bucket = int(min(round(float(x[0] - start) / float(mostCommonStep)), steps - 1))
            finalValues[bucket] = float(x[1])
        valuesMap[m] = finalValues

    seriesList = []
    for m in sorted(valuesMap.keys()):
        series = TimeSeries(removePrefix(m), start, end, mostCommonStep, valuesMap[m])
        series.pathExpression = pathExpr  # hack to pass expressions through to render functions
        seriesList.append(series)

    return seriesList