def fetchDataFromHyperTable(requestContext, pathExpr):
#TODO: use django settings
MIN_INTERVAL_SECS = 10
COL_INTERVAL_SECS = 60 * 60
log.info('fetching %s' % pathExpr)
pathExpr = addPrefix(pathExpr)
metricData = hypertable_index.findMetric(pathExpr)
metrics = [addPrefix(m[0]) for m in metricData]
metricRate = {}
for m in metricData:
if not m[1]:
log.info("metric %s doesn't specify a rate! Not rendering..." % m[0])
metrics.remove(m[0])
else:
metricRate[m[0]] = m[1]
if not metrics:
return []
startDateTime = requestContext['startTime']
endDateTime = requestContext['endTime']
start, end = int(timestamp(requestContext['startTime'])), int(timestamp(requestContext['endTime']))
startColString = startDateTime.strftime('metric:%Y-%m-%d %H')
endColString = endDateTime.strftime('metric:%Y-%m-%d %H')
cellIntervals = [ CellInterval(m, startColString, True, m, endColString, True) for m in metrics ]
if cellIntervals == None:
return []
nanosStart = start * 10**9L
nanosEnd = end * 10**9L
scan_spec = ScanSpec(None, None, None, 1)
scan_spec.start_time = nanosStart
scan_spec.end_time = nanosEnd
scan_spec.cell_intervals = cellIntervals
scan_spec.versions = COL_INTERVAL_SECS / MIN_INTERVAL_SECS
log.info(startDateTime)
log.info(endDateTime)
log.info(scan_spec)
valuesMap = defaultdict(list)
def processResult(key, family, column, val, ts):
its = long(ts) / 10**9L #nanoseconds -> seconds
valuesMap[key].append((its, val))
HyperTablePool.doScan(scan_spec, "metrics", processResult)
elapsed = end - start
for m in valuesMap.keys():
# resample everything to 'best' granularity
steps = int(end - start) / metricRate[m]
# push final values
finalValues = [None] * steps
for x in valuesMap[m]:
bucket = int(min(round(float(x[0] - start) / float(metricRate[m])), steps - 1))
finalValues[bucket] = float(x[1])
valuesMap[m] = finalValues
seriesList = []
for m in sorted(valuesMap.keys()):
series = TimeSeries(removePrefix(m), start, end, metricRate[m], valuesMap[m])
series.pathExpression = pathExpr # hack to pass expressions through to render functions
seriesList.append(series)
return seriesList
def fetchDataFromHyperTable(requestContext, pathExpr):
# TODO: use django settings
MIN_INTERVAL_SECS = 10
COL_INTERVAL_SECS = 60 * 60
log.info("fetching %s" % pathExpr)
pathExpr = addPrefix(pathExpr)
metrics = [addPrefix(m) for m in hypertable_index.findMetric(pathExpr)]
if not metrics:
return []
startDateTime = requestContext["startTime"]
endDateTime = requestContext["endTime"]
start, end = int(timestamp(requestContext["startTime"])), int(timestamp(requestContext["endTime"]))
startColString = startDateTime.strftime("metric:%Y-%m-%d %H")
endColString = endDateTime.strftime("metric:%Y-%m-%d %H")
cellIntervals = [CellInterval(m, startColString, True, m, endColString, True) for m in metrics]
if cellIntervals == None:
return []
nanosStart = start * 10 ** 9L
nanosEnd = end * 10 ** 9L
scan_spec = ScanSpec(None, None, None, 1)
scan_spec.start_time = nanosStart
scan_spec.end_time = nanosEnd
scan_spec.cell_intervals = cellIntervals
scan_spec.versions = COL_INTERVAL_SECS / MIN_INTERVAL_SECS
log.info(startDateTime)
log.info(endDateTime)
log.info(scan_spec)
valuesMap = defaultdict(list)
sortedVals = {}
metricStep = {}
def processResult(key, family, column, val, ts):
its = long(ts) / 10 ** 9L # nanoseconds -> seconds
valuesMap[key].append((its, val))
HyperTablePool.doScan(scan_spec, "metrics", processResult)
elapsed = end - start
stepsSeen = defaultdict(int)
for m in valuesMap.keys():
# determine step size (the minimum evenly divisible step found)
minStep = elapsed
sortedVals[m] = sorted(valuesMap[m], key=lambda x: x[0])
for i in range(1, len(sortedVals[m])):
step = sortedVals[m][i][0] - sortedVals[m][i - 1][0]
if elapsed % step == 0:
stepsSeen[step] += 1
mostCommonStep = -1
mostCommonCount = 0
for k, v in stepsSeen.iteritems():
if v > mostCommonCount:
mostCommonCount = v
mostCommonStep = k
# hack for no data
if mostCommonStep == -1:
mostCommonStep = 60
for m in valuesMap.keys():
# resample everything to 'best' granularity
steps = int(end - start) / mostCommonStep
# push final values
finalValues = [None] * steps
for x in sortedVals[m]:
bucket = int(min(round(float(x[0] - start) / float(mostCommonStep)), steps - 1))
finalValues[bucket] = float(x[1])
valuesMap[m] = finalValues
seriesList = []
for m in sorted(valuesMap.keys()):
series = TimeSeries(removePrefix(m), start, end, mostCommonStep, valuesMap[m])
series.pathExpression = pathExpr # hack to pass expressions through to render functions
seriesList.append(series)
return seriesList