def find(self, pathExpr):
pathExpr = pathExpr
log.info('searching for: %s' % pathExpr)
if EXPANDABLE_PATH_RE.match(pathExpr):
regex = regexifyPathExpr(pathExpr)
where = 'ROW REGEXP "%s"' % regex
starIndex = pathExpr.find('*')
if starIndex > 0:
where += ' AND ROW =^ "%s"' % pathExpr[0:starIndex]
log.info('where: %s' % where)
return [removePrefix(p) for p in self.findHelper(where)]
else:
return [removePrefix(pathExpr)]
def fetchDataFromHyperTable(requestContext, pathExpr):
if pathExpr.lower().startswith('graphite.'):
pathExpr = pathExpr[9:]
pathExpr = addPrefix(pathExpr)
metrics = [addPrefix(m) for m in HyperStore().find(pathExpr)]
startTime = requestContext['startTime'].strftime('%Y-%m-%d %H:%M:%S')
endTime = requestContext['endTime'].strftime('%Y-%m-%d %H:%M:%S')
start, end, step = timestamp(requestContext['startTime']), timestamp(requestContext['endTime']), 10
buckets = (end - start) / step
where = ' OR '.join(['ROW = "%s"' % m for m in metrics])
query = 'SELECT metric FROM metrics WHERE (%s) AND "%s" < TIMESTAMP < "%s"' % (where, startTime, endTime)
log.info(query)
valuesMap = {}
for m in metrics:
valuesMap[m] = [None for x in xrange(0, buckets)]
def processResult(key, family, column, val, ts):
its = long(ts) / 1000000000L
bucket = int((its - start) / step)
if bucket >= 0 or bucket < buckets:
if valuesMap[key][bucket]:
valuesMap[key][bucket] = float(val)
else:
valuesMap[key][bucket] = float(val)
HyperTablePool.doQuery(query, processResult)
seriesList = []
for m in metrics:
series = TimeSeries(removePrefix(m), start, end, step, valuesMap[m])
series.pathExpression = pathExpr # hack to pass expressions through to render functions
seriesList.append(series)
return seriesList
def processResult(key, family, column, val, ts): metrics.append(removePrefix(key))
def search(self, query):
qre = '(?i)%s' % re.sub('\*', '.*', re.sub('\.', '\.', query))
return [removePrefix(p) for p in self.findByRegex(qre)]
def fetchDataFromHyperTable(requestContext, pathExpr):
#TODO: use django settings
MIN_INTERVAL_SECS = 10
COL_INTERVAL_SECS = 60 * 60
log.info('fetching %s' % pathExpr)
pathExpr = addPrefix(pathExpr)
metricData = hypertable_index.findMetric(pathExpr)
metrics = [addPrefix(m[0]) for m in metricData]
metricRate = {}
for m in metricData:
if not m[1]:
log.info("metric %s doesn't specify a rate! Not rendering..." % m[0])
metrics.remove(m[0])
else:
metricRate[m[0]] = m[1]
if not metrics:
return []
startDateTime = requestContext['startTime']
endDateTime = requestContext['endTime']
start, end = int(timestamp(requestContext['startTime'])), int(timestamp(requestContext['endTime']))
startColString = startDateTime.strftime('metric:%Y-%m-%d %H')
endColString = endDateTime.strftime('metric:%Y-%m-%d %H')
cellIntervals = [ CellInterval(m, startColString, True, m, endColString, True) for m in metrics ]
if cellIntervals == None:
return []
nanosStart = start * 10**9L
nanosEnd = end * 10**9L
scan_spec = ScanSpec(None, None, None, 1)
scan_spec.start_time = nanosStart
scan_spec.end_time = nanosEnd
scan_spec.cell_intervals = cellIntervals
scan_spec.versions = COL_INTERVAL_SECS / MIN_INTERVAL_SECS
log.info(startDateTime)
log.info(endDateTime)
log.info(scan_spec)
valuesMap = defaultdict(list)
def processResult(key, family, column, val, ts):
its = long(ts) / 10**9L #nanoseconds -> seconds
valuesMap[key].append((its, val))
HyperTablePool.doScan(scan_spec, "metrics", processResult)
elapsed = end - start
for m in valuesMap.keys():
# resample everything to 'best' granularity
steps = int(end - start) / metricRate[m]
# push final values
finalValues = [None] * steps
for x in valuesMap[m]:
bucket = int(min(round(float(x[0] - start) / float(metricRate[m])), steps - 1))
finalValues[bucket] = float(x[1])
valuesMap[m] = finalValues
seriesList = []
for m in sorted(valuesMap.keys()):
series = TimeSeries(removePrefix(m), start, end, metricRate[m], valuesMap[m])
series.pathExpression = pathExpr # hack to pass expressions through to render functions
seriesList.append(series)
return seriesList
def fetchDataFromHyperTable(requestContext, pathExpr):
# TODO: use django settings
MIN_INTERVAL_SECS = 10
COL_INTERVAL_SECS = 60 * 60
log.info("fetching %s" % pathExpr)
pathExpr = addPrefix(pathExpr)
metrics = [addPrefix(m) for m in hypertable_index.findMetric(pathExpr)]
if not metrics:
return []
startDateTime = requestContext["startTime"]
endDateTime = requestContext["endTime"]
start, end = int(timestamp(requestContext["startTime"])), int(timestamp(requestContext["endTime"]))
startColString = startDateTime.strftime("metric:%Y-%m-%d %H")
endColString = endDateTime.strftime("metric:%Y-%m-%d %H")
cellIntervals = [CellInterval(m, startColString, True, m, endColString, True) for m in metrics]
if cellIntervals == None:
return []
nanosStart = start * 10 ** 9L
nanosEnd = end * 10 ** 9L
scan_spec = ScanSpec(None, None, None, 1)
scan_spec.start_time = nanosStart
scan_spec.end_time = nanosEnd
scan_spec.cell_intervals = cellIntervals
scan_spec.versions = COL_INTERVAL_SECS / MIN_INTERVAL_SECS
log.info(startDateTime)
log.info(endDateTime)
log.info(scan_spec)
valuesMap = defaultdict(list)
sortedVals = {}
metricStep = {}
def processResult(key, family, column, val, ts):
its = long(ts) / 10 ** 9L # nanoseconds -> seconds
valuesMap[key].append((its, val))
HyperTablePool.doScan(scan_spec, "metrics", processResult)
elapsed = end - start
stepsSeen = defaultdict(int)
for m in valuesMap.keys():
# determine step size (the minimum evenly divisible step found)
minStep = elapsed
sortedVals[m] = sorted(valuesMap[m], key=lambda x: x[0])
for i in range(1, len(sortedVals[m])):
step = sortedVals[m][i][0] - sortedVals[m][i - 1][0]
if elapsed % step == 0:
stepsSeen[step] += 1
mostCommonStep = -1
mostCommonCount = 0
for k, v in stepsSeen.iteritems():
if v > mostCommonCount:
mostCommonCount = v
mostCommonStep = k
# hack for no data
if mostCommonStep == -1:
mostCommonStep = 60
for m in valuesMap.keys():
# resample everything to 'best' granularity
steps = int(end - start) / mostCommonStep
# push final values
finalValues = [None] * steps
for x in sortedVals[m]:
bucket = int(min(round(float(x[0] - start) / float(mostCommonStep)), steps - 1))
finalValues[bucket] = float(x[1])
valuesMap[m] = finalValues
seriesList = []
for m in sorted(valuesMap.keys()):
series = TimeSeries(removePrefix(m), start, end, mostCommonStep, valuesMap[m])
series.pathExpression = pathExpr # hack to pass expressions through to render functions
seriesList.append(series)
return seriesList