def expand_view(request):
"View for expanding a pattern into matching metric paths"
queryParams = request.GET.copy()
queryParams.update(request.POST)
local_only = int( queryParams.get('local', 0) )
group_by_expr = int( queryParams.get('groupByExpr', 0) )
leaves_only = int( queryParams.get('leavesOnly', 0) )
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
results = {}
for query in queryParams.getlist('query'):
results[query] = set()
for node in STORE.find(query, local=local_only, headers=forward_headers):
if node.is_leaf or not leaves_only:
results[query].add( node.path )
# Convert our results to sorted lists because sets aren't json-friendly
if group_by_expr:
for query, matches in results.items():
results[query] = sorted(matches)
else:
results = sorted( reduce(set.union, results.values(), set()) )
result = {
'results' : results
}
response = json_response_for(request, result, jsonp=jsonp)
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def expand_view(request):
"View for expanding a pattern into matching metric paths"
queryParams = request.GET.copy()
queryParams.update(request.POST)
local_only = int(queryParams.get('local', 0))
group_by_expr = int(queryParams.get('groupByExpr', 0))
leaves_only = int(queryParams.get('leavesOnly', 0))
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
results = {}
for query in queryParams.getlist('query'):
results[query] = set()
for node in STORE.find(query,
local=local_only,
headers=forward_headers):
if node.is_leaf or not leaves_only:
results[query].add(node.path)
# Convert our results to sorted lists because sets aren't json-friendly
if group_by_expr:
for query, matches in results.items():
results[query] = sorted(matches)
else:
results = sorted(reduce(set.union, results.values(), set()))
result = {'results': results}
response = json_response_for(request, result, jsonp=jsonp)
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def test_extractForwardHeaders(self):
class DummyRequest(object):
META = {
'HTTP_X_TEST1': 'test',
}
headers = extractForwardHeaders(DummyRequest())
self.assertEqual(headers, {'X-Test1': 'test'})
def test_extractForwardHeaders(self):
class DummyRequest(object):
META = {
'HTTP_X_TEST1': 'test',
}
headers = extractForwardHeaders(DummyRequest())
self.assertEqual(headers, {'X-Test1': 'test'})
def index_json(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
try:
jsonp = queryParams.get('jsonp', False)
requestContext = {
'localOnly': int(queryParams.get('local', 0)),
'forwardHeaders': extractForwardHeaders(request),
}
matches = STORE.get_index(requestContext)
except Exception:
log.exception()
return json_response_for(request, [], jsonp=jsonp, status=500)
return json_response_for(request, matches, jsonp=jsonp)
def index_json(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
try:
jsonp = queryParams.get('jsonp', False)
requestContext = {
'localOnly': int( queryParams.get('local', 0) ),
'forwardHeaders': extractForwardHeaders(request),
}
matches = STORE.get_index(requestContext)
except Exception:
log.exception()
return json_response_for(request, [], jsonp=jsonp, status=500)
return json_response_for(request, matches, jsonp=jsonp)
def renderView(request):
start = time()
(graphOptions, requestOptions) = parseOptions(request)
useCache = 'noCache' not in requestOptions
cacheTimeout = requestOptions['cacheTimeout']
# TODO: Make that a namedtuple or a class.
requestContext = {
'startTime': requestOptions['startTime'],
'endTime': requestOptions['endTime'],
'now': requestOptions['now'],
'localOnly': requestOptions['localOnly'],
'template': requestOptions['template'],
'tzinfo': requestOptions['tzinfo'],
'forwardHeaders': extractForwardHeaders(request),
'data': [],
'prefetched': {},
}
data = requestContext['data']
# First we check the request cache
if useCache:
requestKey = hashRequest(request)
cachedResponse = cache.get(requestKey)
if cachedResponse:
log.cache('Request-Cache hit [%s]' % requestKey)
log.rendering('Returned cached response in %.6f' %
(time() - start))
return cachedResponse
else:
log.cache('Request-Cache miss [%s]' % requestKey)
# Now we prepare the requested data
if requestOptions['graphType'] == 'pie':
for target in requestOptions['targets']:
if target.find(':') >= 0:
try:
name, value = target.split(':', 1)
value = float(value)
except:
raise ValueError("Invalid target '%s'" % target)
data.append((name, value))
else:
seriesList = evaluateTarget(requestContext, target)
for series in seriesList:
func = PieFunctions[requestOptions['pieMode']]
data.append((series.name, func(requestContext, series)
or 0))
elif requestOptions['graphType'] == 'line':
# Let's see if at least our data is cached
if useCache:
targets = requestOptions['targets']
startTime = requestOptions['startTime']
endTime = requestOptions['endTime']
dataKey = hashData(targets, startTime, endTime)
cachedData = cache.get(dataKey)
if cachedData:
log.cache("Data-Cache hit [%s]" % dataKey)
else:
log.cache("Data-Cache miss [%s]" % dataKey)
else:
cachedData = None
if cachedData is not None:
requestContext['data'] = data = cachedData
else: # Have to actually retrieve the data now
targets = requestOptions['targets']
if settings.REMOTE_PREFETCH_DATA and not requestOptions.get(
'localOnly'):
prefetchRemoteData(requestContext, targets)
for target in targets:
if not target.strip():
continue
t = time()
seriesList = evaluateTarget(requestContext, target)
log.rendering("Retrieval of %s took %.6f" %
(target, time() - t))
data.extend(seriesList)
if useCache:
cache.add(dataKey, data, cacheTimeout)
# If data is all we needed, we're done
format = requestOptions.get('format')
if format == 'csv':
response = HttpResponse(content_type='text/csv')
writer = csv.writer(response, dialect='excel')
for series in data:
for i, value in enumerate(series):
timestamp = datetime.fromtimestamp(
series.start + (i * series.step),
requestOptions['tzinfo'])
writer.writerow(
(series.name, timestamp.strftime("%Y-%m-%d %H:%M:%S"),
value))
return response
if format == 'json':
jsonStart = time()
series_data = []
if 'maxDataPoints' in requestOptions and any(data):
startTime = min([series.start for series in data])
endTime = max([series.end for series in data])
timeRange = endTime - startTime
maxDataPoints = requestOptions['maxDataPoints']
for series in data:
numberOfDataPoints = timeRange / series.step
if maxDataPoints < numberOfDataPoints:
valuesPerPoint = math.ceil(
float(numberOfDataPoints) / float(maxDataPoints))
secondsPerPoint = int(valuesPerPoint * series.step)
# Nudge start over a little bit so that the consolidation bands align with each call
# removing 'jitter' seen when refreshing.
nudge = secondsPerPoint + (
series.start % series.step) - (series.start %
secondsPerPoint)
series.start = series.start + nudge
valuesToLose = int(nudge / series.step)
for r in range(1, valuesToLose):
del series[0]
series.consolidate(valuesPerPoint)
timestamps = range(int(series.start),
int(series.end) + 1,
int(secondsPerPoint))
else:
timestamps = range(int(series.start),
int(series.end) + 1,
int(series.step))
datapoints = zip(series, timestamps)
series_data.append(
dict(target=series.name,
tags=series.tags,
datapoints=datapoints))
elif 'noNullPoints' in requestOptions and any(data):
for series in data:
values = []
for (index, v) in enumerate(series):
if v is not None:
timestamp = series.start + (index * series.step)
values.append((v, timestamp))
if len(values) > 0:
series_data.append(
dict(target=series.name,
tags=series.tags,
datapoints=values))
else:
for series in data:
timestamps = range(int(series.start),
int(series.end) + 1, int(series.step))
datapoints = zip(series, timestamps)
series_data.append(
dict(target=series.name,
tags=series.tags,
datapoints=datapoints))
output = json.dumps(
series_data,
indent=(2 if requestOptions['pretty'] else None)).replace(
'None,',
'null,').replace('NaN,',
'null,').replace('Infinity,', '1e9999,')
if 'jsonp' in requestOptions:
response = HttpResponse(content="%s(%s)" %
(requestOptions['jsonp'], output),
content_type='text/javascript')
else:
response = HttpResponse(content=output,
content_type='application/json')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('JSON rendering time %6f' % (time() - jsonStart))
log.rendering('Total request processing time %6f' %
(time() - start))
return response
if format == 'dygraph':
labels = ['Time']
result = '{}'
if data:
datapoints = [[
ts
] for ts in range(data[0].start, data[0].end, data[0].step)]
for series in data:
labels.append(series.name)
for i, point in enumerate(series):
if point is None:
point = 'null'
elif point == float('inf'):
point = 'Infinity'
elif point == float('-inf'):
point = '-Infinity'
elif math.isnan(point):
point = 'null'
datapoints[i].append(point)
line_template = '[%%s000%s]' % ''.join([', %s'] * len(data))
lines = [
line_template % tuple(points) for points in datapoints
]
result = '{"labels" : %s, "data" : [%s]}' % (
json.dumps(labels), ', '.join(lines))
response = HttpResponse(content=result,
content_type='application/json')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('Total dygraph rendering time %.6f' %
(time() - start))
return response
if format == 'rickshaw':
series_data = []
for series in data:
timestamps = range(series.start, series.end, series.step)
datapoints = [{
'x': x,
'y': y
} for x, y in zip(timestamps, series)]
series_data.append(
dict(target=series.name, datapoints=datapoints))
if 'jsonp' in requestOptions:
response = HttpResponse(
content="%s(%s)" %
(requestOptions['jsonp'], json.dumps(series_data)),
mimetype='text/javascript')
else:
response = HttpResponse(content=json.dumps(series_data),
content_type='application/json')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('Total rickshaw rendering time %.6f' %
(time() - start))
return response
if format == 'raw':
response = HttpResponse(content_type='text/plain')
for series in data:
response.write(
"%s,%d,%d,%d|" %
(series.name, series.start, series.end, series.step))
response.write(','.join(map(repr, series)))
response.write('\n')
log.rendering('Total rawData rendering time %.6f' %
(time() - start))
return response
if format == 'svg':
graphOptions['outputFormat'] = 'svg'
elif format == 'pdf':
graphOptions['outputFormat'] = 'pdf'
if format == 'pickle':
response = HttpResponse(content_type='application/pickle')
seriesInfo = [series.getInfo() for series in data]
pickle.dump(seriesInfo, response, protocol=-1)
log.rendering('Total pickle rendering time %.6f' %
(time() - start))
return response
# We've got the data, now to render it
graphOptions['data'] = data
if settings.REMOTE_RENDERING: # Rendering on other machines is faster in some situations
image = delegateRendering(requestOptions['graphType'], graphOptions,
requestContext['forwardHeaders'])
else:
image = doImageRender(requestOptions['graphClass'], graphOptions)
useSVG = graphOptions.get('outputFormat') == 'svg'
if useSVG and 'jsonp' in requestOptions:
response = HttpResponse(content="%s(%s)" %
(requestOptions['jsonp'], json.dumps(image)),
content_type='text/javascript')
elif graphOptions.get('outputFormat') == 'pdf':
response = buildResponse(image, 'application/x-pdf')
else:
response = buildResponse(image,
'image/svg+xml' if useSVG else 'image/png')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('Total rendering time %.6f seconds' % (time() - start))
return response
def parseOptions(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
# Start with some defaults
graphOptions = {'width': 330, 'height': 250}
requestOptions = {}
graphType = queryParams.get('graphType', 'line')
assert graphType in GraphTypes, "Invalid graphType '%s', must be one of %s" % (
graphType, GraphTypes.keys())
graphClass = GraphTypes[graphType]
# Fill in the requestOptions
requestOptions['graphType'] = graphType
requestOptions['graphClass'] = graphClass
requestOptions['pieMode'] = queryParams.get('pieMode', 'average')
cacheTimeout = int(
queryParams.get('cacheTimeout', settings.DEFAULT_CACHE_DURATION))
requestOptions['targets'] = []
requestOptions['forwardHeaders'] = extractForwardHeaders(request)
# Extract the targets out of the queryParams
mytargets = []
# Normal format: ?target=path.1&target=path.2
if len(queryParams.getlist('target')) > 0:
mytargets = queryParams.getlist('target')
# Rails/PHP/jQuery common practice format: ?target[]=path.1&target[]=path.2
elif len(queryParams.getlist('target[]')) > 0:
mytargets = queryParams.getlist('target[]')
# Collect the targets
for target in mytargets:
requestOptions['targets'].append(target)
template = dict()
for key, val in queryParams.items():
if key.startswith("template["):
template[key[9:-1]] = val
requestOptions['template'] = template
if 'pickle' in queryParams:
requestOptions['format'] = 'pickle'
if 'rawData' in queryParams:
requestOptions['format'] = 'raw'
if 'format' in queryParams:
requestOptions['format'] = queryParams['format']
if 'jsonp' in queryParams:
requestOptions['jsonp'] = queryParams['jsonp']
requestOptions['pretty'] = bool(queryParams.get('pretty'))
if 'noCache' in queryParams:
requestOptions['noCache'] = True
if 'maxDataPoints' in queryParams and queryParams['maxDataPoints'].isdigit(
):
requestOptions['maxDataPoints'] = int(queryParams['maxDataPoints'])
if 'noNullPoints' in queryParams:
requestOptions['noNullPoints'] = True
requestOptions['localOnly'] = queryParams.get('local') == '1'
# Fill in the graphOptions
format = requestOptions.get('format')
if format == 'svg':
graphOptions['outputFormat'] = 'svg'
elif format == 'pdf':
graphOptions['outputFormat'] = 'pdf'
else:
graphOptions['outputFormat'] = 'png'
for opt in graphClass.customizable:
if opt in queryParams:
val = queryParams[opt]
if (val.isdigit() or
(val.startswith('-')
and val[1:].isdigit())) and 'color' not in opt.lower():
val = int(val)
elif '.' in val and (val.replace('.', '', 1).isdigit() or
(val.startswith('-')
and val[1:].replace('.', '', 1).isdigit())):
val = float(val)
elif val.lower() in ('true', 'false'):
val = val.lower() == 'true'
elif val.lower() == 'default' or val == '':
continue
graphOptions[opt] = val
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
tzinfo = pytz.timezone(queryParams['tz'])
except pytz.UnknownTimeZoneError:
pass
requestOptions['tzinfo'] = tzinfo
# Get the time interval for time-oriented graph types
if graphType == 'line' or graphType == 'pie':
if 'now' in queryParams:
now = parseATTime(queryParams['now'], tzinfo)
else:
now = datetime.now(tzinfo)
if 'until' in queryParams:
untilTime = parseATTime(queryParams['until'], tzinfo, now)
else:
untilTime = now
if 'from' in queryParams:
fromTime = parseATTime(queryParams['from'], tzinfo, now)
else:
fromTime = parseATTime('-1d', tzinfo, now)
startTime = min(fromTime, untilTime)
endTime = max(fromTime, untilTime)
assert startTime != endTime, "Invalid empty time range"
requestOptions['startTime'] = startTime
requestOptions['endTime'] = endTime
timeRange = endTime - startTime
queryTime = timeRange.days * 86400 + timeRange.seconds # convert the time delta to seconds
if settings.DEFAULT_CACHE_POLICY and not queryParams.get(
'cacheTimeout'):
timeouts = [
timeout for period, timeout in settings.DEFAULT_CACHE_POLICY
if period <= queryTime
]
cacheTimeout = max(timeouts or (0, ))
requestOptions['now'] = now
if cacheTimeout == 0:
requestOptions['noCache'] = True
requestOptions['cacheTimeout'] = cacheTimeout
requestOptions['xFilesFactor'] = float(
queryParams.get('xFilesFactor', settings.DEFAULT_XFILES_FACTOR))
return (graphOptions, requestOptions)
def find_view(request):
"View for finding metrics matching a given pattern"
queryParams = request.GET.copy()
queryParams.update(request.POST)
format = queryParams.get('format', 'treejson')
leaves_only = queryParamAsInt(queryParams, 'leavesOnly', 0)
local_only = queryParamAsInt(queryParams, 'local', 0)
wildcards = queryParamAsInt(queryParams, 'wildcards', 0)
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
value = queryParams['tz']
tzinfo = pytz.timezone(value)
except pytz.UnknownTimeZoneError:
pass
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param tz: {err}'.format(
value=repr(value), err=str(e)))
if 'now' in queryParams:
try:
value = queryParams['now']
now = parseATTime(value, tzinfo)
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param now: {err}'.format(
value=repr(value), err=str(e)))
else:
now = datetime.now(tzinfo)
if 'from' in queryParams and str(queryParams['from']) != '-1':
try:
value = queryParams['from']
fromTime = int(epoch(parseATTime(value, tzinfo, now)))
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param from: {err}'.format(
value=repr(value), err=str(e)))
else:
fromTime = -1
if 'until' in queryParams and str(queryParams['until']) != '-1':
try:
value = queryParams['until']
untilTime = int(epoch(parseATTime(value, tzinfo, now)))
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param until: {err}'.format(
value=repr(value), err=str(e)))
else:
untilTime = -1
nodePosition = queryParamAsInt(queryParams, 'position', -1)
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = queryParamAsInt(queryParams, 'automatic_variants', 0)
try:
query = str(queryParams['query'])
except KeyError:
raise InputParameterError('Missing required parameter \'query\'')
if query == '':
raise InputParameterError('Required parameter \'query\' is empty')
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i, part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list(
STORE.find(
query,
fromTime,
untilTime,
local=local_only,
headers=forward_headers,
leaves_only=leaves_only,
))
except Exception:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' %
(query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info(
"received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d"
% (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
profile = getProfile(request)
content = tree_json(matches,
base_path,
wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'nodelist':
content = nodes_by_position(matches, nodePosition)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'msgpack':
content = msgpack_nodes(matches)
response = HttpResponse(content, content_type='application/x-msgpack')
elif format == 'json':
content = json_nodes(matches)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path,
name=node.name,
is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name': '*'}
results.append(wildcardNode)
response = json_response_for(request, {'metrics': results},
jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type='text/plain')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def _requestContext(request):
return {
'forwardHeaders': extractForwardHeaders(request),
}
def renderView(request):
start = time()
(graphOptions, requestOptions) = parseOptions(request)
useCache = 'noCache' not in requestOptions
cacheTimeout = requestOptions['cacheTimeout']
# TODO: Make that a namedtuple or a class.
requestContext = {
'startTime' : requestOptions['startTime'],
'endTime' : requestOptions['endTime'],
'now': requestOptions['now'],
'localOnly' : requestOptions['localOnly'],
'template' : requestOptions['template'],
'tzinfo' : requestOptions['tzinfo'],
'forwardHeaders': extractForwardHeaders(request),
'data' : [],
'prefetched' : {},
}
data = requestContext['data']
# First we check the request cache
if useCache:
requestKey = hashRequest(request)
cachedResponse = cache.get(requestKey)
if cachedResponse:
log.cache('Request-Cache hit [%s]' % requestKey)
log.rendering('Returned cached response in %.6f' % (time() - start))
return cachedResponse
else:
log.cache('Request-Cache miss [%s]' % requestKey)
# Now we prepare the requested data
if requestOptions['graphType'] == 'pie':
for target in requestOptions['targets']:
if target.find(':') >= 0:
try:
name,value = target.split(':',1)
value = float(value)
except:
raise ValueError("Invalid target '%s'" % target)
data.append( (name,value) )
else:
seriesList = evaluateTarget(requestContext, target)
for series in seriesList:
func = PieFunctions[requestOptions['pieMode']]
data.append( (series.name, func(requestContext, series) or 0 ))
elif requestOptions['graphType'] == 'line':
# Let's see if at least our data is cached
if useCache:
targets = requestOptions['targets']
startTime = requestOptions['startTime']
endTime = requestOptions['endTime']
dataKey = hashData(targets, startTime, endTime)
cachedData = cache.get(dataKey)
if cachedData:
log.cache("Data-Cache hit [%s]" % dataKey)
else:
log.cache("Data-Cache miss [%s]" % dataKey)
else:
cachedData = None
if cachedData is not None:
requestContext['data'] = data = cachedData
else: # Have to actually retrieve the data now
targets = requestOptions['targets']
if settings.REMOTE_PREFETCH_DATA and not requestOptions.get('localOnly'):
prefetchRemoteData(requestContext, targets)
for target in targets:
if not target.strip():
continue
t = time()
seriesList = evaluateTarget(requestContext, target)
log.rendering("Retrieval of %s took %.6f" % (target, time() - t))
data.extend(seriesList)
if useCache:
cache.add(dataKey, data, cacheTimeout)
# If data is all we needed, we're done
format = requestOptions.get('format')
if format == 'csv':
response = HttpResponse(content_type='text/csv')
writer = csv.writer(response, dialect='excel')
for series in data:
for i, value in enumerate(series):
timestamp = datetime.fromtimestamp(series.start + (i * series.step), requestOptions['tzinfo'])
writer.writerow((series.name, timestamp.strftime("%Y-%m-%d %H:%M:%S"), value))
return response
if format == 'json':
jsonStart = time()
series_data = []
if 'maxDataPoints' in requestOptions and any(data):
startTime = min([series.start for series in data])
endTime = max([series.end for series in data])
timeRange = endTime - startTime
maxDataPoints = requestOptions['maxDataPoints']
for series in data:
numberOfDataPoints = timeRange/series.step
if maxDataPoints < numberOfDataPoints:
valuesPerPoint = math.ceil(float(numberOfDataPoints) / float(maxDataPoints))
secondsPerPoint = int(valuesPerPoint * series.step)
# Nudge start over a little bit so that the consolidation bands align with each call
# removing 'jitter' seen when refreshing.
nudge = secondsPerPoint + (series.start % series.step) - (series.start % secondsPerPoint)
series.start = series.start + nudge
valuesToLose = int(nudge/series.step)
for r in range(1, valuesToLose):
del series[0]
series.consolidate(valuesPerPoint)
timestamps = range(int(series.start), int(series.end) + 1, int(secondsPerPoint))
else:
timestamps = range(int(series.start), int(series.end) + 1, int(series.step))
datapoints = zip(series, timestamps)
series_data.append(dict(target=series.name, tags=series.tags, datapoints=datapoints))
elif 'noNullPoints' in requestOptions and any(data):
for series in data:
values = []
for (index,v) in enumerate(series):
if v is not None:
timestamp = series.start + (index * series.step)
values.append((v,timestamp))
if len(values) > 0:
series_data.append(dict(target=series.name, tags=series.tags, datapoints=values))
else:
for series in data:
timestamps = range(int(series.start), int(series.end) + 1, int(series.step))
datapoints = zip(series, timestamps)
series_data.append(dict(target=series.name, tags=series.tags, datapoints=datapoints))
output = json.dumps(series_data, indent=(2 if requestOptions['pretty'] else None)).replace('None,', 'null,').replace('NaN,', 'null,').replace('Infinity,', '1e9999,')
if 'jsonp' in requestOptions:
response = HttpResponse(
content="%s(%s)" % (requestOptions['jsonp'], output),
content_type='text/javascript')
else:
response = HttpResponse(
content=output,
content_type='application/json')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('JSON rendering time %6f' % (time() - jsonStart))
log.rendering('Total request processing time %6f' % (time() - start))
return response
if format == 'dygraph':
labels = ['Time']
result = '{}'
if data:
datapoints = [[ts] for ts in range(data[0].start, data[0].end, data[0].step)]
for series in data:
labels.append(series.name)
for i, point in enumerate(series):
if point is None:
point = 'null'
elif point == float('inf'):
point = 'Infinity'
elif point == float('-inf'):
point = '-Infinity'
elif math.isnan(point):
point = 'null'
datapoints[i].append(point)
line_template = '[%%s000%s]' % ''.join([', %s'] * len(data))
lines = [line_template % tuple(points) for points in datapoints]
result = '{"labels" : %s, "data" : [%s]}' % (json.dumps(labels), ', '.join(lines))
response = HttpResponse(content=result, content_type='application/json')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('Total dygraph rendering time %.6f' % (time() - start))
return response
if format == 'rickshaw':
series_data = []
for series in data:
timestamps = range(series.start, series.end, series.step)
datapoints = [{'x' : x, 'y' : y} for x, y in zip(timestamps, series)]
series_data.append( dict(target=series.name, datapoints=datapoints) )
if 'jsonp' in requestOptions:
response = HttpResponse(
content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data)),
mimetype='text/javascript')
else:
response = HttpResponse(content=json.dumps(series_data),
content_type='application/json')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('Total rickshaw rendering time %.6f' % (time() - start))
return response
if format == 'raw':
response = HttpResponse(content_type='text/plain')
for series in data:
response.write( "%s,%d,%d,%d|" % (series.name, series.start, series.end, series.step) )
response.write( ','.join(map(repr,series)) )
response.write('\n')
log.rendering('Total rawData rendering time %.6f' % (time() - start))
return response
if format == 'svg':
graphOptions['outputFormat'] = 'svg'
elif format == 'pdf':
graphOptions['outputFormat'] = 'pdf'
if format == 'pickle':
response = HttpResponse(content_type='application/pickle')
seriesInfo = [series.getInfo() for series in data]
pickle.dump(seriesInfo, response, protocol=-1)
log.rendering('Total pickle rendering time %.6f' % (time() - start))
return response
# We've got the data, now to render it
graphOptions['data'] = data
if settings.REMOTE_RENDERING: # Rendering on other machines is faster in some situations
image = delegateRendering(requestOptions['graphType'], graphOptions, requestContext['forwardHeaders'])
else:
image = doImageRender(requestOptions['graphClass'], graphOptions)
useSVG = graphOptions.get('outputFormat') == 'svg'
if useSVG and 'jsonp' in requestOptions:
response = HttpResponse(
content="%s(%s)" % (requestOptions['jsonp'], json.dumps(image)),
content_type='text/javascript')
elif graphOptions.get('outputFormat') == 'pdf':
response = buildResponse(image, 'application/x-pdf')
else:
response = buildResponse(image, 'image/svg+xml' if useSVG else 'image/png')
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('Total rendering time %.6f seconds' % (time() - start))
return response
def _requestContext(request, queryParams):
return {
'forwardHeaders': extractForwardHeaders(request),
'localOnly': queryParams.get('local') == '1',
}
def find_view(request):
"View for finding metrics matching a given pattern"
queryParams = request.GET.copy()
queryParams.update(request.POST)
format = queryParams.get('format', 'treejson')
leaves_only = int( queryParams.get('leavesOnly', 0) )
local_only = int( queryParams.get('local', 0) )
wildcards = int( queryParams.get('wildcards', 0) )
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
tzinfo = pytz.timezone(queryParams['tz'])
except pytz.UnknownTimeZoneError:
pass
if 'now' in queryParams:
now = parseATTime(queryParams['now'], tzinfo)
else:
now = datetime.now(tzinfo)
if 'from' in queryParams and str(queryParams['from']) != '-1':
fromTime = int(epoch(parseATTime(queryParams['from'], tzinfo, now)))
else:
fromTime = -1
if 'until' in queryParams and str(queryParams['from']) != '-1':
untilTime = int(epoch(parseATTime(queryParams['until'], tzinfo, now)))
else:
untilTime = -1
nodePosition = int( queryParams.get('position', -1) )
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = int( queryParams.get('automatic_variants', 0) )
try:
query = str(queryParams['query'])
except KeyError:
return HttpResponseBadRequest(content="Missing required parameter 'query'",
content_type='text/plain')
if query == '':
return HttpResponseBadRequest(content="Required parameter 'query' is empty",
content_type='text/plain')
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i,part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list(STORE.find(
query, fromTime, untilTime,
local=local_only,
headers=forward_headers,
leaves_only=leaves_only,
))
except Exception:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' % (query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info("received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d" % (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
profile = getProfile(request)
content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'nodelist':
content = nodes_by_position(matches, nodePosition)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'msgpack':
content = msgpack_nodes(matches)
response = HttpResponse(content, content_type='application/x-msgpack')
elif format == 'json':
content = json_nodes(matches)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name' : '*'}
results.append(wildcardNode)
response = json_response_for(request, { 'metrics' : results }, jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type='text/plain')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def _requestContext(request):
return {
'forwardHeaders': extractForwardHeaders(request),
}
def parseOptions(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
# Start with some defaults
graphOptions = {'width' : 330, 'height' : 250}
requestOptions = {}
graphType = queryParams.get('graphType','line')
if graphType not in GraphTypes:
raise AssertionError("Invalid graphType '%s', must be one of %s"
% (graphType,list(GraphTypes)))
graphClass = GraphTypes[graphType]
# Fill in the requestOptions
requestOptions['graphType'] = graphType
requestOptions['graphClass'] = graphClass
requestOptions['pieMode'] = queryParams.get('pieMode', 'average')
cacheTimeout = int( queryParams.get('cacheTimeout', settings.DEFAULT_CACHE_DURATION) )
requestOptions['targets'] = []
requestOptions['forwardHeaders'] = extractForwardHeaders(request)
# Extract the targets out of the queryParams
mytargets = []
# Normal format: ?target=path.1&target=path.2
if len(queryParams.getlist('target')) > 0:
mytargets = queryParams.getlist('target')
# Rails/PHP/jQuery common practice format: ?target[]=path.1&target[]=path.2
elif len(queryParams.getlist('target[]')) > 0:
mytargets = queryParams.getlist('target[]')
# Collect the targets
for target in mytargets:
requestOptions['targets'].append(target)
template = dict()
for key, val in queryParams.items():
if key.startswith("template["):
template[key[9:-1]] = val
requestOptions['template'] = template
if 'pickle' in queryParams:
requestOptions['format'] = 'pickle'
if 'rawData' in queryParams:
requestOptions['format'] = 'raw'
if 'format' in queryParams:
requestOptions['format'] = queryParams['format']
if 'jsonp' in queryParams:
requestOptions['jsonp'] = queryParams['jsonp']
requestOptions['pretty'] = bool(queryParams.get('pretty'))
if 'noCache' in queryParams:
requestOptions['noCache'] = True
if 'maxDataPoints' in queryParams and queryParams['maxDataPoints'].isdigit():
requestOptions['maxDataPoints'] = int(queryParams['maxDataPoints'])
if 'noNullPoints' in queryParams:
requestOptions['noNullPoints'] = True
requestOptions['localOnly'] = queryParams.get('local') == '1'
# Fill in the graphOptions
format = requestOptions.get('format')
if format == 'svg':
graphOptions['outputFormat'] = 'svg'
elif format == 'pdf':
graphOptions['outputFormat'] = 'pdf'
else:
graphOptions['outputFormat'] = 'png'
for opt in graphClass.customizable:
if opt in queryParams:
val = queryParams[opt]
if (val.isdigit() or (val.startswith('-') and val[1:].isdigit())) and 'color' not in opt.lower():
val = int(val)
elif '.' in val and (val.replace('.','',1).isdigit() or (val.startswith('-') and val[1:].replace('.','',1).isdigit())):
val = float(val)
elif val.lower() in ('true','false'):
val = val.lower() == 'true'
elif val.lower() == 'default' or val == '':
continue
graphOptions[opt] = val
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
tzinfo = pytz.timezone(queryParams['tz'])
except pytz.UnknownTimeZoneError:
pass
requestOptions['tzinfo'] = tzinfo
# Get the time interval for time-oriented graph types
if graphType == 'line' or graphType == 'pie':
if 'now' in queryParams:
now = parseATTime(queryParams['now'], tzinfo)
else:
now = datetime.now(tzinfo)
if 'until' in queryParams:
untilTime = parseATTime(queryParams['until'], tzinfo, now)
else:
untilTime = now
if 'from' in queryParams:
fromTime = parseATTime(queryParams['from'], tzinfo, now)
else:
fromTime = parseATTime('-1d', tzinfo, now)
startTime = min(fromTime, untilTime)
endTime = max(fromTime, untilTime)
assert startTime != endTime, "Invalid empty time range"
requestOptions['startTime'] = startTime
requestOptions['endTime'] = endTime
timeRange = endTime - startTime
queryTime = timeRange.days * 86400 + timeRange.seconds # convert the time delta to seconds
if settings.DEFAULT_CACHE_POLICY and not queryParams.get('cacheTimeout'):
timeouts = [timeout for period,timeout in settings.DEFAULT_CACHE_POLICY if period <= queryTime]
cacheTimeout = max(timeouts or (0,))
requestOptions['now'] = now
if cacheTimeout == 0:
requestOptions['noCache'] = True
requestOptions['cacheTimeout'] = cacheTimeout
requestOptions['xFilesFactor'] = float( queryParams.get('xFilesFactor', settings.DEFAULT_XFILES_FACTOR) )
return (graphOptions, requestOptions)
def _requestContext(request, queryParams):
return {
'forwardHeaders': extractForwardHeaders(request),
'localOnly': queryParams.get('local') == '1',
}