def context_view(request):
if request.method == 'GET':
contexts = []
if not 'metric' not in request.GET:
return HttpResponse('{ "error" : "missing required parameter \"metric\"" }', mimetype='text/json')
for metric in request.GET.getlist('metric'):
try:
context = STORE.get(metric).context
except:
contexts.append({ 'metric' : metric, 'error' : 'failed to retrieve context', 'traceback' : traceback.format_exc() })
else:
contexts.append({ 'metric' : metric, 'context' : context })
content = json.dumps( { 'contexts' : contexts } )
return HttpResponse(content, mimetype='text/json')
elif request.method == 'POST':
if 'metric' not in request.POST:
return HttpResponse('{ "error" : "missing required parameter \"metric\"" }', mimetype='text/json')
newContext = dict( item for item in request.POST.items() if item[0] != 'metric' )
for metric in request.POST.getlist('metric'):
STORE.get(metric).updateContext(newContext)
return HttpResponse('{ "success" : true }', mimetype='text/json')
else:
return HttpResponseBadRequest("invalid method, must be GET or POST")
def context_view(request):
if request.method == 'GET':
contexts = []
if not 'metric' not in request.GET:
return HttpResponse('{ "error" : "missing required parameter \"metric\"" }', content_type='application/json')
for metric in request.GET.getlist('metric'):
try:
context = STORE.get(metric).context
except:
contexts.append({ 'metric' : metric, 'error' : 'failed to retrieve context', 'traceback' : traceback.format_exc() })
else:
contexts.append({ 'metric' : metric, 'context' : context })
return json_response_for(request, { 'contexts' : contexts })
elif request.method == 'POST':
if 'metric' not in request.POST:
return HttpResponse('{ "error" : "missing required parameter \"metric\"" }', content_type='application/json')
newContext = dict( item for item in request.POST.items() if item[0] != 'metric' )
for metric in request.POST.getlist('metric'):
STORE.get(metric).updateContext(newContext)
return HttpResponse('{ "success" : true }', content_type='application/json')
else:
return HttpResponseBadRequest("invalid method, must be GET or POST")
def _fetchData(pathExpr,startTime, endTime, requestContext, seriesList):
matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly'])
fetches = [(node, node.fetch(startTime, endTime)) for node in matching_nodes if node.is_leaf]
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError as e:
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (node.path, e))
(start, end, step) = timeInfo
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
seriesList.append(series)
# Prune empty series with duplicate metric paths to avoid showing empty graph elements for old whisper data
names = set([ s.name for s in seriesList ])
for name in names:
series_with_duplicate_names = [ s for s in seriesList if s.name == name ]
empty_duplicates = [ s for s in series_with_duplicate_names if not nonempty(s) ]
if series_with_duplicate_names == empty_duplicates and len(empty_duplicates) > 0: # if they're all empty
empty_duplicates.pop() # make sure we leave one in seriesList
for series in empty_duplicates:
seriesList.remove(series)
return seriesList
def expand_view(request):
"View for expanding a pattern into matching metric paths"
queryParams = request.GET.copy()
queryParams.update(request.POST)
local_only = int(queryParams.get('local', 0))
group_by_expr = int(queryParams.get('groupByExpr', 0))
leaves_only = int(queryParams.get('leavesOnly', 0))
jsonp = queryParams.get('jsonp', False)
results = {}
for query in queryParams.getlist('query'):
results[query] = set()
for node in STORE.find(query, local=local_only):
if node.is_leaf or not leaves_only:
results[query].add(node.path)
# Convert our results to sorted lists because sets aren't json-friendly
if group_by_expr:
for query, matches in results.items():
results[query] = sorted(matches)
else:
results = sorted(reduce(set.union, results.values(), set()))
result = {'results': results}
response = json_response_for(request, result, jsonp=jsonp)
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def expand_view(request):
"View for expanding a pattern into matching metric paths"
local_only = int(request.REQUEST.get('local', 0))
group_by_expr = int(request.REQUEST.get('groupByExpr', 0))
leaves_only = int(request.REQUEST.get('leavesOnly', 0))
results = {}
for query in request.REQUEST.getlist('query'):
results[query] = set()
for node in STORE.find(query, local=local_only):
if node.is_leaf or not leaves_only:
results[query].add(node.metric_path)
# Convert our results to sorted lists because sets aren't json-friendly
if group_by_expr:
for query, matches in results.items():
results[query] = sorted(matches)
else:
results = sorted(reduce(set.union, results.values(), set()))
result = {'results': results}
response = HttpResponse(json.dumps(result), mimetype='application/json')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def prefetchData(requestContext, pathExpressions):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching than doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by fetchData.
"""
if not pathExpressions:
return
start = time.time()
log.debug("Fetching data for [%s]" % (', '.join(pathExpressions)))
(startTime, endTime, now) = timebounds(requestContext)
prefetched = collections.defaultdict(list)
for result in STORE.fetch(pathExpressions, startTime, endTime, now, requestContext):
if result is None:
continue
prefetched[result['pathExpression']].append((
result['name'],
(
result['time_info'],
result['values'],
),
))
if not requestContext.get('prefetched'):
requestContext['prefetched'] = {}
requestContext['prefetched'][(startTime, endTime, now)] = prefetched
log.rendering("Fetched data for [%s] in %fs" % (', '.join(pathExpressions), time.time() - start))
def expand_view(request):
"View for expanding a pattern into matching metric paths"
local_only = int(request.REQUEST.get("local", 0))
group_by_expr = int(request.REQUEST.get("groupByExpr", 0))
leaves_only = int(request.REQUEST.get("leavesOnly", 0))
results = {}
for query in request.REQUEST.getlist("query"):
results[query] = set()
for node in STORE.find(query, tenant, local=local_only):
if node.is_leaf or not leaves_only:
results[query].add(node.path)
# Convert our results to sorted lists because sets aren't json-friendly
if group_by_expr:
for query, matches in results.items():
results[query] = sorted(matches)
else:
results = sorted(reduce(set.union, results.values(), set()))
result = {"results": results}
response = json_response_for(request, result)
response["Pragma"] = "no-cache"
response["Cache-Control"] = "no-cache"
return response
def prefetchData(requestContext, pathExpressions):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching than doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by fetchData.
"""
if not pathExpressions:
return
start = time.time()
log.debug("Fetching data for [%s]" % (', '.join(pathExpressions)))
(startTime, endTime, now) = timebounds(requestContext)
prefetched = collections.defaultdict(list)
for result in STORE.fetch(pathExpressions, startTime, endTime, now, requestContext):
if result is None:
continue
prefetched[result['pathExpression']].append((
result['name'],
(
result['time_info'],
result['values'],
),
))
if not requestContext.get('prefetched'):
requestContext['prefetched'] = {}
requestContext['prefetched'][(startTime, endTime, now)] = prefetched
log.rendering("Fetched data for [%s] in %fs" % (', '.join(pathExpressions), time.time() - start))
def _fetchData(pathExpr, startTime, endTime, requestContext, seriesList):
matching_nodes = STORE.find(pathExpr,
startTime,
endTime,
local=requestContext['localOnly'])
fetches = [(node, node.fetch(startTime, endTime))
for node in matching_nodes if node.is_leaf]
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info(
"render.datalib.fetchData :: no results for %s.fetch(%s, %s)"
% (node, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError, e:
e = sys.exc_info()[1]
raise Exception(
"could not parse timeInfo/values from metric '%s': %s" %
(node.path, e))
(start, end, step) = timeInfo
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
seriesList.append(series)
def fetchData(requestContext, pathExpr):
seriesList = []
startTime = int( time.mktime( requestContext['startTime'].timetuple() ) )
endTime = int( time.mktime( requestContext['endTime'].timetuple() ) )
matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly'])
fetches = [(node, node.fetch(startTime, endTime)) for node in matching_nodes if node.is_leaf]
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError, e:
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (node.path, e))
(start, end, step) = timeInfo
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
seriesList.append(series)
def _fetchData(pathExpr, startTime, endTime, now, requestContext, seriesList):
result_queue = []
remote_done = False
if settings.REMOTE_PREFETCH_DATA:
prefetched = requestContext['prefetched'].get((startTime, endTime, now), None)
if prefetched is not None:
for result in prefetched[pathExpr]:
result_queue.append(result)
# Since we pre-fetched remote data only, now we can get local data only.
remote_done = True
local = remote_done or requestContext['localOnly']
matching_nodes = STORE.find(
pathExpr, startTime, endTime,
local=local,
headers=requestContext['forwardHeaders'],
leaves_only=True,
)
for node in matching_nodes:
result_queue.append(
(node.path, node.fetch(startTime, endTime, now, requestContext)))
return _merge_results(pathExpr, startTime, endTime, result_queue, seriesList)
def _fetchData(pathExpr, startTime, endTime, now, requestContext, seriesList):
result_queue = []
remote_done = False
if settings.REMOTE_PREFETCH_DATA:
prefetched = requestContext['prefetched'].get((startTime, endTime, now), None)
if prefetched is not None:
for result in prefetched[pathExpr]:
result_queue.append(result)
# Since we pre-fetched remote data only, now we can get local data only.
remote_done = True
local = remote_done or requestContext['localOnly']
matching_nodes = STORE.find(
pathExpr, startTime, endTime,
local=local,
headers=requestContext['forwardHeaders'],
leaves_only=True,
)
for node in matching_nodes:
result_queue.append(
(node.path, node.fetch(startTime, endTime, now, requestContext)))
return _merge_results(pathExpr, startTime, endTime, result_queue, seriesList)
def expand_view(request):
"View for expanding a pattern into matching metric paths"
local_only = int( request.REQUEST.get('local', 0) )
group_by_expr = int( request.REQUEST.get('groupByExpr', 0) )
leaves_only = int( request.REQUEST.get('leavesOnly', 0) )
jsonp = request.REQUEST.get('jsonp', False)
results = {}
for query in request.REQUEST.getlist('query'):
results[query] = set()
for node in STORE.find(query, local=local_only):
if node.is_leaf or not leaves_only:
results[query].add( node.path )
# Convert our results to sorted lists because sets aren't json-friendly
if group_by_expr:
for query, matches in results.items():
results[query] = sorted(matches)
else:
results = sorted( reduce(set.union, results.values(), set()) )
result = {
'results' : results
}
response = json_response_for(request, result, jsonp=jsonp)
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def _fetchData(pathExpr, startTime, endTime, requestContext, seriesList):
matching_nodes = STORE.find(pathExpr,
startTime,
endTime,
local=requestContext['localOnly'],
reqkey=requestContext['request_key'])
matching_nodes = list(matching_nodes)
if len(matching_nodes) > 1:
request_hash = md5("%s_%s_%s" %
(pathExpr, startTime, endTime)).hexdigest()
cached_result = cache.get(request_hash)
if cached_result:
log.info(
"DEBUG:fetchData: got result from cache for %s_%s_%s" %
(pathExpr, startTime, endTime))
fetches = cached_result
else:
log.info("DEBUG:fetchData: no cache for %s_%s_%s" %
(pathExpr, startTime, endTime))
fetches = MultiReader(
matching_nodes,
reqkey=requestContext['request_key']).fetch(
startTime, endTime)
try:
cache.add(request_hash, fetches)
except Exception as err:
log.exception("Failed save data in memcached: %s" %
str(err))
elif len(matching_nodes) == 1:
fetches = [(matching_nodes[0],
matching_nodes[0].fetch(startTime, endTime))]
else:
fetches = []
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info(
"render.datalib.fetchData :: no results for %s.fetch(%s, %s)"
% (node, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError, e:
e = sys.exc_info()[1]
raise Exception(
"could not parse timeInfo/values from metric '%s': %s" %
(node.path, e))
(start, end, step) = timeInfo
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
seriesList.append(series)
def find_metric(request):
"""Autocomplete helper on metric names."""
try:
query = str(request.REQUEST["q"])
except:
return HttpResponseBadRequest(content="Missing required parameter 'q'", content_type="text/plain")
matches = list(STORE.find(query + "*"))
content = "\n".join([node.path for node in matches])
response = HttpResponse(content, content_type="text/plain")
return response
def find_metric(request):
"""Autocomplete helper on metric names."""
try:
query = str( request.REQUEST['q'] )
except:
return HttpResponseBadRequest(
content="Missing required parameter 'q'", mimetype="text/plain")
matches = list( STORE.find(query+"*") )
content = "\n".join([node.metric_path for node in matches ])
response = HttpResponse(content, mimetype='text/plain')
return response
def prefetchRemoteData(requestContext, targets):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching that doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by datalib.
"""
log.rendering("Prefetching remote data")
pathExpressions = extractPathExpressions(targets)
results = STORE.fetch_remote(pathExpressions, requestContext)
# TODO: instead of doing that it would be wait better to use
# the shared cache to cache pathExpr instead of full queries.
requestContext['prefetched'] = PrefetchedData(results)
def prefetchData(requestContext, pathExpressions):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching than doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by fetchData.
"""
if not pathExpressions:
return
start = time.time()
log.debug("Fetching data for [%s]" % (', '.join(pathExpressions)))
(startTime, endTime, now) = timebounds(requestContext)
prefetched = collections.defaultdict(list)
for result in STORE.fetch(pathExpressions, startTime, endTime, now,
requestContext):
if result is None:
continue
prefetched[result['pathExpression']].append((
result['name'],
(
result['time_info'],
result['values'],
),
))
# Several third-party readers including rrdtool and biggraphite return values in a
# generator which can only be iterated on once. These must be converted to a list.
for pathExpression, items in prefetched.items():
for i, (name, (time_info, values)) in enumerate(items):
if isinstance(values, types.GeneratorType):
prefetched[pathExpression][i] = (name, (time_info,
list(values)))
if not requestContext.get('prefetched'):
requestContext['prefetched'] = {}
if (startTime, endTime, now) in requestContext['prefetched']:
requestContext['prefetched'][(startTime, endTime,
now)].update(prefetched)
else:
requestContext['prefetched'][(startTime, endTime, now)] = prefetched
log.rendering("Fetched data for [%s] in %fs" %
(', '.join(pathExpressions), time.time() - start))
def prefetchRemoteData(requestContext, targets):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching that doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by datalib.
"""
pathExpressions = extractPathExpressions(targets)
log.rendering("Prefetching remote data for [%s]" % (', '.join(pathExpressions)))
(startTime, endTime, now) = timebounds(requestContext)
results = STORE.fetch_remote(pathExpressions, startTime, endTime, now, requestContext)
requestContext['prefetched'][(startTime, endTime, now)] = PrefetchedData(results)
def prefetchRemoteData(requestContext, targets):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching that doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by datalib.
"""
log.rendering("Prefetching remote data")
pathExpressions = extractPathExpressions(targets)
(startTime, endTime, now) = timebounds(requestContext)
results = STORE.fetch_remote(pathExpressions, startTime, endTime, now, requestContext)
requestContext['prefetched'][(startTime, endTime, now)] = PrefetchedData(results)
def autoCompleteTags(request, queryParams):
if request.method not in ['GET', 'POST']:
return HttpResponse(status=405)
exprs = []
# Normal format: ?expr=tag1=value1&expr=tag2=value2
if len(queryParams.getlist('expr')) > 0:
exprs = queryParams.getlist('expr')
# Rails/PHP/jQuery common practice format: ?expr[]=tag1=value1&expr[]=tag2=value2
elif len(queryParams.getlist('expr[]')) > 0:
exprs = queryParams.getlist('expr[]')
return STORE.tagdb_auto_complete_tags(
exprs,
tagPrefix=queryParams.get('tagPrefix'),
limit=queryParams.get('limit'),
requestContext=_requestContext(request, queryParams))
def index_json(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
try:
jsonp = queryParams.get('jsonp', False)
requestContext = {
'localOnly': int(queryParams.get('local', 0)),
'forwardHeaders': extractForwardHeaders(request),
}
matches = STORE.get_index(requestContext)
except Exception:
log.exception()
return json_response_for(request, [], jsonp=jsonp, status=500)
return json_response_for(request, matches, jsonp=jsonp)
def autoCompleteTags(request, queryParams):
if request.method not in ['GET', 'POST']:
return HttpResponse(status=405)
exprs = []
# Normal format: ?expr=tag1=value1&expr=tag2=value2
if len(queryParams.getlist('expr')) > 0:
exprs = queryParams.getlist('expr')
# Rails/PHP/jQuery common practice format: ?expr[]=tag1=value1&expr[]=tag2=value2
elif len(queryParams.getlist('expr[]')) > 0:
exprs = queryParams.getlist('expr[]')
return STORE.tagdb_auto_complete_tags(
exprs,
tagPrefix=queryParams.get('tagPrefix'),
limit=queryParams.get('limit'),
requestContext=_requestContext(request)
)
def index_json(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
try:
jsonp = queryParams.get('jsonp', False)
requestContext = {
'localOnly': int( queryParams.get('local', 0) ),
'forwardHeaders': extractForwardHeaders(request),
}
matches = STORE.get_index(requestContext)
except Exception:
log.exception()
return json_response_for(request, [], jsonp=jsonp, status=500)
return json_response_for(request, matches, jsonp=jsonp)
def prefetchData(requestContext, pathExpressions):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching than doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by fetchData.
"""
if not pathExpressions:
return
start = time.time()
log.debug("Fetching data for [%s]" % (', '.join(pathExpressions)))
(startTime, endTime, now) = timebounds(requestContext)
prefetched = collections.defaultdict(list)
for result in STORE.fetch(pathExpressions, startTime, endTime, now, requestContext):
if result is None:
continue
prefetched[result['pathExpression']].append((
result['name'],
(
result['time_info'],
result['values'],
),
))
# Several third-party readers including rrdtool and biggraphite return values in a
# generator which can only be iterated on once. These must be converted to a list.
for pathExpression, items in prefetched.items():
for i, (name, (time_info, values)) in enumerate(items):
if isinstance(values, types.GeneratorType):
prefetched[pathExpression][i] = (name, (time_info, list(values)))
if not requestContext.get('prefetched'):
requestContext['prefetched'] = {}
requestContext['prefetched'][(startTime, endTime, now)] = prefetched
log.rendering("Fetched data for [%s] in %fs" % (', '.join(pathExpressions), time.time() - start))
def completePath(path, shortnames=False):
# Have to extract the path expression from the command
for prefix in ('draw ', 'add ', 'remove '):
if path.startswith(prefix):
path = path[len(prefix):]
break
pattern = re.sub('\w+\(', '', path).replace(')', '') + '*'
results = []
for match in STORE.find(pattern):
if shortnames:
results.append(match.name)
else:
results.append(match.metric_path)
list_items = ["<li>%s</li>" % r for r in results]
list_element = "<ul>" + '\n'.join(list_items) + "</ul>"
return list_element
def prefetchRemoteData(requestContext, targets):
"""Prefetch a bunch of path expressions and stores them in the context.
The idea is that this will allow more batching that doing a query
each time evaluateTarget() needs to fetch a path. All the prefetched
data is stored in the requestContext, to be accessed later by datalib.
"""
# only prefetch if there is at least one active remote finder
# this is to avoid the overhead of tagdb lookups in extractPathExpressions
if len([finder for finder in STORE.finders if not getattr(finder, 'local', True) and not getattr(finder, 'disabled', False)]) < 1:
return
pathExpressions = extractPathExpressions(targets)
log.rendering("Prefetching remote data for [%s]" % (', '.join(pathExpressions)))
(startTime, endTime, now) = timebounds(requestContext)
results = STORE.fetch_remote(pathExpressions, startTime, endTime, now, requestContext)
requestContext['prefetched'][(startTime, endTime, now)] = PrefetchedData(results)
def completePath(path, shortnames=False):
# Have to extract the path expression from the command
for prefix in ('draw ','add ','remove '):
if path.startswith(prefix):
path = path[len(prefix):]
break
pattern = re.sub('\w+\(','',path).replace(')','') + '*'
results = []
for match in STORE.find(pattern):
if shortnames:
results.append(match.name)
else:
results.append(match.metric_path)
list_items = ["<li>%s</li>" % r for r in results]
list_element = "<ul>" + '\n'.join(list_items) + "</ul>"
return list_element
def fetchData(requestContext, pathExpr):
seriesList = []
startTime = int( time.mktime( requestContext['startTime'].timetuple() ) )
endTime = int( time.mktime( requestContext['endTime'].timetuple() ) )
matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly'])
fetches = [(node, node.fetch(startTime, endTime)) for node in matching_nodes if node.is_leaf]
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime))
continue
(timeInfo, values) = results
(start, end, step) = timeInfo
values = [value if value else 0 for value in values] # hack to turn nulls into 0s
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
seriesList.append(series)
# Prune empty series with duplicate metric paths to avoid showing empty graph elements for old whisper data
names = set([ series.name for series in seriesList ])
for name in names:
series_with_duplicate_names = [ series for series in seriesList if series.name == name ]
empty_duplicates = [ series for series in series_with_duplicate_names if not nonempty(series) ]
if series_with_duplicate_names == empty_duplicates and len(empty_duplicates) > 0: # if they're all empty
empty_duplicates.pop() # make sure we leave one in seriesList
for series in empty_duplicates:
seriesList.remove(series)
return seriesList
def fetchData(requestContext, pathExpr):
seriesList = []
startTime = int(time.mktime(requestContext["startTime"].timetuple()))
endTime = int(time.mktime(requestContext["endTime"].timetuple()))
matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext["localOnly"])
fetches = [(node, node.fetch(startTime, endTime)) for node in matching_nodes if node.is_leaf]
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime))
continue
(timeInfo, values) = results
(start, end, step) = timeInfo
values = [value if value else 0 for value in values] # hack to turn nulls into 0s
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr # hack to pass expressions through to render functions
seriesList.append(series)
# Prune empty series with duplicate metric paths to avoid showing empty graph elements for old whisper data
names = set([series.name for series in seriesList])
for name in names:
series_with_duplicate_names = [series for series in seriesList if series.name == name]
empty_duplicates = [series for series in series_with_duplicate_names if not nonempty(series)]
if series_with_duplicate_names == empty_duplicates and len(empty_duplicates) > 0: # if they're all empty
empty_duplicates.pop() # make sure we leave one in seriesList
for series in empty_duplicates:
seriesList.remove(series)
return seriesList
def _fetchData(pathExpr,startTime, endTime, requestContext, seriesList):
matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly'], reqkey=requestContext['request_key'])
matching_nodes = list(matching_nodes)
if len(matching_nodes) > 1:
request_hash = md5("%s_%s_%s" % (pathExpr, startTime, endTime)).hexdigest()
cached_result = cache.get(request_hash)
if cached_result:
log.info("DEBUG:fetchData: got result from cache for %s_%s_%s" % (pathExpr, startTime, endTime))
fetches = cached_result
else:
log.info("DEBUG:fetchData: no cache for %s_%s_%s" % (pathExpr, startTime, endTime))
fetches = MultiReader(matching_nodes, reqkey=requestContext['request_key']).fetch(startTime, endTime)
try:
cache.add(request_hash, fetches)
except Exception as err:
log.exception("Failed save data in memcached: %s" % str(err))
elif len(matching_nodes) == 1:
fetches = [(matching_nodes[0], matching_nodes[0].fetch(startTime, endTime))]
else:
fetches = []
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError, e:
e = sys.exc_info()[1]
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (node.path, e))
(start, end, step) = timeInfo
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
seriesList.append(series)
def autoCompleteValues(request, queryParams):
if request.method not in ['GET', 'POST']:
return HttpResponse(status=405)
exprs = []
# Normal format: ?expr=tag1=value1&expr=tag2=value2
if len(queryParams.getlist('expr')) > 0:
exprs = queryParams.getlist('expr')
# Rails/PHP/jQuery common practice format: ?expr[]=tag1=value1&expr[]=tag2=value2
elif len(queryParams.getlist('expr[]')) > 0:
exprs = queryParams.getlist('expr[]')
tag = queryParams.get('tag')
if not tag:
raise HttpError('no tag specified', status=400)
return STORE.tagdb_auto_complete_values(
exprs,
tag,
valuePrefix=queryParams.get('valuePrefix'),
limit=queryParams.get('limit'),
requestContext=_requestContext(request))
def autoCompleteValues(request, queryParams):
if request.method not in ['GET', 'POST']:
return HttpResponse(status=405)
exprs = []
# Normal format: ?expr=tag1=value1&expr=tag2=value2
if len(queryParams.getlist('expr')) > 0:
exprs = queryParams.getlist('expr')
# Rails/PHP/jQuery common practice format: ?expr[]=tag1=value1&expr[]=tag2=value2
elif len(queryParams.getlist('expr[]')) > 0:
exprs = queryParams.getlist('expr[]')
tag = queryParams.get('tag')
if not tag:
raise HttpError('no tag specified', status=400)
return STORE.tagdb_auto_complete_values(
exprs,
tag,
valuePrefix=queryParams.get('valuePrefix'),
limit=queryParams.get('limit'),
requestContext=_requestContext(request, queryParams)
)
def find_view(request):
"View for finding metrics matching a given pattern"
profile = getProfile(request)
queryParams = request.GET.copy()
queryParams.update(request.POST)
format = queryParams.get('format', 'treejson')
local_only = int(queryParams.get('local', 0))
wildcards = int(queryParams.get('wildcards', 0))
fromTime = int(queryParams.get('from', -1))
untilTime = int(queryParams.get('until', -1))
nodePosition = int(queryParams.get('position', -1))
jsonp = queryParams.get('jsonp', False)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = int(queryParams.get('automatic_variants', 0))
try:
query = str(queryParams['query'])
except:
return HttpResponseBadRequest(
content="Missing required parameter 'query'",
content_type='text/plain')
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i, part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list(STORE.find(query, fromTime, untilTime,
local=local_only))
except:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' %
(query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info(
"received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d"
% (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
content = tree_json(matches,
base_path,
wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'nodelist':
content = nodes_by_position(matches, nodePosition)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path,
name=node.name,
is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name': '*'}
results.append(wildcardNode)
response = json_response_for(request, {'metrics': results},
jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type='text/plain')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def find_view(request):
"View for finding metrics matching a given pattern"
profile = getProfile(request)
format = request.REQUEST.get("format", "treejson")
local_only = int(request.REQUEST.get("local", 0))
wildcards = int(request.REQUEST.get("wildcards", 0))
fromTime = int(request.REQUEST.get("from", -1))
untilTime = int(request.REQUEST.get("until", -1))
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = int(request.REQUEST.get("automatic_variants", 0))
try:
query = str(request.REQUEST["query"])
except:
return HttpResponseBadRequest(content="Missing required parameter 'query'", content_type="text/plain")
if "." in query:
base_path = query.rsplit(".", 1)[0] + "."
else:
base_path = ""
if format == "completer":
query = query.replace("..", "*.")
if not query.endswith("*"):
query += "*"
if automatic_variants:
query_parts = query.split(".")
for i, part in enumerate(query_parts):
if "," in part and "{" not in part:
query_parts[i] = "{%s}" % part
query = ".".join(query_parts)
tenant = request.session["tenant"]
try:
matches = list(STORE.find(query, tenant, fromTime, untilTime, local=local_only))
except:
log.exception()
raise
log.info("find_view query=%s local_only=%s matches=%d" % (query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info(
"received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d"
% (query, fromTime, untilTime, local_only, format, len(matches))
)
if format == "treejson":
content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content)
elif format == "pickle":
content = pickle_nodes(matches)
response = HttpResponse(content, content_type="application/pickle")
elif format == "completer":
results = []
for node in matches:
node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info["path"] += "."
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {"name": "*"}
results.append(wildcardNode)
response = json_response_for(request, {"metrics": results})
else:
return HttpResponseBadRequest(content="Invalid value for 'format' parameter", content_type="text/plain")
response["Pragma"] = "no-cache"
response["Cache-Control"] = "no-cache"
return response
def find_view(request):
"View for finding metrics matching a given pattern"
queryParams = request.GET.copy()
queryParams.update(request.POST)
format = queryParams.get('format', 'treejson')
leaves_only = int( queryParams.get('leavesOnly', 0) )
local_only = int( queryParams.get('local', 0) )
wildcards = int( queryParams.get('wildcards', 0) )
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
tzinfo = pytz.timezone(queryParams['tz'])
except pytz.UnknownTimeZoneError:
pass
if 'now' in queryParams:
now = parseATTime(queryParams['now'], tzinfo)
else:
now = datetime.now(tzinfo)
if 'from' in queryParams and str(queryParams['from']) != '-1':
fromTime = int(epoch(parseATTime(queryParams['from'], tzinfo, now)))
else:
fromTime = -1
if 'until' in queryParams and str(queryParams['from']) != '-1':
untilTime = int(epoch(parseATTime(queryParams['until'], tzinfo, now)))
else:
untilTime = -1
nodePosition = int( queryParams.get('position', -1) )
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = int( queryParams.get('automatic_variants', 0) )
try:
query = str(queryParams['query'])
except KeyError:
return HttpResponseBadRequest(content="Missing required parameter 'query'",
content_type='text/plain')
if query == '':
return HttpResponseBadRequest(content="Required parameter 'query' is empty",
content_type='text/plain')
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i,part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list(STORE.find(
query, fromTime, untilTime,
local=local_only,
headers=forward_headers,
leaves_only=leaves_only,
))
except Exception:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' % (query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info("received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d" % (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
profile = getProfile(request)
content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'nodelist':
content = nodes_by_position(matches, nodePosition)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'msgpack':
content = msgpack_nodes(matches)
response = HttpResponse(content, content_type='application/x-msgpack')
elif format == 'json':
content = json_nodes(matches)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name' : '*'}
results.append(wildcardNode)
response = json_response_for(request, { 'metrics' : results }, jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type='text/plain')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def find_view(request):
"View for finding metrics matching a given pattern"
queryParams = request.GET.copy()
queryParams.update(request.POST)
format = queryParams.get('format', 'treejson')
leaves_only = queryParamAsInt(queryParams, 'leavesOnly', 0)
local_only = queryParamAsInt(queryParams, 'local', 0)
wildcards = queryParamAsInt(queryParams, 'wildcards', 0)
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
value = queryParams['tz']
tzinfo = pytz.timezone(value)
except pytz.UnknownTimeZoneError:
pass
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param tz: {err}'.format(
value=repr(value), err=str(e)))
if 'now' in queryParams:
try:
value = queryParams['now']
now = parseATTime(value, tzinfo)
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param now: {err}'.format(
value=repr(value), err=str(e)))
else:
now = datetime.now(tzinfo)
if 'from' in queryParams and str(queryParams['from']) != '-1':
try:
value = queryParams['from']
fromTime = int(epoch(parseATTime(value, tzinfo, now)))
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param from: {err}'.format(
value=repr(value), err=str(e)))
else:
fromTime = -1
if 'until' in queryParams and str(queryParams['until']) != '-1':
try:
value = queryParams['until']
untilTime = int(epoch(parseATTime(value, tzinfo, now)))
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param until: {err}'.format(
value=repr(value), err=str(e)))
else:
untilTime = -1
nodePosition = queryParamAsInt(queryParams, 'position', -1)
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = queryParamAsInt(queryParams, 'automatic_variants', 0)
try:
query = str(queryParams['query'])
except KeyError:
raise InputParameterError('Missing required parameter \'query\'')
if query == '':
raise InputParameterError('Required parameter \'query\' is empty')
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i, part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list(
STORE.find(
query,
fromTime,
untilTime,
local=local_only,
headers=forward_headers,
leaves_only=leaves_only,
))
except Exception:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' %
(query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info(
"received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d"
% (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
profile = getProfile(request)
content = tree_json(matches,
base_path,
wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'nodelist':
content = nodes_by_position(matches, nodePosition)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'msgpack':
content = msgpack_nodes(matches)
response = HttpResponse(content, content_type='application/x-msgpack')
elif format == 'json':
content = json_nodes(matches)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path,
name=node.name,
is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name': '*'}
results.append(wildcardNode)
response = json_response_for(request, {'metrics': results},
jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type='text/plain')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def _fetchData(pathExpr, startTime, endTime, now, requestContext, seriesList):
matching_nodes = STORE.find(pathExpr,
startTime,
endTime,
local=requestContext['localOnly'],
headers=requestContext.get('forwardHeaders'))
fetches = [(node.path, node.fetch(startTime, endTime, now, requestContext))
for node in matching_nodes if node.is_leaf]
for path, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info(
"render.datalib.fetchData :: no results for %s.fetch(%s, %s)" %
(path, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError as e:
raise Exception(
"could not parse timeInfo/values from metric '%s': %s" %
(path, e))
(start, end, step) = timeInfo
series = TimeSeries(path, start, end, step, values)
# hack to pass expressions through to render functions
series.pathExpression = pathExpr
# Used as a cache to avoid recounting series None values below.
series_best_nones = {}
if series.name in seriesList:
# This counts the Nones in each series, and is unfortunately O(n) for each
# series, which may be worth further optimization. The value of doing this
# at all is to avoid the "flipping" effect of loading a graph multiple times
# and having inconsistent data returned if one of the backing stores has
# inconsistent data. This is imperfect as a validity test, but in practice
# nicely keeps us using the "most complete" dataset available. Think of it
# as a very weak CRDT resolver.
candidate_nones = 0
if not settings.REMOTE_STORE_MERGE_RESULTS:
candidate_nones = len([val for val in values if val is None])
known = seriesList[series.name]
# To avoid repeatedly recounting the 'Nones' in series we've already seen,
# cache the best known count so far in a dict.
if known.name in series_best_nones:
known_nones = series_best_nones[known.name]
else:
known_nones = len([val for val in known if val is None])
if known_nones > candidate_nones:
if settings.REMOTE_STORE_MERGE_RESULTS:
# This series has potential data that might be missing from
# earlier series. Attempt to merge in useful data and update
# the cache count.
log.info("Merging multiple TimeSeries for %s" % known.name)
for i, j in enumerate(known):
if j is None and series[i] is not None:
known[i] = series[i]
known_nones -= 1
# Store known_nones in our cache
series_best_nones[known.name] = known_nones
else:
# Not merging data -
# we've found a series better than what we've already seen. Update
# the count cache and replace the given series in the array.
series_best_nones[known.name] = candidate_nones
seriesList[known.name] = series
else:
# In case if we are merging data - the existing series has no gaps and
# there is nothing to merge together. Save ourselves some work here.
#
# OR - if we picking best serie:
#
# We already have this series in the seriesList, and the
# candidate is 'worse' than what we already have, we don't need
# to compare anything else. Save ourselves some work here.
break
# If we looked at this series above, and it matched a 'known'
# series already, then it's already in the series list (or ignored).
# If not, append it here.
else:
seriesList[series.name] = series
# Stabilize the order of the results by ordering the resulting series by name.
# This returns the result ordering to the behavior observed pre PR#1010.
return [seriesList[k] for k in sorted(seriesList)]
def _fetchData(pathExpr,startTime, endTime, requestContext, seriesList):
matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly'])
fetches = [(node, node.fetch(startTime, endTime)) for node in matching_nodes if node.is_leaf]
for node, results in fetches:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError as e:
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (node.path, e))
(start, end, step) = timeInfo
series = TimeSeries(node.path, start, end, step, values)
series.pathExpression = pathExpr #hack to pass expressions through to render functions
# Used as a cache to avoid recounting series None values below.
series_best_nones = {}
if series.name in seriesList:
# This counts the Nones in each series, and is unfortunately O(n) for each
# series, which may be worth further optimization. The value of doing this
# at all is to avoid the "flipping" effect of loading a graph multiple times
# and having inconsistent data returned if one of the backing stores has
# inconsistent data. This is imperfect as a validity test, but in practice
# nicely keeps us using the "most complete" dataset available. Think of it
# as a very weak CRDT resolver.
candidate_nones = 0
if not settings.REMOTE_STORE_MERGE_RESULTS:
candidate_nones = len(
[val for val in series['values'] if val is None])
known = seriesList[series.name]
# To avoid repeatedly recounting the 'Nones' in series we've already seen,
# cache the best known count so far in a dict.
if known.name in series_best_nones:
known_nones = series_best_nones[known.name]
else:
known_nones = len([val for val in known if val is None])
if known_nones > candidate_nones:
if settings.REMOTE_STORE_MERGE_RESULTS:
# This series has potential data that might be missing from
# earlier series. Attempt to merge in useful data and update
# the cache count.
log.info("Merging multiple TimeSeries for %s" % known.name)
for i, j in enumerate(known):
if j is None and series[i] is not None:
known[i] = series[i]
known_nones -= 1
# Store known_nones in our cache
series_best_nones[known.name] = known_nones
else:
# Not merging data -
# we've found a series better than what we've already seen. Update
# the count cache and replace the given series in the array.
series_best_nones[known.name] = candidate_nones
seriesList[known.name] = series
else:
# In case if we are merging data - the existing series has no gaps and there is nothing to merge
# together. Save ourselves some work here.
#
# OR - if we picking best serie:
#
# We already have this series in the seriesList, and the
# candidate is 'worse' than what we already have, we don't need
# to compare anything else. Save ourselves some work here.
break
# If we looked at this series above, and it matched a 'known'
# series already, then it's already in the series list (or ignored).
# If not, append it here.
else:
seriesList[series.name] = series
# Stabilize the order of the results by ordering the resulting series by name.
# This returns the result ordering to the behavior observed pre PR#1010.
return [seriesList[k] for k in sorted(seriesList)]
def find_view(request):
"View for finding metrics matching a given pattern"
profile = getProfile(request)
format = request.REQUEST.get('format', 'treejson')
local_only = int( request.REQUEST.get('local', 0) )
wildcards = int( request.REQUEST.get('wildcards', 0) )
fromTime = int( request.REQUEST.get('from', -1) )
untilTime = int( request.REQUEST.get('until', -1) )
jsonp = request.REQUEST.get('jsonp', False)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = int( request.REQUEST.get('automatic_variants', 0) )
try:
query = str( request.REQUEST['query'] )
except:
return HttpResponseBadRequest(content="Missing required parameter 'query'",
content_type="text/plain")
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i,part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list( STORE.find(query, fromTime, untilTime, local=local_only) )
except:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' % (query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info("received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d" % (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name' : '*'}
results.append(wildcardNode)
response = json_response_for(request, { 'metrics' : results }, jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type="text/plain")
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def _fetchData(pathExpr, startTime, endTime, now, requestContext, seriesList):
if settings.REMOTE_PREFETCH_DATA:
matching_nodes = [node for node in STORE.find(pathExpr, startTime, endTime, local=True)]
# inflight_requests is only present if at least one remote store
# has been queried
if 'inflight_requests' in requestContext:
fetches = requestContext['inflight_requests']
else:
fetches = {}
def result_queue_generator():
for node in matching_nodes:
if node.is_leaf:
yield (node.path, node.fetch(startTime, endTime, now, requestContext))
log.info(
'render.datalib.fetchData:: result_queue_generator got {count} fetches'
.format(count=len(fetches)),
)
for key, fetch in fetches.iteritems():
log.info(
'render.datalib.fetchData:: getting results of {host}'
.format(host=key),
)
if isinstance(fetch, FetchInProgress):
fetch = fetch.waitForResults()
if fetch is None:
log.info('render.datalib.fetchData:: fetch is None')
continue
for result in fetch:
if result['pathExpression'] == pathExpr:
yield (
result['path'],
(
(result['start'], result['end'], result['step']),
result['values'],
),
)
result_queue = result_queue_generator()
else:
matching_nodes = [node for node in STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly'])]
result_queue = [
(node.path, node.fetch(startTime, endTime, now, requestContext))
for node in matching_nodes
if node.is_leaf
]
log.info("render.datalib.fetchData :: starting to merge")
for path, results in result_queue:
if isinstance(results, FetchInProgress):
results = results.waitForResults()
if not results:
log.info("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (path, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError as e:
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (path, e))
(start, end, step) = timeInfo
series = TimeSeries(path, start, end, step, values)
# hack to pass expressions through to render functions
series.pathExpression = pathExpr
# Used as a cache to avoid recounting series None values below.
series_best_nones = {}
if series.name in seriesList:
# This counts the Nones in each series, and is unfortunately O(n) for each
# series, which may be worth further optimization. The value of doing this
# at all is to avoid the "flipping" effect of loading a graph multiple times
# and having inconsistent data returned if one of the backing stores has
# inconsistent data. This is imperfect as a validity test, but in practice
# nicely keeps us using the "most complete" dataset available. Think of it
# as a very weak CRDT resolver.
candidate_nones = 0
if not settings.REMOTE_STORE_MERGE_RESULTS:
candidate_nones = len(
[val for val in values if val is None])
known = seriesList[series.name]
# To avoid repeatedly recounting the 'Nones' in series we've already seen,
# cache the best known count so far in a dict.
if known.name in series_best_nones:
known_nones = series_best_nones[known.name]
else:
known_nones = len([val for val in known if val is None])
if known_nones > candidate_nones:
if settings.REMOTE_STORE_MERGE_RESULTS:
# This series has potential data that might be missing from
# earlier series. Attempt to merge in useful data and update
# the cache count.
log.info("Merging multiple TimeSeries for %s" % known.name)
for i, j in enumerate(known):
if j is None and series[i] is not None:
known[i] = series[i]
known_nones -= 1
# Store known_nones in our cache
series_best_nones[known.name] = known_nones
else:
# Not merging data -
# we've found a series better than what we've already seen. Update
# the count cache and replace the given series in the array.
series_best_nones[known.name] = candidate_nones
seriesList[known.name] = series
else:
if settings.REMOTE_PREFETCH_DATA:
# if we're using REMOTE_PREFETCH_DATA we can save some time by skipping
# find, but that means we don't know how many nodes to expect so we
# have to iterate over all returned results
continue
# In case if we are merging data - the existing series has no gaps and
# there is nothing to merge together. Save ourselves some work here.
#
# OR - if we picking best serie:
#
# We already have this series in the seriesList, and the
# candidate is 'worse' than what we already have, we don't need
# to compare anything else. Save ourselves some work here.
break
else:
# If we looked at this series above, and it matched a 'known'
# series already, then it's already in the series list (or ignored).
# If not, append it here.
seriesList[series.name] = series
# Stabilize the order of the results by ordering the resulting series by name.
# This returns the result ordering to the behavior observed pre PR#1010.
return [seriesList[k] for k in sorted(seriesList)]