def _extract():
for result, node, pattern in results_nodes_and_patterns:
time_info, values = wait_for_result(result)
yield {
'pathExpression': pattern,
'path': node.path,
'time_info': time_info,
'values': values,
}
def _extract():
for result, node, pattern in results_nodes_and_patterns:
time_info, values = wait_for_result(result)
yield {
'pathExpression': pattern,
'path': node.path,
'name': node.path,
'time_info': time_info,
'values': values,
}
def _fetch_data(self):
prefetched = collections.defaultdict(list)
for result in self._results:
fetched = wait_for_result(result)
if fetched is None:
continue
for result in fetched:
prefetched[result['pathExpression']].append((
result['name'],
(
result['time_info'],
result['values'],
),
))
self._prefetched = prefetched
def _fetch_data(self):
prefetched = collections.defaultdict(list)
for result in self._results:
fetched = wait_for_result(result)
if fetched is None:
continue
for result in fetched:
prefetched[result['pathExpression']].append((
result['name'],
(
result['time_info'],
result['values'],
),
))
self._prefetched = prefetched
def _merge_results(pathExpr, startTime, endTime, result_queue, seriesList):
log.debug("render.datalib.fetchData :: starting to merge")
for path, results in result_queue:
results = wait_for_result(results)
if not results:
log.debug("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (path, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError as e:
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (path, e))
(start, end, step) = timeInfo
series = TimeSeries(path, start, end, step, values)
# hack to pass expressions through to render functions
series.pathExpression = pathExpr
# Used as a cache to avoid recounting series None values below.
series_best_nones = {}
if series.name in seriesList:
# This counts the Nones in each series, and is unfortunately O(n) for each
# series, which may be worth further optimization. The value of doing this
# at all is to avoid the "flipping" effect of loading a graph multiple times
# and having inconsistent data returned if one of the backing stores has
# inconsistent data. This is imperfect as a validity test, but in practice
# nicely keeps us using the "most complete" dataset available. Think of it
# as a very weak CRDT resolver.
candidate_nones = 0
if not settings.REMOTE_STORE_MERGE_RESULTS:
candidate_nones = len(
[val for val in values if val is None])
known = seriesList[series.name]
# To avoid repeatedly recounting the 'Nones' in series we've already seen,
# cache the best known count so far in a dict.
if known.name in series_best_nones:
known_nones = series_best_nones[known.name]
else:
known_nones = len([val for val in known if val is None])
if known_nones > candidate_nones and len(series):
if settings.REMOTE_STORE_MERGE_RESULTS:
# This series has potential data that might be missing from
# earlier series. Attempt to merge in useful data and update
# the cache count.
log.debug("Merging multiple TimeSeries for %s" % known.name)
for i, j in enumerate(known):
if j is None and series[i] is not None:
known[i] = series[i]
known_nones -= 1
# Store known_nones in our cache
series_best_nones[known.name] = known_nones
else:
# Not merging data -
# we've found a series better than what we've already seen. Update
# the count cache and replace the given series in the array.
series_best_nones[known.name] = candidate_nones
seriesList[known.name] = series
else:
if settings.REMOTE_PREFETCH_DATA:
# if we're using REMOTE_PREFETCH_DATA we can save some time by skipping
# find, but that means we don't know how many nodes to expect so we
# have to iterate over all returned results
continue
# In case if we are merging data - the existing series has no gaps and
# there is nothing to merge together. Save ourselves some work here.
#
# OR - if we picking best serie:
#
# We already have this series in the seriesList, and the
# candidate is 'worse' than what we already have, we don't need
# to compare anything else. Save ourselves some work here.
break
else:
# If we looked at this series above, and it matched a 'known'
# series already, then it's already in the series list (or ignored).
# If not, append it here.
seriesList[series.name] = series
# Stabilize the order of the results by ordering the resulting series by name.
# This returns the result ordering to the behavior observed pre PR#1010.
return [seriesList[k] for k in sorted(seriesList)]
def _merge_results(pathExpr, startTime, endTime, result_queue, seriesList):
log.debug("render.datalib.fetchData :: starting to merge")
for path, results in result_queue:
results = wait_for_result(results)
if not results:
log.debug("render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (path, startTime, endTime))
continue
try:
(timeInfo, values) = results
except ValueError as e:
raise Exception("could not parse timeInfo/values from metric '%s': %s" % (path, e))
(start, end, step) = timeInfo
series = TimeSeries(path, start, end, step, values)
# hack to pass expressions through to render functions
series.pathExpression = pathExpr
# Used as a cache to avoid recounting series None values below.
series_best_nones = {}
if series.name in seriesList:
# This counts the Nones in each series, and is unfortunately O(n) for each
# series, which may be worth further optimization. The value of doing this
# at all is to avoid the "flipping" effect of loading a graph multiple times
# and having inconsistent data returned if one of the backing stores has
# inconsistent data. This is imperfect as a validity test, but in practice
# nicely keeps us using the "most complete" dataset available. Think of it
# as a very weak CRDT resolver.
candidate_nones = 0
if not settings.REMOTE_STORE_MERGE_RESULTS:
candidate_nones = len(
[val for val in values if val is None])
known = seriesList[series.name]
# To avoid repeatedly recounting the 'Nones' in series we've already seen,
# cache the best known count so far in a dict.
if known.name in series_best_nones:
known_nones = series_best_nones[known.name]
else:
known_nones = len([val for val in known if val is None])
if known_nones > candidate_nones and len(series):
if settings.REMOTE_STORE_MERGE_RESULTS:
# This series has potential data that might be missing from
# earlier series. Attempt to merge in useful data and update
# the cache count.
log.debug("Merging multiple TimeSeries for %s" % known.name)
for i, j in enumerate(known):
if j is None and series[i] is not None:
known[i] = series[i]
known_nones -= 1
# Store known_nones in our cache
series_best_nones[known.name] = known_nones
else:
# Not merging data -
# we've found a series better than what we've already seen. Update
# the count cache and replace the given series in the array.
series_best_nones[known.name] = candidate_nones
seriesList[known.name] = series
else:
if settings.REMOTE_PREFETCH_DATA:
# if we're using REMOTE_PREFETCH_DATA we can save some time by skipping
# find, but that means we don't know how many nodes to expect so we
# have to iterate over all returned results
continue
# In case if we are merging data - the existing series has no gaps and
# there is nothing to merge together. Save ourselves some work here.
#
# OR - if we picking best serie:
#
# We already have this series in the seriesList, and the
# candidate is 'worse' than what we already have, we don't need
# to compare anything else. Save ourselves some work here.
break
else:
# If we looked at this series above, and it matched a 'known'
# series already, then it's already in the series list (or ignored).
# If not, append it here.
seriesList[series.name] = series
# Stabilize the order of the results by ordering the resulting series by name.
# This returns the result ordering to the behavior observed pre PR#1010.
return [seriesList[k] for k in sorted(seriesList)]