def glob_metrics(self, glob, start_time=None, end_time=None):
"""Return a sorted list of metrics matching this glob."""
super(_ElasticSearchAccessor, self).glob_metrics(glob, start_time, end_time)
tracing.add_attr_to_trace("glob", str(glob))
if glob == "":
return []
components = self.__glob_parser.parse(glob)
glob_depth = _get_depth_from_components(components)
search = self._create_search_query(start_time, end_time)
has_globstar, search = self._search_metrics_from_components(
glob, components, search
)
if has_globstar:
search = search.filter("range", depth={"gte": glob_depth})
else:
search = search.filter("term", depth=glob_depth)
search = search.extra(from_=0, size=MAX_QUERY_SIZE)
# TODO (t.chataigner) try to move the sort in the ES search and return a generator.
log.debug(json.dumps(search.to_dict(), default=str))
documents = search.execute()
results = [self._document_to_metric(document)
for document in self._deduplicate_documents(documents)]
results.sort(key=lambda metric: metric.name)
return results
def glob_metrics(self, glob, start_time=None, end_time=None):
"""Return a sorted list of metrics matching this glob."""
super(_ElasticSearchAccessor,
self).glob_metrics(glob, start_time, end_time)
tracing.add_attr_to_trace("glob", str(glob))
if glob == "":
return []
components = self.__glob_parser.parse(glob)
glob_depth = _get_depth_from_components(components)
search = self._create_search_query(start_time, end_time)
has_globstar, search = self._search_metrics_from_components(
glob, components, search)
if has_globstar:
search = search.filter("range", depth={"gte": glob_depth})
else:
search = search.filter("term", depth=glob_depth)
search = search.extra(from_=0, size=MAX_QUERY_SIZE)
# TODO (t.chataigner) try to move the sort in the ES search and return a generator.
log.debug(json.dumps(search.to_dict(), default=str))
documents = search.execute()
results = [
self._document_to_metric(document)
for document in self._deduplicate_documents(documents)
]
results.sort(key=lambda metric: metric.name)
return results
def glob_directory_names(self, glob, start_time=None, end_time=None):
"""See the real Accessor for a description."""
super(_ElasticSearchAccessor,
self).glob_directory_names(glob, start_time, end_time)
tracing.add_attr_to_trace("glob", str(glob))
if glob == "":
return []
components = self.__glob_parser.parse(glob)
search = self._create_directory_search_query(start_time, end_time)
try:
search = self._search_directory_from_components(
glob, components, search)
except InvalidArgumentError:
return []
log.debug(json.dumps(search.to_dict(), default=str))
response = search.execute()
# This may not be the same behavior as other drivers.
# It returns the glob with the list of possible last component for a directory.
# It doesn't return the list of fully defined directory names.
if "distinct_dirs" not in response.aggregations:
# This happend when there is no index to search for the query.
return []
buckets = response.aggregations.distinct_dirs.buckets
if len(components) == 1:
results = [b.key for b in buckets]
else:
glob_base = glob.rsplit(".", 1)[0]
results = ["%s.%s" % (glob_base, b.key) for b in buckets]
results.sort()
return iter(results)
def __update_read_on(self, metric):
tracing.add_attr_to_trace("metric.name", metric.name)
data = {"doc": {"read_on": datetime.datetime.now()}}
# Get the latest version of the metric and update it.
document = self.__get_document(metric.name)
self.__update_document(data, document.meta.index, metric.id)
READ_ON.inc()
def __update_read_on(self, metric):
tracing.add_attr_to_trace("metric.name", metric.name)
data = {"doc": {"read_on": datetime.datetime.now()}}
# Get the latest version of the metric and update it.
document = self.__get_document(metric.name)
self.__update_document(data, document.meta.index, metric.id)
READ_ON.inc()
def glob_directory_names(self, glob, start_time=None, end_time=None):
"""See the real Accessor for a description."""
super(_ElasticSearchAccessor, self).glob_directory_names(
glob, start_time, end_time
)
tracing.add_attr_to_trace("glob", str(glob))
if glob == "":
return []
components = self.__glob_parser.parse(glob)
search = self._create_search_query(start_time, end_time)
# There are no "directory" documents, only "metric" documents. Hence appending the
# AnySequence after the provided glob: we search for metrics under that path.
has_globstar, search = self._search_metrics_from_components(
glob, components + [[bg_glob.AnySequence()]], search
)
if has_globstar:
# TODO (t.chataigner) Add a log or raise exception.
return []
glob_depth = _get_depth_from_components(components)
# Use (glob_depth + 1) to filter only directories and
# exclude metrics whose depth is glob_depth.
search = search.filter("range", depth={"gte": glob_depth + 1})
search = search.extra(from_=0, size=0) # Do not return metrics.
search.aggs.bucket(
"distinct_dirs", "terms", field="p%d" % glob_depth, size=MAX_QUERY_SIZE
)
log.debug(json.dumps(search.to_dict(), default=str))
response = search.execute()
# This may not be the same behavior as other drivers.
# It returns the glob with the list of possible last component for a directory.
# It doesn't return the list of fully defined directory names.
if "distinct_dirs" not in response.aggregations:
# This happend when there is no index to search for the query.
return []
buckets = response.aggregations.distinct_dirs.buckets
if glob_depth == 0:
results = [b.key for b in buckets]
else:
glob_base = glob.rsplit(".", 1)[0]
results = ["%s.%s" % (glob_base, b.key) for b in buckets]
results.sort()
return iter(results)
def glob_directory_names(self, glob, start_time=None, end_time=None):
"""See the real Accessor for a description."""
super(_ElasticSearchAccessor,
self).glob_directory_names(glob, start_time, end_time)
tracing.add_attr_to_trace("glob", str(glob))
if glob == "":
return []
components = self.__glob_parser.parse(glob)
search = self._create_search_query(start_time, end_time)
# There are no "directory" documents, only "metric" documents. Hence appending the
# AnySequence after the provided glob: we search for metrics under that path.
has_globstar, search = self._search_metrics_from_components(
glob, components + [[bg_glob.AnySequence()]], search)
if has_globstar:
# TODO (t.chataigner) Add a log or raise exception.
return []
glob_depth = _get_depth_from_components(components)
# Use (glob_depth + 1) to filter only directories and
# exclude metrics whose depth is glob_depth.
search = search.filter("range", depth={"gte": glob_depth + 1})
search = search.extra(from_=0, size=0) # Do not return metrics.
search.aggs.bucket("distinct_dirs",
"terms",
field="p%d" % glob_depth,
size=MAX_QUERY_SIZE)
log.debug(json.dumps(search.to_dict(), default=str))
response = search.execute()
# This may not be the same behavior as other drivers.
# It returns the glob with the list of possible last component for a directory.
# It doesn't return the list of fully defined directory names.
if "distinct_dirs" not in response.aggregations:
# This happend when there is no index to search for the query.
return []
buckets = response.aggregations.distinct_dirs.buckets
if glob_depth == 0:
results = [b.key for b in buckets]
else:
glob_base = glob.rsplit(".", 1)[0]
results = ["%s.%s" % (glob_base, b.key) for b in buckets]
results.sort()
return iter(results)
def update_metric(self, name, updated_metadata):
"""See bg_accessor.Accessor."""
super(_ElasticSearchAccessor, self).update_metric(name, updated_metadata)
tracing.add_attr_to_trace("metric.name", name)
name = bg_metric.sanitize_metric_name(name)
metric = self.get_metric(name)
if metric is None:
raise InvalidArgumentError("Unknown metric '%s'" % name)
updated_metric = bg_metric.make_metric(
name,
updated_metadata,
created_on=metric.created_on,
updated_on=datetime.datetime.now(),
read_on=metric.read_on,
)
self.create_metric(updated_metric)
def update_metric(self, name, updated_metadata):
"""See bg_accessor.Accessor."""
super(_ElasticSearchAccessor,
self).update_metric(name, updated_metadata)
tracing.add_attr_to_trace("metric.name", name)
name = bg_metric.sanitize_metric_name(name)
metric = self.get_metric(name)
if metric is None:
raise InvalidArgumentError("Unknown metric '%s'" % name)
updated_metric = bg_metric.make_metric_with_defaults(
name,
updated_metadata,
created_on=metric.created_on,
updated_on=datetime.datetime.now(),
read_on=metric.read_on,
)
self.create_metric(updated_metric)
def __call__(self, request):
"""Code to be executed for each request before request execution."""
tracing_methods = getattr(settings, 'BG_TRACING_METHODS', BG_TRACING_METHODS_DEFAULT)
tracing_whitelist = getattr(settings, 'BG_TRACING_TARGET_WHITELIST',
BG_TRACING_TARGET_WHITELIST_DEFAULT)
if request.method not in tracing_methods:
tracing.stop_trace()
# Improve trace data
if request.method == "GET":
target = str(request.GET.get('target', 'unknown'))
if (tracing_whitelist) and (target not in tracing_whitelist):
print("%s/%s" % (tracing_whitelist, target))
tracing.stop_trace()
tracing.add_attr_to_trace('graphite.target', target)
tracing.add_attr_to_trace('graphite.format',
str(request.GET.get('format', 'unknown')))
response = self.get_response(request)
return response
def read_points():
read_start = time.time()
cached_datapoints = self.__get_cached_datapoints(stage)
tracing.add_attr_to_trace('metric.name', self._metric_name)
tracing.add_attr_to_trace('points.num', points_num)
# TODO: Consider wrapping an array (using NaN for None) for
# speed&memory efficiency
points = [None] * points_num
for ts, point in ts_and_points:
index = stage.step(ts) - start_step
points[index] = point
if cached_datapoints:
points = self._merge_cached_points(
stage,
start_time,
step,
aggregation_method,
points,
cached_datapoints,
raw_step=raw_step,
)
now = time.time()
log.rendering(
"fetch(%s, %d, %d) - %d points - read: %f secs - total: %f secs"
% (
self._metric_name,
start_time,
end_time,
len(points),
now - read_start,
now - fetch_start,
)
)
return (start_time, end_time, stage.precision), points
def read_points():
read_start = time.time()
cached_datapoints = self.__get_cached_datapoints(stage)
tracing.add_attr_to_trace('metric.name', self._metric_name)
tracing.add_attr_to_trace('points.num', points_num)
# TODO: Consider wrapping an array (using NaN for None) for
# speed&memory efficiency
points = [None] * points_num
for ts, point in ts_and_points:
index = stage.step(ts) - start_step
points[index] = point
if cached_datapoints:
points = self._merge_cached_points(
stage,
start_time,
step,
aggregation_method,
points,
cached_datapoints,
raw_step=raw_step,
)
now = time.time()
log.rendering(
"fetch(%s, %d, %d) - %d points - read: %f secs - total: %f secs"
% (
self._metric_name,
start_time,
end_time,
len(points),
now - read_start,
now - fetch_start,
))
return (start_time, end_time, stage.precision), points
def fetch_async(self, start_time, end_time, now=None, requestContext=None):
"""Fetch point for a given interval as per the Graphite API.
Args:
start_time: Timestamp to fetch points from, will constrained by retention policy.
end_time: Timestamp to fetch points until, will constrained by retention policy.
now: Current timestamp as a float, defaults to time.time(), for tests.
Returns:
A callable that returns a tuple made of (rounded start time,
rounded end time, stage precision), points
Points is a list for which missing points are set to None.
"""
tracing.add_attr_to_trace('metric.name', self._metric_name)
fetch_start = time.time()
log.rendering("fetch(%s, %d, %d) - start" %
(self._metric_name, start_time, end_time))
self.__refresh_metric()
if now is None:
now = time.time()
metadata = self.__get_metadata()
start_time, end_time, stage = self.__get_time_info(
start_time, end_time, now)
start_step = stage.step(start_time)
points_num = stage.step(end_time) - start_step
step = stage.precision
aggregation_method = metadata.aggregator.carbon_name
raw_step = metadata.retention.stage0.precision
if not self._metric:
# The metric doesn't exist, let's fail gracefully.
ts_and_points = []
else:
# This returns a generator which we can iterate on later.
ts_and_points = self._accessor.fetch_points(
self._metric, start_time, end_time, stage)
def read_points():
read_start = time.time()
cached_datapoints = self.__get_cached_datapoints(stage)
tracing.add_attr_to_trace('metric.name', self._metric_name)
tracing.add_attr_to_trace('points.num', points_num)
# TODO: Consider wrapping an array (using NaN for None) for
# speed&memory efficiency
points = [None] * points_num
for ts, point in ts_and_points:
index = stage.step(ts) - start_step
points[index] = point
if cached_datapoints:
points = self._merge_cached_points(
stage,
start_time,
step,
aggregation_method,
points,
cached_datapoints,
raw_step=raw_step,
)
now = time.time()
log.rendering(
"fetch(%s, %d, %d) - %d points - read: %f secs - total: %f secs"
% (
self._metric_name,
start_time,
end_time,
len(points),
now - read_start,
now - fetch_start,
))
return (start_time, end_time, stage.precision), points
log.rendering("fetch(%s, %d, %d) - started" %
(self._metric_name, start_time, end_time))
return tracing.trace_simple(read_points)
def fetch_async(self, start_time, end_time, now=None, requestContext=None):
"""Fetch point for a given interval as per the Graphite API.
Args:
start_time: Timestamp to fetch points from, will constrained by retention policy.
end_time: Timestamp to fetch points until, will constrained by retention policy.
now: Current timestamp as a float, defaults to time.time(), for tests.
Returns:
A callable that returns a tuple made of (rounded start time,
rounded end time, stage precision), points
Points is a list for which missing points are set to None.
"""
tracing.add_attr_to_trace('metric.name', self._metric_name)
fetch_start = time.time()
log.rendering(
"fetch(%s, %d, %d) - start" % (self._metric_name, start_time, end_time)
)
self.__refresh_metric()
if now is None:
now = time.time()
metadata = self.__get_metadata()
start_time, end_time, stage = self.__get_time_info(start_time, end_time, now)
start_step = stage.step(start_time)
points_num = stage.step(end_time) - start_step
step = stage.precision
aggregation_method = metadata.aggregator.carbon_name
raw_step = metadata.retention.stage0.precision
if not self._metric:
# The metric doesn't exist, let's fail gracefully.
ts_and_points = []
else:
# This returns a generator which we can iterate on later.
ts_and_points = self._accessor.fetch_points(
self._metric, start_time, end_time, stage
)
def read_points():
read_start = time.time()
cached_datapoints = self.__get_cached_datapoints(stage)
tracing.add_attr_to_trace('metric.name', self._metric_name)
tracing.add_attr_to_trace('points.num', points_num)
# TODO: Consider wrapping an array (using NaN for None) for
# speed&memory efficiency
points = [None] * points_num
for ts, point in ts_and_points:
index = stage.step(ts) - start_step
points[index] = point
if cached_datapoints:
points = self._merge_cached_points(
stage,
start_time,
step,
aggregation_method,
points,
cached_datapoints,
raw_step=raw_step,
)
now = time.time()
log.rendering(
"fetch(%s, %d, %d) - %d points - read: %f secs - total: %f secs"
% (
self._metric_name,
start_time,
end_time,
len(points),
now - read_start,
now - fetch_start,
)
)
return (start_time, end_time, stage.precision), points
log.rendering(
"fetch(%s, %d, %d) - started" % (self._metric_name, start_time, end_time)
)
return tracing.trace_simple(read_points)