def get_queue_metrics(
self,
company_id: str,
from_date: float,
to_date: float,
interval: int,
queue_ids: Sequence[str],
) -> dict:
"""
Get the company queue metrics in the specified time range.
Returned as date histograms of average values per queue and metric type.
The from_date is extended by 'metrics_before_from_date' seconds from
queues.conf due to possibly small amount of points. The default extension is 3600s
In case no queue ids are specified the avg across all the
company queues is calculated for each metric
"""
# self._log_current_metrics(company, queue_ids=queue_ids)
if from_date >= to_date:
raise bad_request.FieldsValueError(
"from_date must be less than to_date")
seconds_before = config.get("services.queues.metrics_before_from_date",
3600)
must_terms = [
QueryBuilder.dates_range(from_date - seconds_before, to_date)
]
if queue_ids:
must_terms.append(QueryBuilder.terms("queue", queue_ids))
es_req = {
"size": 0,
"query": {
"bool": {
"must": must_terms
}
},
"aggs": self._get_dates_agg(interval),
}
with translate_errors_context(), TimingContext("es",
"get_queue_metrics"):
res = self._search_company_metrics(company_id, es_req)
if "aggregations" not in res:
return {}
date_metrics = [
dict(
timestamp=d["key"],
queue_metrics=self._extract_queue_metrics(
d["queues"]["buckets"]),
) for d in res["aggregations"]["dates"]["buckets"]
if d["doc_count"] > 0
]
if queue_ids:
return self._datetime_histogram_per_queue(date_metrics)
return self._average_datetime_histogram(date_metrics)
def get_worker_stats(self, company_id: str,
request: GetStatsRequest) -> dict:
"""
Get statistics for company workers metrics in the specified time range
Returned as date histograms for different aggregation types
grouped by worker, metric type (and optionally metric variant)
Buckets with no metrics are not returned
Note: all the statistics are retrieved as one ES query
"""
if request.from_date >= request.to_date:
raise bad_request.FieldsValueError(
"from_date must be less than to_date")
def get_dates_agg() -> dict:
es_to_agg_types = (
("avg", AggregationType.avg.value),
("min", AggregationType.min.value),
("max", AggregationType.max.value),
)
return {
"dates": {
"date_histogram": {
"field": "timestamp",
"interval": f"{request.interval}s",
"min_doc_count": 1,
},
"aggs": {
agg_type: {
es_agg: {
"field": "value"
}
}
for es_agg, agg_type in es_to_agg_types
},
}
}
def get_variants_agg() -> dict:
return {
"variants": {
"terms": {
"field": "variant"
},
"aggs": get_dates_agg()
}
}
es_req = {
"size": 0,
"aggs": {
"workers": {
"terms": {
"field": "worker"
},
"aggs": {
"metrics": {
"terms": {
"field": "metric"
},
"aggs":
get_variants_agg()
if request.split_by_variant else get_dates_agg(),
}
},
}
},
}
query_terms = [
QueryBuilder.dates_range(request.from_date, request.to_date),
QueryBuilder.terms("metric", {item.key
for item in request.items}),
]
if request.worker_ids:
query_terms.append(QueryBuilder.terms("worker",
request.worker_ids))
es_req["query"] = {"bool": {"must": query_terms}}
with translate_errors_context(), TimingContext("es",
"get_worker_stats"):
data = self._search_company_stats(company_id, es_req)
return self._extract_results(data, request.items,
request.split_by_variant)
def get_activity_report(
self,
company_id: str,
from_date: float,
to_date: float,
interval: int,
active_only: bool,
) -> Sequence[dict]:
"""
Get statistics for company workers metrics in the specified time range
Returned as date histograms for different aggregation types
grouped by worker, metric type (and optionally metric variant)
Note: all the statistics are retrieved using one ES query
"""
if from_date >= to_date:
raise bad_request.FieldsValueError(
"from_date must be less than to_date")
must = [QueryBuilder.dates_range(from_date, to_date)]
if active_only:
must.append({"exists": {"field": "task"}})
es_req = {
"size": 0,
"aggs": {
"dates": {
"date_histogram": {
"field": "timestamp",
"interval": f"{interval}s",
},
"aggs": {
"workers_count": {
"cardinality": {
"field": "worker"
}
}
},
}
},
"query": {
"bool": {
"must": must
}
},
}
with translate_errors_context(), TimingContext(
"es", "get_worker_activity_report"):
data = self._search_company_stats(company_id, es_req)
if "aggregations" not in data:
return {}
ret = [
dict(date=date["key"], count=date["workers_count"]["value"])
for date in data["aggregations"]["dates"]["buckets"]
]
if ret and ret[-1]["date"] > (to_date - 0.9 * interval):
# remove last interval if it's incomplete. Allow 10% tolerance
ret.pop()
return ret