def get_queue_metrics(
self,
company_id: str,
from_date: float,
to_date: float,
interval: int,
queue_ids: Sequence[str],
) -> dict:
"""
Get the company queue metrics in the specified time range.
Returned as date histograms of average values per queue and metric type.
The from_date is extended by 'metrics_before_from_date' seconds from
queues.conf due to possibly small amount of points. The default extension is 3600s
In case no queue ids are specified the avg across all the
company queues is calculated for each metric
"""
# self._log_current_metrics(company, queue_ids=queue_ids)
if from_date >= to_date:
raise bad_request.FieldsValueError(
"from_date must be less than to_date")
seconds_before = config.get("services.queues.metrics_before_from_date",
3600)
must_terms = [
QueryBuilder.dates_range(from_date - seconds_before, to_date)
]
if queue_ids:
must_terms.append(QueryBuilder.terms("queue", queue_ids))
es_req = {
"size": 0,
"query": {
"bool": {
"must": must_terms
}
},
"aggs": self._get_dates_agg(interval),
}
with translate_errors_context(), TimingContext("es",
"get_queue_metrics"):
res = self._search_company_metrics(company_id, es_req)
if "aggregations" not in res:
return {}
date_metrics = [
dict(
timestamp=d["key"],
queue_metrics=self._extract_queue_metrics(
d["queues"]["buckets"]),
) for d in res["aggregations"]["dates"]["buckets"]
if d["doc_count"] > 0
]
if queue_ids:
return self._datetime_histogram_per_queue(date_metrics)
return self._average_datetime_histogram(date_metrics)
def _get_active_workers(cls, company_id, from_timestamp: int,
to_timestamp: int) -> dict:
es_req = {
"size": 0,
"query": QueryBuilder.dates_range(from_timestamp, to_timestamp),
"aggs": {
"workers": {
"terms": {
"field": "worker"
},
"aggs": {
"last_activity_time": {
"max": {
"field": "timestamp"
}
}
},
}
},
}
res = cls._run_worker_stats_query(company_id, es_req)
buckets = safe_get(res, "aggregations/workers/buckets", default=[])
return {
b["key"]: {
"last_activity_time": b["last_activity_time"]["value"]
}
for b in buckets
}
def get_worker_stats_keys(self, company_id: str,
worker_ids: Optional[Sequence[str]]) -> dict:
"""
Get dictionary of metric types grouped by categories
:param company_id: company id
:param worker_ids: optional list of workers to get metric types from.
If not specified them metrics for all the company workers returned
:return:
"""
es_req = {
"size": 0,
"aggs": {
"categories": {
"terms": {
"field": "category"
},
"aggs": {
"metrics": {
"terms": {
"field": "metric"
}
}
},
}
},
}
if worker_ids:
es_req["query"] = QueryBuilder.terms("worker", worker_ids)
res = self._search_company_stats(company_id, es_req)
if not res["hits"]["total"]:
raise bad_request.WorkerStatsNotFound(
f"No statistic metrics found for the company {company_id} and workers {worker_ids}"
)
return {
category["key"]:
[metric["key"] for metric in category["metrics"]["buckets"]]
for category in res["aggregations"]["categories"]["buckets"]
}
def _get_resource_stats_per_agent(cls, company_id: str, key: str) -> dict:
agent_resource_threshold_sec = timedelta(hours=config.get(
"apiserver.statistics.report_interval_hours", 24)).total_seconds()
to_timestamp = int(time.time())
from_timestamp = to_timestamp - int(agent_resource_threshold_sec)
es_req = {
"size": 0,
"query": QueryBuilder.dates_range(from_timestamp, to_timestamp),
"aggs": {
"workers": {
"terms": {
"field": "worker"
},
"aggs": {
"categories": {
"terms": {
"field": "category"
},
"aggs": {
"count": {
"cardinality": {
"field": "variant"
}
}
},
},
"metrics": {
"terms": {
"field": "metric"
},
"aggs": {
"min": {
"min": {
"field": "value"
}
},
"max": {
"max": {
"field": "value"
}
},
"avg": {
"avg": {
"field": "value"
}
},
},
},
},
}
},
}
res = cls._run_worker_stats_query(company_id, es_req)
def _get_cardinality_fields(categories: Sequence[dict]) -> dict:
names = {"cpu": "num_cores"}
return {
names[c["key"]]: safe_get(c, "count/value")
for c in categories if c["key"] in names
}
def _get_metric_fields(metrics: Sequence[dict]) -> dict:
names = {
"cpu_usage": "cpu_usage",
"memory_used": "mem_used_gb",
"memory_free": "mem_free_gb",
}
return {
names[m["key"]]: {
"min": safe_get(m, "min/value"),
"max": safe_get(m, "max/value"),
"avg": safe_get(m, "avg/value"),
}
for m in metrics if m["key"] in names
}
buckets = safe_get(res, "aggregations/workers/buckets", default=[])
return {
b["key"]: {
key: {
"interval_sec":
agent_resource_threshold_sec,
**_get_cardinality_fields(
safe_get(b, "categories/buckets", [])),
**_get_metric_fields(safe_get(b, "metrics/buckets", [])),
}
}
for b in buckets
}
def get_worker_stats(self, company_id: str,
request: GetStatsRequest) -> dict:
"""
Get statistics for company workers metrics in the specified time range
Returned as date histograms for different aggregation types
grouped by worker, metric type (and optionally metric variant)
Buckets with no metrics are not returned
Note: all the statistics are retrieved as one ES query
"""
if request.from_date >= request.to_date:
raise bad_request.FieldsValueError(
"from_date must be less than to_date")
def get_dates_agg() -> dict:
es_to_agg_types = (
("avg", AggregationType.avg.value),
("min", AggregationType.min.value),
("max", AggregationType.max.value),
)
return {
"dates": {
"date_histogram": {
"field": "timestamp",
"interval": f"{request.interval}s",
"min_doc_count": 1,
},
"aggs": {
agg_type: {
es_agg: {
"field": "value"
}
}
for es_agg, agg_type in es_to_agg_types
},
}
}
def get_variants_agg() -> dict:
return {
"variants": {
"terms": {
"field": "variant"
},
"aggs": get_dates_agg()
}
}
es_req = {
"size": 0,
"aggs": {
"workers": {
"terms": {
"field": "worker"
},
"aggs": {
"metrics": {
"terms": {
"field": "metric"
},
"aggs":
get_variants_agg()
if request.split_by_variant else get_dates_agg(),
}
},
}
},
}
query_terms = [
QueryBuilder.dates_range(request.from_date, request.to_date),
QueryBuilder.terms("metric", {item.key
for item in request.items}),
]
if request.worker_ids:
query_terms.append(QueryBuilder.terms("worker",
request.worker_ids))
es_req["query"] = {"bool": {"must": query_terms}}
with translate_errors_context(), TimingContext("es",
"get_worker_stats"):
data = self._search_company_stats(company_id, es_req)
return self._extract_results(data, request.items,
request.split_by_variant)
def get_activity_report(
self,
company_id: str,
from_date: float,
to_date: float,
interval: int,
active_only: bool,
) -> Sequence[dict]:
"""
Get statistics for company workers metrics in the specified time range
Returned as date histograms for different aggregation types
grouped by worker, metric type (and optionally metric variant)
Note: all the statistics are retrieved using one ES query
"""
if from_date >= to_date:
raise bad_request.FieldsValueError(
"from_date must be less than to_date")
must = [QueryBuilder.dates_range(from_date, to_date)]
if active_only:
must.append({"exists": {"field": "task"}})
es_req = {
"size": 0,
"aggs": {
"dates": {
"date_histogram": {
"field": "timestamp",
"interval": f"{interval}s",
},
"aggs": {
"workers_count": {
"cardinality": {
"field": "worker"
}
}
},
}
},
"query": {
"bool": {
"must": must
}
},
}
with translate_errors_context(), TimingContext(
"es", "get_worker_activity_report"):
data = self._search_company_stats(company_id, es_req)
if "aggregations" not in data:
return {}
ret = [
dict(date=date["key"], count=date["workers_count"]["value"])
for date in data["aggregations"]["dates"]["buckets"]
]
if ret and ret[-1]["date"] > (to_date - 0.9 * interval):
# remove last interval if it's incomplete. Allow 10% tolerance
ret.pop()
return ret