def _simulate_workers(self) -> Sequence[str]:
"""
Two workers writing the same metrics. One for 4 seconds. Another one for 2
The first worker reports a task
:return: worker ids
"""
task_id = self._create_running_task(task_name="task-1")
workers = [f"test_{uuid4().hex}", f"test_{uuid4().hex}"]
workers_stats = [(
dict(cpu_usage=[10, 20], memory_used=50),
dict(cpu_usage=[5], memory_used=30),
)] * 4
workers_activity = [
(workers[0], workers[1]),
(workers[0], workers[1]),
(workers[0], ),
(workers[0], ),
]
for ws, stats in zip(workers_activity, workers_stats):
for w, s in zip(ws, stats):
data = dict(
worker=w,
timestamp=int(utc_now_tz_aware().timestamp() * 1000),
machine_stats=s,
)
if w == workers[0]:
data["task"] = task_id
self.api.workers.status_report(**data)
time.sleep(1)
return workers
def _create_temp_worker(self, worker, queue):
self.api.workers.register(worker=worker, queues=[queue])
task = self._create_temp_running_task(f"temp task for worker {worker}")
self.api.workers.status_report(
worker=worker,
timestamp=int(utc_now_tz_aware().timestamp() * 1000),
machine_stats=dict(cpu_usage=[10, 20]),
task=task["id"],
)
return dict(name=worker, ip="127.0.0.1", task=task)
def test_queue_metrics(self):
queue_id = self._temp_queue("TestTempQueue")
task1 = self._create_temp_queued_task("temp task 1", queue_id)
time.sleep(1)
task2 = self._create_temp_queued_task("temp task 2", queue_id)
self.api.queues.get_next_task(queue=queue_id)
self.api.queues.remove_task(queue=queue_id, task=task2["id"])
to_date = utc_now_tz_aware()
from_date = to_date - timedelta(hours=1)
res = self.api.queues.get_queue_metrics(
queue_ids=[queue_id],
from_date=from_date.timestamp(),
to_date=to_date.timestamp(),
interval=5,
)
self.assertMetricQueues(res["queues"], queue_id)
def assertMetricQueues(self, queues_data, queue_id):
self.assertEqual(len(queues_data), 1)
queue_res = queues_data[0]
self.assertEqual(queue_res.queue, queue_id)
dates_len = len(queue_res["dates"])
self.assertTrue(2 >= dates_len >= 1)
for prop in ("avg_waiting_times", "queue_lengths"):
self.assertEqual(len(queue_res[prop]), dates_len)
dates_in_sec = [d / 1000 for d in queue_res["dates"]]
self.assertGreater(
dates_in_sec[0], (utc_now_tz_aware() - timedelta(seconds=15)).timestamp()
)
if dates_len > 1:
self.assertAlmostEqual(dates_in_sec[1] - dates_in_sec[0], 5, places=0)
def test_get_activity_report(self):
# test no workers data
# run on an empty es db since we have no way
# to pass non existing workers to this api
# res = self.api.workers.get_activity_report(
# from_date=from_date.timestamp(),
# to_date=to_date.timestamp(),
# interval=20,
# )
self._simulate_workers()
to_date = utc_now_tz_aware()
from_date = to_date - timedelta(minutes=10)
# no variants
res = self.api.workers.get_activity_report(
from_date=from_date.timestamp(),
to_date=to_date.timestamp(),
interval=20)
self.assertWorkerSeries(res["total"], 2)
self.assertWorkerSeries(res["active"], 1)
self.assertTotalSeriesGreaterThenActive(res["total"], res["active"])
def test_get_stats(self):
workers = self._simulate_workers()
to_date = utc_now_tz_aware()
from_date = to_date - timedelta(days=1)
# no variants
res = self.api.workers.get_stats(
items=[
dict(key="cpu_usage", aggregation="avg"),
dict(key="cpu_usage", aggregation="max"),
dict(key="memory_used", aggregation="max"),
dict(key="memory_used", aggregation="min"),
],
from_date=from_date.timestamp(),
to_date=to_date.timestamp(),
# split_by_variant=True,
interval=1,
worker_ids=workers,
)
self.assertWorkersInStats(workers, res["workers"])
assert all({"cpu_usage", "memory_used"} == set(
map(attrgetter("metric"), worker["metrics"]))
for worker in res["workers"])
def _check_dates_and_stats(metric, stats, worker_id) -> bool:
return set(map(
attrgetter("aggregation"), metric["stats"])) == stats and len(
metric["dates"]) == (4 if worker_id == workers[0] else 2)
assert all(
_check_dates_and_stats(metric, metric_stats, worker["worker"])
for worker in res["workers"]
for metric, metric_stats in zip(worker["metrics"], (
{"avg", "max"}, {"max", "min"})))
# split by variants
res = self.api.workers.get_stats(
items=[dict(key="cpu_usage", aggregation="avg")],
from_date=from_date.timestamp(),
to_date=to_date.timestamp(),
split_by_variant=True,
interval=1,
worker_ids=workers,
)
self.assertWorkersInStats(workers, res["workers"])
def _check_metric_and_variants(worker):
return (all(
_check_dates_and_stats(metric, {"avg"}, worker["worker"])
for metric in worker["metrics"])
and set(map(attrgetter("variant"), worker["metrics"]))
== {"0", "1"} if worker["worker"] == workers[0] else {"0"})
assert all(
_check_metric_and_variants(worker) for worker in res["workers"])
res = self.api.workers.get_stats(
items=[dict(key="cpu_usage", aggregation="avg")],
from_date=from_date.timestamp(),
to_date=to_date.timestamp(),
interval=1,
worker_ids=["Non existing worker id"],
)
assert not res["workers"]