def show_queue_items():
from celery.app.control import Inspect
# Inspect all nodes.
i = Inspect()
# Show the items that have an ETA or are scheduled for later processing
i.scheduled()
# Show tasks that are currently active.
i.active()
# Show tasks that have been claimed by workers
i.reserved()
def get_task_info_list(celery_app: Celery) -> list[CeleryTaskInfo]:
inspector = Inspect(app=celery_app)
scheduled_tasks = inspector.scheduled()
revoked_tasks = inspector.revoked()
if scheduled_tasks is None:
print('Worker not found. Exiting.')
return []
res = []
for host, tasks in scheduled_tasks.items():
revoked = revoked_tasks.get(host, [])
for task_data in tasks:
is_revoked = task_data['request']['id'] in revoked
res.append(
CeleryTaskInfo(
task_data['request']['name'],
task_data['request']['args'],
task_data['eta'],
is_revoked,
))
return res
def test_list_task():
# create tasks
for _ in range(0, 15):
longtime_add.apply_async([1, 2, 4], queue="queue2", countdown=5)
# Inspect all nodes.
i = Inspect(app=app)
# Show the items that have an ETA or are scheduled for later processing
print(i.scheduled())
# # Show tasks that are currently active.
print(i.active())
# # Show tasks that have been claimed by workers
print(i.reserved())
def is_already_running():
inspect = Inspect(app=current_app.extensions["invenio-celery"].celery)
filtered_results = filter(
None, [inspect.scheduled(), inspect.active()]
)
for results in filtered_results:
for result in results.values():
for task in result:
request = task["request"]
matches_name = (
request["name"]
== extend_active_loans_location_closure.name
)
matches_location = request["args"][0] == location_pid
if matches_name and matches_location:
return True
return False
def get_known_task_ids_from_celery_api(app) -> Set[str]:
task_ids = set()
i = Inspect(app=app)
def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]):
if worker_tasks:
for worker_name, worker_tasks in worker_tasks.items():
if not worker_tasks:
continue
for t in worker_tasks:
if not t:
continue
task_ids.add(t.get('id'))
collect_task_ids(i.active())
collect_task_ids(i.scheduled())
collect_task_ids(i.reserved())
return task_ids
def get_celery_stats() -> CeleryStats:
task_ids = set()
all_worker_kinds = set()
busy_worker_kinds = set()
i = Inspect(app=app)
all_workers = set()
workers_with_tasks = set()
def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]):
if worker_tasks:
for worker_name, worker_tasks in worker_tasks.items():
all_workers.add(worker_name)
if not worker_tasks:
continue
workers_with_tasks.add(worker_name)
for t in worker_tasks:
if not t:
continue
task_ids.add(t.get('id'))
collect_task_ids(i.active())
collect_task_ids(i.scheduled())
collect_task_ids(i.reserved())
workers_without_tasks = all_workers.difference(workers_with_tasks)
all_worker_kinds = {
worker_name[:worker_name.index('@')]
for worker_name in all_workers
}
free_workers_available_kinds = {
worker_name[:worker_name.index('@')]
for worker_name in workers_without_tasks
}
all_worker_kinds.difference_update(IGNORE_WORKER_KINDS)
free_workers_available_kinds.difference_update(IGNORE_WORKER_KINDS)
return CeleryStats(tasks_on_workers=task_ids,
free_workers_available_of_any_kind=all_worker_kinds
and (free_workers_available_kinds == all_worker_kinds))
def do_scheduled(request):
""" Get a list of all tasks waiting to be scheduled """
inspect = Inspect(app=celery)
return inspect.scheduled()
def drop_celery_tasks(
task_name: str,
queue_name,
celery_app: Celery,
redis_client: StrictRedis,
in_workers: bool = False,
):
"""
Drop all **tasks queued** that match the `task_name` and `queue_name` passed as parameter. There is no
celery command available atm for this purpose, therefore we need to
read the celery queue (Redis backend), identify the IDs of the tasks and then revoke them.
Params:
#:param task_name: Path to the celery task.
#:param queue_name: Name of the queue from which you which to delete the the queued tasks.
#:param celery_app: Main celery application.
#:param redis_client: Redis client.
#:param in_workers: Specify whether the tasks pre-fetched or fetched by the workers should be revoked. If the value
is set to `1`, it will revoke active, scheduled, and reserved tasks fetched by the workers.
The tasks that are currently executing will not be terminated, instead the new tasks in the queue will not be
accepted. Use with caution, this option might take a while to execute and is not recommended for prod env.
More information in: https://docs.celeryproject.org/en/stable/userguide/monitoring.html.
For reference a Redis item on the queue looks like:
"{\"body\": \"gAIpfXEAfXEBKFgJAAAAY2FsbGJhY2tzcQJOWAgAAABlcnJiYWNrc3EDTlgFAAAAY2hhaW5xBE5YBQAAAGNob3JkcQ
VOdYdxBi4=\", \"content-encoding\": \"binary\", \"content-type\": \"application/x-python-serialize\",
\"headers\": {\"lang\": \"py\", \"task\": \"hi.tasks.on\",
\"id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"shadow\": null, \"eta\": null,
\"expires\": null, \"group\": null, \"retries\": 0, \"timelimit\": [null, null],
\"root_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"parent_id\": null, \"argsrepr\": \"()\",
\"kwargsrepr\": \"{}\", \"origin\": \"gen1@c60fdf6f1554\", \"span_map\":
{\"uber-trace-id\": \"635914c782f0c52f:8a07796eaedf05d1:0:1\"}},
\"properties\": {\"correlation_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"reply_to\":
\"ac8ee0ea-4d30-3065-97da-5a527f7a1fc5\", \"delivery_mode\": 2, \"delivery_info\":
{\"exchange\": \"\", \"routing_key\": \"default\"}, \"priority\": 0,
\"body_encoding\": \"base64\", \"delivery_tag\": \"5626fd36-bfc6-4ac5-b137-943a6067fcf1\"}}"
"""
def _get_tasks_id(workers: list, tasks_ids: list, task_name: str):
"""
Get task ids with the given name included inside the given `workers` tasks.
{'worker1.example.com': [
{'name': 'tasks.sleeptask', 'id': '32666e9b-809c-41fa-8e93-5ae0c80afbbf',
'args': '(8,)', 'kwargs': '{}'}]
}
"""
for worker in workers:
if not workers[worker]:
continue
for _task in workers[worker]:
if _task["name"].split(".")[-1] == task_name:
tasks_ids.append(_task["id"])
i = Inspect(app=celery_app) # Inspect all nodes.
registered = i.registered()
if not registered:
raise Exception("No registered tasks found")
if not any(task_name == _task
for _task in chain(*list(registered.values()))):
logging.error(
f"Command could not be executed, because task is not registered: {task_name}"
)
return
tasks_ids = []
# Revoke tasks already in the broker.
if in_workers:
_get_tasks_id(i.active(), tasks_ids, task_name)
_get_tasks_id(i.scheduled(), tasks_ids, task_name)
_get_tasks_id(i.reserved(), tasks_ids, task_name)
if tasks_ids:
for task_id in tasks_ids:
Control(app=celery_app).revoke(task_id)
else:
logging.info(
f"No active/scheduled/registered task found with the name {task_name}"
)
# revoke tasks in the redis queue.
queue_length = redis_client.llen(queue_name)
if queue_length == 0:
logging.info(f"No items found in queue: {queue_name}")
return
n = 0
batch_size = 10
while True:
items = redis_client.lrange(queue_name, n, n + batch_size)
n += batch_size
if not items:
break
for item in items:
try:
queued_item = json.loads(item)
except JSONDecodeError as e:
logging.info(f"Error decoding item from queue: {e.msg}")
continue
header = queued_item.get("headers")
if header and header["task"] == task_name:
task_id = queued_item["headers"]["id"]
logging.info(f"revoking task id {task_id}")
Control(app=celery_app).revoke(task_id)