def show_queue_items():
from celery.app.control import Inspect
# Inspect all nodes.
i = Inspect()
# Show the items that have an ETA or are scheduled for later processing
i.scheduled()
# Show tasks that are currently active.
i.active()
# Show tasks that have been claimed by workers
i.reserved()
def get_active_tasks(self):
if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'):
app = Celery('awx')
app.config_from_object('django.conf:settings', namespace='CELERY')
inspector = Inspect(app=app)
active_task_queues = inspector.active()
else:
logger.warn("Ignoring celery task inspector")
active_task_queues = None
queues = None
if active_task_queues is not None:
queues = {}
for queue in active_task_queues:
active_tasks = set()
map(lambda at: active_tasks.add(at['id']), active_task_queues[queue])
# celery worker name is of the form [email protected]
queue_name = queue.split('@')
queue_name = queue_name[1 if len(queue_name) > 1 else 0]
queues[queue_name] = active_tasks
else:
if not hasattr(settings, 'CELERY_UNIT_TEST'):
return (None, None)
return (active_task_queues, queues)
def _schedule_processing_blocks(self):
"""Schedule Processing Blocks for execution."""
LOG.info('Starting to Schedule Processing Blocks.')
while True:
time.sleep(0.5)
if not self._queue:
continue
if self._num_pbcs >= self._max_pbcs:
LOG.warning('Resource limit reached!')
continue
_inspect = Inspect(app=APP)
if self._queue and _inspect.active() is not None:
next_pb = self._queue[-1]
LOG.info('Considering %s for execution...', next_pb[2])
utc_now = datetime.datetime.utcnow()
time_in_queue = (utc_now -
datetime_from_isoformat(next_pb[4]))
if time_in_queue.total_seconds() >= 10:
item = self._queue.get()
LOG.info('------------------------------------')
LOG.info('>>> Executing %s! <<<', item)
LOG.info('------------------------------------')
execute_processing_block.delay(item)
self._num_pbcs += 1
else:
LOG.info('Waiting for resources for %s', next_pb[2])
def test_list_task():
# create tasks
for _ in range(0, 15):
longtime_add.apply_async([1, 2, 4], queue="queue2", countdown=5)
# Inspect all nodes.
i = Inspect(app=app)
# Show the items that have an ETA or are scheduled for later processing
print(i.scheduled())
# # Show tasks that are currently active.
print(i.active())
# # Show tasks that have been claimed by workers
print(i.reserved())
def is_already_running():
inspect = Inspect(app=current_app.extensions["invenio-celery"].celery)
filtered_results = filter(
None, [inspect.scheduled(), inspect.active()]
)
for results in filtered_results:
for result in results.values():
for task in result:
request = task["request"]
matches_name = (
request["name"]
== extend_active_loans_location_closure.name
)
matches_location = request["args"][0] == location_pid
if matches_name and matches_location:
return True
return False
def get_known_task_ids_from_celery_api(app) -> Set[str]:
task_ids = set()
i = Inspect(app=app)
def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]):
if worker_tasks:
for worker_name, worker_tasks in worker_tasks.items():
if not worker_tasks:
continue
for t in worker_tasks:
if not t:
continue
task_ids.add(t.get('id'))
collect_task_ids(i.active())
collect_task_ids(i.scheduled())
collect_task_ids(i.reserved())
return task_ids
def get_celery_stats() -> CeleryStats:
task_ids = set()
all_worker_kinds = set()
busy_worker_kinds = set()
i = Inspect(app=app)
all_workers = set()
workers_with_tasks = set()
def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]):
if worker_tasks:
for worker_name, worker_tasks in worker_tasks.items():
all_workers.add(worker_name)
if not worker_tasks:
continue
workers_with_tasks.add(worker_name)
for t in worker_tasks:
if not t:
continue
task_ids.add(t.get('id'))
collect_task_ids(i.active())
collect_task_ids(i.scheduled())
collect_task_ids(i.reserved())
workers_without_tasks = all_workers.difference(workers_with_tasks)
all_worker_kinds = {
worker_name[:worker_name.index('@')]
for worker_name in all_workers
}
free_workers_available_kinds = {
worker_name[:worker_name.index('@')]
for worker_name in workers_without_tasks
}
all_worker_kinds.difference_update(IGNORE_WORKER_KINDS)
free_workers_available_kinds.difference_update(IGNORE_WORKER_KINDS)
return CeleryStats(tasks_on_workers=task_ids,
free_workers_available_of_any_kind=all_worker_kinds
and (free_workers_available_kinds == all_worker_kinds))
def do_active(request):
""" Get all active tasks by worker """
inspect = Inspect(app=celery)
return inspect.active()
def drop_celery_tasks(
task_name: str,
queue_name,
celery_app: Celery,
redis_client: StrictRedis,
in_workers: bool = False,
):
"""
Drop all **tasks queued** that match the `task_name` and `queue_name` passed as parameter. There is no
celery command available atm for this purpose, therefore we need to
read the celery queue (Redis backend), identify the IDs of the tasks and then revoke them.
Params:
#:param task_name: Path to the celery task.
#:param queue_name: Name of the queue from which you which to delete the the queued tasks.
#:param celery_app: Main celery application.
#:param redis_client: Redis client.
#:param in_workers: Specify whether the tasks pre-fetched or fetched by the workers should be revoked. If the value
is set to `1`, it will revoke active, scheduled, and reserved tasks fetched by the workers.
The tasks that are currently executing will not be terminated, instead the new tasks in the queue will not be
accepted. Use with caution, this option might take a while to execute and is not recommended for prod env.
More information in: https://docs.celeryproject.org/en/stable/userguide/monitoring.html.
For reference a Redis item on the queue looks like:
"{\"body\": \"gAIpfXEAfXEBKFgJAAAAY2FsbGJhY2tzcQJOWAgAAABlcnJiYWNrc3EDTlgFAAAAY2hhaW5xBE5YBQAAAGNob3JkcQ
VOdYdxBi4=\", \"content-encoding\": \"binary\", \"content-type\": \"application/x-python-serialize\",
\"headers\": {\"lang\": \"py\", \"task\": \"hi.tasks.on\",
\"id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"shadow\": null, \"eta\": null,
\"expires\": null, \"group\": null, \"retries\": 0, \"timelimit\": [null, null],
\"root_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"parent_id\": null, \"argsrepr\": \"()\",
\"kwargsrepr\": \"{}\", \"origin\": \"gen1@c60fdf6f1554\", \"span_map\":
{\"uber-trace-id\": \"635914c782f0c52f:8a07796eaedf05d1:0:1\"}},
\"properties\": {\"correlation_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"reply_to\":
\"ac8ee0ea-4d30-3065-97da-5a527f7a1fc5\", \"delivery_mode\": 2, \"delivery_info\":
{\"exchange\": \"\", \"routing_key\": \"default\"}, \"priority\": 0,
\"body_encoding\": \"base64\", \"delivery_tag\": \"5626fd36-bfc6-4ac5-b137-943a6067fcf1\"}}"
"""
def _get_tasks_id(workers: list, tasks_ids: list, task_name: str):
"""
Get task ids with the given name included inside the given `workers` tasks.
{'worker1.example.com': [
{'name': 'tasks.sleeptask', 'id': '32666e9b-809c-41fa-8e93-5ae0c80afbbf',
'args': '(8,)', 'kwargs': '{}'}]
}
"""
for worker in workers:
if not workers[worker]:
continue
for _task in workers[worker]:
if _task["name"].split(".")[-1] == task_name:
tasks_ids.append(_task["id"])
i = Inspect(app=celery_app) # Inspect all nodes.
registered = i.registered()
if not registered:
raise Exception("No registered tasks found")
if not any(task_name == _task
for _task in chain(*list(registered.values()))):
logging.error(
f"Command could not be executed, because task is not registered: {task_name}"
)
return
tasks_ids = []
# Revoke tasks already in the broker.
if in_workers:
_get_tasks_id(i.active(), tasks_ids, task_name)
_get_tasks_id(i.scheduled(), tasks_ids, task_name)
_get_tasks_id(i.reserved(), tasks_ids, task_name)
if tasks_ids:
for task_id in tasks_ids:
Control(app=celery_app).revoke(task_id)
else:
logging.info(
f"No active/scheduled/registered task found with the name {task_name}"
)
# revoke tasks in the redis queue.
queue_length = redis_client.llen(queue_name)
if queue_length == 0:
logging.info(f"No items found in queue: {queue_name}")
return
n = 0
batch_size = 10
while True:
items = redis_client.lrange(queue_name, n, n + batch_size)
n += batch_size
if not items:
break
for item in items:
try:
queued_item = json.loads(item)
except JSONDecodeError as e:
logging.info(f"Error decoding item from queue: {e.msg}")
continue
header = queued_item.get("headers")
if header and header["task"] == task_name:
task_id = queued_item["headers"]["id"]
logging.info(f"revoking task id {task_id}")
Control(app=celery_app).revoke(task_id)
def get_active():
i = Inspect()
active_tasks = i.active()
return jsonify(active_tasks)
def workers():
'''
获取所有worker的基本信息，返回格式为：
{'active': {u'celery@ubuntu': []},
'active_queues': {u'celery@ubuntu': [{u'alias': None,
u'auto_delete': False,
u'binding_arguments': None,
u'bindings': [],
u'consumer_arguments': None,
u'durable': True,
u'exchange': {u'arguments': None,
u'auto_delete': False,
u'delivery_mode': None,
u'durable': True,
u'name': u'celery',
u'no_declare': False,
u'passive': False,
u'type': u'direct'},
u'exclusive': False,
u'expires': None,
u'max_length': None,
u'max_length_bytes': None,
u'max_priority': None,
u'message_ttl': None,
u'name': u'celery',
u'no_ack': False,
u'no_declare': None,
u'queue_arguments': None,
u'routing_key': u'celery'}]},
'conf': {u'celery@ubuntu': {u'broker_url': u'amqp://*****:*****@192.168.205.169:5672/DynamicSchedule',
u'include': [u'algorithmModel.algorithm_entry',
u'celery.app.builtins',
u'app_view'],
u'result_backend': u'rpc:///'}},
'registered': {u'celery@ubuntu': [u'algorithmModel.algorithm_entry.optimization',
u'app_view.add']},
'reserved': {u'celery@ubuntu': []},
'revoked': {u'celery@ubuntu': []},
'scheduled': {u'celery@ubuntu': []},
'stats': {u'celery@ubuntu': {u'broker': {u'alternates': [],
u'connect_timeout': 4,
u'failover_strategy': u'round-robin',
u'heartbeat': 120.0,
u'hostname': u'192.168.205.169',
u'insist': False,
u'login_method': u'AMQPLAIN',
u'port': 5672,
u'ssl': False,
u'transport': u'amqp',
u'transport_options': {},
u'uri_prefix': None,
u'userid': u'test',
u'virtual_host': u'DynamicSchedule'},
u'clock': u'3062',
u'pid': 5599,
u'pool': {u'max-concurrency': 4,
u'max-tasks-per-child': u'N/A',
u'processes': [5608, 5609, 5610, 5613],
u'put-guarded-by-semaphore': False,
u'timeouts': [0, 0],
u'writes': {u'all': u'',
u'avg': u'0.00%',
u'inqueues': {u'active': 0, u'total': 4},
u'raw': u'',
u'strategy': u'fair',
u'total': 0}},
u'prefetch_count': 16,
u'rusage': {u'idrss': 0,
u'inblock': 6104,
u'isrss': 0,
u'ixrss': 0,
u'majflt': 0,
u'maxrss': 53660,
u'minflt': 21858,
u'msgrcv': 0,
u'msgsnd': 0,
u'nivcsw': 664,
u'nsignals': 0,
u'nswap': 0,
u'nvcsw': 16273,
u'oublock': 8,
u'stime': 6.916,
u'utime': 4.228},
u'total': {}}}}
'''
INSPECT_METHODS = ('stats', 'active_queues', 'registered', 'scheduled',
'active', 'reserved', 'revoked', 'conf')
inspect = Inspect(app=celery)
active_worker = inspect.active()
if not active_worker:
return "no worker active"
result = dict()
for methods in INSPECT_METHODS:
result[methods] = getattr(inspect, methods)
return json.dumps(result, ensure_ascii=False)