def _schedule_processing_blocks(self):
"""Schedule Processing Blocks for execution."""
LOG.info('Starting to Schedule Processing Blocks.')
while True:
time.sleep(0.5)
if not self._queue:
continue
if self._num_pbcs >= self._max_pbcs:
LOG.warning('Resource limit reached!')
continue
_inspect = Inspect(app=APP)
if self._queue and _inspect.active() is not None:
next_pb = self._queue[-1]
LOG.info('Considering %s for execution...', next_pb[2])
utc_now = datetime.datetime.utcnow()
time_in_queue = (utc_now -
datetime_from_isoformat(next_pb[4]))
if time_in_queue.total_seconds() >= 10:
item = self._queue.get()
LOG.info('------------------------------------')
LOG.info('>>> Executing %s! <<<', item)
LOG.info('------------------------------------')
execute_processing_block.delay(item)
self._num_pbcs += 1
else:
LOG.info('Waiting for resources for %s', next_pb[2])
def get_active_tasks(self):
if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'):
app = Celery('awx')
app.config_from_object('django.conf:settings', namespace='CELERY')
inspector = Inspect(app=app)
active_task_queues = inspector.active()
else:
logger.warn("Ignoring celery task inspector")
active_task_queues = None
queues = None
if active_task_queues is not None:
queues = {}
for queue in active_task_queues:
active_tasks = set()
map(lambda at: active_tasks.add(at['id']), active_task_queues[queue])
# celery worker name is of the form [email protected]
queue_name = queue.split('@')
queue_name = queue_name[1 if len(queue_name) > 1 else 0]
queues[queue_name] = active_tasks
else:
if not hasattr(settings, 'CELERY_UNIT_TEST'):
return (None, None)
return (active_task_queues, queues)
def get_task_info_list(celery_app: Celery) -> list[CeleryTaskInfo]:
inspector = Inspect(app=celery_app)
scheduled_tasks = inspector.scheduled()
revoked_tasks = inspector.revoked()
if scheduled_tasks is None:
print('Worker not found. Exiting.')
return []
res = []
for host, tasks in scheduled_tasks.items():
revoked = revoked_tasks.get(host, [])
for task_data in tasks:
is_revoked = task_data['request']['id'] in revoked
res.append(
CeleryTaskInfo(
task_data['request']['name'],
task_data['request']['args'],
task_data['eta'],
is_revoked,
))
return res
def workers_shutdown(worker_id):
inspect = Inspect(destination=[worker_id], app=celery)
test_ping = inspect.ping()
if test_ping != None:
celery.control.broadcast("shutdown", destination=[worker_id])
shut_ping = inspect.ping()
if shut_ping == None:
return "the worker {} was shutdown".format(worker_id)
def handle(self, *args, **options):
celeryApp = celery.app.app_or_default()
activeQueues = Inspect(app=celeryApp).active_queues()
self.maxNameLen = max([len(name) for name in activeQueues.keys()])
if self.maxNameLen < len('Worker Name'):
self.maxNameLen = len('Worker Name')
self.printRow(['Worker Name', 'Queues'])
print('-' * 80)
for workerName, queues in activeQueues.items():
self.printRow([workerName, ', '.join([q['name'] for q in queues])])
def test_list_task():
# create tasks
for _ in range(0, 15):
longtime_add.apply_async([1, 2, 4], queue="queue2", countdown=5)
# Inspect all nodes.
i = Inspect(app=app)
# Show the items that have an ETA or are scheduled for later processing
print(i.scheduled())
# # Show tasks that are currently active.
print(i.active())
# # Show tasks that have been claimed by workers
print(i.reserved())
def is_already_running():
inspect = Inspect(app=current_app.extensions["invenio-celery"].celery)
filtered_results = filter(
None, [inspect.scheduled(), inspect.active()]
)
for results in filtered_results:
for result in results.values():
for task in result:
request = task["request"]
matches_name = (
request["name"]
== extend_active_loans_location_closure.name
)
matches_location = request["args"][0] == location_pid
if matches_name and matches_location:
return True
return False
def get_known_task_ids_from_celery_api(app) -> Set[str]:
task_ids = set()
i = Inspect(app=app)
def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]):
if worker_tasks:
for worker_name, worker_tasks in worker_tasks.items():
if not worker_tasks:
continue
for t in worker_tasks:
if not t:
continue
task_ids.add(t.get('id'))
collect_task_ids(i.active())
collect_task_ids(i.scheduled())
collect_task_ids(i.reserved())
return task_ids
def get_context_data(self, *args, object_list=None, **kwargs):
context = super().get_context_data()
# 需要启动celery，如果不启动，则返回空列表
try:
context['registered_tasks'] = list(
Inspect(app=current_app).registered().values())[0]
except Exception as e:
context['registered_tasks'] = []
return context
def get_celery_stats() -> CeleryStats:
task_ids = set()
all_worker_kinds = set()
busy_worker_kinds = set()
i = Inspect(app=app)
all_workers = set()
workers_with_tasks = set()
def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]):
if worker_tasks:
for worker_name, worker_tasks in worker_tasks.items():
all_workers.add(worker_name)
if not worker_tasks:
continue
workers_with_tasks.add(worker_name)
for t in worker_tasks:
if not t:
continue
task_ids.add(t.get('id'))
collect_task_ids(i.active())
collect_task_ids(i.scheduled())
collect_task_ids(i.reserved())
workers_without_tasks = all_workers.difference(workers_with_tasks)
all_worker_kinds = {
worker_name[:worker_name.index('@')]
for worker_name in all_workers
}
free_workers_available_kinds = {
worker_name[:worker_name.index('@')]
for worker_name in workers_without_tasks
}
all_worker_kinds.difference_update(IGNORE_WORKER_KINDS)
free_workers_available_kinds.difference_update(IGNORE_WORKER_KINDS)
return CeleryStats(tasks_on_workers=task_ids,
free_workers_available_of_any_kind=all_worker_kinds
and (free_workers_available_kinds == all_worker_kinds))
def show_queue_items():
from celery.app.control import Inspect
# Inspect all nodes.
i = Inspect()
# Show the items that have an ETA or are scheduled for later processing
i.scheduled()
# Show tasks that are currently active.
i.active()
# Show tasks that have been claimed by workers
i.reserved()
def drop_celery_tasks(
task_name: str,
queue_name,
celery_app: Celery,
redis_client: StrictRedis,
in_workers: bool = False,
):
"""
Drop all **tasks queued** that match the `task_name` and `queue_name` passed as parameter. There is no
celery command available atm for this purpose, therefore we need to
read the celery queue (Redis backend), identify the IDs of the tasks and then revoke them.
Params:
#:param task_name: Path to the celery task.
#:param queue_name: Name of the queue from which you which to delete the the queued tasks.
#:param celery_app: Main celery application.
#:param redis_client: Redis client.
#:param in_workers: Specify whether the tasks pre-fetched or fetched by the workers should be revoked. If the value
is set to `1`, it will revoke active, scheduled, and reserved tasks fetched by the workers.
The tasks that are currently executing will not be terminated, instead the new tasks in the queue will not be
accepted. Use with caution, this option might take a while to execute and is not recommended for prod env.
More information in: https://docs.celeryproject.org/en/stable/userguide/monitoring.html.
For reference a Redis item on the queue looks like:
"{\"body\": \"gAIpfXEAfXEBKFgJAAAAY2FsbGJhY2tzcQJOWAgAAABlcnJiYWNrc3EDTlgFAAAAY2hhaW5xBE5YBQAAAGNob3JkcQ
VOdYdxBi4=\", \"content-encoding\": \"binary\", \"content-type\": \"application/x-python-serialize\",
\"headers\": {\"lang\": \"py\", \"task\": \"hi.tasks.on\",
\"id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"shadow\": null, \"eta\": null,
\"expires\": null, \"group\": null, \"retries\": 0, \"timelimit\": [null, null],
\"root_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"parent_id\": null, \"argsrepr\": \"()\",
\"kwargsrepr\": \"{}\", \"origin\": \"gen1@c60fdf6f1554\", \"span_map\":
{\"uber-trace-id\": \"635914c782f0c52f:8a07796eaedf05d1:0:1\"}},
\"properties\": {\"correlation_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"reply_to\":
\"ac8ee0ea-4d30-3065-97da-5a527f7a1fc5\", \"delivery_mode\": 2, \"delivery_info\":
{\"exchange\": \"\", \"routing_key\": \"default\"}, \"priority\": 0,
\"body_encoding\": \"base64\", \"delivery_tag\": \"5626fd36-bfc6-4ac5-b137-943a6067fcf1\"}}"
"""
def _get_tasks_id(workers: list, tasks_ids: list, task_name: str):
"""
Get task ids with the given name included inside the given `workers` tasks.
{'worker1.example.com': [
{'name': 'tasks.sleeptask', 'id': '32666e9b-809c-41fa-8e93-5ae0c80afbbf',
'args': '(8,)', 'kwargs': '{}'}]
}
"""
for worker in workers:
if not workers[worker]:
continue
for _task in workers[worker]:
if _task["name"].split(".")[-1] == task_name:
tasks_ids.append(_task["id"])
i = Inspect(app=celery_app) # Inspect all nodes.
registered = i.registered()
if not registered:
raise Exception("No registered tasks found")
if not any(task_name == _task
for _task in chain(*list(registered.values()))):
logging.error(
f"Command could not be executed, because task is not registered: {task_name}"
)
return
tasks_ids = []
# Revoke tasks already in the broker.
if in_workers:
_get_tasks_id(i.active(), tasks_ids, task_name)
_get_tasks_id(i.scheduled(), tasks_ids, task_name)
_get_tasks_id(i.reserved(), tasks_ids, task_name)
if tasks_ids:
for task_id in tasks_ids:
Control(app=celery_app).revoke(task_id)
else:
logging.info(
f"No active/scheduled/registered task found with the name {task_name}"
)
# revoke tasks in the redis queue.
queue_length = redis_client.llen(queue_name)
if queue_length == 0:
logging.info(f"No items found in queue: {queue_name}")
return
n = 0
batch_size = 10
while True:
items = redis_client.lrange(queue_name, n, n + batch_size)
n += batch_size
if not items:
break
for item in items:
try:
queued_item = json.loads(item)
except JSONDecodeError as e:
logging.info(f"Error decoding item from queue: {e.msg}")
continue
header = queued_item.get("headers")
if header and header["task"] == task_name:
task_id = queued_item["headers"]["id"]
logging.info(f"revoking task id {task_id}")
Control(app=celery_app).revoke(task_id)
def do_active(request):
""" Get all active tasks by worker """
inspect = Inspect(app=celery)
return inspect.active()
def workers():
'''
获取所有worker的基本信息，返回格式为：
{'active': {u'celery@ubuntu': []},
'active_queues': {u'celery@ubuntu': [{u'alias': None,
u'auto_delete': False,
u'binding_arguments': None,
u'bindings': [],
u'consumer_arguments': None,
u'durable': True,
u'exchange': {u'arguments': None,
u'auto_delete': False,
u'delivery_mode': None,
u'durable': True,
u'name': u'celery',
u'no_declare': False,
u'passive': False,
u'type': u'direct'},
u'exclusive': False,
u'expires': None,
u'max_length': None,
u'max_length_bytes': None,
u'max_priority': None,
u'message_ttl': None,
u'name': u'celery',
u'no_ack': False,
u'no_declare': None,
u'queue_arguments': None,
u'routing_key': u'celery'}]},
'conf': {u'celery@ubuntu': {u'broker_url': u'amqp://*****:*****@192.168.205.169:5672/DynamicSchedule',
u'include': [u'algorithmModel.algorithm_entry',
u'celery.app.builtins',
u'app_view'],
u'result_backend': u'rpc:///'}},
'registered': {u'celery@ubuntu': [u'algorithmModel.algorithm_entry.optimization',
u'app_view.add']},
'reserved': {u'celery@ubuntu': []},
'revoked': {u'celery@ubuntu': []},
'scheduled': {u'celery@ubuntu': []},
'stats': {u'celery@ubuntu': {u'broker': {u'alternates': [],
u'connect_timeout': 4,
u'failover_strategy': u'round-robin',
u'heartbeat': 120.0,
u'hostname': u'192.168.205.169',
u'insist': False,
u'login_method': u'AMQPLAIN',
u'port': 5672,
u'ssl': False,
u'transport': u'amqp',
u'transport_options': {},
u'uri_prefix': None,
u'userid': u'test',
u'virtual_host': u'DynamicSchedule'},
u'clock': u'3062',
u'pid': 5599,
u'pool': {u'max-concurrency': 4,
u'max-tasks-per-child': u'N/A',
u'processes': [5608, 5609, 5610, 5613],
u'put-guarded-by-semaphore': False,
u'timeouts': [0, 0],
u'writes': {u'all': u'',
u'avg': u'0.00%',
u'inqueues': {u'active': 0, u'total': 4},
u'raw': u'',
u'strategy': u'fair',
u'total': 0}},
u'prefetch_count': 16,
u'rusage': {u'idrss': 0,
u'inblock': 6104,
u'isrss': 0,
u'ixrss': 0,
u'majflt': 0,
u'maxrss': 53660,
u'minflt': 21858,
u'msgrcv': 0,
u'msgsnd': 0,
u'nivcsw': 664,
u'nsignals': 0,
u'nswap': 0,
u'nvcsw': 16273,
u'oublock': 8,
u'stime': 6.916,
u'utime': 4.228},
u'total': {}}}}
'''
INSPECT_METHODS = ('stats', 'active_queues', 'registered', 'scheduled',
'active', 'reserved', 'revoked', 'conf')
inspect = Inspect(app=celery)
active_worker = inspect.active()
if not active_worker:
return "no worker active"
result = dict()
for methods in INSPECT_METHODS:
result[methods] = getattr(inspect, methods)
return json.dumps(result, ensure_ascii=False)
def do_reserved(request):
""" Get all tasks that have been reserved but are not executing """
inspect = Inspect(app=celery)
return inspect.reserved()
def do_registered(request):
""" Get a list of all registered tasks """
inspect = Inspect(app=celery)
return inspect.registered()
def is_available():
"""Boolean for whether celery has nodes available for async tasks."""
return True if Inspect(app=celery_app).ping() else False
def visit_requests(func):
schedule = Inspect(app=current_app).scheduled()
return [[func(item['request']) for item in v] for k, v in schedule.items()]
from flask import Flask
from pymongo import MongoClient
from .Celery import make_celery
from .config import MONGODB
from flask_cors import *
from celery.app.control import Control,Inspect
app=Flask(__name__)
CORS(app, supports_credentials=True)
app.config.from_pyfile('config.py')
celery=make_celery(app)
control=Control(app=celery)
inspect=Inspect(app=celery)
mongo=MongoClient(MONGODB)
db=mongo.celery
tasks=db.tasks
mirrors=db.mirrors
from .view import *
def do_scheduled(request):
""" Get a list of all tasks waiting to be scheduled """
inspect = Inspect(app=celery)
return inspect.scheduled()
def get_active():
i = Inspect()
active_tasks = i.active()
return jsonify(active_tasks)