def _schedule_processing_blocks(self): """Schedule Processing Blocks for execution.""" LOG.info('Starting to Schedule Processing Blocks.') while True: time.sleep(0.5) if not self._queue: continue if self._num_pbcs >= self._max_pbcs: LOG.warning('Resource limit reached!') continue _inspect = Inspect(app=APP) if self._queue and _inspect.active() is not None: next_pb = self._queue[-1] LOG.info('Considering %s for execution...', next_pb[2]) utc_now = datetime.datetime.utcnow() time_in_queue = (utc_now - datetime_from_isoformat(next_pb[4])) if time_in_queue.total_seconds() >= 10: item = self._queue.get() LOG.info('------------------------------------') LOG.info('>>> Executing %s! <<<', item) LOG.info('------------------------------------') execute_processing_block.delay(item) self._num_pbcs += 1 else: LOG.info('Waiting for resources for %s', next_pb[2])
def get_active_tasks(self): if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'): app = Celery('awx') app.config_from_object('django.conf:settings', namespace='CELERY') inspector = Inspect(app=app) active_task_queues = inspector.active() else: logger.warn("Ignoring celery task inspector") active_task_queues = None queues = None if active_task_queues is not None: queues = {} for queue in active_task_queues: active_tasks = set() map(lambda at: active_tasks.add(at['id']), active_task_queues[queue]) # celery worker name is of the form [email protected] queue_name = queue.split('@') queue_name = queue_name[1 if len(queue_name) > 1 else 0] queues[queue_name] = active_tasks else: if not hasattr(settings, 'CELERY_UNIT_TEST'): return (None, None) return (active_task_queues, queues)
def get_task_info_list(celery_app: Celery) -> list[CeleryTaskInfo]: inspector = Inspect(app=celery_app) scheduled_tasks = inspector.scheduled() revoked_tasks = inspector.revoked() if scheduled_tasks is None: print('Worker not found. Exiting.') return [] res = [] for host, tasks in scheduled_tasks.items(): revoked = revoked_tasks.get(host, []) for task_data in tasks: is_revoked = task_data['request']['id'] in revoked res.append( CeleryTaskInfo( task_data['request']['name'], task_data['request']['args'], task_data['eta'], is_revoked, )) return res
def workers_shutdown(worker_id): inspect = Inspect(destination=[worker_id], app=celery) test_ping = inspect.ping() if test_ping != None: celery.control.broadcast("shutdown", destination=[worker_id]) shut_ping = inspect.ping() if shut_ping == None: return "the worker {} was shutdown".format(worker_id)
def handle(self, *args, **options): celeryApp = celery.app.app_or_default() activeQueues = Inspect(app=celeryApp).active_queues() self.maxNameLen = max([len(name) for name in activeQueues.keys()]) if self.maxNameLen < len('Worker Name'): self.maxNameLen = len('Worker Name') self.printRow(['Worker Name', 'Queues']) print('-' * 80) for workerName, queues in activeQueues.items(): self.printRow([workerName, ', '.join([q['name'] for q in queues])])
def test_list_task(): # create tasks for _ in range(0, 15): longtime_add.apply_async([1, 2, 4], queue="queue2", countdown=5) # Inspect all nodes. i = Inspect(app=app) # Show the items that have an ETA or are scheduled for later processing print(i.scheduled()) # # Show tasks that are currently active. print(i.active()) # # Show tasks that have been claimed by workers print(i.reserved())
def is_already_running(): inspect = Inspect(app=current_app.extensions["invenio-celery"].celery) filtered_results = filter( None, [inspect.scheduled(), inspect.active()] ) for results in filtered_results: for result in results.values(): for task in result: request = task["request"] matches_name = ( request["name"] == extend_active_loans_location_closure.name ) matches_location = request["args"][0] == location_pid if matches_name and matches_location: return True return False
def get_known_task_ids_from_celery_api(app) -> Set[str]: task_ids = set() i = Inspect(app=app) def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]): if worker_tasks: for worker_name, worker_tasks in worker_tasks.items(): if not worker_tasks: continue for t in worker_tasks: if not t: continue task_ids.add(t.get('id')) collect_task_ids(i.active()) collect_task_ids(i.scheduled()) collect_task_ids(i.reserved()) return task_ids
def get_context_data(self, *args, object_list=None, **kwargs): context = super().get_context_data() # 需要启动celery,如果不启动,则返回空列表 try: context['registered_tasks'] = list( Inspect(app=current_app).registered().values())[0] except Exception as e: context['registered_tasks'] = [] return context
def get_celery_stats() -> CeleryStats: task_ids = set() all_worker_kinds = set() busy_worker_kinds = set() i = Inspect(app=app) all_workers = set() workers_with_tasks = set() def collect_task_ids(worker_tasks: Dict[str, List[Dict[str, Any]]]): if worker_tasks: for worker_name, worker_tasks in worker_tasks.items(): all_workers.add(worker_name) if not worker_tasks: continue workers_with_tasks.add(worker_name) for t in worker_tasks: if not t: continue task_ids.add(t.get('id')) collect_task_ids(i.active()) collect_task_ids(i.scheduled()) collect_task_ids(i.reserved()) workers_without_tasks = all_workers.difference(workers_with_tasks) all_worker_kinds = { worker_name[:worker_name.index('@')] for worker_name in all_workers } free_workers_available_kinds = { worker_name[:worker_name.index('@')] for worker_name in workers_without_tasks } all_worker_kinds.difference_update(IGNORE_WORKER_KINDS) free_workers_available_kinds.difference_update(IGNORE_WORKER_KINDS) return CeleryStats(tasks_on_workers=task_ids, free_workers_available_of_any_kind=all_worker_kinds and (free_workers_available_kinds == all_worker_kinds))
def show_queue_items(): from celery.app.control import Inspect # Inspect all nodes. i = Inspect() # Show the items that have an ETA or are scheduled for later processing i.scheduled() # Show tasks that are currently active. i.active() # Show tasks that have been claimed by workers i.reserved()
def drop_celery_tasks( task_name: str, queue_name, celery_app: Celery, redis_client: StrictRedis, in_workers: bool = False, ): """ Drop all **tasks queued** that match the `task_name` and `queue_name` passed as parameter. There is no celery command available atm for this purpose, therefore we need to read the celery queue (Redis backend), identify the IDs of the tasks and then revoke them. Params: #:param task_name: Path to the celery task. #:param queue_name: Name of the queue from which you which to delete the the queued tasks. #:param celery_app: Main celery application. #:param redis_client: Redis client. #:param in_workers: Specify whether the tasks pre-fetched or fetched by the workers should be revoked. If the value is set to `1`, it will revoke active, scheduled, and reserved tasks fetched by the workers. The tasks that are currently executing will not be terminated, instead the new tasks in the queue will not be accepted. Use with caution, this option might take a while to execute and is not recommended for prod env. More information in: https://docs.celeryproject.org/en/stable/userguide/monitoring.html. For reference a Redis item on the queue looks like: "{\"body\": \"gAIpfXEAfXEBKFgJAAAAY2FsbGJhY2tzcQJOWAgAAABlcnJiYWNrc3EDTlgFAAAAY2hhaW5xBE5YBQAAAGNob3JkcQ VOdYdxBi4=\", \"content-encoding\": \"binary\", \"content-type\": \"application/x-python-serialize\", \"headers\": {\"lang\": \"py\", \"task\": \"hi.tasks.on\", \"id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"shadow\": null, \"eta\": null, \"expires\": null, \"group\": null, \"retries\": 0, \"timelimit\": [null, null], \"root_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"parent_id\": null, \"argsrepr\": \"()\", \"kwargsrepr\": \"{}\", \"origin\": \"gen1@c60fdf6f1554\", \"span_map\": {\"uber-trace-id\": \"635914c782f0c52f:8a07796eaedf05d1:0:1\"}}, \"properties\": {\"correlation_id\": \"9fbcc18e-45d5-4b9f-b667-bd351568a361\", \"reply_to\": \"ac8ee0ea-4d30-3065-97da-5a527f7a1fc5\", \"delivery_mode\": 2, \"delivery_info\": {\"exchange\": \"\", \"routing_key\": \"default\"}, \"priority\": 0, \"body_encoding\": \"base64\", \"delivery_tag\": \"5626fd36-bfc6-4ac5-b137-943a6067fcf1\"}}" """ def _get_tasks_id(workers: list, tasks_ids: list, task_name: str): """ Get task ids with the given name included inside the given `workers` tasks. {'worker1.example.com': [ {'name': 'tasks.sleeptask', 'id': '32666e9b-809c-41fa-8e93-5ae0c80afbbf', 'args': '(8,)', 'kwargs': '{}'}] } """ for worker in workers: if not workers[worker]: continue for _task in workers[worker]: if _task["name"].split(".")[-1] == task_name: tasks_ids.append(_task["id"]) i = Inspect(app=celery_app) # Inspect all nodes. registered = i.registered() if not registered: raise Exception("No registered tasks found") if not any(task_name == _task for _task in chain(*list(registered.values()))): logging.error( f"Command could not be executed, because task is not registered: {task_name}" ) return tasks_ids = [] # Revoke tasks already in the broker. if in_workers: _get_tasks_id(i.active(), tasks_ids, task_name) _get_tasks_id(i.scheduled(), tasks_ids, task_name) _get_tasks_id(i.reserved(), tasks_ids, task_name) if tasks_ids: for task_id in tasks_ids: Control(app=celery_app).revoke(task_id) else: logging.info( f"No active/scheduled/registered task found with the name {task_name}" ) # revoke tasks in the redis queue. queue_length = redis_client.llen(queue_name) if queue_length == 0: logging.info(f"No items found in queue: {queue_name}") return n = 0 batch_size = 10 while True: items = redis_client.lrange(queue_name, n, n + batch_size) n += batch_size if not items: break for item in items: try: queued_item = json.loads(item) except JSONDecodeError as e: logging.info(f"Error decoding item from queue: {e.msg}") continue header = queued_item.get("headers") if header and header["task"] == task_name: task_id = queued_item["headers"]["id"] logging.info(f"revoking task id {task_id}") Control(app=celery_app).revoke(task_id)
def do_active(request): """ Get all active tasks by worker """ inspect = Inspect(app=celery) return inspect.active()
def workers(): ''' 获取所有worker的基本信息,返回格式为: {'active': {u'celery@ubuntu': []}, 'active_queues': {u'celery@ubuntu': [{u'alias': None, u'auto_delete': False, u'binding_arguments': None, u'bindings': [], u'consumer_arguments': None, u'durable': True, u'exchange': {u'arguments': None, u'auto_delete': False, u'delivery_mode': None, u'durable': True, u'name': u'celery', u'no_declare': False, u'passive': False, u'type': u'direct'}, u'exclusive': False, u'expires': None, u'max_length': None, u'max_length_bytes': None, u'max_priority': None, u'message_ttl': None, u'name': u'celery', u'no_ack': False, u'no_declare': None, u'queue_arguments': None, u'routing_key': u'celery'}]}, 'conf': {u'celery@ubuntu': {u'broker_url': u'amqp://*****:*****@192.168.205.169:5672/DynamicSchedule', u'include': [u'algorithmModel.algorithm_entry', u'celery.app.builtins', u'app_view'], u'result_backend': u'rpc:///'}}, 'registered': {u'celery@ubuntu': [u'algorithmModel.algorithm_entry.optimization', u'app_view.add']}, 'reserved': {u'celery@ubuntu': []}, 'revoked': {u'celery@ubuntu': []}, 'scheduled': {u'celery@ubuntu': []}, 'stats': {u'celery@ubuntu': {u'broker': {u'alternates': [], u'connect_timeout': 4, u'failover_strategy': u'round-robin', u'heartbeat': 120.0, u'hostname': u'192.168.205.169', u'insist': False, u'login_method': u'AMQPLAIN', u'port': 5672, u'ssl': False, u'transport': u'amqp', u'transport_options': {}, u'uri_prefix': None, u'userid': u'test', u'virtual_host': u'DynamicSchedule'}, u'clock': u'3062', u'pid': 5599, u'pool': {u'max-concurrency': 4, u'max-tasks-per-child': u'N/A', u'processes': [5608, 5609, 5610, 5613], u'put-guarded-by-semaphore': False, u'timeouts': [0, 0], u'writes': {u'all': u'', u'avg': u'0.00%', u'inqueues': {u'active': 0, u'total': 4}, u'raw': u'', u'strategy': u'fair', u'total': 0}}, u'prefetch_count': 16, u'rusage': {u'idrss': 0, u'inblock': 6104, u'isrss': 0, u'ixrss': 0, u'majflt': 0, u'maxrss': 53660, u'minflt': 21858, u'msgrcv': 0, u'msgsnd': 0, u'nivcsw': 664, u'nsignals': 0, u'nswap': 0, u'nvcsw': 16273, u'oublock': 8, u'stime': 6.916, u'utime': 4.228}, u'total': {}}}} ''' INSPECT_METHODS = ('stats', 'active_queues', 'registered', 'scheduled', 'active', 'reserved', 'revoked', 'conf') inspect = Inspect(app=celery) active_worker = inspect.active() if not active_worker: return "no worker active" result = dict() for methods in INSPECT_METHODS: result[methods] = getattr(inspect, methods) return json.dumps(result, ensure_ascii=False)
def do_reserved(request): """ Get all tasks that have been reserved but are not executing """ inspect = Inspect(app=celery) return inspect.reserved()
def do_registered(request): """ Get a list of all registered tasks """ inspect = Inspect(app=celery) return inspect.registered()
def is_available(): """Boolean for whether celery has nodes available for async tasks.""" return True if Inspect(app=celery_app).ping() else False
def visit_requests(func): schedule = Inspect(app=current_app).scheduled() return [[func(item['request']) for item in v] for k, v in schedule.items()]
from flask import Flask from pymongo import MongoClient from .Celery import make_celery from .config import MONGODB from flask_cors import * from celery.app.control import Control,Inspect app=Flask(__name__) CORS(app, supports_credentials=True) app.config.from_pyfile('config.py') celery=make_celery(app) control=Control(app=celery) inspect=Inspect(app=celery) mongo=MongoClient(MONGODB) db=mongo.celery tasks=db.tasks mirrors=db.mirrors from .view import *
def do_scheduled(request): """ Get a list of all tasks waiting to be scheduled """ inspect = Inspect(app=celery) return inspect.scheduled()
def get_active(): i = Inspect() active_tasks = i.active() return jsonify(active_tasks)