def get_consumer_set(connection, queues=None, **options): """Get the :class:`carrot.messaging.ConsumerSet`` for a queue configuration. Defaults to the queues in ``CELERY_QUEUES``. """ queues = queues or conf.get_queues() cset = ConsumerSet(connection) for queue_name, queue_options in queues.items(): queue_options = dict(queue_options) queue_options["routing_key"] = queue_options.pop("binding_key", None) consumer = Consumer(connection, queue=queue_name, backend=cset.backend, **queue_options) cset.consumers.append(consumer) return cset
def get_consumer_set(connection, queues=None, **options): """Get the :class:`carrot.messaging.ConsumerSet`` for a queue configuration. Defaults to the queues in :const:`CELERY_QUEUES`. """ queues = queues or conf.get_queues() cset = ConsumerSet(connection) for queue_name, queue_options in queues.items(): queue_options = dict(queue_options) queue_options["routing_key"] = queue_options.pop("binding_key", None) consumer = Consumer(connection, queue=queue_name, backend=cset.backend, **queue_options) cset.consumers.append(consumer) return cset
def startup_info(self): tasklist = "" if self.loglevel <= logging.INFO: include_builtins = self.loglevel <= logging.DEBUG tasklist = self.tasklist(include_builtins=include_builtins) queues = conf.get_queues() return STARTUP_INFO_FMT % { "conninfo": info.format_broker_info(), "queues": info.format_queues(queues, indent=8), "concurrency": self.concurrency, "loglevel": conf.LOG_LEVELS[self.loglevel], "logfile": self.logfile or "[stderr]", "celerybeat": self.run_clockservice and "ON" or "OFF", "events": self.events and "ON" or "OFF", "tasks": tasklist, "loader": get_full_cls_name(self.loader.__class__), }
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None, task_id=None, publisher=None, connection=None, connect_timeout=None, router=None, expires=None, **options): """Run a task asynchronously by the celery daemon(s). :param task: The :class:`~celery.task.base.Task` to run. :keyword args: The positional arguments to pass on to the task (a :class:`list` or :class:`tuple`). :keyword kwargs: The keyword arguments to pass on to the task (a :class:`dict`) :keyword countdown: Number of seconds into the future that the task should execute. Defaults to immediate delivery (Do not confuse that with the ``immediate`` setting, they are unrelated). :keyword eta: A :class:`~datetime.datetime` object that describes the absolute time and date of when the task should execute. May not be specified if ``countdown`` is also supplied. (Do not confuse this with the ``immediate`` setting, they are unrelated). :keyword expires: Either a :class:`int`, describing the number of seconds, or a :class:`~datetime.datetime` object that describes the absolute time and date of when the task should expire. The task will not be executed after the expiration time. :keyword connection: Re-use existing broker connection instead of establishing a new one. The ``connect_timeout`` argument is not respected if this is set. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword routing_key: The routing key used to route the task to a worker server. Defaults to the tasks :attr:`~celery.task.base.Task.exchange` attribute. :keyword exchange: The named exchange to send the task to. Defaults to the tasks :attr:`~celery.task.base.Task.exchange` attribute. :keyword exchange_type: The exchange type to initalize the exchange as if not already declared. Defaults to the tasks :attr:`~celery.task.base.Task.exchange_type` attribute. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. (Do not confuse this parameter with the ``countdown`` and ``eta`` settings, as they are unrelated). Defaults to the tasks :attr:`~celery.task.base.Task.immediate` attribute. :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. Defaults to the tasks :attr:`~celery.task.base.Task.mandatory` attribute. :keyword priority: The task priority, a number between ``0`` and ``9``. Defaults to the tasks :attr:`~celery.task.base.Task.priority` attribute. :keyword serializer: A string identifying the default serialization method to use. Defaults to the :setting:`CELERY_TASK_SERIALIZER` setting. Can be ``pickle`` ``json``, ``yaml``, or any custom serialization methods that have been registered with :mod:`carrot.serialization.registry`. Defaults to the tasks :attr:`~celery.task.base.Task.serializer` attribute. **Note**: If the :setting:`CELERY_ALWAYS_EAGER` setting is set, it will be replaced by a local :func:`apply` call instead. """ router = router or Router(conf.ROUTES, conf.get_queues(), conf.CREATE_MISSING_QUEUES) if conf.ALWAYS_EAGER: return apply(task, args, kwargs, task_id=task_id) task = tasks[task.name] # get instance from registry options = dict(extract_exec_options(task), **options) options = router.route(options, task.name, args, kwargs) exchange = options.get("exchange") exchange_type = options.get("exchange_type") expires = expires or task.expires publish = publisher or task.get_publisher( connection, exchange=exchange, exchange_type=exchange_type) try: task_id = publish.delay_task(task.name, args, kwargs, task_id=task_id, countdown=countdown, eta=eta, expires=expires, **options) finally: publisher or publish.close() return task.AsyncResult(task_id)
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None, task_id=None, publisher=None, connection=None, connect_timeout=None, router=None, **options): """Run a task asynchronously by the celery daemon(s). :param task: The :class:`~celery.task.base.Task` to run. :keyword args: The positional arguments to pass on to the task (a :class:`list` or :class:`tuple`). :keyword kwargs: The keyword arguments to pass on to the task (a :class:`dict`) :keyword countdown: Number of seconds into the future that the task should execute. Defaults to immediate delivery (Do not confuse that with the ``immediate`` setting, they are unrelated). :keyword eta: A :class:`~datetime.datetime` object that describes the absolute time when the task should execute. May not be specified if ``countdown`` is also supplied. (Do not confuse this with the ``immediate`` setting, they are unrelated). :keyword connection: Re-use existing broker connection instead of establishing a new one. The ``connect_timeout`` argument is not respected if this is set. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword routing_key: The routing key used to route the task to a worker server. Defaults to the tasks :attr:`~celery.task.base.Task.exchange` attribute. :keyword exchange: The named exchange to send the task to. Defaults to the tasks :attr:`~celery.task.base.Task.exchange` attribute. :keyword exchange_type: The exchange type to initalize the exchange as if not already declared. Defaults to the tasks :attr:`~celery.task.base.Task.exchange_type` attribute. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. (Do not confuse this parameter with the ``countdown`` and ``eta`` settings, as they are unrelated). Defaults to the tasks :attr:`~celery.task.base.Task.immediate` attribute. :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. Defaults to the tasks :attr:`~celery.task.base.Task.mandatory` attribute. :keyword priority: The task priority, a number between ``0`` and ``9``. Defaults to the tasks :attr:`~celery.task.base.Task.priority` attribute. :keyword serializer: A string identifying the default serialization method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting. Can be ``pickle`` ``json``, ``yaml``, or any custom serialization methods that have been registered with :mod:`carrot.serialization.registry`. Defaults to the tasks :attr:`~celery.task.base.Task.serializer` attribute. **Note**: If the ``CELERY_ALWAYS_EAGER`` setting is set, it will be replaced by a local :func:`apply` call instead. """ router = router or Router(conf.ROUTES, conf.get_queues()) if conf.ALWAYS_EAGER: return apply(task, args, kwargs, task_id=task_id) task = tasks[task.name] # get instance from registry options = dict(extract_exec_options(task), **options) options = router.route(options, task.name, args, kwargs) exchange = options.get("exchange") exchange_type = options.get("exchange_type") publish = publisher or task.get_publisher(connection, exchange=exchange, exchange_type=exchange_type) try: task_id = publish.delay_task(task.name, args, kwargs, task_id=task_id, countdown=countdown, eta=eta, **options) finally: publisher or publish.close() return task.AsyncResult(task_id)
from itertools import count from carrot.connection import BrokerConnection from carrot.messaging import Publisher, Consumer, ConsumerSet as _ConsumerSet from celery import conf from celery import signals from celery.utils import gen_unique_id, mitemgetter, noop from celery.utils.functional import wraps MSG_OPTIONS = ("mandatory", "priority", "immediate", "routing_key", "serializer", "delivery_mode") get_msg_options = mitemgetter(*MSG_OPTIONS) extract_msg_options = lambda d: dict(zip(MSG_OPTIONS, get_msg_options(d))) default_queue = conf.get_queues()[conf.DEFAULT_QUEUE] _queues_declared = False _exchanges_declared = set() class TaskPublisher(Publisher): """Publish tasks.""" exchange = default_queue["exchange"] exchange_type = default_queue["exchange_type"] routing_key = conf.DEFAULT_ROUTING_KEY serializer = conf.TASK_SERIALIZER auto_declare = False def __init__(self, *args, **kwargs):
from itertools import count from carrot.connection import BrokerConnection from carrot.messaging import Publisher, Consumer, ConsumerSet as _ConsumerSet from celery import conf from celery import signals from celery.utils import gen_unique_id, mitemgetter, noop from celery.utils.functional import wraps MSG_OPTIONS = ("mandatory", "priority", "immediate", "routing_key", "serializer", "delivery_mode") get_msg_options = mitemgetter(*MSG_OPTIONS) extract_msg_options = lambda d: dict(zip(MSG_OPTIONS, get_msg_options(d))) default_queue = conf.get_queues()[conf.DEFAULT_QUEUE] _queues_declared = False _exchanges_declared = set() class TaskPublisher(Publisher): """Publish tasks.""" exchange = default_queue["exchange"] exchange_type = default_queue["exchange_type"] routing_key = conf.DEFAULT_ROUTING_KEY serializer = conf.TASK_SERIALIZER auto_declare = False def __init__(self, *args, **kwargs): super(TaskPublisher, self).__init__(*args, **kwargs)