def get_consumer_set(connection, queues=None, **options):
"""Get the :class:`carrot.messaging.ConsumerSet`` for a queue
configuration.
Defaults to the queues in ``CELERY_QUEUES``.
"""
queues = queues or conf.get_queues()
cset = ConsumerSet(connection)
for queue_name, queue_options in queues.items():
queue_options = dict(queue_options)
queue_options["routing_key"] = queue_options.pop("binding_key", None)
consumer = Consumer(connection, queue=queue_name, backend=cset.backend, **queue_options)
cset.consumers.append(consumer)
return cset
def get_consumer_set(connection, queues=None, **options):
"""Get the :class:`carrot.messaging.ConsumerSet`` for a queue
configuration.
Defaults to the queues in :const:`CELERY_QUEUES`.
"""
queues = queues or conf.get_queues()
cset = ConsumerSet(connection)
for queue_name, queue_options in queues.items():
queue_options = dict(queue_options)
queue_options["routing_key"] = queue_options.pop("binding_key", None)
consumer = Consumer(connection,
queue=queue_name,
backend=cset.backend,
**queue_options)
cset.consumers.append(consumer)
return cset
def startup_info(self):
tasklist = ""
if self.loglevel <= logging.INFO:
include_builtins = self.loglevel <= logging.DEBUG
tasklist = self.tasklist(include_builtins=include_builtins)
queues = conf.get_queues()
return STARTUP_INFO_FMT % {
"conninfo": info.format_broker_info(),
"queues": info.format_queues(queues, indent=8),
"concurrency": self.concurrency,
"loglevel": conf.LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": self.run_clockservice and "ON" or "OFF",
"events": self.events and "ON" or "OFF",
"tasks": tasklist,
"loader": get_full_cls_name(self.loader.__class__),
}
def startup_info(self):
tasklist = ""
if self.loglevel <= logging.INFO:
include_builtins = self.loglevel <= logging.DEBUG
tasklist = self.tasklist(include_builtins=include_builtins)
queues = conf.get_queues()
return STARTUP_INFO_FMT % {
"conninfo": info.format_broker_info(),
"queues": info.format_queues(queues, indent=8),
"concurrency": self.concurrency,
"loglevel": conf.LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": self.run_clockservice and "ON" or "OFF",
"events": self.events and "ON" or "OFF",
"tasks": tasklist,
"loader": get_full_cls_name(self.loader.__class__),
}
def apply_async(task,
args=None,
kwargs=None,
countdown=None,
eta=None,
task_id=None,
publisher=None,
connection=None,
connect_timeout=None,
router=None,
expires=None,
**options):
"""Run a task asynchronously by the celery daemon(s).
:param task: The :class:`~celery.task.base.Task` to run.
:keyword args: The positional arguments to pass on to the
task (a :class:`list` or :class:`tuple`).
:keyword kwargs: The keyword arguments to pass on to the
task (a :class:`dict`)
:keyword countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:keyword eta: A :class:`~datetime.datetime` object that describes the
absolute time and date of when the task should execute. May not be
specified if ``countdown`` is also supplied. (Do not confuse this
with the ``immediate`` setting, they are unrelated).
:keyword expires: Either a :class:`int`, describing the number of seconds,
or a :class:`~datetime.datetime` object that describes the absolute time
and date of when the task should expire.
The task will not be executed after the expiration time.
:keyword connection: Re-use existing broker connection instead
of establishing a new one. The ``connect_timeout`` argument is
not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword routing_key: The routing key used to route the task to a worker
server. Defaults to the tasks :attr:`~celery.task.base.Task.exchange`
attribute.
:keyword exchange: The named exchange to send the task to. Defaults to
the tasks :attr:`~celery.task.base.Task.exchange` attribute.
:keyword exchange_type: The exchange type to initalize the exchange as
if not already declared. Defaults to the tasks
:attr:`~celery.task.base.Task.exchange_type` attribute.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated). Defaults to the tasks
:attr:`~celery.task.base.Task.immediate` attribute.
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task. Defaults to the tasks
:attr:`~celery.task.base.Task.mandatory` attribute.
:keyword priority: The task priority, a number between ``0`` and ``9``.
Defaults to the tasks :attr:`~celery.task.base.Task.priority` attribute.
:keyword serializer: A string identifying the default serialization
method to use. Defaults to the :setting:`CELERY_TASK_SERIALIZER` setting.
Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
methods that have been registered with
:mod:`carrot.serialization.registry`. Defaults to the tasks
:attr:`~celery.task.base.Task.serializer` attribute.
**Note**: If the :setting:`CELERY_ALWAYS_EAGER` setting is set, it will be
replaced by a local :func:`apply` call instead.
"""
router = router or Router(conf.ROUTES, conf.get_queues(),
conf.CREATE_MISSING_QUEUES)
if conf.ALWAYS_EAGER:
return apply(task, args, kwargs, task_id=task_id)
task = tasks[task.name] # get instance from registry
options = dict(extract_exec_options(task), **options)
options = router.route(options, task.name, args, kwargs)
exchange = options.get("exchange")
exchange_type = options.get("exchange_type")
expires = expires or task.expires
publish = publisher or task.get_publisher(
connection, exchange=exchange, exchange_type=exchange_type)
try:
task_id = publish.delay_task(task.name,
args,
kwargs,
task_id=task_id,
countdown=countdown,
eta=eta,
expires=expires,
**options)
finally:
publisher or publish.close()
return task.AsyncResult(task_id)
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
task_id=None, publisher=None, connection=None, connect_timeout=None,
router=None, **options):
"""Run a task asynchronously by the celery daemon(s).
:param task: The :class:`~celery.task.base.Task` to run.
:keyword args: The positional arguments to pass on to the
task (a :class:`list` or :class:`tuple`).
:keyword kwargs: The keyword arguments to pass on to the
task (a :class:`dict`)
:keyword countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:keyword eta: A :class:`~datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword connection: Re-use existing broker connection instead
of establishing a new one. The ``connect_timeout`` argument is
not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword routing_key: The routing key used to route the task to a worker
server. Defaults to the tasks :attr:`~celery.task.base.Task.exchange`
attribute.
:keyword exchange: The named exchange to send the task to. Defaults to
the tasks :attr:`~celery.task.base.Task.exchange` attribute.
:keyword exchange_type: The exchange type to initalize the exchange as
if not already declared. Defaults to the tasks
:attr:`~celery.task.base.Task.exchange_type` attribute.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated). Defaults to the tasks
:attr:`~celery.task.base.Task.immediate` attribute.
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task. Defaults to the tasks
:attr:`~celery.task.base.Task.mandatory` attribute.
:keyword priority: The task priority, a number between ``0`` and ``9``.
Defaults to the tasks :attr:`~celery.task.base.Task.priority` attribute.
:keyword serializer: A string identifying the default serialization
method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting.
Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
methods that have been registered with
:mod:`carrot.serialization.registry`. Defaults to the tasks
:attr:`~celery.task.base.Task.serializer` attribute.
**Note**: If the ``CELERY_ALWAYS_EAGER`` setting is set, it will be
replaced by a local :func:`apply` call instead.
"""
router = router or Router(conf.ROUTES, conf.get_queues())
if conf.ALWAYS_EAGER:
return apply(task, args, kwargs, task_id=task_id)
task = tasks[task.name] # get instance from registry
options = dict(extract_exec_options(task), **options)
options = router.route(options, task.name, args, kwargs)
exchange = options.get("exchange")
exchange_type = options.get("exchange_type")
publish = publisher or task.get_publisher(connection, exchange=exchange,
exchange_type=exchange_type)
try:
task_id = publish.delay_task(task.name, args, kwargs, task_id=task_id,
countdown=countdown, eta=eta, **options)
finally:
publisher or publish.close()
return task.AsyncResult(task_id)
from itertools import count
from carrot.connection import BrokerConnection
from carrot.messaging import Publisher, Consumer, ConsumerSet as _ConsumerSet
from celery import conf
from celery import signals
from celery.utils import gen_unique_id, mitemgetter, noop
from celery.utils.functional import wraps
MSG_OPTIONS = ("mandatory", "priority", "immediate", "routing_key", "serializer", "delivery_mode")
get_msg_options = mitemgetter(*MSG_OPTIONS)
extract_msg_options = lambda d: dict(zip(MSG_OPTIONS, get_msg_options(d)))
default_queue = conf.get_queues()[conf.DEFAULT_QUEUE]
_queues_declared = False
_exchanges_declared = set()
class TaskPublisher(Publisher):
"""Publish tasks."""
exchange = default_queue["exchange"]
exchange_type = default_queue["exchange_type"]
routing_key = conf.DEFAULT_ROUTING_KEY
serializer = conf.TASK_SERIALIZER
auto_declare = False
def __init__(self, *args, **kwargs):
from itertools import count
from carrot.connection import BrokerConnection
from carrot.messaging import Publisher, Consumer, ConsumerSet as _ConsumerSet
from celery import conf
from celery import signals
from celery.utils import gen_unique_id, mitemgetter, noop
from celery.utils.functional import wraps
MSG_OPTIONS = ("mandatory", "priority", "immediate", "routing_key",
"serializer", "delivery_mode")
get_msg_options = mitemgetter(*MSG_OPTIONS)
extract_msg_options = lambda d: dict(zip(MSG_OPTIONS, get_msg_options(d)))
default_queue = conf.get_queues()[conf.DEFAULT_QUEUE]
_queues_declared = False
_exchanges_declared = set()
class TaskPublisher(Publisher):
"""Publish tasks."""
exchange = default_queue["exchange"]
exchange_type = default_queue["exchange_type"]
routing_key = conf.DEFAULT_ROUTING_KEY
serializer = conf.TASK_SERIALIZER
auto_declare = False
def __init__(self, *args, **kwargs):
super(TaskPublisher, self).__init__(*args, **kwargs)
