def send_task(name, args=None, kwargs=None, countdown=None, eta=None,
task_id=None, publisher=None, connection=None, connect_timeout=None,
result_cls=AsyncResult, expires=None, **options):
"""Send task by name.
Useful if you don't have access to the :class:`~celery.task.base.Task`
class.
:param name: Name of task to execute.
Supports the same arguments as :func:`apply_async`.
"""
exchange = options.get("exchange")
exchange_type = options.get("exchange_type")
publish = publisher or TaskPublisher(connection, exchange=exchange,
exchange_type=exchange_type)
try:
task_id = publish.delay_task(name, args, kwargs, task_id=task_id,
countdown=countdown, eta=eta,
expires=expires, **options)
finally:
publisher or publish.close()
return result_cls(task_id)
def get_publisher(self,
connection=None,
exchange=None,
connect_timeout=conf.BROKER_CONNECTION_TIMEOUT,
exchange_type=None):
"""Get a celery task message publisher.
:rtype :class:`celery.messaging.TaskPublisher`:
Please be sure to close the AMQP connection when you're done
with this object, i.e.:
>>> publisher = self.get_publisher()
>>> # do something with publisher
>>> publisher.connection.close()
"""
if exchange is None:
exchange = self.exchange
if exchange_type is None:
exchange_type = self.exchange_type
connection = connection or self.establish_connection(connect_timeout)
return TaskPublisher(connection=connection,
exchange=exchange,
exchange_type=exchange_type,
routing_key=self.routing_key)
def run(self, connect_timeout=conf.AMQP_CONNECTION_TIMEOUT):
"""Run all tasks in the taskset.
:returns: A :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(RefreshFeedTask, args=[
... (["http://foo.com/rss"], {}),
... (["http://bar.com/rss"], {}),
... ])
>>> result = ts.run()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
taskset_id = gen_unique_id()
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
subtasks = [apply(self.task, args, kwargs)
for args, kwargs in self.arguments]
return TaskSetResult(taskset_id, subtasks)
conn = DjangoBrokerConnection(connect_timeout=connect_timeout)
publisher = TaskPublisher(connection=conn,
exchange=self.task.exchange)
subtasks = [apply_async(self.task, args, kwargs,
taskset_id=taskset_id, publisher=publisher)
for args, kwargs in self.arguments]
publisher.close()
conn.close()
return TaskSetResult(taskset_id, subtasks)
def apply_async(self,
connection=None,
connect_timeout=conf.BROKER_CONNECTION_TIMEOUT):
"""Run all tasks in the taskset.
Returns a :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(tasks=(
... RefreshFeedTask.subtask(["http://foo.com/rss"]),
... RefreshFeedTask.subtask(["http://bar.com/rss"]),
... ))
>>> result = ts.apply_async()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
if conf.ALWAYS_EAGER:
return self.apply()
taskset_id = gen_unique_id()
publisher = TaskPublisher(connection=connection)
try:
results = [
task.apply_async(taskset_id=taskset_id, publisher=publisher)
for task in self.tasks
]
finally:
publisher.close()
return TaskSetResult(taskset_id, results)
def apply_async(self, connection=None,
connect_timeout=conf.BROKER_CONNECTION_TIMEOUT):
"""Run all tasks in the taskset.
Returns a :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(tasks=(
... RefreshFeedTask.subtask(["http://foo.com/rss"]),
... RefreshFeedTask.subtask(["http://bar.com/rss"]),
... ))
>>> result = ts.apply_async()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
if conf.ALWAYS_EAGER:
return self.apply()
taskset_id = gen_unique_id()
publisher = TaskPublisher(connection=connection)
try:
results = [task.apply_async(taskset_id=taskset_id,
publisher=publisher)
for task in self.tasks]
finally:
publisher.close()
return TaskSetResult(taskset_id, results)
def run(self):
"""Run all tasks in the taskset.
:returns: A :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(RefreshFeedTask, [
... ["http://foo.com/rss", {}],
... ["http://bar.com/rss", {}],
... )
>>> result = ts.run()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
taskset_id = str(uuid.uuid4())
conn = DjangoAMQPConnection()
publisher = TaskPublisher(connection=conn)
subtask_ids = [
publisher.delay_task_in_set(
task_name=self.task_name, taskset_id=taskset_id, task_args=arg, task_kwargs=kwarg
)
for arg, kwarg in self.arguments
]
publisher.close()
conn.close()
return TaskSetResult(taskset_id, subtask_ids)
def apply_async(
task, args=None, kwargs=None, routing_key=None, immediate=None, mandatory=None, connect_timeout=None, priority=None
):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
"""
if not args:
args = []
if not kwargs:
kwargs = []
message_opts = {"routing_key": routing_key, "immediate": immediate, "mandatory": mandatory, "priority": priority}
for option_name, option_value in message_opts.items():
message_opts[option_name] = getattr(task, option_name, option_value)
conn = DjangoAMQPConnection(connect_timeout=connect_timeout)
publisher = TaskPublisher(connection=conn)
task_id = publisher.delay_task(task.name, args, kwargs, **message_opts)
publisher.close()
conn.close()
return AsyncResult(task_id)
def _producer(self):
connection = establish_connection()
publisher = TaskPublisher(connection)
inqueue = self.inqueue
while 1:
task, args, kwargs, options, receipt = inqueue.get()
result = task.apply_async(args,
kwargs,
publisher=publisher,
**options)
receipt.finished(result)
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
routing_key=None, exchange=None,
immediate=None, mandatory=None, priority=None, connection=None,
connect_timeout=AMQP_CONNECTION_TIMEOUT, **opts):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:param countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:param eta: A :class:`datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword exchange: The named exchange to send the task to. Defaults to
:attr:`celery.task.base.Task.exchange`.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated).
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connection: Re-use existing AMQP connection.
The ``connect_timeout`` argument is not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
"""
args = args or []
kwargs = kwargs or {}
routing_key = routing_key or getattr(task, "routing_key", None)
exchange = exchange or getattr(task, "exchange", None)
immediate = immediate or getattr(task, "immediate", None)
mandatory = mandatory or getattr(task, "mandatory", None)
priority = priority or getattr(task, "priority", None)
taskset_id = opts.get("taskset_id")
publisher = opts.get("publisher")
if countdown:
eta = datetime.now() + timedelta(seconds=countdown)
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
return apply(task, args, kwargs)
need_to_close_connection = False
if not publisher:
if not connection:
connection = DjangoAMQPConnection(connect_timeout=connect_timeout)
need_to_close_connection = True
publisher = TaskPublisher(connection=connection)
delay_task = publisher.delay_task
if taskset_id:
delay_task = curry(publisher.delay_task_in_set, taskset_id)
task_id = delay_task(task.name, args, kwargs,
routing_key=routing_key, exchange=exchange,
mandatory=mandatory, immediate=immediate,
priority=priority, eta=eta)
if need_to_close_connection:
publisher.close()
connection.close()
return AsyncResult(task_id)
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
routing_key=None, exchange=None, task_id=None,
immediate=None, mandatory=None, priority=None, connection=None,
connect_timeout=AMQP_CONNECTION_TIMEOUT, serializer=None, **opts):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:param countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:param eta: A :class:`datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword exchange: The named exchange to send the task to. Defaults to
:attr:`celery.task.base.Task.exchange`.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated).
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connection: Re-use existing AMQP connection.
The ``connect_timeout`` argument is not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
:keyword serializer: A string identifying the default serialization
method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting.
Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
methods that have been registered with
:mod:`carrot.serialization.registry`.
**Note**: If the ``CELERY_ALWAYS_EAGER`` setting is set, it will be
replaced by a local :func:`apply` call instead.
"""
args = args or []
kwargs = kwargs or {}
routing_key = routing_key or getattr(task, "routing_key", None)
exchange = exchange or getattr(task, "exchange", None)
if immediate is None:
immediate = getattr(task, "immediate", None)
if mandatory is None:
mandatory = getattr(task, "mandatory", None)
if priority is None:
priority = getattr(task, "priority", None)
serializer = serializer or getattr(task, "serializer", None)
taskset_id = opts.get("taskset_id")
publisher = opts.get("publisher")
retries = opts.get("retries", 0)
if countdown:
eta = datetime.now() + timedelta(seconds=countdown)
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
return apply(task, args, kwargs)
need_to_close_connection = False
if not publisher:
if not connection:
connection = DjangoBrokerConnection(
connect_timeout=connect_timeout)
need_to_close_connection = True
publisher = TaskPublisher(connection=connection)
delay_task = publisher.delay_task
if taskset_id:
delay_task = curry(publisher.delay_task_in_set, taskset_id)
task_id = delay_task(task.name, args, kwargs,
task_id=task_id, retries=retries,
routing_key=routing_key, exchange=exchange,
mandatory=mandatory, immediate=immediate,
serializer=serializer, priority=priority,
eta=eta)
if need_to_close_connection:
publisher.close()
connection.close()
return AsyncResult(task_id)
