def send_task(name, args=None, kwargs=None, countdown=None, eta=None, task_id=None, publisher=None, connection=None, connect_timeout=None, result_cls=AsyncResult, expires=None, **options): """Send task by name. Useful if you don't have access to the :class:`~celery.task.base.Task` class. :param name: Name of task to execute. Supports the same arguments as :func:`apply_async`. """ exchange = options.get("exchange") exchange_type = options.get("exchange_type") publish = publisher or TaskPublisher(connection, exchange=exchange, exchange_type=exchange_type) try: task_id = publish.delay_task(name, args, kwargs, task_id=task_id, countdown=countdown, eta=eta, expires=expires, **options) finally: publisher or publish.close() return result_cls(task_id)
def get_publisher(self, connection=None, exchange=None, connect_timeout=conf.BROKER_CONNECTION_TIMEOUT, exchange_type=None): """Get a celery task message publisher. :rtype :class:`celery.messaging.TaskPublisher`: Please be sure to close the AMQP connection when you're done with this object, i.e.: >>> publisher = self.get_publisher() >>> # do something with publisher >>> publisher.connection.close() """ if exchange is None: exchange = self.exchange if exchange_type is None: exchange_type = self.exchange_type connection = connection or self.establish_connection(connect_timeout) return TaskPublisher(connection=connection, exchange=exchange, exchange_type=exchange_type, routing_key=self.routing_key)
def run(self, connect_timeout=conf.AMQP_CONNECTION_TIMEOUT): """Run all tasks in the taskset. :returns: A :class:`celery.result.TaskSetResult` instance. Example >>> ts = TaskSet(RefreshFeedTask, args=[ ... (["http://foo.com/rss"], {}), ... (["http://bar.com/rss"], {}), ... ]) >>> result = ts.run() >>> result.taskset_id "d2c9b261-8eff-4bfb-8459-1e1b72063514" >>> result.subtask_ids ["b4996460-d959-49c8-aeb9-39c530dcde25", "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"] >>> result.waiting() True >>> time.sleep(10) >>> result.ready() True >>> result.successful() True >>> result.failed() False >>> result.join() [True, True] """ taskset_id = gen_unique_id() from celery.conf import ALWAYS_EAGER if ALWAYS_EAGER: subtasks = [apply(self.task, args, kwargs) for args, kwargs in self.arguments] return TaskSetResult(taskset_id, subtasks) conn = DjangoBrokerConnection(connect_timeout=connect_timeout) publisher = TaskPublisher(connection=conn, exchange=self.task.exchange) subtasks = [apply_async(self.task, args, kwargs, taskset_id=taskset_id, publisher=publisher) for args, kwargs in self.arguments] publisher.close() conn.close() return TaskSetResult(taskset_id, subtasks)
def apply_async(self, connection=None, connect_timeout=conf.BROKER_CONNECTION_TIMEOUT): """Run all tasks in the taskset. Returns a :class:`celery.result.TaskSetResult` instance. Example >>> ts = TaskSet(tasks=( ... RefreshFeedTask.subtask(["http://foo.com/rss"]), ... RefreshFeedTask.subtask(["http://bar.com/rss"]), ... )) >>> result = ts.apply_async() >>> result.taskset_id "d2c9b261-8eff-4bfb-8459-1e1b72063514" >>> result.subtask_ids ["b4996460-d959-49c8-aeb9-39c530dcde25", "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"] >>> result.waiting() True >>> time.sleep(10) >>> result.ready() True >>> result.successful() True >>> result.failed() False >>> result.join() [True, True] """ if conf.ALWAYS_EAGER: return self.apply() taskset_id = gen_unique_id() publisher = TaskPublisher(connection=connection) try: results = [ task.apply_async(taskset_id=taskset_id, publisher=publisher) for task in self.tasks ] finally: publisher.close() return TaskSetResult(taskset_id, results)
def apply_async(self, connection=None, connect_timeout=conf.BROKER_CONNECTION_TIMEOUT): """Run all tasks in the taskset. Returns a :class:`celery.result.TaskSetResult` instance. Example >>> ts = TaskSet(tasks=( ... RefreshFeedTask.subtask(["http://foo.com/rss"]), ... RefreshFeedTask.subtask(["http://bar.com/rss"]), ... )) >>> result = ts.apply_async() >>> result.taskset_id "d2c9b261-8eff-4bfb-8459-1e1b72063514" >>> result.subtask_ids ["b4996460-d959-49c8-aeb9-39c530dcde25", "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"] >>> result.waiting() True >>> time.sleep(10) >>> result.ready() True >>> result.successful() True >>> result.failed() False >>> result.join() [True, True] """ if conf.ALWAYS_EAGER: return self.apply() taskset_id = gen_unique_id() publisher = TaskPublisher(connection=connection) try: results = [task.apply_async(taskset_id=taskset_id, publisher=publisher) for task in self.tasks] finally: publisher.close() return TaskSetResult(taskset_id, results)
def run(self): """Run all tasks in the taskset. :returns: A :class:`celery.result.TaskSetResult` instance. Example >>> ts = TaskSet(RefreshFeedTask, [ ... ["http://foo.com/rss", {}], ... ["http://bar.com/rss", {}], ... ) >>> result = ts.run() >>> result.taskset_id "d2c9b261-8eff-4bfb-8459-1e1b72063514" >>> result.subtask_ids ["b4996460-d959-49c8-aeb9-39c530dcde25", "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"] >>> result.waiting() True >>> time.sleep(10) >>> result.ready() True >>> result.successful() True >>> result.failed() False >>> result.join() [True, True] """ taskset_id = str(uuid.uuid4()) conn = DjangoAMQPConnection() publisher = TaskPublisher(connection=conn) subtask_ids = [ publisher.delay_task_in_set( task_name=self.task_name, taskset_id=taskset_id, task_args=arg, task_kwargs=kwarg ) for arg, kwarg in self.arguments ] publisher.close() conn.close() return TaskSetResult(taskset_id, subtask_ids)
def apply_async( task, args=None, kwargs=None, routing_key=None, immediate=None, mandatory=None, connect_timeout=None, priority=None ): """Run a task asynchronously by the celery daemon(s). :param task: The task to run (a callable object, or a :class:`Task` instance :param args: The positional arguments to pass on to the task (a ``list``). :param kwargs: The keyword arguments to pass on to the task (a ``dict``) :keyword routing_key: The routing key used to route the task to a worker server. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword priority: The task priority, a number between ``0`` and ``9``. """ if not args: args = [] if not kwargs: kwargs = [] message_opts = {"routing_key": routing_key, "immediate": immediate, "mandatory": mandatory, "priority": priority} for option_name, option_value in message_opts.items(): message_opts[option_name] = getattr(task, option_name, option_value) conn = DjangoAMQPConnection(connect_timeout=connect_timeout) publisher = TaskPublisher(connection=conn) task_id = publisher.delay_task(task.name, args, kwargs, **message_opts) publisher.close() conn.close() return AsyncResult(task_id)
def _producer(self): connection = establish_connection() publisher = TaskPublisher(connection) inqueue = self.inqueue while 1: task, args, kwargs, options, receipt = inqueue.get() result = task.apply_async(args, kwargs, publisher=publisher, **options) receipt.finished(result)
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None, routing_key=None, exchange=None, immediate=None, mandatory=None, priority=None, connection=None, connect_timeout=AMQP_CONNECTION_TIMEOUT, **opts): """Run a task asynchronously by the celery daemon(s). :param task: The task to run (a callable object, or a :class:`Task` instance :param args: The positional arguments to pass on to the task (a ``list``). :param kwargs: The keyword arguments to pass on to the task (a ``dict``) :param countdown: Number of seconds into the future that the task should execute. Defaults to immediate delivery (Do not confuse that with the ``immediate`` setting, they are unrelated). :param eta: A :class:`datetime.datetime` object that describes the absolute time when the task should execute. May not be specified if ``countdown`` is also supplied. (Do not confuse this with the ``immediate`` setting, they are unrelated). :keyword routing_key: The routing key used to route the task to a worker server. :keyword exchange: The named exchange to send the task to. Defaults to :attr:`celery.task.base.Task.exchange`. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. (Do not confuse this parameter with the ``countdown`` and ``eta`` settings, as they are unrelated). :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. :keyword connection: Re-use existing AMQP connection. The ``connect_timeout`` argument is not respected if this is set. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword priority: The task priority, a number between ``0`` and ``9``. """ args = args or [] kwargs = kwargs or {} routing_key = routing_key or getattr(task, "routing_key", None) exchange = exchange or getattr(task, "exchange", None) immediate = immediate or getattr(task, "immediate", None) mandatory = mandatory or getattr(task, "mandatory", None) priority = priority or getattr(task, "priority", None) taskset_id = opts.get("taskset_id") publisher = opts.get("publisher") if countdown: eta = datetime.now() + timedelta(seconds=countdown) from celery.conf import ALWAYS_EAGER if ALWAYS_EAGER: return apply(task, args, kwargs) need_to_close_connection = False if not publisher: if not connection: connection = DjangoAMQPConnection(connect_timeout=connect_timeout) need_to_close_connection = True publisher = TaskPublisher(connection=connection) delay_task = publisher.delay_task if taskset_id: delay_task = curry(publisher.delay_task_in_set, taskset_id) task_id = delay_task(task.name, args, kwargs, routing_key=routing_key, exchange=exchange, mandatory=mandatory, immediate=immediate, priority=priority, eta=eta) if need_to_close_connection: publisher.close() connection.close() return AsyncResult(task_id)
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None, routing_key=None, exchange=None, task_id=None, immediate=None, mandatory=None, priority=None, connection=None, connect_timeout=AMQP_CONNECTION_TIMEOUT, serializer=None, **opts): """Run a task asynchronously by the celery daemon(s). :param task: The task to run (a callable object, or a :class:`Task` instance :param args: The positional arguments to pass on to the task (a ``list``). :param kwargs: The keyword arguments to pass on to the task (a ``dict``) :param countdown: Number of seconds into the future that the task should execute. Defaults to immediate delivery (Do not confuse that with the ``immediate`` setting, they are unrelated). :param eta: A :class:`datetime.datetime` object that describes the absolute time when the task should execute. May not be specified if ``countdown`` is also supplied. (Do not confuse this with the ``immediate`` setting, they are unrelated). :keyword routing_key: The routing key used to route the task to a worker server. :keyword exchange: The named exchange to send the task to. Defaults to :attr:`celery.task.base.Task.exchange`. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. (Do not confuse this parameter with the ``countdown`` and ``eta`` settings, as they are unrelated). :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. :keyword connection: Re-use existing AMQP connection. The ``connect_timeout`` argument is not respected if this is set. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword priority: The task priority, a number between ``0`` and ``9``. :keyword serializer: A string identifying the default serialization method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting. Can be ``pickle`` ``json``, ``yaml``, or any custom serialization methods that have been registered with :mod:`carrot.serialization.registry`. **Note**: If the ``CELERY_ALWAYS_EAGER`` setting is set, it will be replaced by a local :func:`apply` call instead. """ args = args or [] kwargs = kwargs or {} routing_key = routing_key or getattr(task, "routing_key", None) exchange = exchange or getattr(task, "exchange", None) if immediate is None: immediate = getattr(task, "immediate", None) if mandatory is None: mandatory = getattr(task, "mandatory", None) if priority is None: priority = getattr(task, "priority", None) serializer = serializer or getattr(task, "serializer", None) taskset_id = opts.get("taskset_id") publisher = opts.get("publisher") retries = opts.get("retries", 0) if countdown: eta = datetime.now() + timedelta(seconds=countdown) from celery.conf import ALWAYS_EAGER if ALWAYS_EAGER: return apply(task, args, kwargs) need_to_close_connection = False if not publisher: if not connection: connection = DjangoBrokerConnection( connect_timeout=connect_timeout) need_to_close_connection = True publisher = TaskPublisher(connection=connection) delay_task = publisher.delay_task if taskset_id: delay_task = curry(publisher.delay_task_in_set, taskset_id) task_id = delay_task(task.name, args, kwargs, task_id=task_id, retries=retries, routing_key=routing_key, exchange=exchange, mandatory=mandatory, immediate=immediate, serializer=serializer, priority=priority, eta=eta) if need_to_close_connection: publisher.close() connection.close() return AsyncResult(task_id)