def fun_takes_kwargs(fun, kwlist=[]): """With a function, and a list of keyword arguments, returns arguments in the list which the function takes. If the object has an ``argspec`` attribute that is used instead of using the :meth:`inspect.getargspec`` introspection. :param fun: The function to inspect arguments of. :param kwlist: The list of keyword arguments. Examples >>> def foo(self, x, y, logfile=None, loglevel=None): ... return x * y >>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"]) ["logfile", "loglevel"] >>> def foo(self, x, y, **kwargs): >>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"]) ["logfile", "loglevel", "task_id"] """ argspec = getattr(fun, "argspec", getargspec(fun)) args, _varargs, keywords, _defaults = argspec if keywords != None: return kwlist return filter(curry(operator.contains, args), kwlist)
def test_apply_raises_404_on_unregistered_task(self): conf.ALWAYS_EAGER = True try: name = "xxx.does.not.exist" action = curry(self.client.get, task_apply(kwargs={ "task_name": name}) + "?x=4&y=4") self.assertRaises(TemplateDoesNotExist, action) finally: conf.ALWAYS_EAGER = False
def apply_async(self, target, args=None, kwargs=None, callbacks=None, errbacks=None, on_ack=noop): """Equivalent of the :func:``apply`` built-in function. All ``callbacks`` and ``errbacks`` should complete immediately since otherwise the thread which handles the result will get blocked. """ args = args or [] kwargs = kwargs or {} callbacks = callbacks or [] errbacks = errbacks or [] on_ready = curry(self.on_ready, callbacks, errbacks, on_ack) self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % ( target, args, kwargs)) self.replace_dead_workers() return self._pool.apply_async(target, args, kwargs, callback=on_ready)
from anyjson import deserialize as JSON_load from billiard.utils.functional import curry from celery import conf from celery import states from celery.utils import gen_unique_id, get_full_cls_name from celery.backends import default_backend from celery.exceptions import RetryTaskError from celery.decorators import task from celery.datastructures import ExceptionInfo def reversestar(name, **kwargs): return reverse(name, kwargs=kwargs) task_is_successful = curry(reversestar, "celery-is_task_successful") task_status = curry(reversestar, "celery-task_status") task_apply = curry(reverse, "celery-apply") scratch = {} @task() def mytask(x, y): ret = scratch["result"] = int(x) * int(y) return ret def create_exception(name, base=Exception): return type(name, (base, ), {}) def catch_exception(exception):
def get_backend_cls(backend): """Get backend class by name/alias""" if backend not in _backend_cache: _backend_cache[backend] = get_cls_by_name(backend, BACKEND_ALIASES) return _backend_cache[backend] """ .. function:: get_default_backend_cls() Get the backend class specified in :setting:`CELERY_RESULT_BACKEND`. """ get_default_backend_cls = curry(get_backend_cls, conf.RESULT_BACKEND) """ .. class:: DefaultBackend The default backend class used for storing task results and status, specified in :setting:`CELERY_RESULT_BACKEND`. """ DefaultBackend = get_default_backend_cls() """ .. data:: default_backend An instance of :class:`DefaultBackend`.
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None, routing_key=None, exchange=None, task_id=None, immediate=None, mandatory=None, priority=None, connection=None, connect_timeout=AMQP_CONNECTION_TIMEOUT, serializer=None, **opts): """Run a task asynchronously by the celery daemon(s). :param task: The task to run (a callable object, or a :class:`Task` instance :param args: The positional arguments to pass on to the task (a ``list``). :param kwargs: The keyword arguments to pass on to the task (a ``dict``) :param countdown: Number of seconds into the future that the task should execute. Defaults to immediate delivery (Do not confuse that with the ``immediate`` setting, they are unrelated). :param eta: A :class:`datetime.datetime` object that describes the absolute time when the task should execute. May not be specified if ``countdown`` is also supplied. (Do not confuse this with the ``immediate`` setting, they are unrelated). :keyword routing_key: The routing key used to route the task to a worker server. :keyword exchange: The named exchange to send the task to. Defaults to :attr:`celery.task.base.Task.exchange`. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. (Do not confuse this parameter with the ``countdown`` and ``eta`` settings, as they are unrelated). :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. :keyword connection: Re-use existing AMQP connection. The ``connect_timeout`` argument is not respected if this is set. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword priority: The task priority, a number between ``0`` and ``9``. :keyword serializer: A string identifying the default serialization method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting. Can be ``pickle`` ``json``, ``yaml``, or any custom serialization methods that have been registered with :mod:`carrot.serialization.registry`. **Note**: If the ``CELERY_ALWAYS_EAGER`` setting is set, it will be replaced by a local :func:`apply` call instead. """ args = args or [] kwargs = kwargs or {} routing_key = routing_key or getattr(task, "routing_key", None) exchange = exchange or getattr(task, "exchange", None) if immediate is None: immediate = getattr(task, "immediate", None) if mandatory is None: mandatory = getattr(task, "mandatory", None) if priority is None: priority = getattr(task, "priority", None) serializer = serializer or getattr(task, "serializer", None) taskset_id = opts.get("taskset_id") publisher = opts.get("publisher") retries = opts.get("retries", 0) if countdown: eta = datetime.now() + timedelta(seconds=countdown) from celery.conf import ALWAYS_EAGER if ALWAYS_EAGER: return apply(task, args, kwargs) need_to_close_connection = False if not publisher: if not connection: connection = DjangoBrokerConnection( connect_timeout=connect_timeout) need_to_close_connection = True publisher = TaskPublisher(connection=connection) delay_task = publisher.delay_task if taskset_id: delay_task = curry(publisher.delay_task_in_set, taskset_id) task_id = delay_task(task.name, args, kwargs, task_id=task_id, retries=retries, routing_key=routing_key, exchange=exchange, mandatory=mandatory, immediate=immediate, serializer=serializer, priority=priority, eta=eta) if need_to_close_connection: publisher.close() connection.close() return AsyncResult(task_id)
import os import sys sys.path.insert(0, os.getcwd()) import time import unittest from itertools import chain, izip from billiard.utils.functional import curry from celery.task.base import Task from celery.utils import gen_unique_id from celery.tests.utils import skip_if_environ from celery.worker import buckets from celery.registry import TaskRegistry skip_if_disabled = curry(skip_if_environ("SKIP_RLIMITS")) class MockJob(object): def __init__(self, task_id, task_name, args, kwargs): self.task_id = task_id self.task_name = task_name self.args = args self.kwargs = kwargs def __eq__(self, other): if isinstance(other, self.__class__): return bool(self.task_id == other.task_id \ and self.task_name == other.task_name \ and self.args == other.args \
def get_backend_cls(backend): """Get backend class by name/alias""" if backend not in _backend_cache: _backend_cache[backend] = _get_backend_cls(backend) return _backend_cache[backend] """ .. function:: get_default_backend_cls() Get the backend class specified in :setting:`CELERY_BACKEND`. """ get_default_backend_cls = curry(get_backend_cls, conf.CELERY_BACKEND) """ .. class:: DefaultBackend The default backend class used for storing task results and status, specified in :setting:`CELERY_BACKEND`. """ DefaultBackend = get_default_backend_cls() """ .. data:: default_backend An instance of :class:`DefaultBackend`.