def fun_takes_kwargs(fun, kwlist=[]):
"""With a function, and a list of keyword arguments, returns arguments
in the list which the function takes.
If the object has an ``argspec`` attribute that is used instead
of using the :meth:`inspect.getargspec`` introspection.
:param fun: The function to inspect arguments of.
:param kwlist: The list of keyword arguments.
Examples
>>> def foo(self, x, y, logfile=None, loglevel=None):
... return x * y
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel"]
>>> def foo(self, x, y, **kwargs):
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel", "task_id"]
"""
argspec = getattr(fun, "argspec", getargspec(fun))
args, _varargs, keywords, _defaults = argspec
if keywords != None:
return kwlist
return filter(curry(operator.contains, args), kwlist)
def test_apply_raises_404_on_unregistered_task(self):
conf.ALWAYS_EAGER = True
try:
name = "xxx.does.not.exist"
action = curry(self.client.get, task_apply(kwargs={
"task_name": name}) + "?x=4&y=4")
self.assertRaises(TemplateDoesNotExist, action)
finally:
conf.ALWAYS_EAGER = False
def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, on_ack=noop):
"""Equivalent of the :func:``apply`` built-in function.
All ``callbacks`` and ``errbacks`` should complete immediately since
otherwise the thread which handles the result will get blocked.
"""
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks, on_ack)
self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
self.replace_dead_workers()
return self._pool.apply_async(target, args, kwargs,
callback=on_ready)
from anyjson import deserialize as JSON_load
from billiard.utils.functional import curry
from celery import conf
from celery import states
from celery.utils import gen_unique_id, get_full_cls_name
from celery.backends import default_backend
from celery.exceptions import RetryTaskError
from celery.decorators import task
from celery.datastructures import ExceptionInfo
def reversestar(name, **kwargs):
return reverse(name, kwargs=kwargs)
task_is_successful = curry(reversestar, "celery-is_task_successful")
task_status = curry(reversestar, "celery-task_status")
task_apply = curry(reverse, "celery-apply")
scratch = {}
@task()
def mytask(x, y):
ret = scratch["result"] = int(x) * int(y)
return ret
def create_exception(name, base=Exception):
return type(name, (base, ), {})
def catch_exception(exception):
def get_backend_cls(backend):
"""Get backend class by name/alias"""
if backend not in _backend_cache:
_backend_cache[backend] = get_cls_by_name(backend, BACKEND_ALIASES)
return _backend_cache[backend]
"""
.. function:: get_default_backend_cls()
Get the backend class specified in :setting:`CELERY_RESULT_BACKEND`.
"""
get_default_backend_cls = curry(get_backend_cls, conf.RESULT_BACKEND)
"""
.. class:: DefaultBackend
The default backend class used for storing task results and status,
specified in :setting:`CELERY_RESULT_BACKEND`.
"""
DefaultBackend = get_default_backend_cls()
"""
.. data:: default_backend
An instance of :class:`DefaultBackend`.
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
routing_key=None, exchange=None, task_id=None,
immediate=None, mandatory=None, priority=None, connection=None,
connect_timeout=AMQP_CONNECTION_TIMEOUT, serializer=None, **opts):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:param countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:param eta: A :class:`datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword exchange: The named exchange to send the task to. Defaults to
:attr:`celery.task.base.Task.exchange`.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated).
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connection: Re-use existing AMQP connection.
The ``connect_timeout`` argument is not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
:keyword serializer: A string identifying the default serialization
method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting.
Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
methods that have been registered with
:mod:`carrot.serialization.registry`.
**Note**: If the ``CELERY_ALWAYS_EAGER`` setting is set, it will be
replaced by a local :func:`apply` call instead.
"""
args = args or []
kwargs = kwargs or {}
routing_key = routing_key or getattr(task, "routing_key", None)
exchange = exchange or getattr(task, "exchange", None)
if immediate is None:
immediate = getattr(task, "immediate", None)
if mandatory is None:
mandatory = getattr(task, "mandatory", None)
if priority is None:
priority = getattr(task, "priority", None)
serializer = serializer or getattr(task, "serializer", None)
taskset_id = opts.get("taskset_id")
publisher = opts.get("publisher")
retries = opts.get("retries", 0)
if countdown:
eta = datetime.now() + timedelta(seconds=countdown)
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
return apply(task, args, kwargs)
need_to_close_connection = False
if not publisher:
if not connection:
connection = DjangoBrokerConnection(
connect_timeout=connect_timeout)
need_to_close_connection = True
publisher = TaskPublisher(connection=connection)
delay_task = publisher.delay_task
if taskset_id:
delay_task = curry(publisher.delay_task_in_set, taskset_id)
task_id = delay_task(task.name, args, kwargs,
task_id=task_id, retries=retries,
routing_key=routing_key, exchange=exchange,
mandatory=mandatory, immediate=immediate,
serializer=serializer, priority=priority,
eta=eta)
if need_to_close_connection:
publisher.close()
connection.close()
return AsyncResult(task_id)
import os
import sys
sys.path.insert(0, os.getcwd())
import time
import unittest
from itertools import chain, izip
from billiard.utils.functional import curry
from celery.task.base import Task
from celery.utils import gen_unique_id
from celery.tests.utils import skip_if_environ
from celery.worker import buckets
from celery.registry import TaskRegistry
skip_if_disabled = curry(skip_if_environ("SKIP_RLIMITS"))
class MockJob(object):
def __init__(self, task_id, task_name, args, kwargs):
self.task_id = task_id
self.task_name = task_name
self.args = args
self.kwargs = kwargs
def __eq__(self, other):
if isinstance(other, self.__class__):
return bool(self.task_id == other.task_id \
and self.task_name == other.task_name \
and self.args == other.args \
def get_backend_cls(backend):
"""Get backend class by name/alias"""
if backend not in _backend_cache:
_backend_cache[backend] = _get_backend_cls(backend)
return _backend_cache[backend]
"""
.. function:: get_default_backend_cls()
Get the backend class specified in :setting:`CELERY_BACKEND`.
"""
get_default_backend_cls = curry(get_backend_cls, conf.CELERY_BACKEND)
"""
.. class:: DefaultBackend
The default backend class used for storing task results and status,
specified in :setting:`CELERY_BACKEND`.
"""
DefaultBackend = get_default_backend_cls()
"""
.. data:: default_backend
An instance of :class:`DefaultBackend`.
