def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, accept_callback=None, timeout_callback=None,
**compat):
"""Equivalent of the :func:``apply`` built-in function.
All ``callbacks`` and ``errbacks`` should complete immediately since
otherwise the thread which handles the result will get blocked.
"""
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks)
on_worker_error = curry(self.on_worker_error, errbacks)
self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
return self._pool.apply_async(target, args, kwargs,
callback=on_ready,
accept_callback=accept_callback,
timeout_callback=timeout_callback,
error_callback=on_worker_error,
waitforslot=self.putlocks)
def apply_async(self,
target,
args=None,
kwargs=None,
callbacks=None,
errbacks=None,
accept_callback=None,
timeout_callback=None,
**compat):
"""Equivalent of the :func:``apply`` built-in function.
All ``callbacks`` and ``errbacks`` should complete immediately since
otherwise the thread which handles the result will get blocked.
"""
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks)
on_worker_error = curry(self.on_worker_error, errbacks)
self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" %
(target, args, kwargs))
return self._pool.apply_async(target,
args,
kwargs,
callback=on_ready,
accept_callback=accept_callback,
timeout_callback=timeout_callback,
error_callback=on_worker_error,
waitforslot=self.putlocks)
def find_nearest_pickleable_exception(exc):
"""With an exception instance, iterate over its super classes (by mro)
and find the first super exception that is pickleable. It does
not go below :exc:`Exception` (i.e. it skips :exc:`Exception`,
:class:`BaseException` and :class:`object`). If that happens
you should use :exc:`UnpickleableException` instead.
:param exc: An exception instance.
:returns: the nearest exception if it's not :exc:`Exception` or below,
if it is it returns ``None``.
:rtype: :exc:`Exception`
"""
unwanted = (Exception, BaseException, object)
is_unwanted = lambda exc: any(map(curry(operator.is_, exc), unwanted))
mro_ = getattr(exc.__class__, "mro", lambda: [])
for supercls in mro_():
if is_unwanted(supercls):
# only BaseException and object, from here on down,
# we don't care about these.
return None
try:
exc_args = getattr(exc, "args", [])
superexc = supercls(*exc_args)
pickle.dumps(superexc)
except:
pass
else:
return superexc
return None
def fun_takes_kwargs(fun, kwlist=[]):
"""With a function, and a list of keyword arguments, returns arguments
in the list which the function takes.
If the object has an ``argspec`` attribute that is used instead
of using the :meth:`inspect.getargspec`` introspection.
:param fun: The function to inspect arguments of.
:param kwlist: The list of keyword arguments.
Examples
>>> def foo(self, x, y, logfile=None, loglevel=None):
... return x * y
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel"]
>>> def foo(self, x, y, **kwargs):
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel", "task_id"]
"""
argspec = getattr(fun, "argspec", getargspec(fun))
args, _varargs, keywords, _defaults = argspec
if keywords != None:
return kwlist
return filter(curry(operator.contains, args), kwlist)
def fun_takes_kwargs(fun, kwlist=[]):
"""With a function, and a list of keyword arguments, returns arguments
in the list which the function takes.
If the object has an ``argspec`` attribute that is used instead
of using the :meth:`inspect.getargspec`` introspection.
:param fun: The function to inspect arguments of.
:param kwlist: The list of keyword arguments.
Examples
>>> def foo(self, x, y, logfile=None, loglevel=None):
... return x * y
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel"]
>>> def foo(self, x, y, **kwargs):
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel", "task_id"]
"""
argspec = getattr(fun, "argspec", getargspec(fun))
args, _varargs, keywords, _defaults = argspec
if keywords != None:
return kwlist
return filter(curry(operator.contains, args), kwlist)
def apply_async(self,
target,
args=None,
kwargs=None,
callbacks=None,
errbacks=None,
accept_callback=None,
**compat):
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks)
self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" %
(target, args, kwargs))
req = WorkRequest(do_work,
(target, args, kwargs, on_ready, accept_callback))
self._pool.putRequest(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._pool._results_queue.queue.clear()
return req
def test_apply_raises_404_on_unregistered_task(self):
conf.ALWAYS_EAGER = True
try:
name = "xxx.does.not.exist"
action = curry(self.client.get, task_apply(kwargs={
"task_name": name}) + "?x=4&y=4")
self.assertRaises(TemplateDoesNotExist, action)
finally:
conf.ALWAYS_EAGER = False
def test_apply_raises_404_on_unregistered_task(self):
conf.ALWAYS_EAGER = True
try:
name = "xxx.does.not.exist"
action = curry(self.client.get,
task_apply(kwargs={"task_name": name}) + "?x=4&y=4")
self.assertRaises(TemplateDoesNotExist, action)
finally:
conf.ALWAYS_EAGER = False
def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, accept_callback=None, **compat):
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks)
self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
req = WorkRequest(do_work, (target, args, kwargs, on_ready,
accept_callback))
self._pool.putRequest(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._pool._results_queue.queue.clear()
return req
def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, on_ack=noop):
"""Equivalent of the :func:``apply`` built-in function.
All ``callbacks`` and ``errbacks`` should complete immediately since
otherwise the thread which handles the result will get blocked.
"""
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks, on_ack)
self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
self.replace_dead_workers()
return self._pool.apply_async(target, args, kwargs,
callback=on_ready)
def fun_takes_kwargs(fun, kwlist=[]):
"""With a function, and a list of keyword arguments, returns arguments
in the list which the function takes.
:param fun: The function to inspect arguments of.
:param kwlist: The list of keyword arguments.
Examples
>>> def foo(self, x, y, logfile=None, loglevel=None):
... return x * y
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel"]
>>> def foo(self, x, y, **kwargs):
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel", "task_id"]
"""
args, _varargs, keywords, _defaults = getargspec(fun)
if keywords != None:
return kwlist
return filter(curry(operator.contains, args), kwlist)
from celery.backends import default_backend
from celery.exceptions import RetryTaskError
from celery.datastructures import ExceptionInfo
from celery.decorators import task
from celery.utils import gen_unique_id, get_full_cls_name
from celery.utils.functional import curry
from djcelery.views import task_webhook
from djcelery.tests.req import MockRequest
def reversestar(name, **kwargs):
return reverse(name, kwargs=kwargs)
task_is_successful = curry(reversestar, "celery-is_task_successful")
task_status = curry(reversestar, "celery-task_status")
task_apply = curry(reverse, "celery-apply")
registered_tasks = curry(reverse, "celery-tasks")
scratch = {}
@task()
def mytask(x, y):
ret = scratch["result"] = int(x) * int(y)
return ret
def create_exception(name, base=Exception):
return type(name, (base, ), {})
sys.path.insert(0, os.getcwd())
import time
import unittest2 as unittest
from itertools import chain, izip
from celery.task.base import Task
from celery.utils import timeutils
from celery.utils import gen_unique_id
from celery.utils.functional import curry
from celery.worker import buckets
from celery.registry import TaskRegistry
from celery.tests.utils import skip_if_environ
skip_if_disabled = curry(skip_if_environ("SKIP_RLIMITS"))
class MockJob(object):
def __init__(self, task_id, task_name, args, kwargs):
self.task_id = task_id
self.task_name = task_name
self.args = args
self.kwargs = kwargs
def __eq__(self, other):
if isinstance(other, self.__class__):
return bool(self.task_id == other.task_id \
and self.task_name == other.task_name \
and self.args == other.args \
from celery.utils.functional import curry
from celery import conf
from celery import states
from celery.utils import gen_unique_id, get_full_cls_name
from celery.backends import default_backend
from celery.exceptions import RetryTaskError
from celery.decorators import task
from celery.datastructures import ExceptionInfo
def reversestar(name, **kwargs):
return reverse(name, kwargs=kwargs)
task_is_successful = curry(reversestar, "celery-is_task_successful")
task_status = curry(reversestar, "celery-task_status")
task_apply = curry(reverse, "celery-apply")
scratch = {}
@task()
def mytask(x, y):
ret = scratch["result"] = int(x) * int(y)
return ret
def create_exception(name, base=Exception):
return type(name, (base, ), {})
def get_backend_cls(backend):
"""Get backend class by name/alias"""
if backend not in _backend_cache:
aliases = dict(BACKEND_ALIASES, **current_loader().override_backends)
_backend_cache[backend] = get_cls_by_name(backend, aliases)
return _backend_cache[backend]
"""
.. function:: get_default_backend_cls()
Get the backend class specified in the ``CELERY_RESULT_BACKEND`` setting.
"""
get_default_backend_cls = curry(get_backend_cls, conf.RESULT_BACKEND)
"""
.. class:: DefaultBackend
The default backend class used for storing task results and status,
specified in the ``CELERY_RESULT_BACKEND`` setting.
"""
DefaultBackend = get_default_backend_cls()
"""
.. data:: default_backend
An instance of :class:`DefaultBackend`.
import sys
import time
import unittest2 as unittest
from itertools import chain, izip
from celery.registry import TaskRegistry
from celery.task.base import Task
from celery.utils import timeutils
from celery.utils import gen_unique_id
from celery.utils.functional import curry
from celery.worker import buckets
from celery.tests.utils import skip_if_environ
skip_if_disabled = curry(skip_if_environ("SKIP_RLIMITS"))
class MockJob(object):
def __init__(self, task_id, task_name, args, kwargs):
self.task_id = task_id
self.task_name = task_name
self.args = args
self.kwargs = kwargs
def __eq__(self, other):
if isinstance(other, self.__class__):
return bool(self.task_id == other.task_id \
and self.task_name == other.task_name \
and self.args == other.args \
and self.kwargs == other.kwargs)
def get_backend_cls(backend):
"""Get backend class by name/alias"""
if backend not in _backend_cache:
aliases = dict(BACKEND_ALIASES, **current_loader().override_backends)
_backend_cache[backend] = get_cls_by_name(backend, aliases)
return _backend_cache[backend]
"""
.. function:: get_default_backend_cls()
Get the backend class specified in the ``CELERY_RESULT_BACKEND`` setting.
"""
get_default_backend_cls = curry(get_backend_cls, conf.RESULT_BACKEND)
"""
.. class:: DefaultBackend
The default backend class used for storing task results and status,
specified in the ``CELERY_RESULT_BACKEND`` setting.
"""
DefaultBackend = get_default_backend_cls()
"""
.. data:: default_backend
An instance of :class:`DefaultBackend`.
"""
default_backend = DefaultBackend()
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
routing_key=None, exchange=None, task_id=None,
immediate=None, mandatory=None, priority=None, connection=None,
connect_timeout=AMQP_CONNECTION_TIMEOUT, serializer=None, **opts):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:param countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:param eta: A :class:`datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword exchange: The named exchange to send the task to. Defaults to
:attr:`celery.task.base.Task.exchange`.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated).
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connection: Re-use existing AMQP connection.
The ``connect_timeout`` argument is not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
:keyword serializer: A string identifying the default serialization
method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting.
Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
methods that have been registered with
:mod:`carrot.serialization.registry`.
"""
args = args or []
kwargs = kwargs or {}
routing_key = routing_key or getattr(task, "routing_key", None)
exchange = exchange or getattr(task, "exchange", None)
if immediate is None:
immediate = getattr(task, "immediate", None)
if mandatory is None:
mandatory = getattr(task, "mandatory", None)
if priority is None:
priority = getattr(task, "priority", None)
serializer = serializer or getattr(task, "serializer", None)
taskset_id = opts.get("taskset_id")
publisher = opts.get("publisher")
retries = opts.get("retries", 0)
if countdown:
eta = datetime.now() + timedelta(seconds=countdown)
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
return apply(task, args, kwargs)
need_to_close_connection = False
if not publisher:
if not connection:
connection = DjangoBrokerConnection(
connect_timeout=connect_timeout)
need_to_close_connection = True
publisher = TaskPublisher(connection=connection)
delay_task = publisher.delay_task
if taskset_id:
delay_task = curry(publisher.delay_task_in_set, taskset_id)
task_id = delay_task(task.name, args, kwargs,
task_id=task_id, retries=retries,
routing_key=routing_key, exchange=exchange,
mandatory=mandatory, immediate=immediate,
serializer=serializer, priority=priority,
eta=eta)
if need_to_close_connection:
publisher.close()
connection.close()
return AsyncResult(task_id)