def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, accept_callback=None, timeout_callback=None,
**compat):
"""Equivalent of the :func:`apply` built-in function.
All `callbacks` and `errbacks` should complete immediately since
otherwise the thread which handles the result will get blocked.
"""
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = partial(self.on_ready, callbacks, errbacks)
on_worker_error = partial(self.on_worker_error, errbacks)
self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
return self._pool.apply_async(target, args, kwargs,
callback=on_ready,
accept_callback=accept_callback,
timeout_callback=timeout_callback,
error_callback=on_worker_error,
waitforslot=self.putlocks)
def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, accept_callback=None, timeout_callback=None,
**compat):
"""Equivalent of the :func:`apply` built-in function.
All `callbacks` and `errbacks` should complete immediately since
otherwise the thread which handles the result will get blocked.
"""
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = partial(self.on_ready, callbacks, errbacks)
on_worker_error = partial(self.on_worker_error, errbacks)
self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
return self.on_apply(target, args, kwargs,
callback=on_ready,
accept_callback=accept_callback,
timeout_callback=timeout_callback,
error_callback=on_worker_error,
waitforslot=self.putlocks)
def fun_takes_kwargs(fun, kwlist=[]):
"""With a function, and a list of keyword arguments, returns arguments
in the list which the function takes.
If the object has an `argspec` attribute that is used instead
of using the :meth:`inspect.getargspec` introspection.
:param fun: The function to inspect arguments of.
:param kwlist: The list of keyword arguments.
Examples
>>> def foo(self, x, y, logfile=None, loglevel=None):
... return x * y
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel"]
>>> def foo(self, x, y, **kwargs):
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel", "task_id"]
"""
argspec = getattr(fun, "argspec", getargspec(fun))
args, _varargs, keywords, _defaults = argspec
if keywords != None:
return kwlist
return filter(partial(operator.contains, args), kwlist)
def fun_takes_kwargs(fun, kwlist=[]):
"""With a function, and a list of keyword arguments, returns arguments
in the list which the function takes.
If the object has an `argspec` attribute that is used instead
of using the :meth:`inspect.getargspec` introspection.
:param fun: The function to inspect arguments of.
:param kwlist: The list of keyword arguments.
Examples
>>> def foo(self, x, y, logfile=None, loglevel=None):
... return x * y
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel"]
>>> def foo(self, x, y, **kwargs):
>>> fun_takes_kwargs(foo, ["logfile", "loglevel", "task_id"])
["logfile", "loglevel", "task_id"]
"""
argspec = getattr(fun, "argspec", getargspec(fun))
args, _varargs, keywords, _defaults = argspec
if keywords != None:
return kwlist
return filter(partial(operator.contains, args), kwlist)
def test_apply_raises_404_on_unregistered_task(self):
default_app.conf.CELERY_ALWAYS_EAGER = True
try:
name = "xxx.does.not.exist"
action = partial(self.client.get, task_apply(kwargs={
"task_name": name}) + "?x=4&y=4")
self.assertRaises(TemplateDoesNotExist, action)
finally:
default_app.conf.CELERY_ALWAYS_EAGER = False
def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, accept_callback=None, **compat):
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = partial(self.on_ready, callbacks, errbacks)
self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
req = WorkRequest(do_work, (target, args, kwargs, on_ready,
accept_callback))
self._pool.putRequest(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._pool._results_queue.queue.clear()
return req
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, working_directory=None, **kwargs):
kwargs.pop("app", None)
beat = partial(self.app.Beat, logfile=logfile, pidfile=pidfile,
**kwargs)
if not detach:
return beat().run()
context, on_stop = create_daemon_context(
logfile=logfile,
pidfile=pidfile,
uid=uid,
gid=gid,
umask=umask,
working_directory=working_directory)
context.open()
try:
beat().run()
finally:
on_stop()
class MyError(Exception):
# On Py2.4 repr(exc) includes the object id, so comparing
# texts is pointless when the id the "same" KeyError does not match.
def __repr__(self):
return "<%s: %r>" % (self.__class__.__name__, self.args)
class MyRetryTaskError(Exception):
def __repr__(self):
return "<%s: %r>" % (self.__class__.__name__, self.args)
task_is_successful = partial(reversestar, "celery-is_task_successful")
task_status = partial(reversestar, "celery-task_status")
task_apply = partial(reverse, "celery-apply")
registered_tasks = partial(reverse, "celery-tasks")
scratch = {}
@task()
def mytask(x, y):
ret = scratch["result"] = int(x) * int(y)
return ret
def create_exception(name, base=Exception):
return type(name, (base, ), {})
import sys
import time
from celery.tests.utils import unittest
from itertools import chain, izip
from celery.registry import TaskRegistry
from celery.task.base import Task
from celery.utils import timeutils
from celery.utils import gen_unique_id
from celery.utils.functional import partial
from celery.worker import buckets
from celery.tests.utils import skip_if_environ
skip_if_disabled = partial(skip_if_environ("SKIP_RLIMITS"))
class MockJob(object):
def __init__(self, task_id, task_name, args, kwargs):
self.task_id = task_id
self.task_name = task_name
self.args = args
self.kwargs = kwargs
def __eq__(self, other):
if isinstance(other, self.__class__):
return bool(self.task_id == other.task_id \
and self.task_name == other.task_name \
and self.args == other.args \
import sys
import time
from celery.tests.utils import unittest
from itertools import chain, izip
from celery.registry import TaskRegistry
from celery.task.base import Task
from celery.utils import timeutils
from celery.utils import gen_unique_id
from celery.utils.functional import partial
from celery.worker import buckets
from celery.tests.utils import skip_if_environ
skip_if_disabled = partial(skip_if_environ("SKIP_RLIMITS"))
class MockJob(object):
def __init__(self, task_id, task_name, args, kwargs):
self.task_id = task_id
self.task_name = task_name
self.args = args
self.kwargs = kwargs
def __eq__(self, other):
if isinstance(other, self.__class__):
return bool(self.task_id == other.task_id \
and self.task_name == other.task_name \
and self.args == other.args \
and self.kwargs == other.kwargs)
