def setUp(self):
self._threads_at_setup = list(threading.enumerate())
from celery import _state
from celery import result
result.task_join_will_block = \
_state.task_join_will_block = lambda: False
self._current_app = current_app()
self._default_app = _state.default_app
trap = Trap()
self._prev_tls = _state._tls
_state.set_default_app(trap)
class NonTLS(object):
current_app = trap
_state._tls = NonTLS()
self.app = self.Celery(set_as_current=False)
if not self.contained:
self.app.set_current()
root = logging.getLogger()
self.__rootlevel = root.level
self.__roothandlers = root.handlers
_state._set_task_join_will_block(False)
try:
self.setup()
except:
self._teardown_app()
raise
def app(request):
from celery import _state
prev_current_app = current_app()
prev_default_app = _state.default_app
prev_finalizers = set(_state._on_app_finalizers)
prev_apps = weakref.WeakSet(_state._apps)
trap = Trap()
prev_tls = _state._tls
_state.set_default_app(trap)
class NonTLS(object):
current_app = trap
_state._tls = NonTLS()
app = TestApp(set_as_current=False)
is_not_contained = any([
not getattr(request.module, 'app_contained', True),
not getattr(request.cls, 'app_contained', True),
not getattr(request.function, 'app_contained', True)
])
if is_not_contained:
app.set_current()
def fin():
_state.set_default_app(prev_default_app)
_state._tls = prev_tls
_state._tls.current_app = prev_current_app
if app is not prev_current_app:
app.close()
_state._on_app_finalizers = prev_finalizers
_state._apps = prev_apps
request.addfinalizer(fin)
return app
def _send_call(self, my_task):
"""Sends Celery asynchronous call and stores async call information for
retrieval laster"""
args, kwargs, queue = [], {}, None
if self.args:
args = _eval_args(self.args, my_task)
if self.kwargs:
kwargs = _eval_kwargs(self.kwargs, my_task)
if self.call_server_id:
queue = 'server.{0}'.format(valueof(my_task, self.call_server_id))
elif self.call_queue:
queue = valueof(my_task, self.call_queue)
LOG.debug("%s (task id %s) calling %s",
self.name,
my_task.id,
self.call,
extra=dict(data=dict(args=args, kwargs=kwargs)))
# Add current workflow information
kwargs['workflow'] = {'data': my_task.workflow.data}
async_call = current_app().send_task(self.call,
args=args,
kwargs=kwargs,
queue=queue)
my_task.internal_data['task_id'] = async_call.task_id
my_task.internal_data['async_call'] = async_call
LOG.debug("'%s' called: %s", self.call, async_call.task_id)
def app(request):
from celery import _state
prev_current_app = current_app()
prev_default_app = _state.default_app
prev_finalizers = set(_state._on_app_finalizers)
prev_apps = weakref.WeakSet(_state._apps)
trap = Trap()
prev_tls = _state._tls
_state.set_default_app(trap)
class NonTLS(object):
current_app = trap
_state._tls = NonTLS()
app = TestApp(set_as_current=False)
is_not_contained = any([
not getattr(request.module, 'app_contained', True),
not getattr(request.cls, 'app_contained', True),
not getattr(request.function, 'app_contained', True)
])
if is_not_contained:
app.set_current()
def fin():
_state.set_default_app(prev_default_app)
_state._tls = prev_tls
_state._tls.current_app = prev_current_app
if app is not prev_current_app:
app.close()
_state._on_app_finalizers = prev_finalizers
_state._apps = prev_apps
request.addfinalizer(fin)
return app
def setup_test_environment():
"""Our own setup that hijacks Jinja template rendering."""
global IS_SETUP
if IS_SETUP:
return
IS_SETUP = True
if HAS_JINJA2:
old_render = jinja2.Template.render
def instrumented_render(self, *args, **kwargs):
context = dict(*args, **kwargs)
test.signals.template_rendered.send(sender=self, template=self, context=context)
return old_render(self, *args, **kwargs)
jinja2.Template.render = instrumented_render
try:
from celery.app import current_app
current_app().conf.CELERY_ALWAYS_EAGER = True
except ImportError:
pass
try:
import async_signals
async_signals.stop_the_machine()
settings.ASYNC_SIGNALS = False
except ImportError:
pass
def app(request):
"""Fixture creating a Celery application instance."""
from celery import _state
prev_current_app = current_app()
prev_default_app = _state.default_app
prev_finalizers = set(_state._on_app_finalizers)
prev_apps = weakref.WeakSet(_state._apps)
trap = Trap()
prev_tls = _state._tls
_state.set_default_app(trap)
class NonTLS(object):
current_app = trap
_state._tls = NonTLS()
test_app = TestApp(set_as_current=False)
is_not_contained = any([
not getattr(request.module, 'app_contained', True),
not getattr(request.cls, 'app_contained', True),
not getattr(request.function, 'app_contained', True)
])
if is_not_contained:
test_app.set_current()
yield test_app
_state.set_default_app(prev_default_app)
_state._tls = prev_tls
_state._tls.current_app = prev_current_app
if test_app is not prev_current_app:
test_app.close()
_state._on_app_finalizers = prev_finalizers
_state._apps = prev_apps
def aggregate_result_set(self, agg, acc):
"""
Loop on a set of celery AsyncResults and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if not self.results:
return acc
backend = current_app().backend
amqp_backend = backend.__class__.__name__.startswith("AMQP")
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
check_mem_usage() # warn if too much memory is used
result = result_dict["result"]
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
if amqp_backend:
# work around a celery bug
del backend._cache[task_id]
return acc
def aggregate_result_set(self, agg, acc):
"""
Loop on a set of celery AsyncResults and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if isinstance(self.oqtask, types.FunctionType):
# don't use celery
return super(OqTaskManager, self).aggregate_result_set(
agg, acc)
if not self.results:
return acc
backend = current_app().backend
amqp_backend = backend.__class__.__name__.startswith('AMQP')
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
idx = self.task_ids.index(task_id)
self.task_ids.pop(idx)
parallel.check_mem_usage() # warn if too much memory is used
result = result_dict['result']
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
if amqp_backend:
# work around a celery bug
del backend._cache[task_id]
return acc
def setUp(self):
self._threads_at_setup = list(threading.enumerate())
from celery import _state
from celery import result
result.task_join_will_block = \
_state.task_join_will_block = lambda: False
self._current_app = current_app()
self._default_app = _state.default_app
trap = Trap()
self._prev_tls = _state._tls
_state.set_default_app(trap)
class NonTLS(object):
current_app = trap
_state._tls = NonTLS()
self.app = self.Celery(set_as_current=False)
if not self.contained:
self.app.set_current()
root = logging.getLogger()
self.__rootlevel = root.level
self.__roothandlers = root.handlers
_state._set_task_join_will_block(False)
try:
self.setup()
except:
self._teardown_app()
raise
def setup_test_environment():
"""Our own setup that hijacks Jinja template rendering."""
global IS_SETUP
if IS_SETUP:
return
IS_SETUP = True
if HAS_JINJA2:
old_render = jinja2.Template.render
def instrumented_render(self, *args, **kwargs):
context = dict(*args, **kwargs)
test.signals.template_rendered.send(sender=self,
template=self,
context=context)
return old_render(self, *args, **kwargs)
jinja2.Template.render = instrumented_render
try:
from celery.app import current_app
current_app().conf.CELERY_ALWAYS_EAGER = True
except ImportError:
pass
try:
import async_signals
async_signals.stop_the_machine()
settings.ASYNC_SIGNALS = False
except ImportError:
pass
def aggregate_result_set(self, agg, acc):
"""
Loop on a set of celery AsyncResults and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if isinstance(self.oqtask, types.FunctionType):
# don't use celery
return super(OqTaskManager,
self).aggregate_result_set(agg, acc)
if not self.results:
return acc
backend = current_app().backend
amqp_backend = backend.__class__.__name__.startswith('AMQP')
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
idx = self.task_ids.index(task_id)
self.task_ids.pop(idx)
parallel.check_mem_usage() # warn if too much memory is used
result = result_dict['result']
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
if amqp_backend:
# work around a celery bug
del backend._cache[task_id]
return acc
def run_now(modeladmin, request, queryset):
app = current_app()
for pt in queryset:
try:
app.tasks[pt.task].apply_async()
except KeyError:
init_providers()
app.tasks[pt.task].apply_async()
def setUp(self):
if current_app().IS_WINDOWS:
raise SkipTest("curses monitor does not run on Windows")
from celery.events import cursesmon
self.monitor = cursesmon.CursesMonitor(object())
self.win = MockWindow()
self.monitor.win = self.win
def run_now(modeladmin, request, queryset):
app = current_app()
for pt in queryset:
try:
app.tasks[pt.task].apply_async()
except KeyError:
init_providers()
app.tasks[pt.task].apply_async()
def setUp(self):
from celery.app import current_app
from celery.backends.cache import CacheBackend, DummyClient
app = self.app = self._current_app = current_app()
if isinstance(app.backend, CacheBackend):
if isinstance(app.backend.client, DummyClient):
app.backend.client.cache.clear()
app.backend._cache.clear()
self.setup()
def __init__(self, *args, **kwargs):
self.loader = kwargs.get("loader") or current_app().loader
self.hostname = kwargs.get("hostname") or socket.gethostname()
super(WorkerTaskTrace, self).__init__(*args, **kwargs)
self._store_errors = True
if self.task.ignore_result:
self._store_errors = self.task.store_errors_even_if_ignored
self.super = super(WorkerTaskTrace, self)
def map_reduce(task, task_args, agg, acc):
"""
Given a task and an iterable of positional arguments, apply the
task function to the arguments in parallel and return an aggregate
result depending on the initial value of the accumulator
and on the aggregation function. To save memory, the order is
not preserved and there is no list with the intermediated results:
the accumulator is incremented as soon as a task result comes.
NB: if the environment variable OQ_NO_DISTRIBUTE is set the
tasks are run sequentially in the current process and then
map_reduce(task, task_args, agg, acc) is the same as
reduce(agg, itertools.starmap(task, task_args), acc).
Users of map_reduce should be aware of the fact that when
thousands of tasks are spawned and large arguments are passed
or large results are returned they may incur in memory issue:
this is way the calculators limit the queue with the
`concurrent_task` concept.
:param task: a `celery` task callable.
:param task_args: an iterable over positional arguments
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if no_distribute():
for the_args in task_args:
result, exctype = safely_call(task.task_func, the_args)
if exctype:
raise RuntimeError(result)
acc = agg(acc, result)
else:
backend = current_app().backend
unpik = 0
job_id = task_args[0][0]
taskname = task.__name__
mon = LightMonitor("unpickling %s" % taskname, job_id, task)
to_send = 0
pickled_args = []
for args in task_args:
piks = pickle_sequence(args)
pickled_args.append(piks)
to_send += sum(len(p) for p in piks)
logs.LOG.info("Sending %dM", to_send / ONE_MB)
taskset = TaskSet(tasks=map(task.subtask, pickled_args))
for task_id, result_dict in taskset.apply_async().iter_native():
check_mem_usage() # log a warning if too much memory is used
result_pik = result_dict["result"]
with mon:
result, exctype = result_pik.unpickle()
if exctype:
raise RuntimeError(result)
unpik += len(result_pik)
acc = agg(acc, result)
del backend._cache[task_id] # work around a celery bug
logs.LOG.info("Unpickled %dM of received data in %s seconds", unpik / ONE_MB, mon.duration)
return acc
def setUp(self):
from celery.app import current_app
from celery.backends.cache import CacheBackend, DummyClient
app = self.app = self._current_app = current_app()
if isinstance(app.backend, CacheBackend):
if isinstance(app.backend.client, DummyClient):
app.backend.client.cache.clear()
app.backend._cache.clear()
self.setup()
def __init__(self, *args, **kwargs):
self.loader = kwargs.get("loader") or current_app().loader
self.hostname = kwargs.get("hostname") or socket.gethostname()
super(WorkerTaskTrace, self).__init__(*args, **kwargs)
self._store_errors = True
if self.task.ignore_result:
self._store_errors = self.task.store_errors_even_if_ignored
self.super = super(WorkerTaskTrace, self)
def run(self, *args, **kwargs):
app = current_app()
queues = len(app.amqp.queues.keys())
messages_removed = app.control.discard_all()
if messages_removed:
self.out("Purged %s %s from %s known task %s." %
(messages_removed, pluralize(messages_removed, "message"),
queues, pluralize(queues, "queue")))
else:
self.out("No messages purged from %s known %s" %
(queues, pluralize(queues, "queue")))
def run(self, *args, **kwargs):
app = current_app()
queues = len(app.amqp.queues.keys())
messages_removed = app.control.discard_all()
if messages_removed:
self.out("Purged %s %s from %s known task %s." % (
messages_removed, pluralize(messages_removed, "message"),
queues, pluralize(queues, "queue")))
else:
self.out("No messages purged from %s known %s" % (
queues, pluralize(queues, "queue")))
def setUp(self):
from celery.app import current_app
from celery.backends.cache import CacheBackend, DummyClient
app = self.app = self._current_app = current_app()
if isinstance(app.backend, CacheBackend):
if isinstance(app.backend.client, DummyClient):
app.backend.client.cache.clear()
app.backend._cache.clear()
root = logging.getLogger()
self.__rootlevel = root.level
self.__roothandlers = root.handlers
self.setup()
def setUp(self):
from celery.app import current_app
from celery.backends.cache import CacheBackend, DummyClient
app = self.app = self._current_app = current_app()
if isinstance(app.backend, CacheBackend):
if isinstance(app.backend.client, DummyClient):
app.backend.client.cache.clear()
app.backend._cache.clear()
root = logging.getLogger()
self.__rootlevel = root.level
self.__roothandlers = root.handlers
self.setup()
def save(self, update_last_modified=False, *arg, **kwargs):
if not update_last_modified:
app = current_app()
provider = self.provider.split('.')[-1]
for task in app.tasks:
try:
if not app.tasks[task].run_every and task.find(
provider) != -1:
app.tasks[task].apply_async(args=(self, ))
except:
pass
return original_save(self, *arg, **kwargs)
def save_model(self, request, obj, form, change):
''' Saving...
:param request: request object to view
:param obj: Schedule instance
:param form: Form instance
:param change: bool
'''
if 'status' in form.changed_data:
if obj.status == 'scheduled':
if obj.dt_start < now() or (obj.task != None
and obj.task != ""):
#: Don not save()
return
#: create_task
t = enqueue_schedule.apply_async(("admin", obj.id), {},
eta=obj.dt_start)
obj.task = t.id
elif obj.status == "canceled":
if obj.task != None:
app.current_app().control.revoke(obj.task)
super(ScheduleAdmin, self).save_model(request, obj, form, change)
def aggregate_result_set(self, agg, acc):
"""
Loop on a set results and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if not self.results:
return acc
distribute = oq_distribute() # not called for distribute == 'no'
if distribute == 'celery':
backend = current_app().backend
amqp_backend = backend.__class__.__name__.startswith('AMQP')
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
idx = self.task_ids.index(task_id)
self.task_ids.pop(idx)
check_mem_usage() # warn if too much memory is used
result = result_dict['result']
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
if amqp_backend:
# work around a celery bug
del backend._cache[task_id]
return acc
elif distribute == 'futures':
for future in as_completed(self.results):
check_mem_usage()
# log a warning if too much memory is used
result = future.result()
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
return acc
def aggregate_result_set(self, agg, acc):
"""
Loop on a set results and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if not self.results:
return acc
distribute = oq_distribute() # not called for distribute == 'no'
if distribute == 'celery':
backend = current_app().backend
amqp_backend = backend.__class__.__name__.startswith('AMQP')
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
idx = self.task_ids.index(task_id)
self.task_ids.pop(idx)
check_mem_usage() # warn if too much memory is used
result = result_dict['result']
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
if amqp_backend:
# work around a celery bug
del backend._cache[task_id]
return acc
elif distribute == 'futures':
for future in as_completed(self.results):
check_mem_usage()
# log a warning if too much memory is used
result = future.result()
if isinstance(result, BaseException):
raise result
self.received.append(len(result))
acc = agg(acc, result.unpickle())
return acc
def setUp(self):
from celery import _state
self._current_app = current_app()
self._default_app = _state.default_app
trap = Trap()
_state.set_default_app(trap)
_state._tls.current_app = trap
self.app = self.Celery(set_as_current=False)
if not self.contained:
self.app.set_current()
root = logging.getLogger()
self.__rootlevel = root.level
self.__roothandlers = root.handlers
try:
self.setup()
except:
self._teardown_app()
raise
def setUp(self):
self._threads_at_setup = list(threading.enumerate())
from celery import _state
self._current_app = current_app()
self._default_app = _state.default_app
trap = Trap()
_state.set_default_app(trap)
_state._tls.current_app = trap
self.app = self.Celery(set_as_current=False)
if not self.contained:
self.app.set_current()
root = logging.getLogger()
self.__rootlevel = root.level
self.__roothandlers = root.handlers
try:
self.setup()
except:
self._teardown_app()
raise
def aggregate_result_set(self, agg, acc):
"""
Loop on a set of celery AsyncResults and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if not self.results:
return acc
backend = current_app().backend
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
check_mem_usage() # log a warning if too much memory is used
result = result_dict['result']
if isinstance(result, BaseException):
raise result
acc = agg(acc, result.unpickle())
del backend._cache[task_id] # work around a celery bug
return acc
def aggregate_result_set(self, agg, acc):
"""
Loop on a set of celery AsyncResults and update the accumulator
by using the aggregation function.
:param agg: the aggregation function, (acc, val) -> new acc
:param acc: the initial value of the accumulator
:returns: the final value of the accumulator
"""
if not self.results:
return acc
backend = current_app().backend
rset = ResultSet(self.results)
for task_id, result_dict in rset.iter_native():
check_mem_usage() # warn if too much memory is used
result = result_dict['result']
if isinstance(result, BaseException):
raise result
self.received += len(result)
acc = agg(acc, result.unpickle())
del backend._cache[task_id] # work around a celery bug
return acc
def _send_call(self, my_task):
"""Sends Celery asynchronous call and stores async call information for
retrieval laster"""
args, kwargs, queue = [], {}, None
if self.args:
args = _eval_args(self.args, my_task)
if self.kwargs:
kwargs = _eval_kwargs(self.kwargs, my_task)
if self.call_server_id:
queue = 'server.{0}'.format(valueof(my_task, self.call_server_id))
elif self.call_queue:
queue = valueof(my_task, self.call_queue)
LOG.debug("%s (task id %s) calling %s", self.name, my_task.id,
self.call, extra=dict(data=dict(args=args, kwargs=kwargs)))
# Add current workflow information
kwargs['workflow'] = {
'data': my_task.workflow.data
}
async_call = current_app().send_task(self.call, args=args, kwargs=kwargs, queue=queue)
my_task.internal_data['task_id'] = async_call.task_id
my_task.internal_data['async_call'] = async_call
LOG.debug("'%s' called: %s", self.call, async_call.task_id)
def _get_current_app():
from celery.app import current_app
return current_app()
def setUp(self):
from celery.app import current_app
self._current_app = current_app()
self.setup()
def includeme(config):
app = current_app()
convert_celery_options(config.registry.settings)
app.config_from_object(config.registry.settings)
app.config = config
def do_cancel_publish(publish):
''' helper cancel task'''
if publish.task_id is not None:
app.current_app().control.revoke(publish.task_id)
def _get_current_app():
from celery.app import current_app
return current_app()
def handle_revoke(self, *args, **options):
''' revoke specified task
'''
if len(args) == 2:
print "Revoking task ", args[1]
app.current_app().control.revoke(args[1])
def setUp(self):
from celery.app import current_app
self.app = self._current_app = current_app()
self.setup()
def handle_revoke(self,*args,**options):
''' revoke specified task
'''
if len(args) == 2:
print "Revoking task ",args[1]
app.current_app().control.revoke(args[1])
