def setUp(self): self._threads_at_setup = list(threading.enumerate()) from celery import _state from celery import result result.task_join_will_block = \ _state.task_join_will_block = lambda: False self._current_app = current_app() self._default_app = _state.default_app trap = Trap() self._prev_tls = _state._tls _state.set_default_app(trap) class NonTLS(object): current_app = trap _state._tls = NonTLS() self.app = self.Celery(set_as_current=False) if not self.contained: self.app.set_current() root = logging.getLogger() self.__rootlevel = root.level self.__roothandlers = root.handlers _state._set_task_join_will_block(False) try: self.setup() except: self._teardown_app() raise
def app(request): from celery import _state prev_current_app = current_app() prev_default_app = _state.default_app prev_finalizers = set(_state._on_app_finalizers) prev_apps = weakref.WeakSet(_state._apps) trap = Trap() prev_tls = _state._tls _state.set_default_app(trap) class NonTLS(object): current_app = trap _state._tls = NonTLS() app = TestApp(set_as_current=False) is_not_contained = any([ not getattr(request.module, 'app_contained', True), not getattr(request.cls, 'app_contained', True), not getattr(request.function, 'app_contained', True) ]) if is_not_contained: app.set_current() def fin(): _state.set_default_app(prev_default_app) _state._tls = prev_tls _state._tls.current_app = prev_current_app if app is not prev_current_app: app.close() _state._on_app_finalizers = prev_finalizers _state._apps = prev_apps request.addfinalizer(fin) return app
def _send_call(self, my_task): """Sends Celery asynchronous call and stores async call information for retrieval laster""" args, kwargs, queue = [], {}, None if self.args: args = _eval_args(self.args, my_task) if self.kwargs: kwargs = _eval_kwargs(self.kwargs, my_task) if self.call_server_id: queue = 'server.{0}'.format(valueof(my_task, self.call_server_id)) elif self.call_queue: queue = valueof(my_task, self.call_queue) LOG.debug("%s (task id %s) calling %s", self.name, my_task.id, self.call, extra=dict(data=dict(args=args, kwargs=kwargs))) # Add current workflow information kwargs['workflow'] = {'data': my_task.workflow.data} async_call = current_app().send_task(self.call, args=args, kwargs=kwargs, queue=queue) my_task.internal_data['task_id'] = async_call.task_id my_task.internal_data['async_call'] = async_call LOG.debug("'%s' called: %s", self.call, async_call.task_id)
def setup_test_environment(): """Our own setup that hijacks Jinja template rendering.""" global IS_SETUP if IS_SETUP: return IS_SETUP = True if HAS_JINJA2: old_render = jinja2.Template.render def instrumented_render(self, *args, **kwargs): context = dict(*args, **kwargs) test.signals.template_rendered.send(sender=self, template=self, context=context) return old_render(self, *args, **kwargs) jinja2.Template.render = instrumented_render try: from celery.app import current_app current_app().conf.CELERY_ALWAYS_EAGER = True except ImportError: pass try: import async_signals async_signals.stop_the_machine() settings.ASYNC_SIGNALS = False except ImportError: pass
def app(request): """Fixture creating a Celery application instance.""" from celery import _state prev_current_app = current_app() prev_default_app = _state.default_app prev_finalizers = set(_state._on_app_finalizers) prev_apps = weakref.WeakSet(_state._apps) trap = Trap() prev_tls = _state._tls _state.set_default_app(trap) class NonTLS(object): current_app = trap _state._tls = NonTLS() test_app = TestApp(set_as_current=False) is_not_contained = any([ not getattr(request.module, 'app_contained', True), not getattr(request.cls, 'app_contained', True), not getattr(request.function, 'app_contained', True) ]) if is_not_contained: test_app.set_current() yield test_app _state.set_default_app(prev_default_app) _state._tls = prev_tls _state._tls.current_app = prev_current_app if test_app is not prev_current_app: test_app.close() _state._on_app_finalizers = prev_finalizers _state._apps = prev_apps
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith("AMQP") rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): check_mem_usage() # warn if too much memory is used result = result_dict["result"] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if isinstance(self.oqtask, types.FunctionType): # don't use celery return super(OqTaskManager, self).aggregate_result_set( agg, acc) if not self.results: return acc backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith('AMQP') rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) parallel.check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if isinstance(self.oqtask, types.FunctionType): # don't use celery return super(OqTaskManager, self).aggregate_result_set(agg, acc) if not self.results: return acc backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith('AMQP') rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) parallel.check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc
def run_now(modeladmin, request, queryset): app = current_app() for pt in queryset: try: app.tasks[pt.task].apply_async() except KeyError: init_providers() app.tasks[pt.task].apply_async()
def setUp(self): if current_app().IS_WINDOWS: raise SkipTest("curses monitor does not run on Windows") from celery.events import cursesmon self.monitor = cursesmon.CursesMonitor(object()) self.win = MockWindow() self.monitor.win = self.win
def setUp(self): from celery.app import current_app from celery.backends.cache import CacheBackend, DummyClient app = self.app = self._current_app = current_app() if isinstance(app.backend, CacheBackend): if isinstance(app.backend.client, DummyClient): app.backend.client.cache.clear() app.backend._cache.clear() self.setup()
def __init__(self, *args, **kwargs): self.loader = kwargs.get("loader") or current_app().loader self.hostname = kwargs.get("hostname") or socket.gethostname() super(WorkerTaskTrace, self).__init__(*args, **kwargs) self._store_errors = True if self.task.ignore_result: self._store_errors = self.task.store_errors_even_if_ignored self.super = super(WorkerTaskTrace, self)
def map_reduce(task, task_args, agg, acc): """ Given a task and an iterable of positional arguments, apply the task function to the arguments in parallel and return an aggregate result depending on the initial value of the accumulator and on the aggregation function. To save memory, the order is not preserved and there is no list with the intermediated results: the accumulator is incremented as soon as a task result comes. NB: if the environment variable OQ_NO_DISTRIBUTE is set the tasks are run sequentially in the current process and then map_reduce(task, task_args, agg, acc) is the same as reduce(agg, itertools.starmap(task, task_args), acc). Users of map_reduce should be aware of the fact that when thousands of tasks are spawned and large arguments are passed or large results are returned they may incur in memory issue: this is way the calculators limit the queue with the `concurrent_task` concept. :param task: a `celery` task callable. :param task_args: an iterable over positional arguments :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if no_distribute(): for the_args in task_args: result, exctype = safely_call(task.task_func, the_args) if exctype: raise RuntimeError(result) acc = agg(acc, result) else: backend = current_app().backend unpik = 0 job_id = task_args[0][0] taskname = task.__name__ mon = LightMonitor("unpickling %s" % taskname, job_id, task) to_send = 0 pickled_args = [] for args in task_args: piks = pickle_sequence(args) pickled_args.append(piks) to_send += sum(len(p) for p in piks) logs.LOG.info("Sending %dM", to_send / ONE_MB) taskset = TaskSet(tasks=map(task.subtask, pickled_args)) for task_id, result_dict in taskset.apply_async().iter_native(): check_mem_usage() # log a warning if too much memory is used result_pik = result_dict["result"] with mon: result, exctype = result_pik.unpickle() if exctype: raise RuntimeError(result) unpik += len(result_pik) acc = agg(acc, result) del backend._cache[task_id] # work around a celery bug logs.LOG.info("Unpickled %dM of received data in %s seconds", unpik / ONE_MB, mon.duration) return acc
def run(self, *args, **kwargs): app = current_app() queues = len(app.amqp.queues.keys()) messages_removed = app.control.discard_all() if messages_removed: self.out("Purged %s %s from %s known task %s." % (messages_removed, pluralize(messages_removed, "message"), queues, pluralize(queues, "queue"))) else: self.out("No messages purged from %s known %s" % (queues, pluralize(queues, "queue")))
def run(self, *args, **kwargs): app = current_app() queues = len(app.amqp.queues.keys()) messages_removed = app.control.discard_all() if messages_removed: self.out("Purged %s %s from %s known task %s." % ( messages_removed, pluralize(messages_removed, "message"), queues, pluralize(queues, "queue"))) else: self.out("No messages purged from %s known %s" % ( queues, pluralize(queues, "queue")))
def setUp(self): from celery.app import current_app from celery.backends.cache import CacheBackend, DummyClient app = self.app = self._current_app = current_app() if isinstance(app.backend, CacheBackend): if isinstance(app.backend.client, DummyClient): app.backend.client.cache.clear() app.backend._cache.clear() root = logging.getLogger() self.__rootlevel = root.level self.__roothandlers = root.handlers self.setup()
def save(self, update_last_modified=False, *arg, **kwargs): if not update_last_modified: app = current_app() provider = self.provider.split('.')[-1] for task in app.tasks: try: if not app.tasks[task].run_every and task.find( provider) != -1: app.tasks[task].apply_async(args=(self, )) except: pass return original_save(self, *arg, **kwargs)
def save_model(self, request, obj, form, change): ''' Saving... :param request: request object to view :param obj: Schedule instance :param form: Form instance :param change: bool ''' if 'status' in form.changed_data: if obj.status == 'scheduled': if obj.dt_start < now() or (obj.task != None and obj.task != ""): #: Don not save() return #: create_task t = enqueue_schedule.apply_async(("admin", obj.id), {}, eta=obj.dt_start) obj.task = t.id elif obj.status == "canceled": if obj.task != None: app.current_app().control.revoke(obj.task) super(ScheduleAdmin, self).save_model(request, obj, form, change)
def aggregate_result_set(self, agg, acc): """ Loop on a set results and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc distribute = oq_distribute() # not called for distribute == 'no' if distribute == 'celery': backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith('AMQP') rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc elif distribute == 'futures': for future in as_completed(self.results): check_mem_usage() # log a warning if too much memory is used result = future.result() if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) return acc
def setUp(self): from celery import _state self._current_app = current_app() self._default_app = _state.default_app trap = Trap() _state.set_default_app(trap) _state._tls.current_app = trap self.app = self.Celery(set_as_current=False) if not self.contained: self.app.set_current() root = logging.getLogger() self.__rootlevel = root.level self.__roothandlers = root.handlers try: self.setup() except: self._teardown_app() raise
def setUp(self): self._threads_at_setup = list(threading.enumerate()) from celery import _state self._current_app = current_app() self._default_app = _state.default_app trap = Trap() _state.set_default_app(trap) _state._tls.current_app = trap self.app = self.Celery(set_as_current=False) if not self.contained: self.app.set_current() root = logging.getLogger() self.__rootlevel = root.level self.__roothandlers = root.handlers try: self.setup() except: self._teardown_app() raise
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc backend = current_app().backend rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): check_mem_usage() # log a warning if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result acc = agg(acc, result.unpickle()) del backend._cache[task_id] # work around a celery bug return acc
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc backend = current_app().backend rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received += len(result) acc = agg(acc, result.unpickle()) del backend._cache[task_id] # work around a celery bug return acc
def _send_call(self, my_task): """Sends Celery asynchronous call and stores async call information for retrieval laster""" args, kwargs, queue = [], {}, None if self.args: args = _eval_args(self.args, my_task) if self.kwargs: kwargs = _eval_kwargs(self.kwargs, my_task) if self.call_server_id: queue = 'server.{0}'.format(valueof(my_task, self.call_server_id)) elif self.call_queue: queue = valueof(my_task, self.call_queue) LOG.debug("%s (task id %s) calling %s", self.name, my_task.id, self.call, extra=dict(data=dict(args=args, kwargs=kwargs))) # Add current workflow information kwargs['workflow'] = { 'data': my_task.workflow.data } async_call = current_app().send_task(self.call, args=args, kwargs=kwargs, queue=queue) my_task.internal_data['task_id'] = async_call.task_id my_task.internal_data['async_call'] = async_call LOG.debug("'%s' called: %s", self.call, async_call.task_id)
def _get_current_app(): from celery.app import current_app return current_app()
def setUp(self): from celery.app import current_app self._current_app = current_app() self.setup()
def includeme(config): app = current_app() convert_celery_options(config.registry.settings) app.config_from_object(config.registry.settings) app.config = config
def do_cancel_publish(publish): ''' helper cancel task''' if publish.task_id is not None: app.current_app().control.revoke(publish.task_id)
def handle_revoke(self, *args, **options): ''' revoke specified task ''' if len(args) == 2: print "Revoking task ", args[1] app.current_app().control.revoke(args[1])
def setUp(self): from celery.app import current_app self.app = self._current_app = current_app() self.setup()
def handle_revoke(self,*args,**options): ''' revoke specified task ''' if len(args) == 2: print "Revoking task ",args[1] app.current_app().control.revoke(args[1])