def process_initializer(app, hostname): """Pool child process initializer. This will initialize a child pool process to ensure the correct app instance is used and things like logging works. """ platforms.signals.reset(*WORKER_SIGRESET) platforms.signals.ignore(*WORKER_SIGIGNORE) platforms.set_mp_process_title('celeryd', hostname=hostname) # This is for Windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. app.loader.init_worker() app.loader.init_worker_process() app.log.setup(int(os.environ.get('CELERY_LOG_LEVEL', 0) or 0), os.environ.get('CELERY_LOG_FILE') or None, bool(os.environ.get('CELERY_LOG_REDIRECT', False)), str(os.environ.get('CELERY_LOG_REDIRECT_LEVEL'))) if os.environ.get('FORKED_BY_MULTIPROCESSING'): # pool did execv after fork trace.setup_worker_optimizations(app) else: app.set_current() set_default_app(app) app.finalize() trace._tasks = app._tasks # enables fast_trace_task optimization. # rebuild execution handler for all tasks. from celery.app.trace import build_tracer for name, task in items(app.tasks): task.__trace__ = build_tracer(name, task, app.loader, hostname, app=app) signals.worker_process_init.send(sender=None)
def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) assert trace.trace_task_ret is trace._fast_trace_task tid = uuid() message = self.TaskMessage(self.mytask.name, tid, args=[4]) assert len(message.payload) == 3 try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding) assert not failed assert res == repr(4 ** 4) assert runtime is not None assert isinstance(runtime, numbers.Real) finally: reset_worker_optimizations() assert trace.trace_task_ret is trace._trace_task_ret delattr(self.mytask, '__trace__') failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, ) assert not failed assert res == repr(4 ** 4) assert runtime is not None assert isinstance(runtime, numbers.Real)
def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) self.assertIs(trace.trace_task_ret, trace._fast_trace_task) tid = uuid() message = TaskMessage(self.mytask.name, tid, args=[4]) assert len(message.payload) == 3 try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) self.assertTrue(runtime) self.assertIsInstance(runtime, numbers.Real) finally: reset_worker_optimizations() self.assertIs(trace.trace_task_ret, trace._trace_task_ret) delattr(self.mytask, '__trace__') failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, ) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) self.assertTrue(runtime) self.assertIsInstance(runtime, numbers.Real)
def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) self.assertIs(trace.trace_task_ret, trace._fast_trace_task) try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {}) self.assertEqual(res, 4**4) finally: reset_worker_optimizations() self.assertIs(trace.trace_task_ret, trace._trace_task_ret) delattr(self.mytask, '__trace__') res = trace.trace_task_ret( self.mytask.name, uuid(), [4], {}, app=self.app, ) self.assertEqual(res, 4**4)
def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) self.assertIs(trace.trace_task_ret, trace._fast_trace_task) tid = uuid() message = TaskMessage(self.mytask.name, tid, args=[4]) assert len(message.payload) == 3 try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) self.assertIsNotNone(runtime) self.assertIsInstance(runtime, numbers.Real) finally: reset_worker_optimizations() self.assertIs(trace.trace_task_ret, trace._trace_task_ret) delattr(self.mytask, '__trace__') failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, ) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) self.assertIsNotNone(runtime) self.assertIsInstance(runtime, numbers.Real)
def on_before_init(self, **kwargs): trace.setup_worker_optimizations(self.app, self.hostname) # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send(sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs) check_privileges(self.app.conf.accept_content)
def on_before_init(self, **kwargs): trace.setup_worker_optimizations(self.app) # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send( sender=self.hostname, instance=self, conf=self.app.conf, )
def on_before_init(self, **kwargs): trace.setup_worker_optimizations(self.app) # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send( sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs, )
def on_before_init(self, **kwargs): trace.setup_worker_optimizations(self.app) # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send( sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs, ) check_privileges(self.app.conf.CELERY_ACCEPT_CONTENT)
def on_before_init(self, **kwargs): trace.setup_worker_optimizations(self.app, self.hostname) # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send( sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs, ) check_privileges(self.app.conf.CELERY_ACCEPT_CONTENT)
def on_before_init(self, quiet=False, **kwargs): self.quiet = quiet trace.setup_worker_optimizations(self.app, self.hostname) # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send( sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs, ) check_privileges(self.app.conf.accept_content)
def test_stackprotection(self): setup_worker_optimizations(self.app) try: @self.app.task(shared=False, bind=True) def foo(self, i): if i: return foo(0) return self.request assert foo(1).called_directly finally: reset_worker_optimizations()
def test_stackprotection(self): setup_worker_optimizations(self.app) try: @self.app.task(shared=False, bind=True) def foo(self, i): if i: return foo(0) return self.request self.assertTrue(foo(1).called_directly) finally: reset_worker_optimizations()
def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) self.assertIs(trace.trace_task_ret, trace._fast_trace_task) try: self.mytask.__trace__ = build_tracer(self.mytask.name, self.mytask, self.app.loader, "test", app=self.app) res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {}) self.assertEqual(res, 4 ** 4) finally: reset_worker_optimizations() self.assertIs(trace.trace_task_ret, trace._trace_task_ret) delattr(self.mytask, "__trace__") res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app) self.assertEqual(res, 4 ** 4)
def process_initializer(app, hostname): """Pool child process initializer. Initialize the child pool process to ensure the correct app instance is used and things like logging works. """ _set_task_join_will_block(True) platforms.signals.reset(*WORKER_SIGRESET) platforms.signals.ignore(*WORKER_SIGIGNORE) platforms.set_mp_process_title("celeryd", hostname=hostname) # This is for Windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. app.loader.init_worker() app.loader.init_worker_process() logfile = os.environ.get("CELERY_LOG_FILE") or None if logfile and "%i" in logfile.lower(): # logfile path will differ so need to set up logging again. app.log.already_setup = False app.log.setup( int(os.environ.get("CELERY_LOG_LEVEL", 0) or 0), logfile, bool(os.environ.get("CELERY_LOG_REDIRECT", False)), str(os.environ.get("CELERY_LOG_REDIRECT_LEVEL")), hostname=hostname, ) if os.environ.get("FORKED_BY_MULTIPROCESSING"): # pool did execv after fork trace.setup_worker_optimizations(app, hostname) else: app.set_current() set_default_app(app) app.finalize() trace._tasks = app._tasks # enables fast_trace_task optimization. # rebuild execution handler for all tasks. from celery.app.trace import build_tracer for name, task in items(app.tasks): task.__trace__ = build_tracer(name, task, app.loader, hostname, app=app) from celery.worker import state as worker_state worker_state.reset_state() signals.worker_process_init.send(sender=None)
def on_init_blueprint(self): self._custom_logging = self.setup_logging() # apply task execution optimizations # -- This will finalize the app! trace.setup_worker_optimizations(self.app, self.hostname)