def process_initializer(): # There seems to a bug in multiprocessing (backport?) # when detached, where the worker gets EOFErrors from time to time # and the logger is left from the parent process causing a crash. _hijack_multiprocessing_logger() platform.reset_signal("SIGTERM") platform.set_mp_process_title("celeryd") # This is for windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. from celery.loaders import current_loader current_loader().init_worker()
def init_loader(self): # Run the worker init handler. # (Usually imports task modules and such.) from celery.loaders import current_loader self.loader = current_loader() self.loader.init_worker()
def init_loader(self): from celery.loaders import current_loader, load_settings self.loader = current_loader() self.settings = load_settings() if not self.loader.configured: raise ImproperlyConfigured( "Celery needs to be configured to run celeryd.")
def get_registered_tasks(): from celery.loaders import current_loader from ws.tasks import BPMTask current_loader = current_loader() current_loader.import_default_modules() return [key for key, value in task_registry.items()\ if isinstance(value, BPMTask)]
def test_send_email(self): from celery.loaders import current_loader loader = current_loader() old_mail_admins = loader.mail_admins old_enable_mails = mytask.send_error_emails mail_sent = [False] def mock_mail_admins(*args, **kwargs): mail_sent[0] = True loader.mail_admins = mock_mail_admins mytask.send_error_emails = True try: tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) try: raise KeyError("moofoobar") except: einfo = ExceptionInfo(sys.exc_info()) tw.on_failure(einfo) self.assertTrue(mail_sent[0]) mail_sent[0] = False mytask.send_error_emails = False tw.on_failure(einfo) self.assertFalse(mail_sent[0]) finally: loader.mail_admins = old_mail_admins mytask.send_error_emails = old_enable_mails
def __init__(self, *args, **kwargs): self.loader = kwargs.pop("loader", current_loader()) super(WorkerTaskTrace, self).__init__(*args, **kwargs) self._store_errors = True if self.task.ignore_result: self._store_errors = conf.STORE_ERRORS_EVEN_IF_IGNORED self.super = super(WorkerTaskTrace, self)
def process_initializer(hostname): """Initializes the process so it can be used to process tasks. Used for multiprocessing environments. """ map(platforms.reset_signal, WORKER_SIGRESET) map(platforms.ignore_signal, WORKER_SIGIGNORE) platforms.set_mp_process_title("celeryd", hostname=hostname) # This is for windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. from celery.loaders import current_loader current_loader().init_worker() signals.worker_process_init.send(sender=None)
def process_initializer(): """Initializes the process so it can be used to process tasks. Used for multiprocessing environments. """ map(platform.reset_signal, WORKER_SIGRESET) map(platform.ignore_signal, WORKER_SIGIGNORE) platform.set_mp_process_title("celeryd") # This is for windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. from celery.loaders import current_loader current_loader().init_worker() signals.worker_process_init.send(sender=None)
def test_load_settings(self): loader = loaders.current_loader() loaders._settings = None settings = loaders.load_settings() self.assertTrue(loaders._settings) settings = loaders.load_settings() self.assertIs(settings, loaders._settings) self.assertIs(settings, loader.conf)
def test_current_loader(self): warnings.resetwarnings() with catch_warnings(record=True) as log: self.assertIs(loaders.current_loader(), self.app.loader) warning = log[0].message self.assertIsInstance(warning, CPendingDeprecationWarning) self.assertIn("deprecation", warning.args[0])
def __init__(self, *args, **kwargs): self.loader = kwargs.get("loader") or current_loader() self.hostname = kwargs.get("hostname") or socket.gethostname() super(WorkerTaskTrace, self).__init__(*args, **kwargs) self._store_errors = True if self.task.ignore_result: self._store_errors = self.task.store_errors_even_if_ignored self.super = super(WorkerTaskTrace, self)
def run_clockservice(loglevel=conf.CELERYBEAT_LOG_LEVEL, logfile=conf.CELERYBEAT_LOG_FILE, schedule=conf.CELERYBEAT_SCHEDULE_FILENAME, **kwargs): """Starts the celerybeat clock server.""" print("celerybeat %s is starting." % celery.__version__) # Setup logging if not isinstance(loglevel, int): loglevel = conf.LOG_LEVELS[loglevel.upper()] # Run the worker init handler. # (Usually imports task modules and such.) from celery.loaders import current_loader current_loader().init_worker() # Dump configuration to screen so we have some basic information # when users sends e-mails. print(STARTUP_INFO_FMT % { "conninfo": info.format_broker_info(), "logfile": logfile or "@stderr", "loglevel": conf.LOG_LEVELS[loglevel], "schedule": schedule, }) print("celerybeat has started.") arg_start = "manage" in sys.argv[0] and 2 or 1 platform.set_process_title("celerybeat", info=" ".join(sys.argv[arg_start:])) def _run_clock(): from celery.log import setup_logger logger = setup_logger(loglevel, logfile) clockservice = ClockService(logger=logger, schedule_filename=schedule) try: install_sync_handler(clockservice) clockservice.start() except Exception, e: emergency_error(logfile, "celerybeat raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
def configure(): from celery.loaders import current_loader from django.conf import settings if not settings.configured: settings_module = os.environ.get("CELERY_CONFIG_MODULE", "celeryconfig") settings.configure(SETTINGS_MODULE=settings_module, **dict(DEFAULTS, **current_loader().conf)) settings.DEBUG = True
def send_error_email(self, task, context, exc, whitelist=None, enabled=False, fail_silently=True): if enabled and not task.disable_error_emails: if whitelist: if not isinstance(exc, tuple(whitelist)): return subject = self.email_subject.strip() % context body = self.email_body.strip() % context return current_loader().mail_admins(subject, body, fail_silently=fail_silently)
def process_initializer(): """Initializes the process so it can be used to process tasks. Used for multiprocessing environments. """ # There seems to a bug in multiprocessing (backport?) # when detached, where the worker gets EOFErrors from time to time # and the logger is left from the parent process causing a crash. _hijack_multiprocessing_logger() platform.reset_signal("SIGTERM") platform.ignore_signal("SIGINT") platform.set_mp_process_title("celeryd") # This is for windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. from celery.loaders import current_loader current_loader().init_worker() signals.worker_process_init.send(sender=None)
def process_initializer(): """Initializes the process so it can be used to process tasks. Used for multiprocessing environments. """ # There seems to a bug in multiprocessing (backport?) # when detached, where the worker gets EOFErrors from time to time # and the logger is left from the parent process causing a crash. _hijack_multiprocessing_logger() map(platform.reset_signal, WORKER_SIGRESET) map(platform.ignore_signal, WORKER_SIGIGNORE) platform.set_mp_process_title("celeryd") # This is for windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. from celery.loaders import current_loader current_loader().init_worker() signals.worker_process_init.send(sender=None)
def process_initializer(): # There seems to a bug in multiprocessing (backport?) # when detached, where the worker gets EOFErrors from time to time # and the logger is left from the parent process causing a crash. _hijack_multiprocessing_logger() platform.reset_signal("SIGTERM") platform.set_mp_process_title("celeryd") # On Windows we need to run a dummy command 'celeryinit' # for django to fully initialize after fork() if not callable(getattr(os, "fork", None)): from django.core.management import execute_manager settings_mod = os.environ.get("DJANGO_SETTINGS_MODULE", "settings") project_settings = __import__(settings_mod, {}, {}, [""]) execute_manager(project_settings, argv=["manage.py", "celeryinit"]) # This is for windows and other platforms not supporting # fork(). Note that init_worker makes sure it's only # run once per process. from celery.loaders import current_loader current_loader().init_worker()
def test_current_loader(self): with self.assertWarnsRegex(CPendingDeprecationWarning, r'deprecation'): self.assertIs(loaders.current_loader(), self.app.loader)
def get_backend_cls(backend): """Get backend class by name/alias""" if backend not in _backend_cache: aliases = dict(BACKEND_ALIASES, **current_loader().override_backends) _backend_cache[backend] = get_cls_by_name(backend, aliases) return _backend_cache[backend]
def test_current_loader(self): with self.assertPendingDeprecation(): self.assertIs(loaders.current_loader(), self.app.loader)
def test_current_loader(self): self.assertIs(loaders.current_loader(), self.app.loader)
def test_current_loader(self): loader1 = loaders.current_loader() loader2 = loaders.current_loader() self.assertIs(loader1, loader2) self.assertIs(loader2, loaders._loader)
def run_worker(concurrency=conf.CELERYD_CONCURRENCY, loglevel=conf.CELERYD_LOG_LEVEL, logfile=conf.CELERYD_LOG_FILE, hostname=None, discard=False, run_clockservice=False, events=False, **kwargs): """Starts the celery worker server.""" hostname = hostname or socket.gethostname() print("celery@%s v%s is starting." % (hostname, celery.__version__)) from celery.loaders import current_loader, load_settings loader = current_loader() settings = load_settings() if not concurrency: concurrency = multiprocessing.cpu_count() if conf.CELERY_BACKEND == "database" \ and settings.DATABASE_ENGINE == "sqlite3" and \ concurrency > 1: import warnings warnings.warn("The sqlite3 database engine doesn't support " "concurrency. We'll be using a single process only.", UserWarning) concurrency = 1 # Setup logging if not isinstance(loglevel, int): loglevel = conf.LOG_LEVELS[loglevel.upper()] if discard: discarded_count = discard_all() what = discarded_count > 1 and "messages" or "message" print("discard: Erased %d %s from the queue.\n" % ( discarded_count, what)) # Run the worker init handler. # (Usually imports task modules and such.) loader.on_worker_init() # Dump configuration to screen so we have some basic information # when users sends e-mails. tasklist = "" if loglevel <= logging.INFO: from celery.registry import tasks tasklist = tasks.keys() if not loglevel <= logging.DEBUG: tasklist = filter(lambda s: not s.startswith("celery."), tasklist) tasklist = TASK_LIST_FMT % "\n".join(" . %s" % task for task in sorted(tasklist)) print(STARTUP_INFO_FMT % { "conninfo": info.format_broker_info(), "queues": info.format_routing_table(indent=8), "concurrency": concurrency, "loglevel": conf.LOG_LEVELS[loglevel], "logfile": logfile or "[stderr]", "celerybeat": run_clockservice and "ON" or "OFF", "events": events and "ON" or "OFF", "tasks": tasklist, "loader": loader.__class__.__module__, }) print("Celery has started.") set_process_status("Running...") def run_worker(): worker = WorkController(concurrency=concurrency, loglevel=loglevel, logfile=logfile, hostname=hostname, embed_clockservice=run_clockservice, send_events=events) # Install signal handler so SIGHUP restarts the worker. install_worker_restart_handler(worker) from celery import signals signals.worker_init.send(sender=worker) try: worker.start() except Exception, e: emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
def test_loader_is_configured(self): from celery.loaders import current_loader, load_settings loader = current_loader() self.assertIsInstance(loader, celery.FlaskLoader) settings = load_settings() self.assertTrue(loader.configured)
def init_loader(self): # Run the worker init handler. # (Usually imports task modules and such.) from celery.loaders import current_loader current_loader().init_worker()
def test_current_loader(self): with self.assertWarnsRegex(CPendingDeprecationWarning, r"deprecation"): self.assertIs(loaders.current_loader(), self.app.loader)
def init_loader(self): from celery.loaders import current_loader, load_settings self.loader = current_loader() self.settings = load_settings()
def main(): from celery.loaders import current_loader current_loader().init_worker() connect_to_server() reactor.run()