def test_load_settings(self): loader = loaders.current_loader() loaders._settings = None settings = loaders.load_settings() self.assertTrue(loaders._settings) settings = loaders.load_settings() self.assertIs(settings, loaders._settings) self.assertIs(settings, loader.conf)
def __init__( self, redis_host=None, redis_port=None, redis_db=None, redis_timeout=None, redis_connect_retry=None, redis_connect_timeout=None, ): if redis is None: raise ImproperlyConfigured( "You need to install the redis library in order to use " + "Redis result store backend." ) settings = load_settings() self.redis_host = redis_host or getattr(settings, "REDIS_HOST", self.redis_host) self.redis_port = redis_port or getattr(settings, "REDIS_PORT", self.redis_port) self.redis_db = redis_db or getattr(settings, "REDIS_DB", self.redis_db) self.redis_timeout = redis_timeout or getattr(settings, "REDIS_TIMEOUT", self.redis_timeout) self.redis_connect_retry = redis_connect_retry or getattr( settings, "REDIS_CONNECT_RETRY", self.redis_connect_retry ) if self.redis_port: self.redis_port = int(self.redis_port) if not self.redis_host or not self.redis_port: raise ImproperlyConfigured( "In order to use the Redis result store backend, you have to " "set the REDIS_HOST and REDIS_PORT settings" ) super(RedisBackend, self).__init__() self._connection = None
def init_loader(self): from celery.loaders import current_loader, load_settings self.loader = current_loader() self.settings = load_settings() if not self.loader.configured: raise ImproperlyConfigured( "Celery needs to be configured to run celeryd.")
def __init__(self, tyrant_host=None, tyrant_port=None): """Initialize Tokyo Tyrant backend instance. Raises :class:`celery.exceptions.ImproperlyConfigured` if :setting:`TT_HOST` or :setting:`TT_PORT` is not set. """ if not pytyrant: raise ImproperlyConfigured( "You need to install the pytyrant library to use the " + "Tokyo Tyrant backend.") settings = load_settings() self.tyrant_host = tyrant_host or \ getattr(settings, "TT_HOST", self.tyrant_host) self.tyrant_port = tyrant_port or \ getattr(settings, "TT_PORT", self.tyrant_port) if self.tyrant_port: self.tyrant_port = int(self.tyrant_port) if not self.tyrant_host or not self.tyrant_port: raise ImproperlyConfigured( "To use the Tokyo Tyrant backend, you have to " "set the TT_HOST and TT_PORT settings in your settings.py") super(TyrantBackend, self).__init__() self._connection = None
def __init__(self, *args, **kwargs): """Initialize MongoDB backend instance. :raises django.core.exceptions.ImproperlyConfigured: if module :mod:`pymongo` is not available. """ if not pymongo: raise ImproperlyConfigured( "You need to install the pymongo library to use the " "MongoDB backend.") settings = load_settings() config = getattr(settings, "CELERY_MONGODB_BACKEND_SETTINGS", None) if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( "MongoDB backend settings should be grouped in a dict") self.mongodb_host = config.get('host', self.mongodb_host) self.mongodb_port = int(config.get('port', self.mongodb_port)) self.mongodb_user = config.get('user', self.mongodb_user) self.mongodb_password = config.get( 'password', self.mongodb_password) self.mongodb_database = config.get( 'database', self.mongodb_database) self.mongodb_taskmeta_collection = config.get( 'taskmeta_collection', self.mongodb_taskmeta_collection) super(MongoBackend, self).__init__(*args, **kwargs) self._connection = None self._database = None
def test_load_settings(self): warnings.resetwarnings() with catch_warnings(record=True) as log: self.assertIs(loaders.load_settings(), self.app.conf) warning = log[0].message self.assertIsInstance(warning, CPendingDeprecationWarning) self.assertIn("deprecation", warning.args[0])
def test_constructor_raises_if_not_host_or_port(self): from celery.backends import pyredis from celery.loaders import load_settings settings = load_settings() prev_host, settings.REDIS_HOST = settings.REDIS_HOST, None prev_port, settings.REDIS_PORT = settings.REDIS_PORT, None try: self.assertRaises(ImproperlyConfigured, pyredis.RedisBackend) finally: settings.REDIS_HOST = prev_host settings.REDIS_PORT = prev_port
def __init__(self, redis_host=None, redis_port=None, redis_db=None, redis_timeout=None, redis_password=None, redis_connect_retry=None, redis_connect_timeout=None, expires=None): if redis is None: raise ImproperlyConfigured( "You need to install the redis library in order to use " + "Redis result store backend.") settings = load_settings() self.redis_host = redis_host or \ getattr(settings, "REDIS_HOST", self.redis_host) self.redis_port = redis_port or \ getattr(settings, "REDIS_PORT", self.redis_port) self.redis_db = redis_db or \ getattr(settings, "REDIS_DB", self.redis_db) self.redis_password = redis_password or \ getattr(settings, "REDIS_PASSWORD", self.redis_password) self.expires = expires if self.expires is None: self.expires = conf.TASK_RESULT_EXPIRES if isinstance(self.expires, timedelta): self.expires = timeutils.timedelta_seconds(self.expires) if self.expires is not None: self.expires = int(self.expires) for setting_name in self.deprecated_settings: if getattr(settings, setting_name, None) is not None: warnings.warn( "The setting '%s' is no longer supported by the " "python Redis client!" % setting_name.upper(), DeprecationWarning) if self.redis_port: self.redis_port = int(self.redis_port) if not self.redis_host or not self.redis_port: raise ImproperlyConfigured( "In order to use the Redis result store backend, you have to " "set the REDIS_HOST and REDIS_PORT settings") super(RedisBackend, self).__init__() self._connection = None
def __init__(self, servers=None, keyspace=None, column_family=None, cassandra_options=None, **kwargs): """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if the ``CASSANDRA_SERVERS`` setting is not set. """ self.logger = setup_logger("celery.backends.cassandra") self.result_expires = kwargs.get("result_expires") or \ conf.TASK_RESULT_EXPIRES if not pycassa: raise ImproperlyConfigured( "You need to install the pycassa library to use the " "Cassandra backend. See http://github.com/vomjom/pycassa") settings = load_settings() self.servers = servers or \ getattr(settings, "CASSANDRA_SERVERS", self.servers) self.keyspace = keyspace or \ getattr(settings, "CASSANDRA_KEYSPACE", self.keyspace) self.column_family = column_family or \ getattr(settings, "CASSANDRA_COLUMN_FAMILY", self.column_family) self.cassandra_options = dict(cassandra_options or {}, **getattr(settings, "CASSANDRA_OPTIONS", {})) if not self.servers or not self.keyspace or not self.column_family: raise ImproperlyConfigured( "Cassandra backend not configured.") super(CassandraBackend, self).__init__() self._column_family = None
def __init__(self, servers=None, keyspace=None, column_family=None, cassandra_options=None, **kwargs): """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if the :setting:`CASSANDRA_SERVERS` setting is not set. """ self.logger = setup_logger("celery.backends.cassandra") self.result_expires = kwargs.get("result_expires") or \ conf.TASK_RESULT_EXPIRES if not pycassa: raise ImproperlyConfigured( "You need to install the pycassa library to use the " "Cassandra backend. See http://github.com/vomjom/pycassa") settings = load_settings() self.servers = servers or \ getattr(settings, "CASSANDRA_SERVERS", self.servers) self.keyspace = keyspace or \ getattr(settings, "CASSANDRA_KEYSPACE", self.keyspace) self.column_family = column_family or \ getattr(settings, "CASSANDRA_COLUMN_FAMILY", self.column_family) self.cassandra_options = dict(cassandra_options or {}, **getattr(settings, "CASSANDRA_OPTIONS", {})) if not self.servers or not self.keyspace or not self.column_family: raise ImproperlyConfigured( "Cassandra backend not configured.") super(CassandraBackend, self).__init__() self._column_family = None
def __init__(self, *args, **kwargs): """Initialize MongoDB backend instance. :raises celery.exceptions.ImproperlyConfigured: if module :mod:`pymongo` is not available. """ self.result_expires = kwargs.get("result_expires") or \ conf.TASK_RESULT_EXPIRES if not pymongo: raise ImproperlyConfigured( "You need to install the pymongo library to use the " "MongoDB backend.") settings = load_settings() config = getattr(settings, "CELERY_MONGODB_BACKEND_SETTINGS", None) if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( "MongoDB backend settings should be grouped in a dict") self.mongodb_host = config.get("host", self.mongodb_host) self.mongodb_port = int(config.get("port", self.mongodb_port)) self.mongodb_user = config.get("user", self.mongodb_user) self.mongodb_password = config.get( "password", self.mongodb_password) self.mongodb_database = config.get( "database", self.mongodb_database) self.mongodb_taskmeta_collection = config.get( "taskmeta_collection", self.mongodb_taskmeta_collection) super(MongoBackend, self).__init__(*args, **kwargs) self._connection = None self._database = None
def __init__(self, *args, **kwargs): """Initialize MongoDB backend instance. :raises celery.exceptions.ImproperlyConfigured: if module :mod:`pymongo` is not available. """ self.result_expires = kwargs.get("result_expires") or \ conf.TASK_RESULT_EXPIRES if not pymongo: raise ImproperlyConfigured( "You need to install the pymongo library to use the " "MongoDB backend.") settings = load_settings() config = getattr(settings, "CELERY_MONGODB_BACKEND_SETTINGS", None) if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( "MongoDB backend settings should be grouped in a dict") self.mongodb_host = config.get("host", self.mongodb_host) self.mongodb_port = int(config.get("port", self.mongodb_port)) self.mongodb_user = config.get("user", self.mongodb_user) self.mongodb_password = config.get("password", self.mongodb_password) self.mongodb_database = config.get("database", self.mongodb_database) self.mongodb_taskmeta_collection = config.get( "taskmeta_collection", self.mongodb_taskmeta_collection) super(MongoBackend, self).__init__(*args, **kwargs) self._connection = None self._database = None
def run_worker(concurrency=conf.CELERYD_CONCURRENCY, loglevel=conf.CELERYD_LOG_LEVEL, logfile=conf.CELERYD_LOG_FILE, hostname=None, discard=False, run_clockservice=False, events=False, **kwargs): """Starts the celery worker server.""" hostname = hostname or socket.gethostname() print("celery@%s v%s is starting." % (hostname, celery.__version__)) from celery.loaders import current_loader, load_settings loader = current_loader() settings = load_settings() if not concurrency: concurrency = multiprocessing.cpu_count() if conf.CELERY_BACKEND == "database" \ and settings.DATABASE_ENGINE == "sqlite3" and \ concurrency > 1: import warnings warnings.warn("The sqlite3 database engine doesn't support " "concurrency. We'll be using a single process only.", UserWarning) concurrency = 1 # Setup logging if not isinstance(loglevel, int): loglevel = conf.LOG_LEVELS[loglevel.upper()] if discard: discarded_count = discard_all() what = discarded_count > 1 and "messages" or "message" print("discard: Erased %d %s from the queue.\n" % ( discarded_count, what)) # Run the worker init handler. # (Usually imports task modules and such.) loader.on_worker_init() # Dump configuration to screen so we have some basic information # when users sends e-mails. tasklist = "" if loglevel <= logging.INFO: from celery.registry import tasks tasklist = tasks.keys() if not loglevel <= logging.DEBUG: tasklist = filter(lambda s: not s.startswith("celery."), tasklist) tasklist = TASK_LIST_FMT % "\n".join(" . %s" % task for task in sorted(tasklist)) print(STARTUP_INFO_FMT % { "conninfo": info.format_broker_info(), "queues": info.format_routing_table(indent=8), "concurrency": concurrency, "loglevel": conf.LOG_LEVELS[loglevel], "logfile": logfile or "[stderr]", "celerybeat": run_clockservice and "ON" or "OFF", "events": events and "ON" or "OFF", "tasks": tasklist, "loader": loader.__class__.__module__, }) print("Celery has started.") set_process_status("Running...") def run_worker(): worker = WorkController(concurrency=concurrency, loglevel=loglevel, logfile=logfile, hostname=hostname, embed_clockservice=run_clockservice, send_events=events) # Install signal handler so SIGHUP restarts the worker. install_worker_restart_handler(worker) from celery import signals signals.worker_init.send(sender=worker) try: worker.start() except Exception, e: emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
import logging import warnings from datetime import timedelta from celery.loaders import load_settings DEFAULT_P_LOG_FMT = '[%(asctime)s: %(levelname)s/%(processName)s] %(message)s' DEFAULT_LOG_FMT = '[%(asctime)s: %(levelname)s] %(message)s' LOG_LEVELS = dict(logging._levelNames) LOG_LEVELS["FATAL"] = logging.FATAL LOG_LEVELS[logging.FATAL] = "FATAL" settings = load_settings() _DEFAULTS = { "CELERY_BACKEND": "database", "CELERY_ALWAYS_EAGER": False, "CELERY_TASK_RESULT_EXPIRES": timedelta(days=5), "CELERY_SEND_EVENTS": False, "CELERY_IGNORE_RESULT": False, "CELERY_STORE_ERRORS_EVEN_IF_IGNORED": False, "CELERY_TASK_SERIALIZER": "pickle", "CELERY_DISABLE_RATE_LIMITS": False, "CELERY_DEFAULT_ROUTING_KEY": "celery", "CELERY_DEFAULT_QUEUE": "celery", "CELERY_DEFAULT_EXCHANGE": "celery", "CELERY_DEFAULT_EXCHANGE_TYPE": "direct", "CELERY_BROKER_CONNECTION_TIMEOUT": 4, "CELERY_BROKER_CONNECTION_RETRY": True, "CELERY_BROKER_CONNECTION_MAX_RETRIES": 100,
def test_load_settings(self): with self.assertWarnsRegex(CPendingDeprecationWarning, r"deprecation"): self.assertIs(loaders.load_settings(), self.app.conf)
from celery.loaders import load_settings DEFAULT_PROCESS_LOG_FMT = """ [%(asctime)s: %(levelname)s/%(processName)s] %(message)s """.strip() DEFAULT_LOG_FMT = '[%(asctime)s: %(levelname)s] %(message)s' DEFAULT_TASK_LOG_FMT = " ".join(""" [%(asctime)s: %(levelname)s/%(processName)s] [%(task_name)s(%(task_id)s)] %(message)s """.strip().split()) LOG_LEVELS = dict(logging._levelNames) LOG_LEVELS["FATAL"] = logging.FATAL LOG_LEVELS[logging.FATAL] = "FATAL" settings = load_settings() _DEFAULTS = { "CELERY_RESULT_BACKEND": "database", "CELERY_ALWAYS_EAGER": False, "CELERY_EAGER_PROPAGATES_EXCEPTIONS": False, "CELERY_TASK_RESULT_EXPIRES": timedelta(days=1), "CELERY_SEND_EVENTS": False, "CELERY_IGNORE_RESULT": False, "CELERY_STORE_ERRORS_EVEN_IF_IGNORED": False, "CELERY_TASK_SERIALIZER": "pickle", "CELERY_DISABLE_RATE_LIMITS": False, "CELERYD_TASK_TIME_LIMIT": None, "CELERYD_TASK_SOFT_TIME_LIMIT": None, "CELERYD_MAX_TASKS_PER_CHILD": None, "CELERY_ROUTES": None,
def test_load_settings(self): self.assertIs(loaders.load_settings(), self.app.conf)
def init_loader(self): from celery.loaders import current_loader, load_settings self.loader = current_loader() self.settings = load_settings()
def test_load_settings(self): with self.assertPendingDeprecation(): self.assertIs(loaders.load_settings(), self.app.conf)
def test_load_settings(self): with self.assertWarnsRegex(CPendingDeprecationWarning, r'deprecation'): self.assertIs(loaders.load_settings(), self.app.conf)
def test_loader_is_configured(self): from celery.loaders import current_loader, load_settings loader = current_loader() self.assertIsInstance(loader, celery.FlaskLoader) settings = load_settings() self.assertTrue(loader.configured)