def test_load_settings(self):
loader = loaders.current_loader()
loaders._settings = None
settings = loaders.load_settings()
self.assertTrue(loaders._settings)
settings = loaders.load_settings()
self.assertIs(settings, loaders._settings)
self.assertIs(settings, loader.conf)
def __init__(
self,
redis_host=None,
redis_port=None,
redis_db=None,
redis_timeout=None,
redis_connect_retry=None,
redis_connect_timeout=None,
):
if redis is None:
raise ImproperlyConfigured(
"You need to install the redis library in order to use " + "Redis result store backend."
)
settings = load_settings()
self.redis_host = redis_host or getattr(settings, "REDIS_HOST", self.redis_host)
self.redis_port = redis_port or getattr(settings, "REDIS_PORT", self.redis_port)
self.redis_db = redis_db or getattr(settings, "REDIS_DB", self.redis_db)
self.redis_timeout = redis_timeout or getattr(settings, "REDIS_TIMEOUT", self.redis_timeout)
self.redis_connect_retry = redis_connect_retry or getattr(
settings, "REDIS_CONNECT_RETRY", self.redis_connect_retry
)
if self.redis_port:
self.redis_port = int(self.redis_port)
if not self.redis_host or not self.redis_port:
raise ImproperlyConfigured(
"In order to use the Redis result store backend, you have to "
"set the REDIS_HOST and REDIS_PORT settings"
)
super(RedisBackend, self).__init__()
self._connection = None
def init_loader(self):
from celery.loaders import current_loader, load_settings
self.loader = current_loader()
self.settings = load_settings()
if not self.loader.configured:
raise ImproperlyConfigured(
"Celery needs to be configured to run celeryd.")
def init_loader(self):
from celery.loaders import current_loader, load_settings
self.loader = current_loader()
self.settings = load_settings()
if not self.loader.configured:
raise ImproperlyConfigured(
"Celery needs to be configured to run celeryd.")
def __init__(self, tyrant_host=None, tyrant_port=None):
"""Initialize Tokyo Tyrant backend instance.
Raises :class:`celery.exceptions.ImproperlyConfigured` if
:setting:`TT_HOST` or :setting:`TT_PORT` is not set.
"""
if not pytyrant:
raise ImproperlyConfigured(
"You need to install the pytyrant library to use the "
+ "Tokyo Tyrant backend.")
settings = load_settings()
self.tyrant_host = tyrant_host or \
getattr(settings, "TT_HOST", self.tyrant_host)
self.tyrant_port = tyrant_port or \
getattr(settings, "TT_PORT", self.tyrant_port)
if self.tyrant_port:
self.tyrant_port = int(self.tyrant_port)
if not self.tyrant_host or not self.tyrant_port:
raise ImproperlyConfigured(
"To use the Tokyo Tyrant backend, you have to "
"set the TT_HOST and TT_PORT settings in your settings.py")
super(TyrantBackend, self).__init__()
self._connection = None
def __init__(self, *args, **kwargs):
"""Initialize MongoDB backend instance.
:raises django.core.exceptions.ImproperlyConfigured: if
module :mod:`pymongo` is not available.
"""
if not pymongo:
raise ImproperlyConfigured(
"You need to install the pymongo library to use the "
"MongoDB backend.")
settings = load_settings()
config = getattr(settings, "CELERY_MONGODB_BACKEND_SETTINGS", None)
if config is not None:
if not isinstance(config, dict):
raise ImproperlyConfigured(
"MongoDB backend settings should be grouped in a dict")
self.mongodb_host = config.get('host', self.mongodb_host)
self.mongodb_port = int(config.get('port', self.mongodb_port))
self.mongodb_user = config.get('user', self.mongodb_user)
self.mongodb_password = config.get(
'password', self.mongodb_password)
self.mongodb_database = config.get(
'database', self.mongodb_database)
self.mongodb_taskmeta_collection = config.get(
'taskmeta_collection', self.mongodb_taskmeta_collection)
super(MongoBackend, self).__init__(*args, **kwargs)
self._connection = None
self._database = None
def test_load_settings(self):
warnings.resetwarnings()
with catch_warnings(record=True) as log:
self.assertIs(loaders.load_settings(), self.app.conf)
warning = log[0].message
self.assertIsInstance(warning, CPendingDeprecationWarning)
self.assertIn("deprecation", warning.args[0])
def test_load_settings(self):
warnings.resetwarnings()
with catch_warnings(record=True) as log:
self.assertIs(loaders.load_settings(), self.app.conf)
warning = log[0].message
self.assertIsInstance(warning, CPendingDeprecationWarning)
self.assertIn("deprecation", warning.args[0])
def test_constructor_raises_if_not_host_or_port(self):
from celery.backends import pyredis
from celery.loaders import load_settings
settings = load_settings()
prev_host, settings.REDIS_HOST = settings.REDIS_HOST, None
prev_port, settings.REDIS_PORT = settings.REDIS_PORT, None
try:
self.assertRaises(ImproperlyConfigured, pyredis.RedisBackend)
finally:
settings.REDIS_HOST = prev_host
settings.REDIS_PORT = prev_port
def test_constructor_raises_if_not_host_or_port(self):
from celery.backends import pyredis
from celery.loaders import load_settings
settings = load_settings()
prev_host, settings.REDIS_HOST = settings.REDIS_HOST, None
prev_port, settings.REDIS_PORT = settings.REDIS_PORT, None
try:
self.assertRaises(ImproperlyConfigured, pyredis.RedisBackend)
finally:
settings.REDIS_HOST = prev_host
settings.REDIS_PORT = prev_port
def __init__(self,
redis_host=None,
redis_port=None,
redis_db=None,
redis_timeout=None,
redis_password=None,
redis_connect_retry=None,
redis_connect_timeout=None,
expires=None):
if redis is None:
raise ImproperlyConfigured(
"You need to install the redis library in order to use " +
"Redis result store backend.")
settings = load_settings()
self.redis_host = redis_host or \
getattr(settings, "REDIS_HOST", self.redis_host)
self.redis_port = redis_port or \
getattr(settings, "REDIS_PORT", self.redis_port)
self.redis_db = redis_db or \
getattr(settings, "REDIS_DB", self.redis_db)
self.redis_password = redis_password or \
getattr(settings, "REDIS_PASSWORD",
self.redis_password)
self.expires = expires
if self.expires is None:
self.expires = conf.TASK_RESULT_EXPIRES
if isinstance(self.expires, timedelta):
self.expires = timeutils.timedelta_seconds(self.expires)
if self.expires is not None:
self.expires = int(self.expires)
for setting_name in self.deprecated_settings:
if getattr(settings, setting_name, None) is not None:
warnings.warn(
"The setting '%s' is no longer supported by the "
"python Redis client!" % setting_name.upper(),
DeprecationWarning)
if self.redis_port:
self.redis_port = int(self.redis_port)
if not self.redis_host or not self.redis_port:
raise ImproperlyConfigured(
"In order to use the Redis result store backend, you have to "
"set the REDIS_HOST and REDIS_PORT settings")
super(RedisBackend, self).__init__()
self._connection = None
def __init__(self, redis_host=None, redis_port=None, redis_db=None,
redis_timeout=None,
redis_password=None,
redis_connect_retry=None,
redis_connect_timeout=None,
expires=None):
if redis is None:
raise ImproperlyConfigured(
"You need to install the redis library in order to use "
+ "Redis result store backend.")
settings = load_settings()
self.redis_host = redis_host or \
getattr(settings, "REDIS_HOST", self.redis_host)
self.redis_port = redis_port or \
getattr(settings, "REDIS_PORT", self.redis_port)
self.redis_db = redis_db or \
getattr(settings, "REDIS_DB", self.redis_db)
self.redis_password = redis_password or \
getattr(settings, "REDIS_PASSWORD",
self.redis_password)
self.expires = expires
if self.expires is None:
self.expires = conf.TASK_RESULT_EXPIRES
if isinstance(self.expires, timedelta):
self.expires = timeutils.timedelta_seconds(self.expires)
if self.expires is not None:
self.expires = int(self.expires)
for setting_name in self.deprecated_settings:
if getattr(settings, setting_name, None) is not None:
warnings.warn(
"The setting '%s' is no longer supported by the "
"python Redis client!" % setting_name.upper(),
DeprecationWarning)
if self.redis_port:
self.redis_port = int(self.redis_port)
if not self.redis_host or not self.redis_port:
raise ImproperlyConfigured(
"In order to use the Redis result store backend, you have to "
"set the REDIS_HOST and REDIS_PORT settings")
super(RedisBackend, self).__init__()
self._connection = None
def __init__(self, servers=None, keyspace=None, column_family=None,
cassandra_options=None, **kwargs):
"""Initialize Cassandra backend.
Raises :class:`celery.exceptions.ImproperlyConfigured` if
the ``CASSANDRA_SERVERS`` setting is not set.
"""
self.logger = setup_logger("celery.backends.cassandra")
self.result_expires = kwargs.get("result_expires") or \
conf.TASK_RESULT_EXPIRES
if not pycassa:
raise ImproperlyConfigured(
"You need to install the pycassa library to use the "
"Cassandra backend. See http://github.com/vomjom/pycassa")
settings = load_settings()
self.servers = servers or \
getattr(settings, "CASSANDRA_SERVERS", self.servers)
self.keyspace = keyspace or \
getattr(settings, "CASSANDRA_KEYSPACE",
self.keyspace)
self.column_family = column_family or \
getattr(settings, "CASSANDRA_COLUMN_FAMILY",
self.column_family)
self.cassandra_options = dict(cassandra_options or {},
**getattr(settings,
"CASSANDRA_OPTIONS", {}))
if not self.servers or not self.keyspace or not self.column_family:
raise ImproperlyConfigured(
"Cassandra backend not configured.")
super(CassandraBackend, self).__init__()
self._column_family = None
def __init__(self, servers=None, keyspace=None, column_family=None,
cassandra_options=None, **kwargs):
"""Initialize Cassandra backend.
Raises :class:`celery.exceptions.ImproperlyConfigured` if
the :setting:`CASSANDRA_SERVERS` setting is not set.
"""
self.logger = setup_logger("celery.backends.cassandra")
self.result_expires = kwargs.get("result_expires") or \
conf.TASK_RESULT_EXPIRES
if not pycassa:
raise ImproperlyConfigured(
"You need to install the pycassa library to use the "
"Cassandra backend. See http://github.com/vomjom/pycassa")
settings = load_settings()
self.servers = servers or \
getattr(settings, "CASSANDRA_SERVERS", self.servers)
self.keyspace = keyspace or \
getattr(settings, "CASSANDRA_KEYSPACE",
self.keyspace)
self.column_family = column_family or \
getattr(settings, "CASSANDRA_COLUMN_FAMILY",
self.column_family)
self.cassandra_options = dict(cassandra_options or {},
**getattr(settings,
"CASSANDRA_OPTIONS", {}))
if not self.servers or not self.keyspace or not self.column_family:
raise ImproperlyConfigured(
"Cassandra backend not configured.")
super(CassandraBackend, self).__init__()
self._column_family = None
def __init__(self, *args, **kwargs):
"""Initialize MongoDB backend instance.
:raises celery.exceptions.ImproperlyConfigured: if
module :mod:`pymongo` is not available.
"""
self.result_expires = kwargs.get("result_expires") or \
conf.TASK_RESULT_EXPIRES
if not pymongo:
raise ImproperlyConfigured(
"You need to install the pymongo library to use the "
"MongoDB backend.")
settings = load_settings()
config = getattr(settings, "CELERY_MONGODB_BACKEND_SETTINGS", None)
if config is not None:
if not isinstance(config, dict):
raise ImproperlyConfigured(
"MongoDB backend settings should be grouped in a dict")
self.mongodb_host = config.get("host", self.mongodb_host)
self.mongodb_port = int(config.get("port", self.mongodb_port))
self.mongodb_user = config.get("user", self.mongodb_user)
self.mongodb_password = config.get(
"password", self.mongodb_password)
self.mongodb_database = config.get(
"database", self.mongodb_database)
self.mongodb_taskmeta_collection = config.get(
"taskmeta_collection", self.mongodb_taskmeta_collection)
super(MongoBackend, self).__init__(*args, **kwargs)
self._connection = None
self._database = None
def __init__(self, *args, **kwargs):
"""Initialize MongoDB backend instance.
:raises celery.exceptions.ImproperlyConfigured: if
module :mod:`pymongo` is not available.
"""
self.result_expires = kwargs.get("result_expires") or \
conf.TASK_RESULT_EXPIRES
if not pymongo:
raise ImproperlyConfigured(
"You need to install the pymongo library to use the "
"MongoDB backend.")
settings = load_settings()
config = getattr(settings, "CELERY_MONGODB_BACKEND_SETTINGS", None)
if config is not None:
if not isinstance(config, dict):
raise ImproperlyConfigured(
"MongoDB backend settings should be grouped in a dict")
self.mongodb_host = config.get("host", self.mongodb_host)
self.mongodb_port = int(config.get("port", self.mongodb_port))
self.mongodb_user = config.get("user", self.mongodb_user)
self.mongodb_password = config.get("password",
self.mongodb_password)
self.mongodb_database = config.get("database",
self.mongodb_database)
self.mongodb_taskmeta_collection = config.get(
"taskmeta_collection", self.mongodb_taskmeta_collection)
super(MongoBackend, self).__init__(*args, **kwargs)
self._connection = None
self._database = None
def run_worker(concurrency=conf.CELERYD_CONCURRENCY,
loglevel=conf.CELERYD_LOG_LEVEL, logfile=conf.CELERYD_LOG_FILE,
hostname=None,
discard=False, run_clockservice=False, events=False, **kwargs):
"""Starts the celery worker server."""
hostname = hostname or socket.gethostname()
print("celery@%s v%s is starting." % (hostname, celery.__version__))
from celery.loaders import current_loader, load_settings
loader = current_loader()
settings = load_settings()
if not concurrency:
concurrency = multiprocessing.cpu_count()
if conf.CELERY_BACKEND == "database" \
and settings.DATABASE_ENGINE == "sqlite3" and \
concurrency > 1:
import warnings
warnings.warn("The sqlite3 database engine doesn't support "
"concurrency. We'll be using a single process only.",
UserWarning)
concurrency = 1
# Setup logging
if not isinstance(loglevel, int):
loglevel = conf.LOG_LEVELS[loglevel.upper()]
if discard:
discarded_count = discard_all()
what = discarded_count > 1 and "messages" or "message"
print("discard: Erased %d %s from the queue.\n" % (
discarded_count, what))
# Run the worker init handler.
# (Usually imports task modules and such.)
loader.on_worker_init()
# Dump configuration to screen so we have some basic information
# when users sends e-mails.
tasklist = ""
if loglevel <= logging.INFO:
from celery.registry import tasks
tasklist = tasks.keys()
if not loglevel <= logging.DEBUG:
tasklist = filter(lambda s: not s.startswith("celery."), tasklist)
tasklist = TASK_LIST_FMT % "\n".join(" . %s" % task
for task in sorted(tasklist))
print(STARTUP_INFO_FMT % {
"conninfo": info.format_broker_info(),
"queues": info.format_routing_table(indent=8),
"concurrency": concurrency,
"loglevel": conf.LOG_LEVELS[loglevel],
"logfile": logfile or "[stderr]",
"celerybeat": run_clockservice and "ON" or "OFF",
"events": events and "ON" or "OFF",
"tasks": tasklist,
"loader": loader.__class__.__module__,
})
print("Celery has started.")
set_process_status("Running...")
def run_worker():
worker = WorkController(concurrency=concurrency,
loglevel=loglevel,
logfile=logfile,
hostname=hostname,
embed_clockservice=run_clockservice,
send_events=events)
# Install signal handler so SIGHUP restarts the worker.
install_worker_restart_handler(worker)
from celery import signals
signals.worker_init.send(sender=worker)
try:
worker.start()
except Exception, e:
emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % (
e.__class__, e, traceback.format_exc()))
import logging
import warnings
from datetime import timedelta
from celery.loaders import load_settings
DEFAULT_P_LOG_FMT = '[%(asctime)s: %(levelname)s/%(processName)s] %(message)s'
DEFAULT_LOG_FMT = '[%(asctime)s: %(levelname)s] %(message)s'
LOG_LEVELS = dict(logging._levelNames)
LOG_LEVELS["FATAL"] = logging.FATAL
LOG_LEVELS[logging.FATAL] = "FATAL"
settings = load_settings()
_DEFAULTS = {
"CELERY_BACKEND": "database",
"CELERY_ALWAYS_EAGER": False,
"CELERY_TASK_RESULT_EXPIRES": timedelta(days=5),
"CELERY_SEND_EVENTS": False,
"CELERY_IGNORE_RESULT": False,
"CELERY_STORE_ERRORS_EVEN_IF_IGNORED": False,
"CELERY_TASK_SERIALIZER": "pickle",
"CELERY_DISABLE_RATE_LIMITS": False,
"CELERY_DEFAULT_ROUTING_KEY": "celery",
"CELERY_DEFAULT_QUEUE": "celery",
"CELERY_DEFAULT_EXCHANGE": "celery",
"CELERY_DEFAULT_EXCHANGE_TYPE": "direct",
"CELERY_BROKER_CONNECTION_TIMEOUT": 4,
"CELERY_BROKER_CONNECTION_RETRY": True,
"CELERY_BROKER_CONNECTION_MAX_RETRIES": 100,
def test_load_settings(self):
with self.assertWarnsRegex(CPendingDeprecationWarning, r"deprecation"):
self.assertIs(loaders.load_settings(), self.app.conf)
from celery.loaders import load_settings
DEFAULT_PROCESS_LOG_FMT = """
[%(asctime)s: %(levelname)s/%(processName)s] %(message)s
""".strip()
DEFAULT_LOG_FMT = '[%(asctime)s: %(levelname)s] %(message)s'
DEFAULT_TASK_LOG_FMT = " ".join("""
[%(asctime)s: %(levelname)s/%(processName)s]
[%(task_name)s(%(task_id)s)] %(message)s
""".strip().split())
LOG_LEVELS = dict(logging._levelNames)
LOG_LEVELS["FATAL"] = logging.FATAL
LOG_LEVELS[logging.FATAL] = "FATAL"
settings = load_settings()
_DEFAULTS = {
"CELERY_RESULT_BACKEND": "database",
"CELERY_ALWAYS_EAGER": False,
"CELERY_EAGER_PROPAGATES_EXCEPTIONS": False,
"CELERY_TASK_RESULT_EXPIRES": timedelta(days=1),
"CELERY_SEND_EVENTS": False,
"CELERY_IGNORE_RESULT": False,
"CELERY_STORE_ERRORS_EVEN_IF_IGNORED": False,
"CELERY_TASK_SERIALIZER": "pickle",
"CELERY_DISABLE_RATE_LIMITS": False,
"CELERYD_TASK_TIME_LIMIT": None,
"CELERYD_TASK_SOFT_TIME_LIMIT": None,
"CELERYD_MAX_TASKS_PER_CHILD": None,
"CELERY_ROUTES": None,
def test_load_settings(self):
self.assertIs(loaders.load_settings(), self.app.conf)
def test_load_settings(self):
self.assertIs(loaders.load_settings(), self.app.conf)
def init_loader(self):
from celery.loaders import current_loader, load_settings
self.loader = current_loader()
self.settings = load_settings()
def test_load_settings(self):
with self.assertPendingDeprecation():
self.assertIs(loaders.load_settings(), self.app.conf)
def test_load_settings(self):
with self.assertPendingDeprecation():
self.assertIs(loaders.load_settings(), self.app.conf)
def test_load_settings(self):
with self.assertWarnsRegex(CPendingDeprecationWarning, r'deprecation'):
self.assertIs(loaders.load_settings(), self.app.conf)
def test_loader_is_configured(self):
from celery.loaders import current_loader, load_settings
loader = current_loader()
self.assertIsInstance(loader, celery.FlaskLoader)
settings = load_settings()
self.assertTrue(loader.configured)
