def __init__(self, host=None, port=None, db=None, password=None, expires=None, max_connections=None, url=None, **kwargs): super(RedisBackend, self).__init__(**kwargs) conf = self.app.conf if self.redis is None: raise ImproperlyConfigured(REDIS_MISSING) # For compatibility with the old REDIS_* configuration keys. def _get(key): for prefix in 'CELERY_REDIS_{0}', 'REDIS_{0}': try: return conf[prefix.format(key)] except KeyError: pass if host and '://' in host: url, host = host, None self.url = url uhost = uport = upass = udb = None if url: _, uhost, uport, _, upass, udb, _ = _parse_url(url) udb = udb.strip('/') if udb else 0 self.host = uhost or host or _get('HOST') or self.host self.port = int(uport or port or _get('PORT') or self.port) self.db = udb or db or _get('DB') or self.db self.password = upass or password or _get('PASSWORD') or self.password self.expires = self.prepare_expires(expires, type=int) self.max_connections = (max_connections or _get('MAX_CONNECTIONS') or self.max_connections)
def __init__(self, app, expires=None, backend=None, options=None, url=None, **kwargs): options = {} if not options else options super(CacheBackend, self).__init__(app, **kwargs) self.url = url self.options = dict(self.app.conf.cache_backend_options, **options) self.backend = url or backend or self.app.conf.cache_backend if self.backend: self.backend, _, servers = self.backend.partition('://') self.servers = servers.rstrip('/').split(';') self.expires = self.prepare_expires(expires, type=int) try: self.Client, self.key_t = backends[self.backend]() except KeyError: raise ImproperlyConfigured( UNKNOWN_BACKEND.format(self.backend, ', '.join(backends))) self._encode_prefixes() # rencode the keyprefixes
def wait_for_pending(self, result, timeout=None, interval=0.5, no_ack=True, on_message=None, on_interval=None, callback=None, propagate=True): self._ensure_not_eager() if on_message is not None: raise ImproperlyConfigured( 'Backend does not support on_message callback') meta = self.wait_for( result.id, timeout=timeout, interval=interval, on_interval=on_interval, no_ack=no_ack, ) if meta: result._maybe_set_cache(meta) return result.maybe_throw(propagate=propagate, callback=callback)
def __init__(self, dburi=None, expires=None, engine_options=None, url=None, **kwargs): # The `url` argument was added later and is used by # the app to set backend by url (celery.backends.get_backend_by_url) super(DatabaseBackend, self).__init__(**kwargs) conf = self.app.conf self.expires = maybe_timedelta(self.prepare_expires(expires)) self.dburi = url or dburi or conf.CELERY_RESULT_DBURI self.engine_options = dict( engine_options or {}, **conf.CELERY_RESULT_ENGINE_OPTIONS or {}) self.short_lived_sessions = kwargs.get( 'short_lived_sessions', conf.CELERY_RESULT_DB_SHORT_LIVED_SESSIONS, ) tablenames = conf.CELERY_RESULT_DB_TABLENAMES or {} Task.__table__.name = tablenames.get('task', 'celery_taskmeta') TaskSet.__table__.name = tablenames.get('group', 'celery_tasksetmeta') if not self.dburi: raise ImproperlyConfigured( 'Missing connection string! Do you have ' 'CELERY_RESULT_DBURI set to a real value?')
def __init__( self, url=None, container_name=None, retry_initial_backoff_sec=None, retry_increment_base=None, retry_max_attempts=None, *args, **kwargs ): super(AzureBlockBlobBackend, self).__init__(*args, **kwargs) if azurestorage is None: raise ImproperlyConfigured( "You need to install the azure-storage library to use the " "AzureBlockBlob backend" ) conf = self.app.conf self._connection_string = self._parse_url(url) self._container_name = container_name or conf["azureblockblob_container_name"] self._retry_initial_backoff_sec = ( retry_initial_backoff_sec or conf["azureblockblob_retry_initial_backoff_sec"] ) self._retry_increment_base = ( retry_increment_base or conf["azureblockblob_retry_increment_base"] ) self._retry_max_attempts = ( retry_max_attempts or conf["azureblockblob_retry_max_attempts"] )
def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): # The `url` argument was added later and is used by # the app to set backend by url (celery.backends.get_backend_by_url) super(DatabaseBackend, self).__init__( expires_type=maybe_timedelta, **kwargs ) conf = self.app.conf self.dburi = url or dburi or conf.sqlalchemy_dburi self.engine_options = dict( engine_options or {}, **conf.sqlalchemy_engine_options or {}) self.short_lived_sessions = kwargs.get( 'short_lived_sessions', conf.sqlalchemy_short_lived_sessions, ) tablenames = conf.sqlalchemy_table_names or {} Task.__table__.name = tablenames.get('task', 'celery_taskmeta') TaskSet.__table__.name = tablenames.get('group', 'celery_tasksetmeta') if not self.dburi: raise ImproperlyConfigured( 'Missing connection string! Do you have the' ' sqlalchemy_dburi setting set to a real value?')
def __init__(self, app=None, **kwargs): self.options = {} super(MongoBackend, self).__init__(app, **kwargs) if not pymongo: raise ImproperlyConfigured( 'You need to install the pymongo library to use the ' 'MongoDB backend.') # Set option defaults for key, value in items(self._prepare_client_options()): self.options.setdefault(key, value) # update conf with mongo uri data, only if uri was given if self.url: if self.url == 'mongodb://': self.url += 'localhost' uri_data = pymongo.uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection hostslist = [ '{0}:{1}'.format(x[0], x[1]) for x in uri_data['nodelist'] ] self.user = uri_data['username'] self.password = uri_data['password'] self.mongo_host = hostslist if uri_data['database']: # if no database is provided in the uri, use default self.database_name = uri_data['database'] self.options.update(uri_data['options']) # update conf with specific settings config = self.app.conf.get('mongodb_backend_settings') if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( 'MongoDB backend settings should be grouped in a dict') config = dict(config) # do not modify original if 'host' in config or 'port' in config: # these should take over uri conf self.mongo_host = None self.host = config.pop('host', self.host) self.port = config.pop('port', self.port) self.mongo_host = config.pop('mongo_host', self.mongo_host) self.user = config.pop('user', self.user) self.password = config.pop('password', self.password) self.database_name = config.pop('database', self.database_name) self.taskmeta_collection = config.pop( 'taskmeta_collection', self.taskmeta_collection, ) self.groupmeta_collection = config.pop( 'groupmeta_collection', self.groupmeta_collection, ) self.options.update(config.pop('options', {})) self.options.update(config)
def detect_settings(conf, preconf=None, ignore_keys=None, prefix=None, all_keys=None, old_keys=None): preconf = {} if not preconf else preconf ignore_keys = set() if not ignore_keys else ignore_keys all_keys = SETTING_KEYS if not all_keys else all_keys old_keys = _OLD_SETTING_KEYS if not old_keys else old_keys source = conf if conf is None: source, conf = preconf, {} have = set(source.keys()) - ignore_keys is_in_new = have.intersection(all_keys) is_in_old = have.intersection(old_keys) info = None if is_in_new: # have new setting names info, left = _settings_info, is_in_old if is_in_old and len(is_in_old) > len(is_in_new): # Majority of the settings are old. info, left = _old_settings_info, is_in_new if is_in_old: # have old setting names, or a majority of the names are old. if not info: info, left = _old_settings_info, is_in_new if is_in_new and len(is_in_new) > len(is_in_old): # Majority of the settings are new info, left = _settings_info, is_in_old else: # no settings, just use new format. info, left = _settings_info, is_in_old if prefix: # always use new format if prefix is used. info, left = _settings_info, set() # only raise error for keys that the user didn't provide two keys # for (e.g., both ``result_expires`` and ``CELERY_TASK_RESULT_EXPIRES``). really_left = {key for key in left if info.convert[key] not in have} if really_left: # user is mixing old/new, or new/old settings, give renaming # suggestions. raise ImproperlyConfigured( info.mix_error.format(renames='\n'.join( FMT_REPLACE_SETTING.format(replace=key, with_=info.convert[key]) for key in sorted(really_left)))) preconf = {info.convert.get(k, k): v for k, v in preconf.items()} defaults = dict(deepcopy(info.defaults), **preconf) return Settings( preconf, [conf, defaults], (_old_key_to_new, _new_key_to_old), deprecated_settings=is_in_old, prefix=prefix, )
from celery import states from celery.backends.base import BaseBackend from celery.exceptions import ImproperlyConfigured from celery.five import range from celery.utils.timeutils import maybe_timedelta from .models import Task from .models import TaskSet from .session import SessionManager try: from sqlalchemy.exc import DatabaseError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError except ImportError: # pragma: no cover raise ImproperlyConfigured( 'The database result backend requires SQLAlchemy to be installed.' 'See http://pypi.python.org/pypi/SQLAlchemy') logger = logging.getLogger(__name__) __all__ = ['DatabaseBackend'] @contextmanager def session_cleanup(session): try: yield except Exception: session.rollback() raise finally:
def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, connection_pool=None, **kwargs): super().__init__(expires_type=int, **kwargs) _get = self.app.conf.get if self.redis is None: raise ImproperlyConfigured(E_REDIS_MISSING.strip()) if host and '://' in host: url, host = host, None self.max_connections = (max_connections or _get('redis_max_connections') or self.max_connections) self._ConnectionPool = connection_pool socket_timeout = _get('redis_socket_timeout') socket_connect_timeout = _get('redis_socket_connect_timeout') retry_on_timeout = _get('redis_retry_on_timeout') socket_keepalive = _get('redis_socket_keepalive') health_check_interval = _get('redis_backend_health_check_interval') self.connparams = { 'host': _get('redis_host') or 'localhost', 'port': _get('redis_port') or 6379, 'db': _get('redis_db') or 0, 'password': _get('redis_password'), 'max_connections': self.max_connections, 'socket_timeout': socket_timeout and float(socket_timeout), 'retry_on_timeout': retry_on_timeout or False, 'socket_connect_timeout': socket_connect_timeout and float(socket_connect_timeout), } username = _get('redis_username') if username: # We're extra careful to avoid including this configuration value # if it wasn't specified since older versions of py-redis # don't support specifying a username. # Only Redis>6.0 supports username/password authentication. # TODO: Include this in connparams' definition once we drop # support for py-redis<3.4.0. self.connparams['username'] = username if health_check_interval: self.connparams["health_check_interval"] = health_check_interval # absent in redis.connection.UnixDomainSocketConnection if socket_keepalive: self.connparams['socket_keepalive'] = socket_keepalive # "redis_backend_use_ssl" must be a dict with the keys: # 'ssl_cert_reqs', 'ssl_ca_certs', 'ssl_certfile', 'ssl_keyfile' # (the same as "broker_use_ssl") ssl = _get('redis_backend_use_ssl') if ssl: self.connparams.update(ssl) self.connparams['connection_class'] = self.connection_class_ssl if url: self.connparams = self._params_from_url(url, self.connparams) # If we've received SSL parameters via query string or the # redis_backend_use_ssl dict, check ssl_cert_reqs is valid. If set # via query string ssl_cert_reqs will be a string so convert it here if ('connection_class' in self.connparams and issubclass( self.connparams['connection_class'], redis.SSLConnection)): ssl_cert_reqs_missing = 'MISSING' ssl_string_to_constant = { 'CERT_REQUIRED': CERT_REQUIRED, 'CERT_OPTIONAL': CERT_OPTIONAL, 'CERT_NONE': CERT_NONE, 'required': CERT_REQUIRED, 'optional': CERT_OPTIONAL, 'none': CERT_NONE } ssl_cert_reqs = self.connparams.get('ssl_cert_reqs', ssl_cert_reqs_missing) ssl_cert_reqs = ssl_string_to_constant.get(ssl_cert_reqs, ssl_cert_reqs) if ssl_cert_reqs not in ssl_string_to_constant.values(): raise ValueError(E_REDIS_SSL_CERT_REQS_MISSING_INVALID) if ssl_cert_reqs == CERT_OPTIONAL: logger.warning(W_REDIS_SSL_CERT_OPTIONAL) elif ssl_cert_reqs == CERT_NONE: logger.warning(W_REDIS_SSL_CERT_NONE) self.connparams['ssl_cert_reqs'] = ssl_cert_reqs self.url = url self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) self.result_consumer = self.ResultConsumer( self, self.app, self.accept, self._pending_results, self._pending_messages, )
def init_queues(self): try: self.app.select_queues(self.use_queues) except KeyError, exc: raise ImproperlyConfigured(UNKNOWN_QUEUE % (self.use_queues, exc))
def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, connection_pool=None, **kwargs): super(RedisBackend, self).__init__(expires_type=int, **kwargs) _get = self.app.conf.get if self.redis is None: raise ImproperlyConfigured(E_REDIS_MISSING.strip()) if host and '://' in host: url, host = host, None self.max_connections = (max_connections or _get('redis_max_connections') or self.max_connections) self._ConnectionPool = connection_pool socket_timeout = _get('redis_socket_timeout') socket_connect_timeout = _get('redis_socket_connect_timeout') retry_on_timeout = _get('redis_retry_on_timeout') socket_keepalive = _get('redis_socket_keepalive') self.connparams = { 'host': _get('redis_host') or 'localhost', 'port': _get('redis_port') or 6379, 'db': _get('redis_db') or 0, 'password': _get('redis_password'), 'max_connections': self.max_connections, 'socket_timeout': socket_timeout and float(socket_timeout), 'retry_on_timeout': retry_on_timeout or False, 'socket_connect_timeout': socket_connect_timeout and float(socket_connect_timeout), } # absent in redis.connection.UnixDomainSocketConnection if socket_keepalive: self.connparams['socket_keepalive'] = socket_keepalive # "redis_backend_use_ssl" must be a dict with the keys: # 'ssl_cert_reqs', 'ssl_ca_certs', 'ssl_certfile', 'ssl_keyfile' # (the same as "broker_use_ssl") ssl = _get('redis_backend_use_ssl') if ssl: self.connparams.update(ssl) self.connparams['connection_class'] = redis.SSLConnection if url: self.connparams = self._params_from_url(url, self.connparams) # If we've received SSL parameters via query string or the # redis_backend_use_ssl dict, check ssl_cert_reqs is valid. If set # via query string ssl_cert_reqs will be a string so convert it here if ('connection_class' in self.connparams and self.connparams['connection_class'] is redis.SSLConnection): ssl_cert_reqs_missing = 'MISSING' ssl_string_to_constant = { 'CERT_REQUIRED': CERT_REQUIRED, 'CERT_OPTIONAL': CERT_OPTIONAL, 'CERT_NONE': CERT_NONE, 'required': CERT_REQUIRED, 'optional': CERT_OPTIONAL, 'none': CERT_NONE } ssl_cert_reqs = self.connparams.get('ssl_cert_reqs', ssl_cert_reqs_missing) ssl_cert_reqs = ssl_string_to_constant.get(ssl_cert_reqs, ssl_cert_reqs) if ssl_cert_reqs not in ssl_string_to_constant.values(): raise ValueError(E_REDIS_SSL_CERT_REQS_MISSING_INVALID) if ssl_cert_reqs == CERT_OPTIONAL: logger.warning(W_REDIS_SSL_CERT_OPTIONAL) elif ssl_cert_reqs == CERT_NONE: logger.warning(W_REDIS_SSL_CERT_NONE) self.connparams['ssl_cert_reqs'] = ssl_cert_reqs self.url = url self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) self.result_consumer = self.ResultConsumer( self, self.app, self.accept, self._pending_results, self._pending_messages, )
def __init__(self, url=None, table_name=None, *args, **kwargs): super(DynamoDBBackend, self).__init__(*args, **kwargs) self.url = url self.table_name = table_name or self.table_name if not boto3: raise ImproperlyConfigured( 'You need to install the boto3 library to use the ' 'DynamoDB backend.') aws_credentials_given = False aws_access_key_id = None aws_secret_access_key = None if url is not None: scheme, region, port, username, password, table, query = \ parse_url(url) aws_access_key_id = username aws_secret_access_key = password access_key_given = aws_access_key_id is not None secret_key_given = aws_secret_access_key is not None if access_key_given != secret_key_given: raise ImproperlyConfigured( 'You need to specify both the Access Key ID ' 'and Secret.') aws_credentials_given = access_key_given if region == 'localhost': # We are using the downloadable, local version of DynamoDB self.endpoint_url = 'http://localhost:{}'.format(port) self.aws_region = 'us-east-1' logger.warning( 'Using local-only DynamoDB endpoint URL: {}'.format( self.endpoint_url)) else: self.aws_region = region # If endpoint_url is explicitly set use it instead _get = self.app.conf.get config_endpoint_url = _get('dynamodb_endpoint_url') if config_endpoint_url: self.endpoint_url = config_endpoint_url self.read_capacity_units = int( query.get('read', self.read_capacity_units)) self.write_capacity_units = int( query.get('write', self.write_capacity_units)) self.table_name = table or self.table_name self._available_fields = (self._key_field, self._value_field, self._timestamp_field) self._client = None if aws_credentials_given: self._get_client(access_key_id=aws_access_key_id, secret_access_key=aws_secret_access_key)
SETTING_MISSING = """\ You have to configure a special task serializer for signing and verifying tasks: * task_serializer = 'auth' You have to accept only tasks which are serialized with 'auth'. There is no point in signing messages if they are not verified. * accept_content = ['auth'] """ __all__ = ("setup_security", ) try: import cryptography # noqa except ImportError: raise ImproperlyConfigured(CRYPTOGRAPHY_NOT_INSTALLED) def setup_security( allowed_serializers=None, key=None, cert=None, store=None, digest=None, serializer="json", app=None, ): """See :meth:`@Celery.setup_security`.""" if app is None: from celery import current_app
def replace(self, sig): """Replace this task, with a new task inheriting the task id. Execution of the host task ends immediately and no subsequent statements will be run. .. versionadded:: 4.0 Arguments: sig (Signature): signature to replace with. Raises: ~@Ignore: This is always raised when called in asynchronous context. It is best to always use ``return self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord") if isinstance(sig, _chain) and not getattr(sig, "tasks", True): raise ImproperlyConfigured("Cannot replace with an empty chain") # Ensure callbacks or errbacks from the replaced signature are retained if isinstance(sig, group): # Groups get uplifted to a chord so that we can link onto the body sig |= self.app.tasks['celery.accumulate'].s(index=0) for callback in maybe_list(self.request.callbacks) or []: sig.link(callback) for errback in maybe_list(self.request.errbacks) or []: sig.link_error(errback) # If the replacement signature is a chain, we need to push callbacks # down to the final task so they run at the right time even if we # proceed to link further tasks from the original request below if isinstance(sig, _chain) and "link" in sig.options: final_task_links = sig.tasks[-1].options.setdefault("link", []) final_task_links.extend(maybe_list(sig.options["link"])) # We need to freeze the replacement signature with the current task's # ID to ensure that we don't disassociate it from the existing task IDs # which would break previously constructed results objects. sig.freeze(self.request.id) # Ensure the important options from the original signature are retained replaced_task_nesting = self.request.get('replaced_task_nesting', 0) + 1 sig.set(chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, replaced_task_nesting=replaced_task_nesting) # If the task being replaced is part of a chain, we need to re-create # it with the replacement signature - these subsequent tasks will # retain their original task IDs as well for t in reversed(self.request.chain or []): sig |= signature(t, app=self.app) # Stamping sig with parents groups stamped_headers = self.request.stamped_headers if self.request.stamps: groups = self.request.stamps.get("groups") sig.stamp(visitor=GroupStampingVisitor( groups=groups, stamped_headers=stamped_headers)) # Finally, either apply or delay the new signature! if self.request.is_eager: return sig.apply().get() else: sig.delay() raise Ignore('Replaced by new task')
def get_connection(self): if Redis is None: raise ImproperlyConfigured('`redis `library is not installed') return Redis(**self.options)
def join(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True, on_message=None, disable_sync_subtasks=True, on_interval=None): """Gather the results of all tasks as a list in order. Note: This can be an expensive operation for result store backends that must resort to polling (e.g., database). You should consider using :meth:`join_native` if your backend supports it. Warning: Waiting for tasks within a task may lead to deadlocks. Please see :ref:`task-synchronous-subtasks`. Arguments: timeout (float): The number of seconds to wait for results before the operation times out. propagate (bool): If any of the tasks raises an exception, the exception will be re-raised when this flag is set. interval (float): Time to wait (in seconds) before retrying to retrieve a result from the set. Note that this does not have any effect when using the amqp result store backend, as it does not use polling. callback (Callable): Optional callback to be called for every result received. Must have signature ``(task_id, value)`` No results will be returned by this function if a callback is specified. The order of results is also arbitrary when a callback is used. To get access to the result object for a particular id you'll have to generate an index first: ``index = {r.id: r for r in gres.results.values()}`` Or you can create new result objects on the fly: ``result = app.AsyncResult(task_id)`` (both will take advantage of the backend cache anyway). no_ack (bool): Automatic message acknowledgment (Note that if this is set to :const:`False` then the messages *will not be acknowledged*). disable_sync_subtasks (bool): Disable tasks to wait for sub tasks this is the default configuration. CAUTION do not enable this unless you must. Raises: celery.exceptions.TimeoutError: if ``timeout`` isn't :const:`None` and the operation takes longer than ``timeout`` seconds. """ if disable_sync_subtasks: assert_will_not_block() time_start = monotonic() remaining = None if on_message is not None: raise ImproperlyConfigured( 'Backend does not support on_message callback') results = [] for result in self.results: remaining = None if timeout: remaining = timeout - (monotonic() - time_start) if remaining <= 0.0: raise TimeoutError('join operation timed out') value = result.get( timeout=remaining, propagate=propagate, interval=interval, no_ack=no_ack, on_interval=on_interval, ) if callback: callback(result.id, value) else: results.append(value) return results
def _parse_url(cls, url, prefix=AZURE_BLOCK_BLOB_CONNECTION_PREFIX): connection_string = url[len(prefix):] if not connection_string: raise ImproperlyConfigured("Invalid URL") return connection_string
def replace(self, sig): """Replace this task, with a new task inheriting the task id. Execution of the host task ends immediately and no subsequent statements will be run. .. versionadded:: 4.0 Arguments: sig (~@Signature): signature to replace with. Raises: ~@Ignore: This is always raised when called in asynchronous context. It is best to always use ``return self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord" ) if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( link=self.request.callbacks, link_error=self.request.errbacks, ) elif isinstance(sig, _chain): if not sig.tasks: raise ImproperlyConfigured( "Cannot replace with an empty chain" ) if self.request.chain: # We need to freeze the new signature with the current task's ID to # ensure that we don't disassociate the new chain from the existing # task IDs which would break previously constructed results # objects. sig.freeze(self.request.id) if "link" in sig.options: final_task_links = sig.tasks[-1].options.setdefault("link", []) final_task_links.extend(maybe_list(sig.options["link"])) # Construct the new remainder of the task by chaining the signature # we're being replaced by with signatures constructed from the # chain elements in the current request. for t in reversed(self.request.chain): sig |= signature(t, app=self.app) sig.set( chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, ) sig.freeze(self.request.id) if self.request.is_eager: task_result = sig.apply() with allow_join_result(): return task_result.get() else: sig.delay() raise Ignore('Replaced by new task')
def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, connection_pool=None, **kwargs): super(RedisBackend, self).__init__(expires_type=int, **kwargs) _get = self.app.conf.get if self.redis is None: raise ImproperlyConfigured(E_REDIS_MISSING.strip()) if host and '://' in host: url, host = host, None self.max_connections = (max_connections or _get('redis_max_connections') or self.max_connections) self._ConnectionPool = connection_pool socket_timeout = _get('redis_socket_timeout') socket_connect_timeout = _get('redis_socket_connect_timeout') self.connparams = { 'host': _get('redis_host') or 'localhost', 'port': _get('redis_port') or 6379, 'db': _get('redis_db') or 0, 'password': _get('redis_password'), 'max_connections': self.max_connections, 'socket_timeout': socket_timeout and float(socket_timeout), 'socket_connect_timeout': socket_connect_timeout and float(socket_connect_timeout), } # "redis_backend_use_ssl" must be a dict with the keys: # 'ssl_cert_reqs', 'ssl_ca_certs', 'ssl_certfile', 'ssl_keyfile' # (the same as "broker_use_ssl") ssl = _get('redis_backend_use_ssl') if ssl: self.connparams.update(ssl) self.connparams['connection_class'] = redis.SSLConnection if url: self.connparams = self._params_from_url(url, self.connparams) self.url = url self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) self.result_consumer = self.ResultConsumer( self, self.app, self.accept, self._pending_results, self._pending_messages, )
def __init__(self, app=None, **kwargs): self.options = {} super(MongoBackend, self).__init__(app, **kwargs) if not pymongo: raise ImproperlyConfigured( "You need to install the pymongo library to use the " "MongoDB backend.") # Set option defaults for key, value in items(self._prepare_client_options()): self.options.setdefault(key, value) # update conf with mongo uri data, only if uri was given if self.url: self.url = self._ensure_mongodb_uri_compliance(self.url) uri_data = pymongo.uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection hostslist = [ "{0}:{1}".format(x[0], x[1]) for x in uri_data["nodelist"] ] self.user = uri_data["username"] self.password = uri_data["password"] self.mongo_host = hostslist if uri_data["database"]: # if no database is provided in the uri, use default self.database_name = uri_data["database"] self.options.update(uri_data["options"]) # update conf with specific settings config = self.app.conf.get("mongodb_backend_settings") if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( "MongoDB backend settings should be grouped in a dict") config = dict(config) # don't modify original if "host" in config or "port" in config: # these should take over uri conf self.mongo_host = None self.host = config.pop("host", self.host) self.port = config.pop("port", self.port) self.mongo_host = config.pop("mongo_host", self.mongo_host) self.user = config.pop("user", self.user) self.password = config.pop("password", self.password) self.database_name = config.pop("database", self.database_name) self.taskmeta_collection = config.pop( "taskmeta_collection", self.taskmeta_collection, ) self.groupmeta_collection = config.pop( "groupmeta_collection", self.groupmeta_collection, ) self.options.update(config.pop("options", {})) self.options.update(config)
def _parse_url(cls, url, prefix="azureblockblob://"): connection_string = url[len(prefix):] if not connection_string: raise ImproperlyConfigured("Invalid URL") return connection_string
def __init__(self, app=None, url=None, **kwargs): """Initialize MongoDB backend instance. :raises celery.exceptions.ImproperlyConfigured: if module :mod:`pymongo` is not available. """ self.options = {} super(MongoBackend, self).__init__(app, **kwargs) if not pymongo: raise ImproperlyConfigured( 'You need to install the pymongo library to use the ' 'MongoDB backend.') self.url = url # Set option defaults for key, value in items(self._prepare_client_options()): self.options.setdefault(key, value) # update conf with mongo uri data, only if uri was given if self.url: uri_data = pymongo.uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection make_host_str = lambda x: "{0}:{1}".format(x[0], x[1]) hostslist = map(make_host_str, uri_data['nodelist']) self.user = uri_data['username'] self.password = uri_data['password'] self.mongo_host = hostslist if uri_data['database']: # if no database is provided in the uri, use default self.database_name = uri_data['database'] self.options.update(uri_data['options']) # update conf with specific settings config = self.app.conf.get('CELERY_MONGODB_BACKEND_SETTINGS') if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( 'MongoDB backend settings should be grouped in a dict') config = dict(config) # do not modify original if 'host' in config or 'port' in config: # these should take over uri conf self.mongo_host = None self.host = config.pop('host', self.host) self.port = config.pop('port', self.port) self.mongo_host = config.pop('mongo_host', self.mongo_host) self.user = config.pop('user', self.user) self.password = config.pop('password', self.password) self.database_name = config.pop('database', self.database_name) self.taskmeta_collection = config.pop( 'taskmeta_collection', self.taskmeta_collection, ) self.groupmeta_collection = config.pop( 'groupmeta_collection', self.groupmeta_collection, ) self.options.update(config.pop('options', {})) self.options.update(config)
def _find_path(self, url): if not url: raise ImproperlyConfigured( 'You need to configure a path for the File-system backend') if url is not None and url.startswith('file:///'): return url[7:]
from celery import states from celery.backends.base import BaseBackend from celery.exceptions import ImproperlyConfigured from celery.five import range from celery.utils.time import maybe_timedelta from celery.backends.database.models import Task from celery.backends.database.models import TaskSet from celery.backends.database.session import SessionManager try: from sqlalchemy.exc import DatabaseError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError except ImportError: # pragma: no cover raise ImproperlyConfigured( 'The database result backend requires SQLAlchemy to be installed.' 'See https://pypi.org/project/SQLAlchemy/') logger = logging.getLogger(__name__) __all__ = ('DatabaseBackend', ) @contextmanager def session_cleanup(session): try: yield except Exception: session.rollback() raise finally:
def __init__(self, *args, **kwargs): if self.sentinel is None: raise ImproperlyConfigured(E_REDIS_SENTINEL_MISSING.strip()) super().__init__(*args, **kwargs)
def _verify_django_version(django: "ModuleType") -> None: if django.VERSION < (1, 11): raise ImproperlyConfigured('Celery 5.x requires Django 1.11 or later.')
def _verify_django_version(django): if django.VERSION < (1, 11): raise ImproperlyConfigured('Celery 4.x requires Django 1.11 or later.')
def __init__(self, url=None, table_name=None, *args, **kwargs): super(DynamoDBBackend, self).__init__(*args, **kwargs) self.url = url self.table_name = table_name or self.table_name if not boto3: raise ImproperlyConfigured( "You need to install the boto3 library to use the " "DynamoDB backend." ) aws_credentials_given = False aws_access_key_id = None aws_secret_access_key = None if url is not None: scheme, region, port, username, password, table, query = parse_url(url) aws_access_key_id = username aws_secret_access_key = password access_key_given = aws_access_key_id is not None secret_key_given = aws_secret_access_key is not None if access_key_given != secret_key_given: raise ImproperlyConfigured( "You need to specify both the Access Key ID " "and Secret." ) aws_credentials_given = access_key_given if region == "localhost": # We are using the downloadable, local version of DynamoDB self.endpoint_url = "http://localhost:{}".format(port) self.aws_region = "us-east-1" logger.warning( "Using local-only DynamoDB endpoint URL: {}".format( self.endpoint_url ) ) else: self.aws_region = region # If endpoint_url is explicitly set use it instead _get = self.app.conf.get config_endpoint_url = _get("dynamodb_endpoint_url") if config_endpoint_url: self.endpoint_url = config_endpoint_url self.read_capacity_units = int(query.get("read", self.read_capacity_units)) self.write_capacity_units = int( query.get("write", self.write_capacity_units) ) ttl = query.get("ttl_seconds", self.time_to_live_seconds) if ttl: try: self.time_to_live_seconds = int(ttl) except ValueError as e: logger.error('TTL must be a number; got "{ttl}"', exc_info=e) raise e self.table_name = table or self.table_name self._available_fields = ( self._key_field, self._value_field, self._timestamp_field, ) self._client = None if aws_credentials_given: self._get_client( access_key_id=aws_access_key_id, secret_access_key=aws_secret_access_key )