def __init__(self,
app,
connection=None,
exchange=None,
exchange_type=None,
persistent=None,
serializer=None,
auto_delete=True,
**kwargs):
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = (conf.CELERY_RESULT_PERSISTENT
if persistent is None else persistent)
exchange = exchange or conf.CELERY_RESULT_EXCHANGE
exchange_type = exchange_type or conf.CELERY_RESULT_EXCHANGE_TYPE
self.exchange = self._create_exchange(exchange, exchange_type,
self.persistent)
self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER
self.auto_delete = auto_delete
self.expires = None
if 'expires' not in kwargs or kwargs['expires'] is not None:
self.expires = self.prepare_expires(kwargs.get('expires'))
self.queue_arguments = dictfilter({
'x-expires':
maybe_s_to_ms(self.expires),
})
self.mutex = threading.Lock()
def _add_optparse_argument(
parser,
opt,
typemap={"string": text_t, "int": int, "long": long_t, "float": float, "complex": complex, "choice": None},
):
if opt.callback:
opt.type = _optparse_callback_to_type(opt, opt.type)
# argparse checks for existence of this kwarg
if opt.action == "callback":
opt.action = None
# store_true sets value to "('NO', 'DEFAULT')" for some
# crazy reason, so not to set a sane default here.
if opt.action == "store_true" and opt.default is None:
opt.default = False
parser.add_argument(
*opt._long_opts + opt._short_opts,
**dictfilter(
dict(
action=opt.action,
type=typemap.get(opt.type, opt.type),
dest=opt.dest,
nargs=opt.nargs,
choices=opt.choices,
help=opt.help,
metavar=opt.metavar,
default=opt.default,
)
)
)
def _add_optparse_argument(parser, opt, typemap={
'string': text_t,
'int': int,
'long': long_t,
'float': float,
'complex': complex,
'choice': None}):
if opt.callback:
opt.type = _optparse_callback_to_type(opt, opt.type)
# argparse checks for existence of this kwarg
if opt.action == 'callback':
opt.action = None
# store_true sets value to "('NO', 'DEFAULT')" for some
# crazy reason, so not to set a sane default here.
if opt.action == 'store_true' and opt.default is None:
opt.default = False
parser.add_argument(
*opt._long_opts + opt._short_opts,
**dictfilter({
'action': opt.action,
'type': typemap.get(opt.type, opt.type),
'dest': opt.dest,
'nargs': opt.nargs,
'choices': opt.choices,
'help': opt.help,
'metavar': opt.metavar,
'default': opt.default}))
def __init__(
self,
app,
connection=None,
exchange=None,
exchange_type=None,
persistent=None,
serializer=None,
auto_delete=True,
**kwargs
):
super(BaseRPCBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self._out_of_band = {}
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.result_exchange
exchange_type = exchange_type or conf.result_exchange_type
self.exchange = self._create_exchange(exchange, exchange_type, self.delivery_mode)
self.serializer = serializer or conf.result_serializer
self.auto_delete = auto_delete
self.queue_arguments = dictfilter({"x-expires": maybe_s_to_ms(self.expires)})
self.result_consumer = self.ResultConsumer(
self, self.app, self.accept, self._pending_results, self._pending_messages
)
if register_after_fork is not None:
register_after_fork(self, _on_after_fork_cleanup_backend)
def _add_optparse_argument(parser, opt, typemap=None):
typemap = ({
"string": text_t,
"int": int,
"long": long_t,
"float": float,
"complex": complex,
"choice": None,
} if not typemap else typemap)
if opt.callback:
opt.type = _optparse_callback_to_type(opt, opt.type)
# argparse checks for existence of this kwarg
if opt.action == "callback":
opt.action = None
# store_true sets value to "('NO', 'DEFAULT')" for some
# crazy reason, so not to set a sane default here.
if opt.action == "store_true" and opt.default is None:
opt.default = False
parser.add_argument(
*opt._long_opts + opt._short_opts,
**dictfilter({
"action": opt.action,
"type": typemap.get(opt.type, opt.type),
"dest": opt.dest,
"nargs": opt.nargs,
"choices": opt.choices,
"help": opt.help,
"metavar": opt.metavar,
"default": opt.default,
}))
def _params_from_url(self, url, defaults):
scheme, host, port, user, password, path, query = _parse_url(url)
connparams = dict(
defaults, **dictfilter({
'host': host, 'port': port, 'password': password,
'db': query.pop('virtual_host', None)})
)
if scheme == 'socket':
# use 'path' as path to the socket… in this case
# the database number should be given in 'query'
connparams.update({
'connection_class': self.redis.UnixDomainSocketConnection,
'path': '/' + path,
})
# host+port are invalid options when using this connection type.
connparams.pop('host', None)
connparams.pop('port', None)
else:
connparams['db'] = path
# db may be string and start with / like in kombu.
db = connparams.get('db') or 0
db = db.strip('/') if isinstance(db, string_t) else db
connparams['db'] = int(db)
# Query parameters override other parameters
connparams.update(query)
return connparams
def _params_from_url(self, url, defaults):
scheme, host, port, user, password, path, query = _parse_url(url)
connparams = dict(
defaults,
**dictfilter({
'host': host,
'port': port,
'password': password,
'db': query.pop('virtual_host', None)
}))
if scheme == 'socket':
# use 'path' as path to the socket… in this case
# the database number should be given in 'query'
connparams.update({
'connection_class': self.redis.UnixDomainSocketConnection,
'path': '/' + path,
})
# host+port are invalid options when using this connection type.
connparams.pop('host', None)
connparams.pop('port', None)
else:
connparams['db'] = path
# db may be string and start with / like in kombu.
db = connparams.get('db') or 0
db = db.strip('/') if isinstance(db, string_t) else db
connparams['db'] = int(db)
# Query parameters override other parameters
connparams.update(query)
return connparams
def __init__(self,
app,
connection=None,
exchange=None,
exchange_type=None,
persistent=None,
serializer=None,
auto_delete=True,
**kwargs):
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.result_exchange
exchange_type = exchange_type or conf.result_exchange_type
self.exchange = self._create_exchange(
exchange,
exchange_type,
self.delivery_mode,
)
self.serializer = serializer or conf.result_serializer
self.auto_delete = auto_delete
self.queue_arguments = dictfilter({
'x-expires':
maybe_s_to_ms(self.expires),
})
self.result_consumer = self.ResultConsumer(self, self.app, self.accept,
self._pending_results)
if register_after_fork is not None:
register_after_fork(self, _on_after_fork_cleanup_backend)
def __init__(
self,
app,
connection=None,
exchange=None,
exchange_type=None,
persistent=None,
serializer=None,
auto_delete=True,
**kwargs
):
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.CELERY_RESULT_EXCHANGE
exchange_type = exchange_type or conf.CELERY_RESULT_EXCHANGE_TYPE
self.exchange = self._create_exchange(exchange, exchange_type, self.delivery_mode)
self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER
self.auto_delete = auto_delete
self.expires = None
if "expires" not in kwargs or kwargs["expires"] is not None:
self.expires = self.prepare_expires(kwargs.get("expires"))
self.queue_arguments = dictfilter({"x-expires": maybe_s_to_ms(self.expires)})
def __init__(self,
app,
connection=None,
exchange=None,
exchange_type=None,
persistent=None,
serializer=None,
auto_delete=True,
**kwargs):
deprecated.warn(
'The AMQP result backend',
deprecation='4.0',
removal='5.0',
alternative='Please use RPC backend or a persistent backend.')
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.result_exchange
exchange_type = exchange_type or conf.result_exchange_type
self.exchange = self._create_exchange(
exchange,
exchange_type,
self.delivery_mode,
)
self.serializer = serializer or conf.result_serializer
self.auto_delete = auto_delete
self.queue_arguments = dictfilter({
'x-expires':
maybe_s_to_ms(self.expires),
})
def __init__(self,
app,
connection=None,
exchange=None,
exchange_type=None,
persistent=None,
serializer=None,
auto_delete=True,
**kwargs):
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.result_exchange
exchange_type = exchange_type or conf.result_exchange_type
self.exchange = self._create_exchange(
exchange,
exchange_type,
self.delivery_mode,
)
self.serializer = serializer or conf.result_serializer
self.auto_delete = auto_delete
self.queue_arguments = dictfilter({
'x-expires':
maybe_s_to_ms(self.expires),
})
def _params_from_url(self, url, defaults):
scheme, host, port, _, password, path, query = _parse_url(url)
connparams = dict(
defaults,
**dictfilter({
"host": host,
"port": port,
"password": password,
"db": query.pop("virtual_host", None),
}))
if scheme == "socket":
# use 'path' as path to the socket… in this case
# the database number should be given in 'query'
connparams.update({
"connection_class": self.redis.UnixDomainSocketConnection,
"path": "/" + path,
})
# host+port are invalid options when using this connection type.
connparams.pop("host", None)
connparams.pop("port", None)
connparams.pop("socket_connect_timeout")
else:
connparams["db"] = path
ssl_param_keys = [
"ssl_ca_certs",
"ssl_certfile",
"ssl_keyfile",
"ssl_cert_reqs",
]
if scheme == "redis":
# If connparams or query string contain ssl params, raise error
if any(key in connparams for key in ssl_param_keys) or any(
key in query for key in ssl_param_keys):
raise ValueError(E_REDIS_SSL_PARAMS_AND_SCHEME_MISMATCH)
if scheme == "rediss":
connparams["connection_class"] = redis.SSLConnection
# The following parameters, if present in the URL, are encoded. We
# must add the decoded values to connparams.
for ssl_setting in ssl_param_keys:
ssl_val = query.pop(ssl_setting, None)
if ssl_val:
connparams[ssl_setting] = unquote(ssl_val)
# db may be string and start with / like in kombu.
db = connparams.get("db") or 0
db = db.strip("/") if isinstance(db, string_t) else db
connparams["db"] = int(db)
for key, value in query.items():
if key in redis.connection.URL_QUERY_ARGUMENT_PARSERS:
query[key] = redis.connection.URL_QUERY_ARGUMENT_PARSERS[key](
value)
# Query parameters override other parameters
connparams.update(query)
return connparams
def _add_optparse_argument(
parser,
opt,
typemap={
'string': text_t,
'int': int,
'long': long_t,
'float': float,
'complex': complex,
'choice': None
}):
if opt.callback:
opt.type = _optparse_callback_to_type(opt, opt.type)
# argparse checks for existence of this kwarg
if opt.action == 'callback':
opt.action = None
# store_true sets value to "('NO', 'DEFAULT')" for some
# crazy reason, so not to set a sane default here.
if opt.action == 'store_true' and opt.default is None:
opt.default = False
parser.add_argument(
*opt._long_opts + opt._short_opts,
**dictfilter({
'action': opt.action,
'type': typemap.get(opt.type, opt.type),
'dest': opt.dest,
'nargs': opt.nargs,
'choices': opt.choices,
'help': opt.help,
'metavar': opt.metavar,
'default': opt.default
}))
def _get_queue_arguments(self, ttl=None, expires=None):
conf = self.app.conf
return dictfilter(
{
"x-message-ttl": maybe_s_to_ms(ttl if ttl is not None else conf.event_queue_ttl),
"x-expires": maybe_s_to_ms(expires if expires is not None else conf.event_queue_expires),
}
)
def _get_queue_arguments(self):
conf = self.app.conf
return dictfilter(
{
"x-message-ttl": maybe_s_to_ms(conf.CELERY_EVENT_QUEUE_TTL),
"x-expires": maybe_s_to_ms(conf.CELERY_EVENT_QUEUE_EXPIRES),
}
)
def _get_queue_arguments(self):
conf = self.app.conf
return dictfilter({
'x-message-ttl':
maybe_s_to_ms(conf.CELERY_EVENT_QUEUE_TTL),
'x-expires':
maybe_s_to_ms(conf.CELERY_EVENT_QUEUE_EXPIRES),
})
def _params_from_url(self, url, defaults):
scheme, host, port, username, password, path, query = _parse_url(url)
connparams = dict(
defaults,
**dictfilter({
'host': host,
'port': port,
'username': username,
'password': password,
'db': query.pop('virtual_host', None)
}))
if scheme == 'socket':
# use 'path' as path to the socket… in this case
# the database number should be given in 'query'
connparams.update({
'connection_class': self.redis.UnixDomainSocketConnection,
'path': '/' + path,
})
# host+port are invalid options when using this connection type.
connparams.pop('host', None)
connparams.pop('port', None)
connparams.pop('socket_connect_timeout')
else:
connparams['db'] = path
ssl_param_keys = [
'ssl_ca_certs', 'ssl_certfile', 'ssl_keyfile', 'ssl_cert_reqs'
]
if scheme == 'redis':
# If connparams or query string contain ssl params, raise error
if (any(key in connparams for key in ssl_param_keys)
or any(key in query for key in ssl_param_keys)):
raise ValueError(E_REDIS_SSL_PARAMS_AND_SCHEME_MISMATCH)
if scheme == 'rediss':
connparams['connection_class'] = redis.SSLConnection
# The following parameters, if present in the URL, are encoded. We
# must add the decoded values to connparams.
for ssl_setting in ssl_param_keys:
ssl_val = query.pop(ssl_setting, None)
if ssl_val:
connparams[ssl_setting] = unquote(ssl_val)
# db may be string and start with / like in kombu.
db = connparams.get('db') or 0
db = db.strip('/') if isinstance(db, str) else db
connparams['db'] = int(db)
for key, value in query.items():
if key in redis.connection.URL_QUERY_ARGUMENT_PARSERS:
query[key] = redis.connection.URL_QUERY_ARGUMENT_PARSERS[key](
value)
# Query parameters override other parameters
connparams.update(query)
return connparams
def _params_from_url(self, url, defaults):
scheme, host, port, _, password, path, query = _parse_url(url)
connparams = dict(
defaults,
**dictfilter({
'host': host,
'port': port,
'password': password,
'db': query.pop('virtual_host', None)
}))
if scheme == 'socket':
# use 'path' as path to the socket… in this case
# the database number should be given in 'query'
connparams.update({
'connection_class': self.redis.UnixDomainSocketConnection,
'path': '/' + path,
})
# host+port are invalid options when using this connection type.
connparams.pop('host', None)
connparams.pop('port', None)
connparams.pop('socket_connect_timeout')
else:
connparams['db'] = path
if scheme == 'rediss':
connparams['connection_class'] = redis.SSLConnection
# The following parameters, if present in the URL, are encoded. We
# must add the decoded values to connparams.
for ssl_setting in ['ssl_ca_certs', 'ssl_certfile', 'ssl_keyfile']:
ssl_val = query.pop(ssl_setting, None)
if ssl_val:
connparams[ssl_setting] = unquote(ssl_val)
ssl_cert_reqs = query.pop('ssl_cert_reqs', 'MISSING')
if ssl_cert_reqs == 'CERT_REQUIRED':
connparams['ssl_cert_reqs'] = CERT_REQUIRED
elif ssl_cert_reqs == 'CERT_OPTIONAL':
logger.warn(W_REDIS_SSL_CERT_OPTIONAL)
connparams['ssl_cert_reqs'] = CERT_OPTIONAL
elif ssl_cert_reqs == 'CERT_NONE':
logger.warn(W_REDIS_SSL_CERT_NONE)
connparams['ssl_cert_reqs'] = CERT_NONE
else:
raise ValueError(E_REDIS_SSL_CERT_REQS_MISSING)
# db may be string and start with / like in kombu.
db = connparams.get('db') or 0
db = db.strip('/') if isinstance(db, string_t) else db
connparams['db'] = int(db)
for key, value in query.items():
if key in redis.connection.URL_QUERY_ARGUMENT_PARSERS:
query[key] = redis.connection.URL_QUERY_ARGUMENT_PARSERS[key](
value)
# Query parameters override other parameters
connparams.update(query)
return connparams
def _params_from_url(self, url, defaults):
scheme, host, port, _, password, path, query = _parse_url(url)
connparams = dict(
defaults, **dictfilter({
'host': host, 'port': port, 'password': password,
'db': query.pop('virtual_host', None)})
)
if scheme == 'socket':
# use 'path' as path to the socket… in this case
# the database number should be given in 'query'
connparams.update({
'connection_class': self.redis.UnixDomainSocketConnection,
'path': '/' + path,
})
# host+port are invalid options when using this connection type.
connparams.pop('host', None)
connparams.pop('port', None)
connparams.pop('socket_connect_timeout')
else:
connparams['db'] = path
if scheme == 'rediss':
connparams['connection_class'] = redis.SSLConnection
# The following parameters, if present in the URL, are encoded. We
# must add the decoded values to connparams.
for ssl_setting in ['ssl_ca_certs', 'ssl_certfile', 'ssl_keyfile']:
ssl_val = query.pop(ssl_setting, None)
if ssl_val:
connparams[ssl_setting] = unquote(ssl_val)
ssl_cert_reqs = query.pop('ssl_cert_reqs', 'MISSING')
if ssl_cert_reqs == 'CERT_REQUIRED':
connparams['ssl_cert_reqs'] = CERT_REQUIRED
elif ssl_cert_reqs == 'CERT_OPTIONAL':
logger.warn(W_REDIS_SSL_CERT_OPTIONAL)
connparams['ssl_cert_reqs'] = CERT_OPTIONAL
elif ssl_cert_reqs == 'CERT_NONE':
logger.warn(W_REDIS_SSL_CERT_NONE)
connparams['ssl_cert_reqs'] = CERT_NONE
else:
raise ValueError(E_REDIS_SSL_CERT_REQS_MISSING)
# db may be string and start with / like in kombu.
db = connparams.get('db') or 0
db = db.strip('/') if isinstance(db, string_t) else db
connparams['db'] = int(db)
if redis:
for key, value in query.items():
if key in redis.connection.URL_QUERY_ARGUMENT_PARSERS:
query[key] = redis.connection.URL_QUERY_ARGUMENT_PARSERS[key](
value
)
# Query parameters override other parameters
connparams.update(query)
return connparams
def _get_queue_arguments(self, ttl=None, expires=None):
conf = self.app.conf
return dictfilter({
'x-message-ttl':
maybe_s_to_ms(ttl if ttl is not None else conf.event_queue_ttl, ),
'x-expires':
maybe_s_to_ms(
expires
if expires is not None else conf.event_queue_expires, ),
})
def __radd__(self, other):
if not isinstance(other, date):
return NotImplemented
year = self.year or other.year
month = self.month or other.month
day = min(monthrange(year, month)[1], self.day or other.day)
ret = other.replace(**dict(
dictfilter(self._fields()), year=year, month=month, day=day))
if self.weekday is not None:
ret += timedelta(days=(7 - ret.weekday() + self.weekday) % 7)
return ret + timedelta(days=self.days)
def _fields(self, **extra):
return dictfilter(
{
'year': self.year,
'month': self.month,
'day': self.day,
'hour': self.hour,
'minute': self.minute,
'second': self.second,
'microsecond': self.microsecond,
}, **extra)
def _get_queue_arguments(self, ttl=None, expires=None):
conf = self.app.conf
return dictfilter({
'x-message-ttl':
maybe_s_to_ms(
ttl if ttl is not None else conf.CELERY_EVENT_QUEUE_TTL, ),
'x-expires':
maybe_s_to_ms(
expires
if expires is not None else conf.CELERY_EVENT_QUEUE_EXPIRES, ),
})
def _get_queue_arguments(self, ttl=None, expires=None):
conf = self.app.conf
return dictfilter({
'x-message-ttl': maybe_s_to_ms(
ttl if ttl is not None else conf.CELERY_EVENT_QUEUE_TTL,
),
'x-expires': maybe_s_to_ms(
expires if expires is not None
else conf.CELERY_EVENT_QUEUE_EXPIRES,
),
})
def __init__(self, app, connection=None, exchange=None, exchange_type=None,
persistent=None, serializer=None, auto_delete=True, **kwargs):
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.result_exchange
exchange_type = exchange_type or conf.result_exchange_type
self.exchange = self._create_exchange(
exchange, exchange_type, self.delivery_mode,
)
self.serializer = serializer or conf.result_serializer
self.auto_delete = auto_delete
self.queue_arguments = dictfilter({
'x-expires': maybe_s_to_ms(self.expires),
})
def __init__(self, app, connection=None, exchange=None, exchange_type=None,
persistent=None, serializer=None, auto_delete=True, **kwargs):
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = (conf.CELERY_RESULT_PERSISTENT if persistent is None
else persistent)
exchange = exchange or conf.CELERY_RESULT_EXCHANGE
exchange_type = exchange_type or conf.CELERY_RESULT_EXCHANGE_TYPE
self.exchange = self._create_exchange(exchange, exchange_type,
self.persistent)
self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER
self.auto_delete = auto_delete
self.expires = None
if 'expires' not in kwargs or kwargs['expires'] is not None:
self.expires = self.prepare_expires(kwargs.get('expires'))
self.queue_arguments = dictfilter({
'x-expires': maybe_s_to_ms(self.expires),
})
def __init__(self, app, connection=None, exchange=None, exchange_type=None,
persistent=None, serializer=None, auto_delete=True, **kwargs):
deprecated.warn(
'The AMQP result backend', deprecation='4.0', removal='5.0',
alternative='Please use RPC backend or a persistent backend.')
super(AMQPBackend, self).__init__(app, **kwargs)
conf = self.app.conf
self._connection = connection
self.persistent = self.prepare_persistent(persistent)
self.delivery_mode = 2 if self.persistent else 1
exchange = exchange or conf.result_exchange
exchange_type = exchange_type or conf.result_exchange_type
self.exchange = self._create_exchange(
exchange, exchange_type, self.delivery_mode,
)
self.serializer = serializer or conf.result_serializer
self.auto_delete = auto_delete
self.queue_arguments = dictfilter({
'x-expires': maybe_s_to_ms(self.expires),
})
