def test_symbol_by_name_package(self):
from celery.worker import WorkController
self.assertIs(
symbol_by_name('.worker:WorkController', package='celery'),
WorkController,
)
self.assertTrue(symbol_by_name(':group', package='celery'))
def test_symbol_by_name_package(self):
from celery.worker import WorkController
self.assertIs(
symbol_by_name('.worker:WorkController', package='celery'),
WorkController,
)
self.assertTrue(symbol_by_name(':group', package='celery'))
def __init__(self, main=None, loader=None, backend=None,
amqp=None, events=None, log=None, control=None,
set_as_current=True, accept_magic_kwargs=False,
tasks=None, broker=None, include=None, changes=None,
config_source=None, fixups=None, task_cls=None,
autofinalize=True, **kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.task_cls = task_cls or self.task_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self.user_options = defaultdict(set)
self.steps = defaultdict(set)
self.autofinalize = autofinalize
self.configured = False
self._config_source = config_source
self._pending_defaults = deque()
self.finalized = False
self._finalize_mutex = threading.Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# If the class defins a custom __reduce_args__ we need to use
# the old way of pickling apps, which is pickling a list of
# args instead of the new way that pickles a dict of keywords.
self._using_v1_reduce = app_has_custom(self, '__reduce_args__')
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = changes or {}
if broker:
self._preconf['BROKER_URL'] = broker
if backend:
self._preconf['CELERY_RESULT_BACKEND'] = backend
if include:
self._preconf['CELERY_IMPORTS'] = include
# - Apply fixups.
self.fixups = set(self.builtin_fixups) if fixups is None else fixups
# ...store fixup instances in _fixups to keep weakrefs alive.
self._fixups = [symbol_by_name(fixup)(self) for fixup in self.fixups]
if self.set_as_current:
self.set_current()
self.on_init()
_register_app(self)
def __init__(self,
main=None,
loader=None,
backend=None,
amqp=None,
events=None,
log=None,
control=None,
set_as_current=True,
accept_magic_kwargs=False,
tasks=None,
broker=None,
include=None,
**kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.backend_cls = backend or self.backend_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self.user_options = defaultdict(set)
self.steps = defaultdict(set)
self.configured = False
self._pending_defaults = deque()
self.finalized = False
self._finalize_mutex = threading.Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = {}
if broker:
self._preconf['BROKER_URL'] = broker
if include:
self._preconf['CELERY_IMPORTS'] = include
self.fixups = list(
filter(None,
(symbol_by_name(f).include(self) for f in DEFAULT_FIXUPS)))
if self.set_as_current:
self.set_current()
self.on_init()
_register_app(self)
def _load_object(self, path):
# imp = import_from_cwd
if ':' in path:
# Path includes attribute so can just jump
# here (e.g., ``os.path:abspath``).
return symbol_by_name(path, imp=import_from_cwd)
# Not sure if path is just a module name or if it includes an
# attribute name (e.g., ``os.path``, vs, ``os.path.abspath``).
try:
return import_from_cwd(path)
except ImportError:
# Not a module name, so try module + attribute.
return symbol_by_name(path, imp=import_from_cwd)
def _smart_import(self, path, imp=None):
imp = self.import_module if imp is None else imp
if ':' in path:
# Path includes attribute so can just jump here.
# e.g. ``os.path:abspath``.
return symbol_by_name(path, imp=imp)
# Not sure if path is just a module name or if it includes an
# attribute name (e.g. ``os.path``, vs, ``os.path.abspath``).
try:
return imp(path)
except ImportError:
# Not a module name, so try module + attribute.
return symbol_by_name(path, imp=imp)
def _smart_import(self, path, imp=None):
imp = self.import_module if imp is None else imp
if ':' in path:
# Path includes attribute so can just jump here.
# e.g. ``os.path:abspath``.
return symbol_by_name(path, imp=imp)
# Not sure if path is just a module name or if it includes an
# attribute name (e.g. ``os.path``, vs, ``os.path.abspath``).
try:
return imp(path)
except ImportError:
# Not a module name, so try module + attribute.
return symbol_by_name(path, imp=imp)
def subclass_with_self(self,
Class,
name=None,
attribute='app',
reverse=None,
**kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is 'app'.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
__reduce__=__reduce__,
**kw)
return type(name or Class.__name__, (Class, ), attrs)
def __init__(self, main=None, loader=None, backend=None,
amqp=None, events=None, log=None, control=None,
set_as_current=True, accept_magic_kwargs=False,
tasks=None, broker=None, include=None, **kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.backend_cls = backend or self.backend_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self.finalized = False
self._finalize_mutex = Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = {}
if broker:
self._preconf['BROKER_URL'] = broker
if include:
self._preconf['CELERY_IMPORTS'] = include
if self.set_as_current:
self.set_current()
self.on_init()
_register_app(self)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is 'app'.
:keyword reverse: Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
:keyword keep_reduce: If enabled a custom ``__reduce__`` implementation
will not be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
Arguments:
Class (type): The app-compatible class to subclass.
name (str): Custom name for the target class.
attribute (str): Name of the attribute holding the app,
Default is 'app'.
reverse (str): Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation won't be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict(
{attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def hook_subscribe(state, event, url=None, callback=None):
"""Subscribe to webhook."""
subs = subscribers_for_event(event)
_unsubscribe(subs, url)
subscribers_for_event(event).append(
symbol_by_name(callback) if callback else url, )
return {'ok': 'url {0!r} subscribed to event {1!r}'.format(url, event)}
def __init__(self, main=None, loader=None, backend=None,
amqp=None, events=None, log=None, control=None,
set_as_current=True, accept_magic_kwargs=False,
tasks=None, broker=None, include=None, **kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.backend_cls = backend or self.backend_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self.finalized = False
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = {}
if broker:
self._preconf["BROKER_URL"] = broker
if include:
self._preconf["CELERY_IMPORTS"] = include
if self.set_as_current:
self.set_current()
self.on_init()
def subclass_with_self(self, Class,
name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class by setting its app attribute
to this instance.
App-compatible means the class has an 'app' attribute providing
the default app, e.g.: ``class Foo(object): app = None``.
:param Class: The class to subclass.
:keyword name: Custom name for the target subclass.
:keyword attribute: Name of the attribute holding the app.
Default is ``"app"``.
:keyword reverse: Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
:keyword keep_reduce: If enabled a custom ``__reduce__`` implementation
will not be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_keys__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def hook_subscribe(state, event, url=None, callback=None):
subs = subscribers_for_event(event)
_unsubscribe(subs, url)
subscribers_for_event(event).append(
symbol_by_name(callback) if callback else url,
)
return {'ok': 'url {0!r} subscribed to event {1!r}'.format(url, event)}
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is 'app'.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict({attribute: self}, __module__=Class.__module__,
__doc__=Class.__doc__, __reduce__=__reduce__, **kw)
return type(name or Class.__name__, (Class, ), attrs)
def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd):
"""Find app by name."""
from .base import Celery
try:
sym = symbol_by_name(app, imp=imp)
except AttributeError:
# last part was not an attribute, but a module
sym = imp(app)
if isinstance(sym, ModuleType) and ":" not in app:
try:
found = sym.app
if isinstance(found, ModuleType):
raise AttributeError()
except AttributeError:
try:
found = sym.celery
if isinstance(found, ModuleType):
raise AttributeError()
except AttributeError:
if getattr(sym, "__path__", None):
try:
return find_app("{0}.celery".format(app), symbol_by_name=symbol_by_name, imp=imp)
except ImportError:
pass
for suspect in values(vars(sym)):
if isinstance(suspect, Celery):
return suspect
raise
else:
return found
else:
return found
return sym
def run_suite(self, names, suite,
block_timeout=None, no_color=False, **options):
return symbol_by_name(suite)(
self.app,
block_timeout=block_timeout,
no_color=no_color,
).run(names, **options)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class.
App-compatible means that the class has a class attribute that
provides the default app it should use, for example:
``class Foo: app = None``.
Arguments:
Class (type): The app-compatible class to subclass.
name (str): Custom name for the target class.
attribute (str): Name of the attribute holding the app,
Default is 'app'.
reverse (str): Reverse path to this object used for pickling
purposes. For example, to get ``app.AsyncResult``,
use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation won't be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict(
{attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(name or Class.__name__, (Class,), attrs)
def __init__(self,
main=None,
loader=None,
backend=None,
amqp=None,
events=None,
log=None,
control=None,
set_as_current=True,
accept_magic_kwargs=False,
tasks=None,
broker=None,
include=None,
changes=None,
config_source=None,
**kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.backend_cls = backend or self.backend_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self._config_source = config_source
self.configured = False
self._pending_defaults = deque()
self.finalized = False
self._finalize_mutex = threading.Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = changes or {}
if broker:
self._preconf['BROKER_URL'] = broker
if include:
self._preconf['CELERY_IMPORTS'] = include
if self.set_as_current:
self.set_current()
# See Issue #1126
# this is used when pickling the app object so that configuration
# is reread without having to pickle the contents
# (which is often unpickleable anyway)
if self._config_source:
self.config_from_object(self._config_source)
self.on_init()
_register_app(self)
def get_best_json(attr=None, choices=['simplejson', 'json']):
for i, module in enumerate(choices):
try:
sym = ':'.join([module, attr]) if attr else module
return symbol_by_name(sym), _JSON_EXTRA_ARGS.get(module, {})
except (AttributeError, ImportError):
if i + 1 >= len(choices):
raise
def flask_app(self) -> Flask:
if has_app_context():
return unwrap(flask_current_app)
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self.flask_app_factory()
register_after_fork(app, self._setup_after_fork)
return app
def get_backend_cls(backend=None, loader=None):
"""Get backend class by name/alias"""
backend = backend or "disabled"
loader = loader or current_app.loader
aliases = dict(BACKEND_ALIASES, **loader.override_backends)
try:
return symbol_by_name(backend, aliases)
except ValueError, exc:
raise ValueError, ValueError(UNKNOWN_BACKEND % (backend, exc)), sys.exc_info()[2]
def get_best_json(attr=None,
choices=['simplejson', 'json']):
for i, module in enumerate(choices):
try:
sym = ':'.join([module, attr]) if attr else module
return symbol_by_name(sym), _JSON_EXTRA_ARGS.get(module, {})
except (AttributeError, ImportError):
if i + 1 >= len(choices):
raise
def get_backend_cls(backend=None, loader=None):
"""Get backend class by name/alias"""
backend = backend or 'disabled'
loader = loader or current_app.loader
aliases = dict(BACKEND_ALIASES, **loader.override_backends)
try:
return symbol_by_name(backend, aliases)
except ValueError as exc:
reraise(ValueError, ValueError(UNKNOWN_BACKEND.format(backend, exc)),
sys.exc_info()[2])
def get_extension_commands(namespace='celery.commands'):
try:
from pkg_resources import iter_entry_points
except ImportError:
return
for ep in iter_entry_points(namespace):
for attr in ep.attrs:
command(symbol_by_name(':'.join([ep.module_name, attr])),
name=ep.name)
def get_best_json(attr=None,
choices=['kombu.utils.json', 'simplejson', 'json']):
for i, module in enumerate(choices):
try:
return symbol_by_name(
':'.join([module, attr]) if attr else module
)
except (AttributeError, ImportError):
if i + 1 >= len(choices):
raise
def run_suite(self,
names,
suite,
block_timeout=None,
no_color=False,
**options):
return symbol_by_name(suite)(
self.app,
block_timeout=block_timeout,
no_color=no_color,
).run(names, **options)
def get_scheduler(self,
lazy=False,
extension_namespace='celery.beat_schedulers'):
filename = self.schedule_filename
aliases = dict(load_extension_class_names(extension_namespace) or {})
return symbol_by_name(self.scheduler_cls, aliases=aliases)(
app=self.app,
schedule_filename=filename,
max_interval=self.max_interval,
lazy=lazy,
)
def get_backend_cls(backend=None, loader=None):
"""Get backend class by name/alias"""
backend = backend or "disabled"
loader = loader or current_app.loader
aliases = dict(BACKEND_ALIASES, **loader.override_backends)
try:
cls = symbol_by_name(backend, aliases)
except ValueError as exc:
reraise(ImproperlyConfigured, ImproperlyConfigured(UNKNOWN_BACKEND.format(backend, exc)), sys.exc_info()[2])
if isinstance(cls, types.ModuleType):
raise ImproperlyConfigured(UNKNOWN_BACKEND.format(backend, "is a Python module, not a backend class."))
return cls
def traverse_subscribers(it, *args, **kwargs):
stream = deque([it])
while stream:
for node in maybe_list(stream.popleft()):
if isinstance(node, string_types) and node.startswith('!'):
node = symbol_by_name(node[1:])
if isinstance(node, Callable):
node = node(*args, **kwargs)
if is_list(node):
stream.append(node)
elif node:
yield node
def traverse_subscribers(it, *args, **kwargs):
stream = deque([it])
while stream:
for node in maybe_list(stream.popleft()):
if isinstance(node, string_types) and node.startswith('!'):
node = symbol_by_name(node[1:])
if isinstance(node, Callable):
node = node(*args, **kwargs)
if is_list(node):
stream.append(node)
elif node:
yield node
def flask_app(self):
if self._flask_app is None:
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self._flask_app = self.flask_app_factory()
self.app_context = app.app_context()
if 'sentry' in app.extensions:
from raven.contrib.celery import register_signal, register_logger_signal
client = app.extensions['sentry'].client
register_signal(client)
register_logger_signal(client)
return self._flask_app
def __init__(self, app, *args, **kwargs):
self.backend = symbol_by_name(
getattr(app.conf, 'CELERYBEAT_REDUNDANT_BACKEND',
'celery_redundant_scheduler.backends.redis:Backend'))(
app, **getattr(app.conf,
'CELERYBEAT_REDUNDANT_BACKEND_OPTIONS',
{}))
class Entry(self.Entry):
backend = self.backend
self.Entry = Entry
super(RedundantScheduler, self).__init__(app, *args, **kwargs)
def get_backend_cls(backend=None, loader=None):
"""Get backend class by name/alias"""
backend = backend or 'disabled'
loader = loader or current_app.loader
aliases = dict(BACKEND_ALIASES, **loader.override_backends)
try:
cls = symbol_by_name(backend, aliases)
except ValueError as exc:
reraise(ImproperlyConfigured, ImproperlyConfigured(
UNKNOWN_BACKEND.format(backend, exc)), sys.exc_info()[2])
if isinstance(cls, types.ModuleType):
raise ImproperlyConfigured(UNKNOWN_BACKEND.format(
backend, 'is a Python module, not a backend class.'))
return cls
def __init__(self, main=None, loader=None, backend=None,
amqp=None, events=None, log=None, control=None,
set_as_current=True, accept_magic_kwargs=False,
tasks=None, broker=None, include=None, changes=None,
config_source=None,
**kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.backend_cls = backend or self.backend_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self._config_source = config_source
self.configured = False
self._pending_defaults = deque()
self.finalized = False
self._finalize_mutex = threading.Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = changes or {}
if broker:
self._preconf['BROKER_URL'] = broker
if include:
self._preconf['CELERY_IMPORTS'] = include
if self.set_as_current:
self.set_current()
# See Issue #1126
# this is used when pickling the app object so that configuration
# is reread without having to pickle the contents
# (which is often unpickleable anyway)
if self._config_source:
self.config_from_object(self._config_source)
self.on_init()
_register_app(self)
def load_extension_commands(namespace='celery.commands'):
try:
from pkg_resources import iter_entry_points
except ImportError:
return
for ep in iter_entry_points(namespace):
sym = ':'.join([ep.module_name, ep.attrs[0]])
try:
cls = symbol_by_name(sym)
except (ImportError, SyntaxError), exc:
warnings.warn('Cannot load extension %r: %r' % (sym, exc))
else:
heapq.heappush(_get_extension_classes(), ep.name)
command(cls, name=ep.name)
def load(self):
try:
from pkg_resources import iter_entry_points
except ImportError: # pragma: no cover
return
for ep in iter_entry_points(self.namespace):
sym = ":".join([ep.module_name, ep.attrs[0]])
try:
cls = symbol_by_name(sym)
except (ImportError, SyntaxError) as exc:
warnings.warn("Cannot load extension {0!r}: {1!r}".format(sym, exc))
else:
self.add(cls, ep.name)
return self.names
def load_extension_commands(namespace='celery.commands'):
try:
from pkg_resources import iter_entry_points
except ImportError:
return
for ep in iter_entry_points(namespace):
sym = ':'.join([ep.module_name, ep.attrs[0]])
try:
cls = symbol_by_name(sym)
except (ImportError, SyntaxError), exc:
warnings.warn('Cannot load extension %r: %r' % (sym, exc))
else:
heapq.heappush(_get_extension_classes(), ep.name)
command(cls, name=ep.name)
def config_from_object(self, obj, silent=False):
if isinstance(obj, basestring):
try:
if '.' in obj:
obj = symbol_by_name(obj, imp=self.import_from_cwd)
else:
obj = self.import_from_cwd(obj)
except (ImportError, AttributeError):
if silent:
return False
raise
if not hasattr(obj, '__getitem__'):
obj = DictAttribute(obj)
self._conf = obj
return True
def config_from_object(self, obj, silent=False):
if isinstance(obj, basestring):
try:
if '.' in obj:
obj = symbol_by_name(obj, imp=self.import_from_cwd)
else:
obj = self.import_from_cwd(obj)
except (ImportError, AttributeError):
if silent:
return False
raise
if not hasattr(obj, '__getitem__'):
obj = DictAttribute(obj)
self._conf = obj
return True
def load_extension_commands(namespace='celery.commands'):
try:
from pkg_resources import iter_entry_points
except ImportError:
return
for ep in iter_entry_points(namespace):
_get_extension_classes().append(ep.name)
sym = ':'.join([ep.module_name, ep.attrs[0]])
try:
cls = symbol_by_name(sym)
except (ImportError, SyntaxError) as exc:
warnings.warn(
'Cannot load extension {0!r}: {1!r}'.format(sym, exc))
else:
command(cls, name=ep.name)
def flask_app(self):
if has_app_context():
return flask_current_app._get_current_object()
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self.flask_app_factory()
if 'sentry' in app.extensions:
from raven.contrib.celery import register_signal, register_logger_signal
client = app.extensions['sentry'].client
client.tags['process_type'] = 'celery task'
register_signal(client)
register_logger_signal(client)
register_after_fork(app, self._setup_after_fork)
return app
def by_name(backend=None, loader=None,
extension_namespace='celery.result_backends'):
"""Get backend class by name/alias."""
backend = backend or 'disabled'
loader = loader or current_app.loader
aliases = dict(BACKEND_ALIASES, **loader.override_backends)
aliases.update(load_extension_class_names(extension_namespace))
try:
cls = symbol_by_name(backend, aliases)
except ValueError as exc:
reraise(ImproperlyConfigured, ImproperlyConfigured(
UNKNOWN_BACKEND.strip().format(backend, exc)), sys.exc_info()[2])
if isinstance(cls, types.ModuleType):
raise ImproperlyConfigured(UNKNOWN_BACKEND.strip().format(
backend, 'is a Python module, not a backend class.'))
return cls
def flask_app(self):
if has_app_context():
return flask_current_app._get_current_object()
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self.flask_app_factory()
if 'sentry' in app.extensions:
from raven.contrib.celery import register_signal, register_logger_signal
client = app.extensions['sentry'].client
client.tags['process_type'] = 'celery task'
register_signal(client)
register_logger_signal(client)
register_after_fork(app, self._setup_after_fork)
return app
def load(self):
try:
from pkg_resources import iter_entry_points
except ImportError: # pragma: no cover
return
for ep in iter_entry_points(self.namespace):
sym = ':'.join([ep.module_name, ep.attrs[0]])
try:
cls = symbol_by_name(sym)
except (ImportError, SyntaxError) as exc:
warnings.warn(
'Cannot load extension {0!r}: {1!r}'.format(sym, exc))
else:
self.add(cls, ep.name)
return self.names
def subclass_with_self(self, Class, name=None, attribute="app", **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is "app".
"""
Class = symbol_by_name(Class)
return type(name or Class.__name__, (Class, ), dict({attribute: self,
"__module__": Class.__module__, "__doc__": Class.__doc__}, **kw))
def flask_app(self):
if has_app_context():
return unwrap(flask_current_app)
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self.flask_app_factory()
if "sentry" in app.extensions:
from raven.contrib.celery import register_signal, register_logger_signal
client = app.extensions["sentry"].client
client.tags["process_type"] = "celery task"
register_signal(client)
register_logger_signal(client)
register_after_fork(app, self._setup_after_fork)
return app
def flask_app(self):
if has_app_context():
return unwrap(flask_current_app)
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self.flask_app_factory()
if "sentry" in app.extensions:
from raven.contrib.celery import register_signal, register_logger_signal
client = app.extensions["sentry"].client
client.tags["process_type"] = "celery task"
register_signal(client)
register_logger_signal(client)
register_after_fork(app, self._setup_after_fork)
return app
def by_name(backend=None, loader=None,
extension_namespace='celery.result_backends'):
"""Get backend class by name/alias."""
backend = backend or 'disabled'
loader = loader or current_app.loader
aliases = dict(BACKEND_ALIASES, **loader.override_backends)
aliases.update(
load_extension_class_names(extension_namespace) or {})
try:
cls = symbol_by_name(backend, aliases)
except ValueError as exc:
reraise(ImproperlyConfigured, ImproperlyConfigured(
UNKNOWN_BACKEND.strip().format(backend, exc)), sys.exc_info()[2])
if isinstance(cls, types.ModuleType):
raise ImproperlyConfigured(UNKNOWN_BACKEND.strip().format(
backend, 'is a Python module, not a backend class.'))
return cls
def flask_app(self) -> Flask:
if has_app_context():
return unwrap(flask_current_app)
self.flask_app_factory = symbol_by_name(self.flask_app_factory)
app = self.flask_app_factory()
if "sentry" in app.extensions:
# pyre-fixme[21]: Could not find `raven`.
from raven.contrib.celery import register_signal, register_logger_signal
client = app.extensions["sentry"].client
client.tags["process_type"] = "celery task"
register_signal(client)
register_logger_signal(client)
# pyre-fixme[16]: Module `multiprocessing` has no attribute `util`.
register_after_fork(app, self._setup_after_fork)
return app
def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd):
"""Find app by name."""
from .base import Celery
try:
sym = symbol_by_name(app, imp=imp)
except AttributeError:
# last part was not an attribute, but a module
sym = imp(app)
if isinstance(sym, ModuleType) and ':' not in app:
try:
found = sym.app
if isinstance(found, ModuleType):
raise AttributeError()
except AttributeError:
try:
found = sym.celery
if isinstance(found, ModuleType):
raise AttributeError(
"attribute 'celery' is the celery module not the instance of celery"
)
except AttributeError:
if getattr(sym, '__path__', None):
try:
return find_app(
'{0}.celery'.format(app),
symbol_by_name=symbol_by_name,
imp=imp,
)
except ImportError:
pass
for suspect in values(vars(sym)):
if isinstance(suspect, Celery):
return suspect
raise
else:
return found
else:
return found
return sym
def subclass_with_self(self,
Class,
name=None,
attribute='app',
reverse=None,
keep_reduce=False,
**kw):
# type: (type, str, str, str, bool, **Any) -> type
"""Subclass an app-compatible class.
App-compatible means the class has an 'app' attribute providing
the default app, e.g.: ``class Foo(object): app = None``.
Arguments:
Class (Any): The class to subclass.
Keyword Arguments:
name (str): Custom name for the target subclass.
attribute (str): Name of the attribute holding the app.
Default is ``"app"``.
reverse (str): Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation will not be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_keys__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class, ), attrs)
def default(task, app, consumer,
info=logger.info, error=logger.error, task_reserved=task_reserved,
to_system_tz=timezone.to_system, bytes=bytes, buffer_t=buffer_t,
proto1_to_proto2=proto1_to_proto2):
"""Default task execution strategy.
Note:
Strategies are here as an optimization, so sadly
it's not very easy to override.
"""
hostname = consumer.hostname
connection_errors = consumer.connection_errors
_does_info = logger.isEnabledFor(logging.INFO)
# task event related
# (optimized to avoid calling request.send_event)
eventer = consumer.event_dispatcher
events = eventer and eventer.enabled
send_event = eventer.send
task_sends_events = events and task.send_events
call_at = consumer.timer.call_at
apply_eta_task = consumer.apply_eta_task
rate_limits_enabled = not consumer.disable_rate_limits
get_bucket = consumer.task_buckets.__getitem__
handle = consumer.on_task_request
limit_task = consumer._limit_task
limit_post_eta = consumer._limit_post_eta
body_can_be_buffer = consumer.pool.body_can_be_buffer
Request = symbol_by_name(task.Request)
Req = create_request_cls(Request, task, consumer.pool, hostname, eventer)
revoked_tasks = consumer.controller.state.revoked
def task_message_handler(message, body, ack, reject, callbacks,
to_timestamp=to_timestamp):
if body is None and 'args' not in message.payload:
body, headers, decoded, utc = (
message.body, message.headers, False, app.uses_utc_timezone(),
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
if 'args' in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(message,
message.payload)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
req = Req(
message,
on_ack=ack, on_reject=reject, app=app, hostname=hostname,
eventer=eventer, task=task, connection_errors=connection_errors,
body=body, headers=headers, decoded=decoded, utc=utc,
)
if _does_info:
info('Received task: %s', req)
if (req.expires or req.id in revoked_tasks) and req.revoked():
return
signals.task_received.send(sender=consumer, request=req)
if task_sends_events:
send_event(
'task-received',
uuid=req.id, name=req.name,
args=req.argsrepr, kwargs=req.kwargsrepr,
root_id=req.root_id, parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
bucket = None
eta = None
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, app.timezone)
except (OverflowError, ValueError) as exc:
error("Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta, exc, req.info(safe=True), exc_info=True)
req.reject(requeue=False)
if rate_limits_enabled:
bucket = get_bucket(task.name)
if eta and bucket:
consumer.qos.increment_eventually()
return call_at(eta, limit_post_eta, (req, bucket, 1),
priority=6)
if eta:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req,), priority=6)
return task_message_handler
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
return task_message_handler
def __init__(self, main=None, loader=None, backend=None,
amqp=None, events=None, log=None, control=None,
set_as_current=True, accept_magic_kwargs=False,
tasks=None, broker=None, include=None, changes=None,
config_source=None, fixups=None, **kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.backend_cls = backend or self.backend_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.accept_magic_kwargs = accept_magic_kwargs
self.user_options = defaultdict(set)
self._config_source = config_source
self.steps = defaultdict(set)
self.configured = False
self._pending_defaults = deque()
self.finalized = False
self._finalize_mutex = threading.Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# If the class defins a custom __reduce_args__ we need to use
# the old way of pickling apps, which is pickling a list of
# args instead of the new way that pickles a dict of keywords.
self._using_v1_reduce = app_has_custom(self, '__reduce_args__')
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = changes or {}
if broker:
self._preconf['BROKER_URL'] = broker
if include:
self._preconf['CELERY_IMPORTS'] = include
enable_insecure_serializers()
# Apply fixups.
self.fixups = set(fixups or ())
for fixup in self.fixups | BUILTIN_FIXUPS:
symbol_by_name(fixup)(self)
if self.set_as_current:
self.set_current()
# See Issue #1126
# this is used when pickling the app object so that configuration
# is reread without having to pickle the contents
# (which is often unpickleable anyway)
if self._config_source:
self.config_from_object(self._config_source)
self.on_init()
_register_app(self)
def test_symbol_by_name_returns_default(self):
default = object()
self.assertIs(symbol_by_name('xyz.ryx.qedoa.weq:foz',
default=default), default)
def __init__(self, main=None, loader=None, backend=None,
amqp=None, events=None, log=None, control=None,
set_as_current=True, tasks=None, broker=None, include=None,
changes=None, config_source=None, fixups=None, task_cls=None,
autofinalize=True, namespace=None, **kwargs):
self.clock = LamportClock()
self.main = main
self.amqp_cls = amqp or self.amqp_cls
self.events_cls = events or self.events_cls
self.loader_cls = loader or self.loader_cls
self.log_cls = log or self.log_cls
self.control_cls = control or self.control_cls
self.task_cls = task_cls or self.task_cls
self.set_as_current = set_as_current
self.registry_cls = symbol_by_name(self.registry_cls)
self.user_options = defaultdict(set)
self.steps = defaultdict(set)
self.autofinalize = autofinalize
self.namespace = namespace
self.configured = False
self._config_source = config_source
self._pending_defaults = deque()
self._pending_periodic_tasks = deque()
self.finalized = False
self._finalize_mutex = threading.Lock()
self._pending = deque()
self._tasks = tasks
if not isinstance(self._tasks, TaskRegistry):
self._tasks = TaskRegistry(self._tasks or {})
# If the class defines a custom __reduce_args__ we need to use
# the old way of pickling apps, which is pickling a list of
# args instead of the new way that pickles a dict of keywords.
self._using_v1_reduce = app_has_custom(self, '__reduce_args__')
# these options are moved to the config to
# simplify pickling of the app object.
self._preconf = changes or {}
self._preconf_set_by_auto = set()
self.__autoset('broker_url', broker)
self.__autoset('result_backend', backend)
self.__autoset('include', include)
self._conf = Settings(
PendingConfiguration(
self._preconf, self._get_from_conf_and_finalize),
prefix=self.namespace,
)
# - Apply fix-ups.
self.fixups = set(self.builtin_fixups) if fixups is None else fixups
# ...store fixup instances in _fixups to keep weakrefs alive.
self._fixups = [symbol_by_name(fixup)(self) for fixup in self.fixups]
if self.set_as_current:
self.set_current()
# Signals
if self.on_configure is None:
# used to be a method pre 4.0
self.on_configure = Signal()
self.on_after_configure = Signal()
self.on_after_finalize = Signal()
self.on_after_fork = Signal()
self.on_init()
_register_app(self)
def symbol_by_name(self, name, imp=import_from_cwd):
return symbol_by_name(name, imp=imp)