def test_symbol_by_name_package(self): from celery.worker import WorkController self.assertIs( symbol_by_name('.worker:WorkController', package='celery'), WorkController, ) self.assertTrue(symbol_by_name(':group', package='celery'))
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, tasks=None, broker=None, include=None, changes=None, config_source=None, fixups=None, task_cls=None, autofinalize=True, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.task_cls = task_cls or self.task_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.accept_magic_kwargs = accept_magic_kwargs self.user_options = defaultdict(set) self.steps = defaultdict(set) self.autofinalize = autofinalize self.configured = False self._config_source = config_source self._pending_defaults = deque() self.finalized = False self._finalize_mutex = threading.Lock() self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # If the class defins a custom __reduce_args__ we need to use # the old way of pickling apps, which is pickling a list of # args instead of the new way that pickles a dict of keywords. self._using_v1_reduce = app_has_custom(self, '__reduce_args__') # these options are moved to the config to # simplify pickling of the app object. self._preconf = changes or {} if broker: self._preconf['BROKER_URL'] = broker if backend: self._preconf['CELERY_RESULT_BACKEND'] = backend if include: self._preconf['CELERY_IMPORTS'] = include # - Apply fixups. self.fixups = set(self.builtin_fixups) if fixups is None else fixups # ...store fixup instances in _fixups to keep weakrefs alive. self._fixups = [symbol_by_name(fixup)(self) for fixup in self.fixups] if self.set_as_current: self.set_current() self.on_init() _register_app(self)
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, tasks=None, broker=None, include=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.backend_cls = backend or self.backend_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.accept_magic_kwargs = accept_magic_kwargs self.user_options = defaultdict(set) self.steps = defaultdict(set) self.configured = False self._pending_defaults = deque() self.finalized = False self._finalize_mutex = threading.Lock() self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # these options are moved to the config to # simplify pickling of the app object. self._preconf = {} if broker: self._preconf['BROKER_URL'] = broker if include: self._preconf['CELERY_IMPORTS'] = include self.fixups = list( filter(None, (symbol_by_name(f).include(self) for f in DEFAULT_FIXUPS))) if self.set_as_current: self.set_current() self.on_init() _register_app(self)
def _load_object(self, path): # imp = import_from_cwd if ':' in path: # Path includes attribute so can just jump # here (e.g., ``os.path:abspath``). return symbol_by_name(path, imp=import_from_cwd) # Not sure if path is just a module name or if it includes an # attribute name (e.g., ``os.path``, vs, ``os.path.abspath``). try: return import_from_cwd(path) except ImportError: # Not a module name, so try module + attribute. return symbol_by_name(path, imp=import_from_cwd)
def _smart_import(self, path, imp=None): imp = self.import_module if imp is None else imp if ':' in path: # Path includes attribute so can just jump here. # e.g. ``os.path:abspath``. return symbol_by_name(path, imp=imp) # Not sure if path is just a module name or if it includes an # attribute name (e.g. ``os.path``, vs, ``os.path.abspath``). try: return imp(path) except ImportError: # Not a module name, so try module + attribute. return symbol_by_name(path, imp=imp)
def subclass_with_self(self, Class, name=None, attribute='app', reverse=None, **kw): """Subclass an app-compatible class by setting its app attribute to be this app instance. App-compatible means that the class has a class attribute that provides the default app it should use, e.g. ``class Foo: app = None``. :param Class: The app-compatible class to subclass. :keyword name: Custom name for the target class. :keyword attribute: Name of the attribute holding the app, default is 'app'. """ Class = symbol_by_name(Class) reverse = reverse if reverse else Class.__name__ def __reduce__(self): return _unpickle_appattr, (reverse, self.__reduce_args__()) attrs = dict({attribute: self}, __module__=Class.__module__, __doc__=Class.__doc__, __reduce__=__reduce__, **kw) return type(name or Class.__name__, (Class, ), attrs)
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, tasks=None, broker=None, include=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.backend_cls = backend or self.backend_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.accept_magic_kwargs = accept_magic_kwargs self.finalized = False self._finalize_mutex = Lock() self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # these options are moved to the config to # simplify pickling of the app object. self._preconf = {} if broker: self._preconf['BROKER_URL'] = broker if include: self._preconf['CELERY_IMPORTS'] = include if self.set_as_current: self.set_current() self.on_init() _register_app(self)
def subclass_with_self(self, Class, name=None, attribute='app', reverse=None, keep_reduce=False, **kw): """Subclass an app-compatible class by setting its app attribute to be this app instance. App-compatible means that the class has a class attribute that provides the default app it should use, e.g. ``class Foo: app = None``. :param Class: The app-compatible class to subclass. :keyword name: Custom name for the target class. :keyword attribute: Name of the attribute holding the app, default is 'app'. :keyword reverse: Reverse path to this object used for pickling purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``. :keyword keep_reduce: If enabled a custom ``__reduce__`` implementation will not be provided. """ Class = symbol_by_name(Class) reverse = reverse if reverse else Class.__name__ def __reduce__(self): return _unpickle_appattr, (reverse, self.__reduce_args__()) attrs = dict({attribute: self}, __module__=Class.__module__, __doc__=Class.__doc__, **kw) if not keep_reduce: attrs['__reduce__'] = __reduce__ return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def subclass_with_self(self, Class, name=None, attribute='app', reverse=None, keep_reduce=False, **kw): """Subclass an app-compatible class by setting its app attribute to be this app instance. App-compatible means that the class has a class attribute that provides the default app it should use, e.g. ``class Foo: app = None``. Arguments: Class (type): The app-compatible class to subclass. name (str): Custom name for the target class. attribute (str): Name of the attribute holding the app, Default is 'app'. reverse (str): Reverse path to this object used for pickling purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``. keep_reduce (bool): If enabled a custom ``__reduce__`` implementation won't be provided. """ Class = symbol_by_name(Class) reverse = reverse if reverse else Class.__name__ def __reduce__(self): return _unpickle_appattr, (reverse, self.__reduce_args__()) attrs = dict( {attribute: self}, __module__=Class.__module__, __doc__=Class.__doc__, **kw) if not keep_reduce: attrs['__reduce__'] = __reduce__ return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def hook_subscribe(state, event, url=None, callback=None): """Subscribe to webhook.""" subs = subscribers_for_event(event) _unsubscribe(subs, url) subscribers_for_event(event).append( symbol_by_name(callback) if callback else url, ) return {'ok': 'url {0!r} subscribed to event {1!r}'.format(url, event)}
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, tasks=None, broker=None, include=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.backend_cls = backend or self.backend_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.accept_magic_kwargs = accept_magic_kwargs self.finalized = False self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # these options are moved to the config to # simplify pickling of the app object. self._preconf = {} if broker: self._preconf["BROKER_URL"] = broker if include: self._preconf["CELERY_IMPORTS"] = include if self.set_as_current: self.set_current() self.on_init()
def subclass_with_self(self, Class, name=None, attribute='app', reverse=None, keep_reduce=False, **kw): """Subclass an app-compatible class by setting its app attribute to this instance. App-compatible means the class has an 'app' attribute providing the default app, e.g.: ``class Foo(object): app = None``. :param Class: The class to subclass. :keyword name: Custom name for the target subclass. :keyword attribute: Name of the attribute holding the app. Default is ``"app"``. :keyword reverse: Reverse path to this object used for pickling purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``. :keyword keep_reduce: If enabled a custom ``__reduce__`` implementation will not be provided. """ Class = symbol_by_name(Class) reverse = reverse if reverse else Class.__name__ def __reduce__(self): return _unpickle_appattr, (reverse, self.__reduce_keys__()) attrs = dict({attribute: self}, __module__=Class.__module__, __doc__=Class.__doc__, **kw) if not keep_reduce: attrs['__reduce__'] = __reduce__ return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def hook_subscribe(state, event, url=None, callback=None): subs = subscribers_for_event(event) _unsubscribe(subs, url) subscribers_for_event(event).append( symbol_by_name(callback) if callback else url, ) return {'ok': 'url {0!r} subscribed to event {1!r}'.format(url, event)}
def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd): """Find app by name.""" from .base import Celery try: sym = symbol_by_name(app, imp=imp) except AttributeError: # last part was not an attribute, but a module sym = imp(app) if isinstance(sym, ModuleType) and ":" not in app: try: found = sym.app if isinstance(found, ModuleType): raise AttributeError() except AttributeError: try: found = sym.celery if isinstance(found, ModuleType): raise AttributeError() except AttributeError: if getattr(sym, "__path__", None): try: return find_app("{0}.celery".format(app), symbol_by_name=symbol_by_name, imp=imp) except ImportError: pass for suspect in values(vars(sym)): if isinstance(suspect, Celery): return suspect raise else: return found else: return found return sym
def run_suite(self, names, suite, block_timeout=None, no_color=False, **options): return symbol_by_name(suite)( self.app, block_timeout=block_timeout, no_color=no_color, ).run(names, **options)
def subclass_with_self(self, Class, name=None, attribute='app', reverse=None, keep_reduce=False, **kw): """Subclass an app-compatible class. App-compatible means that the class has a class attribute that provides the default app it should use, for example: ``class Foo: app = None``. Arguments: Class (type): The app-compatible class to subclass. name (str): Custom name for the target class. attribute (str): Name of the attribute holding the app, Default is 'app'. reverse (str): Reverse path to this object used for pickling purposes. For example, to get ``app.AsyncResult``, use ``"AsyncResult"``. keep_reduce (bool): If enabled a custom ``__reduce__`` implementation won't be provided. """ Class = symbol_by_name(Class) reverse = reverse if reverse else Class.__name__ def __reduce__(self): return _unpickle_appattr, (reverse, self.__reduce_args__()) attrs = dict( {attribute: self}, __module__=Class.__module__, __doc__=Class.__doc__, **kw) if not keep_reduce: attrs['__reduce__'] = __reduce__ return type(name or Class.__name__, (Class,), attrs)
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, tasks=None, broker=None, include=None, changes=None, config_source=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.backend_cls = backend or self.backend_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.accept_magic_kwargs = accept_magic_kwargs self._config_source = config_source self.configured = False self._pending_defaults = deque() self.finalized = False self._finalize_mutex = threading.Lock() self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # these options are moved to the config to # simplify pickling of the app object. self._preconf = changes or {} if broker: self._preconf['BROKER_URL'] = broker if include: self._preconf['CELERY_IMPORTS'] = include if self.set_as_current: self.set_current() # See Issue #1126 # this is used when pickling the app object so that configuration # is reread without having to pickle the contents # (which is often unpickleable anyway) if self._config_source: self.config_from_object(self._config_source) self.on_init() _register_app(self)
def get_best_json(attr=None, choices=['simplejson', 'json']): for i, module in enumerate(choices): try: sym = ':'.join([module, attr]) if attr else module return symbol_by_name(sym), _JSON_EXTRA_ARGS.get(module, {}) except (AttributeError, ImportError): if i + 1 >= len(choices): raise
def flask_app(self) -> Flask: if has_app_context(): return unwrap(flask_current_app) self.flask_app_factory = symbol_by_name(self.flask_app_factory) app = self.flask_app_factory() register_after_fork(app, self._setup_after_fork) return app
def get_backend_cls(backend=None, loader=None): """Get backend class by name/alias""" backend = backend or "disabled" loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) try: return symbol_by_name(backend, aliases) except ValueError, exc: raise ValueError, ValueError(UNKNOWN_BACKEND % (backend, exc)), sys.exc_info()[2]
def get_backend_cls(backend=None, loader=None): """Get backend class by name/alias""" backend = backend or 'disabled' loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) try: return symbol_by_name(backend, aliases) except ValueError as exc: reraise(ValueError, ValueError(UNKNOWN_BACKEND.format(backend, exc)), sys.exc_info()[2])
def get_extension_commands(namespace='celery.commands'): try: from pkg_resources import iter_entry_points except ImportError: return for ep in iter_entry_points(namespace): for attr in ep.attrs: command(symbol_by_name(':'.join([ep.module_name, attr])), name=ep.name)
def get_best_json(attr=None, choices=['kombu.utils.json', 'simplejson', 'json']): for i, module in enumerate(choices): try: return symbol_by_name( ':'.join([module, attr]) if attr else module ) except (AttributeError, ImportError): if i + 1 >= len(choices): raise
def get_scheduler(self, lazy=False, extension_namespace='celery.beat_schedulers'): filename = self.schedule_filename aliases = dict(load_extension_class_names(extension_namespace) or {}) return symbol_by_name(self.scheduler_cls, aliases=aliases)( app=self.app, schedule_filename=filename, max_interval=self.max_interval, lazy=lazy, )
def get_backend_cls(backend=None, loader=None): """Get backend class by name/alias""" backend = backend or "disabled" loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) try: cls = symbol_by_name(backend, aliases) except ValueError as exc: reraise(ImproperlyConfigured, ImproperlyConfigured(UNKNOWN_BACKEND.format(backend, exc)), sys.exc_info()[2]) if isinstance(cls, types.ModuleType): raise ImproperlyConfigured(UNKNOWN_BACKEND.format(backend, "is a Python module, not a backend class.")) return cls
def traverse_subscribers(it, *args, **kwargs): stream = deque([it]) while stream: for node in maybe_list(stream.popleft()): if isinstance(node, string_types) and node.startswith('!'): node = symbol_by_name(node[1:]) if isinstance(node, Callable): node = node(*args, **kwargs) if is_list(node): stream.append(node) elif node: yield node
def flask_app(self): if self._flask_app is None: self.flask_app_factory = symbol_by_name(self.flask_app_factory) app = self._flask_app = self.flask_app_factory() self.app_context = app.app_context() if 'sentry' in app.extensions: from raven.contrib.celery import register_signal, register_logger_signal client = app.extensions['sentry'].client register_signal(client) register_logger_signal(client) return self._flask_app
def __init__(self, app, *args, **kwargs): self.backend = symbol_by_name( getattr(app.conf, 'CELERYBEAT_REDUNDANT_BACKEND', 'celery_redundant_scheduler.backends.redis:Backend'))( app, **getattr(app.conf, 'CELERYBEAT_REDUNDANT_BACKEND_OPTIONS', {})) class Entry(self.Entry): backend = self.backend self.Entry = Entry super(RedundantScheduler, self).__init__(app, *args, **kwargs)
def get_backend_cls(backend=None, loader=None): """Get backend class by name/alias""" backend = backend or 'disabled' loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) try: cls = symbol_by_name(backend, aliases) except ValueError as exc: reraise(ImproperlyConfigured, ImproperlyConfigured( UNKNOWN_BACKEND.format(backend, exc)), sys.exc_info()[2]) if isinstance(cls, types.ModuleType): raise ImproperlyConfigured(UNKNOWN_BACKEND.format( backend, 'is a Python module, not a backend class.')) return cls
def load_extension_commands(namespace='celery.commands'): try: from pkg_resources import iter_entry_points except ImportError: return for ep in iter_entry_points(namespace): sym = ':'.join([ep.module_name, ep.attrs[0]]) try: cls = symbol_by_name(sym) except (ImportError, SyntaxError), exc: warnings.warn('Cannot load extension %r: %r' % (sym, exc)) else: heapq.heappush(_get_extension_classes(), ep.name) command(cls, name=ep.name)
def load(self): try: from pkg_resources import iter_entry_points except ImportError: # pragma: no cover return for ep in iter_entry_points(self.namespace): sym = ":".join([ep.module_name, ep.attrs[0]]) try: cls = symbol_by_name(sym) except (ImportError, SyntaxError) as exc: warnings.warn("Cannot load extension {0!r}: {1!r}".format(sym, exc)) else: self.add(cls, ep.name) return self.names
def config_from_object(self, obj, silent=False): if isinstance(obj, basestring): try: if '.' in obj: obj = symbol_by_name(obj, imp=self.import_from_cwd) else: obj = self.import_from_cwd(obj) except (ImportError, AttributeError): if silent: return False raise if not hasattr(obj, '__getitem__'): obj = DictAttribute(obj) self._conf = obj return True
def load_extension_commands(namespace='celery.commands'): try: from pkg_resources import iter_entry_points except ImportError: return for ep in iter_entry_points(namespace): _get_extension_classes().append(ep.name) sym = ':'.join([ep.module_name, ep.attrs[0]]) try: cls = symbol_by_name(sym) except (ImportError, SyntaxError) as exc: warnings.warn( 'Cannot load extension {0!r}: {1!r}'.format(sym, exc)) else: command(cls, name=ep.name)
def flask_app(self): if has_app_context(): return flask_current_app._get_current_object() self.flask_app_factory = symbol_by_name(self.flask_app_factory) app = self.flask_app_factory() if 'sentry' in app.extensions: from raven.contrib.celery import register_signal, register_logger_signal client = app.extensions['sentry'].client client.tags['process_type'] = 'celery task' register_signal(client) register_logger_signal(client) register_after_fork(app, self._setup_after_fork) return app
def by_name(backend=None, loader=None, extension_namespace='celery.result_backends'): """Get backend class by name/alias.""" backend = backend or 'disabled' loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) aliases.update(load_extension_class_names(extension_namespace)) try: cls = symbol_by_name(backend, aliases) except ValueError as exc: reraise(ImproperlyConfigured, ImproperlyConfigured( UNKNOWN_BACKEND.strip().format(backend, exc)), sys.exc_info()[2]) if isinstance(cls, types.ModuleType): raise ImproperlyConfigured(UNKNOWN_BACKEND.strip().format( backend, 'is a Python module, not a backend class.')) return cls
def load(self): try: from pkg_resources import iter_entry_points except ImportError: # pragma: no cover return for ep in iter_entry_points(self.namespace): sym = ':'.join([ep.module_name, ep.attrs[0]]) try: cls = symbol_by_name(sym) except (ImportError, SyntaxError) as exc: warnings.warn( 'Cannot load extension {0!r}: {1!r}'.format(sym, exc)) else: self.add(cls, ep.name) return self.names
def subclass_with_self(self, Class, name=None, attribute="app", **kw): """Subclass an app-compatible class by setting its app attribute to be this app instance. App-compatible means that the class has a class attribute that provides the default app it should use, e.g. ``class Foo: app = None``. :param Class: The app-compatible class to subclass. :keyword name: Custom name for the target class. :keyword attribute: Name of the attribute holding the app, default is "app". """ Class = symbol_by_name(Class) return type(name or Class.__name__, (Class, ), dict({attribute: self, "__module__": Class.__module__, "__doc__": Class.__doc__}, **kw))
def flask_app(self): if has_app_context(): return unwrap(flask_current_app) self.flask_app_factory = symbol_by_name(self.flask_app_factory) app = self.flask_app_factory() if "sentry" in app.extensions: from raven.contrib.celery import register_signal, register_logger_signal client = app.extensions["sentry"].client client.tags["process_type"] = "celery task" register_signal(client) register_logger_signal(client) register_after_fork(app, self._setup_after_fork) return app
def by_name(backend=None, loader=None, extension_namespace='celery.result_backends'): """Get backend class by name/alias.""" backend = backend or 'disabled' loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) aliases.update( load_extension_class_names(extension_namespace) or {}) try: cls = symbol_by_name(backend, aliases) except ValueError as exc: reraise(ImproperlyConfigured, ImproperlyConfigured( UNKNOWN_BACKEND.strip().format(backend, exc)), sys.exc_info()[2]) if isinstance(cls, types.ModuleType): raise ImproperlyConfigured(UNKNOWN_BACKEND.strip().format( backend, 'is a Python module, not a backend class.')) return cls
def flask_app(self) -> Flask: if has_app_context(): return unwrap(flask_current_app) self.flask_app_factory = symbol_by_name(self.flask_app_factory) app = self.flask_app_factory() if "sentry" in app.extensions: # pyre-fixme[21]: Could not find `raven`. from raven.contrib.celery import register_signal, register_logger_signal client = app.extensions["sentry"].client client.tags["process_type"] = "celery task" register_signal(client) register_logger_signal(client) # pyre-fixme[16]: Module `multiprocessing` has no attribute `util`. register_after_fork(app, self._setup_after_fork) return app
def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd): """Find app by name.""" from .base import Celery try: sym = symbol_by_name(app, imp=imp) except AttributeError: # last part was not an attribute, but a module sym = imp(app) if isinstance(sym, ModuleType) and ':' not in app: try: found = sym.app if isinstance(found, ModuleType): raise AttributeError() except AttributeError: try: found = sym.celery if isinstance(found, ModuleType): raise AttributeError( "attribute 'celery' is the celery module not the instance of celery" ) except AttributeError: if getattr(sym, '__path__', None): try: return find_app( '{0}.celery'.format(app), symbol_by_name=symbol_by_name, imp=imp, ) except ImportError: pass for suspect in values(vars(sym)): if isinstance(suspect, Celery): return suspect raise else: return found else: return found return sym
def subclass_with_self(self, Class, name=None, attribute='app', reverse=None, keep_reduce=False, **kw): # type: (type, str, str, str, bool, **Any) -> type """Subclass an app-compatible class. App-compatible means the class has an 'app' attribute providing the default app, e.g.: ``class Foo(object): app = None``. Arguments: Class (Any): The class to subclass. Keyword Arguments: name (str): Custom name for the target subclass. attribute (str): Name of the attribute holding the app. Default is ``"app"``. reverse (str): Reverse path to this object used for pickling purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``. keep_reduce (bool): If enabled a custom ``__reduce__`` implementation will not be provided. """ Class = symbol_by_name(Class) reverse = reverse if reverse else Class.__name__ def __reduce__(self): return _unpickle_appattr, (reverse, self.__reduce_keys__()) attrs = dict({attribute: self}, __module__=Class.__module__, __doc__=Class.__doc__, **kw) if not keep_reduce: attrs['__reduce__'] = __reduce__ return type(bytes_if_py2(name or Class.__name__), (Class, ), attrs)
def default(task, app, consumer, info=logger.info, error=logger.error, task_reserved=task_reserved, to_system_tz=timezone.to_system, bytes=bytes, buffer_t=buffer_t, proto1_to_proto2=proto1_to_proto2): """Default task execution strategy. Note: Strategies are here as an optimization, so sadly it's not very easy to override. """ hostname = consumer.hostname connection_errors = consumer.connection_errors _does_info = logger.isEnabledFor(logging.INFO) # task event related # (optimized to avoid calling request.send_event) eventer = consumer.event_dispatcher events = eventer and eventer.enabled send_event = eventer.send task_sends_events = events and task.send_events call_at = consumer.timer.call_at apply_eta_task = consumer.apply_eta_task rate_limits_enabled = not consumer.disable_rate_limits get_bucket = consumer.task_buckets.__getitem__ handle = consumer.on_task_request limit_task = consumer._limit_task limit_post_eta = consumer._limit_post_eta body_can_be_buffer = consumer.pool.body_can_be_buffer Request = symbol_by_name(task.Request) Req = create_request_cls(Request, task, consumer.pool, hostname, eventer) revoked_tasks = consumer.controller.state.revoked def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None and 'args' not in message.payload: body, headers, decoded, utc = ( message.body, message.headers, False, app.uses_utc_timezone(), ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body else: if 'args' in message.payload: body, headers, decoded, utc = hybrid_to_proto2(message, message.payload) else: body, headers, decoded, utc = proto1_to_proto2(message, body) req = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc, ) if _does_info: info('Received task: %s', req) if (req.expires or req.id in revoked_tasks) and req.revoked(): return signals.task_received.send(sender=consumer, request=req) if task_sends_events: send_event( 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) bucket = None eta = None if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, app.timezone) except (OverflowError, ValueError) as exc: error("Couldn't convert ETA %r to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True) req.reject(requeue=False) if rate_limits_enabled: bucket = get_bucket(task.name) if eta and bucket: consumer.qos.increment_eventually() return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6) if eta: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req,), priority=6) return task_message_handler if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback(req) for callback in callbacks] handle(req) return task_message_handler
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, tasks=None, broker=None, include=None, changes=None, config_source=None, fixups=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.backend_cls = backend or self.backend_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.accept_magic_kwargs = accept_magic_kwargs self.user_options = defaultdict(set) self._config_source = config_source self.steps = defaultdict(set) self.configured = False self._pending_defaults = deque() self.finalized = False self._finalize_mutex = threading.Lock() self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # If the class defins a custom __reduce_args__ we need to use # the old way of pickling apps, which is pickling a list of # args instead of the new way that pickles a dict of keywords. self._using_v1_reduce = app_has_custom(self, '__reduce_args__') # these options are moved to the config to # simplify pickling of the app object. self._preconf = changes or {} if broker: self._preconf['BROKER_URL'] = broker if include: self._preconf['CELERY_IMPORTS'] = include enable_insecure_serializers() # Apply fixups. self.fixups = set(fixups or ()) for fixup in self.fixups | BUILTIN_FIXUPS: symbol_by_name(fixup)(self) if self.set_as_current: self.set_current() # See Issue #1126 # this is used when pickling the app object so that configuration # is reread without having to pickle the contents # (which is often unpickleable anyway) if self._config_source: self.config_from_object(self._config_source) self.on_init() _register_app(self)
def test_symbol_by_name_returns_default(self): default = object() self.assertIs(symbol_by_name('xyz.ryx.qedoa.weq:foz', default=default), default)
def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, tasks=None, broker=None, include=None, changes=None, config_source=None, fixups=None, task_cls=None, autofinalize=True, namespace=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls self.control_cls = control or self.control_cls self.task_cls = task_cls or self.task_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) self.user_options = defaultdict(set) self.steps = defaultdict(set) self.autofinalize = autofinalize self.namespace = namespace self.configured = False self._config_source = config_source self._pending_defaults = deque() self._pending_periodic_tasks = deque() self.finalized = False self._finalize_mutex = threading.Lock() self._pending = deque() self._tasks = tasks if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) # If the class defines a custom __reduce_args__ we need to use # the old way of pickling apps, which is pickling a list of # args instead of the new way that pickles a dict of keywords. self._using_v1_reduce = app_has_custom(self, '__reduce_args__') # these options are moved to the config to # simplify pickling of the app object. self._preconf = changes or {} self._preconf_set_by_auto = set() self.__autoset('broker_url', broker) self.__autoset('result_backend', backend) self.__autoset('include', include) self._conf = Settings( PendingConfiguration( self._preconf, self._get_from_conf_and_finalize), prefix=self.namespace, ) # - Apply fix-ups. self.fixups = set(self.builtin_fixups) if fixups is None else fixups # ...store fixup instances in _fixups to keep weakrefs alive. self._fixups = [symbol_by_name(fixup)(self) for fixup in self.fixups] if self.set_as_current: self.set_current() # Signals if self.on_configure is None: # used to be a method pre 4.0 self.on_configure = Signal() self.on_after_configure = Signal() self.on_after_finalize = Signal() self.on_after_fork = Signal() self.on_init() _register_app(self)
def symbol_by_name(self, name, imp=import_from_cwd): return symbol_by_name(name, imp=imp)