def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
try:
self.interface_errors = (
symbol_by_name('django.db.utils.InterfaceError'),
)
except (ImportError, AttributeError):
self._interface_errors = ()
# Database-related exceptions.
DatabaseError = symbol_by_name('django.db:DatabaseError')
try:
import MySQLdb as mysql
_my_database_errors = (mysql.DatabaseError,
mysql.InterfaceError,
mysql.OperationalError)
except ImportError:
_my_database_errors = () # noqa
try:
import psycopg2 as pg
_pg_database_errors = (pg.DatabaseError,
pg.InterfaceError,
pg.OperationalError)
except ImportError:
_pg_database_errors = () # noqa
try:
import sqlite3
_lite_database_errors = (sqlite3.DatabaseError,
sqlite3.InterfaceError,
sqlite3.OperationalError)
except ImportError:
_lite_database_errors = () # noqa
try:
import cx_Oracle as oracle
_oracle_database_errors = (oracle.DatabaseError,
oracle.InterfaceError,
oracle.OperationalError)
except ImportError:
_oracle_database_errors = () # noqa
try:
self._close_old_connections = symbol_by_name(
'django.db:close_old_connections',
)
except (ImportError, AttributeError):
self._close_old_connections = None
self.database_errors = (
(DatabaseError,) +
_my_database_errors +
_pg_database_errors +
_lite_database_errors +
_oracle_database_errors
)
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get("CELERY_DB_REUSE_MAX", None)
self._db = import_module("django.db")
self._cache = import_module("django.core.cache")
self._settings = symbol_by_name("django.conf:settings")
self.interface_errors = (symbol_by_name("django.db.utils.InterfaceError"),)
self.DatabaseError = symbol_by_name("django.db:DatabaseError")
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
self.interface_errors = (
symbol_by_name('django.db.utils.InterfaceError'), )
self.DatabaseError = symbol_by_name('django.db:DatabaseError')
def _firstpass(self, steps):
for step in values(steps):
step.requires = [symbol_by_name(dep) for dep in step.requires]
stream = deque(step.requires for step in values(steps))
while stream:
for node in stream.popleft():
node = symbol_by_name(node)
if node.name not in self.steps:
steps[node.name] = node
stream.append(node.requires)
def _firstpass(self, steps):
for step in values(steps):
step.requires = [symbol_by_name(dep) for dep in step.requires]
stream = deque(step.requires for step in values(steps))
while stream:
for node in stream.popleft():
node = symbol_by_name(node)
if node.name not in self.steps:
steps[node.name] = node
stream.append(node.requires)
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
try:
self.interface_errors = (
symbol_by_name('django.db.utils.InterfaceError'), )
except (ImportError, AttributeError):
self._interface_errors = ()
# Database-related exceptions.
DatabaseError = symbol_by_name('django.db:DatabaseError')
try:
import MySQLdb as mysql
_my_database_errors = (mysql.DatabaseError, mysql.InterfaceError,
mysql.OperationalError)
except ImportError:
_my_database_errors = () # noqa
try:
import psycopg2 as pg
_pg_database_errors = (pg.DatabaseError, pg.InterfaceError,
pg.OperationalError)
except ImportError:
_pg_database_errors = () # noqa
try:
import sqlite3
_lite_database_errors = (sqlite3.DatabaseError,
sqlite3.InterfaceError,
sqlite3.OperationalError)
except ImportError:
_lite_database_errors = () # noqa
try:
import cx_Oracle as oracle
_oracle_database_errors = (oracle.DatabaseError,
oracle.InterfaceError,
oracle.OperationalError)
except ImportError:
_oracle_database_errors = () # noqa
try:
self._close_old_connections = symbol_by_name(
'django.db:close_old_connections', )
except (ImportError, AttributeError):
self._close_old_connections = None
self.database_errors = ((DatabaseError, ) + _my_database_errors +
_pg_database_errors + _lite_database_errors +
_oracle_database_errors)
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
try:
self.interface_errors = (
symbol_by_name('django.db.utils.InterfaceError'),
)
except (ImportError, AttributeError):
self._interface_errors = ()
self.DatabaseError = symbol_by_name('django.db:DatabaseError')
def resolve_transport(transport: str | None = None) -> str | None:
"""Get transport by name.
Arguments:
transport (Union[str, type]): This can be either
an actual transport class, or the fully qualified
path to a transport class, or the alias of a transport.
"""
if isinstance(transport, str):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
from kombu.utils.text import fmatch_best
alt = fmatch_best(transport, TRANSPORT_ALIASES)
if alt:
raise KeyError(
'No such transport: {}. Did you mean {}?'.format(
transport, alt))
raise KeyError(f'No such transport: {transport}')
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def instantiate(name, *args, **kwargs):
"""Instantiate class by name.
See Also:
:func:`symbol_by_name`.
"""
return symbol_by_name(name)(*args, **kwargs)
def subclass_with_self(self, Class,
name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
# type: (type, str, str, str, bool, **Any) -> type
"""Subclass an app-compatible class.
App-compatible means the class has an 'app' attribute providing
the default app, e.g.: ``class Foo(object): app = None``.
Arguments:
Class (Any): The class to subclass.
Keyword Arguments:
name (str): Custom name for the target subclass.
attribute (str): Name of the attribute holding the app.
Default is ``"app"``.
reverse (str): Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation will not be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_keys__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def resolve_transport(transport=None):
"""Get transport by name.
Arguments:
transport (Union[str, type]): This can be either
an actual transport class, or the fully qualified
path to a transport class, or the alias of a transport.
"""
if isinstance(transport, string_t):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
from kombu.utils.text import fmatch_best
alt = fmatch_best(transport, TRANSPORT_ALIASES)
if alt:
raise KeyError(
'No such transport: {0}. Did you mean {1}?'.format(
transport, alt))
raise KeyError('No such transport: {0}'.format(transport))
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def get_worker_instance(name=None):
sys.path.append(os.getcwd())
name = name or os.getenv("QINIU_UFOP_CELERY")
if not name:
# app默认取当前工作路径的app.ufop
name = "app.ufop"
return symbol_by_name(name)
def instantiate(name, *args, **kwargs):
"""Instantiate class by name.
See Also:
:func:`symbol_by_name`.
"""
return symbol_by_name(name)(*args, **kwargs)
def hook_subscribe(state, event, url=None, callback=None):
"""Subscribe to webhook."""
subs = subscribers_for_event(event)
_unsubscribe(subs, url)
subscribers_for_event(event).append(
symbol_by_name(callback) if callback else url,
)
return ok('url {0!r} subscribed to event {1!r}'.format(url, event))
def load_extension_classes(namespace):
for name, class_name in load_extension_class_names(namespace):
try:
cls = symbol_by_name(class_name)
except (ImportError, SyntaxError) as exc:
warnings.warn(
f'Cannot load {namespace} extension {class_name!r}: {exc!r}')
else:
yield name, cls
def load_extension_classes(namespace):
for name, class_name in load_extension_class_names(namespace):
try:
cls = symbol_by_name(class_name)
except (ImportError, SyntaxError) as exc:
warnings.warn('Cannot load {0} extension {1!r}: {2!r}'.format(
namespace, class_name, exc))
else:
yield name, cls
def install(self):
# Need to add project directory to path
sys.path.append(os.getcwd())
self._settings = symbol_by_name("django.conf:settings")
self.app.loader.now = self.now
signals.import_modules.connect(self.on_import_modules)
signals.worker_init.connect(self.on_worker_init)
return self
def install(self):
# Need to add project directory to path
sys.path.append(os.getcwd())
self._settings = symbol_by_name('django.conf:settings')
self.app.loader.now = self.now
signals.import_modules.connect(self.on_import_modules)
signals.worker_init.connect(self.on_worker_init)
return self
def load_extension_classes(namespace):
for name, class_name in load_extension_class_names(namespace):
try:
cls = symbol_by_name(class_name)
except (ImportError, SyntaxError) as exc:
warnings.warn(
'Cannot load {0} extension {1!r}: {2!r}'.format(
namespace, class_name, exc))
else:
yield name, cls
def traverse_subscribers(it, *args, **kwargs):
stream = deque([it])
while stream:
for node in maybe_list(stream.popleft()):
if isinstance(node, string_types) and node.startswith('!'):
node = symbol_by_name(node[1:])
if isinstance(node, Callable):
node = node(*args, **kwargs)
if is_list(node):
stream.append(node)
elif node:
yield node
def install(self):
# Need to add project directory to path.
# The project directory has precedence over system modules,
# so we prepend it to the path.
sys.path.insert(0, os.getcwd())
self._settings = symbol_by_name('django.conf:settings')
self.app.loader.now = self.now
signals.import_modules.connect(self.on_import_modules)
signals.worker_init.connect(self.on_worker_init)
return self
def commands(self):
rv = dict()
dir = os.path.dirname(__file__)
command_names = find_commands(dir)
for name in command_names:
cls = symbol_by_name(
"qiniu_ufop.management.commands.%s:Command" % name)
parser = self.parsers.add_parser(
name, help=cls.__doc__, description=cls.__doc__)
parser.root = self
cmd = cls(parser)
rv[name] = cmd
return rv
def resolve_transport(transport=None):
if isinstance(transport, string_t):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
from kombu.utils.text import fmatch_best
alt = fmatch_best(transport, TRANSPORT_ALIASES)
if alt:
raise KeyError(
'No such transport: {0}. Did you mean {1}?'.format(
transport, alt))
raise KeyError('No such transport: {0}'.format(transport))
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def resolve_transport(transport=None):
if isinstance(transport, string_t):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
from kombu.utils.text import fmatch_best
alt = fmatch_best(transport, TRANSPORT_ALIASES)
if alt:
raise KeyError(
'No such transport: {0}. Did you mean {1}?'.format(
transport, alt))
raise KeyError('No such transport: {0}'.format(transport))
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def subclass_with_self(self,
Class,
name=None,
attribute='app',
reverse=None,
keep_reduce=False,
**kw):
# type: (type, str, str, str, bool, **Any) -> type
"""Subclass an app-compatible class.
App-compatible means the class has an 'app' attribute providing
the default app, e.g.: ``class Foo(object): app = None``.
Arguments:
Class (Any): The class to subclass.
Keyword Arguments:
name (str): Custom name for the target subclass.
attribute (str): Name of the attribute holding the app.
Default is ``"app"``.
reverse (str): Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation will not be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_keys__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class, ), attrs)
def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd):
"""Find app by name."""
from ..base import Corn
try:
sym = symbol_by_name(app, imp=imp)
except AttributeError:
# last part was not an attribute, but a module
sym = imp(app)
if isinstance(sym, ModuleType) and ':' not in app:
try:
found = sym.app
if isinstance(found, ModuleType):
raise AttributeError()
except AttributeError:
try:
found = sym.corn
if isinstance(found, ModuleType):
raise AttributeError()
except AttributeError:
if getattr(sym, '__path__', None):
try:
return find_app(
'{0}.corn'.format(app),
symbol_by_name=symbol_by_name,
imp=imp,
)
except ImportError:
pass
for suspect in values(vars(sym)):
if isinstance(suspect, Corn):
return suspect
raise
else:
return found
else:
return found
return sym
def model_reverser(self):
# type: () -> Callable
return symbol_by_name('thorn.reverse.model_reverser')
def webhook_model(self):
# type: () -> Callable
return symbol_by_name('thorn.decorators.webhook_model')
def test_no_default(self):
with pytest.raises(ImportError):
symbol_by_name('xyz.ryx.qedoa.weq:foz')
def reverse(self):
return symbol_by_name(self.reverse_cls)
def test_imp_reraises_ValueError(self):
imp = Mock()
imp.side_effect = ValueError()
with pytest.raises(ValueError):
symbol_by_name('kombu.Connection', imp=imp)
def result_handler(self):
""":rtype: qiniu_ufop.web.result.ResultHandler"""
cls = self.application.settings.get(
"ufop_dispatcher", "qiniu_ufop.web.result.ResultHandler")
return symbol_by_name(cls)()
class UnitLogging(symbol_by_name(Celery.log_cls)):
"""Sets up logging for the test application."""
def __init__(self, *args, **kwargs):
super(UnitLogging, self).__init__(*args, **kwargs)
self.already_setup = True
def get_implementation(cls):
"""Return pool implementation by name."""
return symbol_by_name(cls, ALIASES)
def _now(self):
return symbol_by_name("django.utils.timezone:now")
def test_instance_returns_instance(self):
instance = object()
assert symbol_by_name(instance) is instance
def get_implementation(cls):
return symbol_by_name(cls, ALIASES)
def load_step(self, step):
step = symbol_by_name(step)
return step.name, step
def load_signals(self, signals):
return {
symbol_by_name(sig): handler
for sig, handler in items(signals)
}
def _get_dispatcher(self, dispatcher=None):
# type: (Union[str, Dispatcher]) -> Dispatcher
if dispatcher is None:
dispatcher = self.settings.THORN_DISPATCHER
return symbol_by_name(dispatcher, self.dispatchers)
def Subscriber(self):
return symbol_by_name(
getattr(self.config, 'THORN_SUBSCRIBER_MODEL', None) or
self.subscriber_cls)
def load_signals(self, signals):
return {
symbol_by_name(sig): handler
for sig, handler in items(signals)
}
def _now(self):
return symbol_by_name('django.utils.timezone:now')
def _now(self):
try:
return symbol_by_name("django.utils.timezone:now")
except (AttributeError, ImportError): # pre django-1.4
return datetime.now
def test_package(self):
assert symbol_by_name('.entity:Exchange', package='kombu') is Exchange
assert symbol_by_name(':Consumer', package='kombu')
def reverse(self):
return symbol_by_name(self.reverse_cls)
def hmac_sign(self):
# type: () -> Callable
return symbol_by_name(self.settings.THORN_HMAC_SIGNER)
def config(self):
return symbol_by_name(self.settings_cls)
def load_step(self, step):
step = symbol_by_name(step)
return step.name, step
def test_no_default(self):
with pytest.raises(ImportError):
symbol_by_name('xyz.ryx.qedoa.weq:foz')
def Subscriber(self):
return symbol_by_name(
getattr(self.config, 'THORN_SUBSCRIBER_MODEL', None)
or self.subscriber_cls)
def test_imp_reraises_ValueError(self):
imp = Mock()
imp.side_effect = ValueError()
with pytest.raises(ValueError):
symbol_by_name('kombu.Connection', imp=imp)
def test_package(self):
assert symbol_by_name('.entity:Exchange', package='kombu') is Exchange
assert symbol_by_name(':Consumer', package='kombu')
def test_returns_default(self):
default = object()
assert symbol_by_name(
'xyz.ryx.qedoa.weq:foz', default=default) is default
def test_instance_returns_instance(self):
instance = object()
assert symbol_by_name(instance) is instance
def test_returns_default(self):
default = object()
assert symbol_by_name('xyz.ryx.qedoa.weq:foz',
default=default) is default
def config(self):
return symbol_by_name(self.settings_cls)
