def test_package(self):
from kombu.entity import Exchange
self.assertIs(
utils.symbol_by_name('.entity:Exchange', package='kombu'),
Exchange,
)
self.assertTrue(utils.symbol_by_name(':Consumer', package='kombu'))
def test_package(self):
from kombu.entity import Exchange
self.assertIs(
utils.symbol_by_name('.entity:Exchange', package='kombu'),
Exchange,
)
self.assertTrue(utils.symbol_by_name(':Consumer', package='kombu'))
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
self._mail_admins = symbol_by_name('django.core.mail:mail_admins')
# Current time and date
try:
self._now = symbol_by_name('django.utils.timezone:now')
except ImportError: # pre django-1.4
self._now = datetime.now # noqa
# Database-related exceptions.
DatabaseError = symbol_by_name('django.db:DatabaseError')
try:
import MySQLdb as mysql
_my_database_errors = (mysql.DatabaseError,
mysql.InterfaceError,
mysql.OperationalError)
except ImportError:
_my_database_errors = () # noqa
try:
import psycopg2 as pg
_pg_database_errors = (pg.DatabaseError,
pg.InterfaceError,
pg.OperationalError)
except ImportError:
_pg_database_errors = () # noqa
try:
import sqlite3
_lite_database_errors = (sqlite3.DatabaseError,
sqlite3.InterfaceError,
sqlite3.OperationalError)
except ImportError:
_lite_database_errors = () # noqa
try:
import cx_Oracle as oracle
_oracle_database_errors = (oracle.DatabaseError,
oracle.InterfaceError,
oracle.OperationalError)
except ImportError:
_oracle_database_errors = () # noqa
try:
self._close_old_connections = symbol_by_name(
'django.db:close_old_connections',
)
except ImportError:
self._close_old_connections = None
self.database_errors = (
(DatabaseError, ) +
_my_database_errors +
_pg_database_errors +
_lite_database_errors +
_oracle_database_errors
)
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
try:
self.interface_errors = (
symbol_by_name('django.db.utils.InterfaceError'),
)
except (ImportError, AttributeError):
self._interface_errors = ()
# Database-related exceptions.
DatabaseError = symbol_by_name('django.db:DatabaseError')
try:
import MySQLdb as mysql
_my_database_errors = (mysql.DatabaseError,
mysql.InterfaceError,
mysql.OperationalError)
except ImportError:
_my_database_errors = () # noqa
try:
import psycopg2 as pg
_pg_database_errors = (pg.DatabaseError,
pg.InterfaceError,
pg.OperationalError)
except ImportError:
_pg_database_errors = () # noqa
try:
import sqlite3
_lite_database_errors = (sqlite3.DatabaseError,
sqlite3.InterfaceError,
sqlite3.OperationalError)
except ImportError:
_lite_database_errors = () # noqa
try:
import cx_Oracle as oracle
_oracle_database_errors = (oracle.DatabaseError,
oracle.InterfaceError,
oracle.OperationalError)
except ImportError:
_oracle_database_errors = () # noqa
try:
self._close_old_connections = symbol_by_name(
'django.db:close_old_connections',
)
except (ImportError, AttributeError):
self._close_old_connections = None
self.database_errors = (
(DatabaseError,) +
_my_database_errors +
_pg_database_errors +
_lite_database_errors +
_oracle_database_errors
)
def __init__(self, app):
self.app = app
self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None)
self._db = import_module('django.db')
self._cache = import_module('django.core.cache')
self._settings = symbol_by_name('django.conf:settings')
try:
self.interface_errors = (
symbol_by_name('django.db.utils.InterfaceError'),
)
except (ImportError, AttributeError):
self._interface_errors = ()
# Database-related exceptions.
DatabaseError = symbol_by_name('django.db:DatabaseError')
try:
import MySQLdb as mysql
_my_database_errors = (mysql.DatabaseError,
mysql.InterfaceError,
mysql.OperationalError)
except ImportError:
_my_database_errors = () # noqa
try:
import psycopg2 as pg
_pg_database_errors = (pg.DatabaseError,
pg.InterfaceError,
pg.OperationalError)
except ImportError:
_pg_database_errors = () # noqa
try:
import sqlite3
_lite_database_errors = (sqlite3.DatabaseError,
sqlite3.InterfaceError,
sqlite3.OperationalError)
except ImportError:
_lite_database_errors = () # noqa
try:
import cx_Oracle as oracle
_oracle_database_errors = (oracle.DatabaseError,
oracle.InterfaceError,
oracle.OperationalError)
except ImportError:
_oracle_database_errors = () # noqa
try:
self._close_old_connections = symbol_by_name(
'django.db:close_old_connections',
)
except (ImportError, AttributeError):
self._close_old_connections = None
self.database_errors = (
(DatabaseError,) +
_my_database_errors +
_pg_database_errors +
_lite_database_errors +
_oracle_database_errors
)
def _firstpass(self, steps):
for step in values(steps):
step.requires = [symbol_by_name(dep) for dep in step.requires]
stream = deque(step.requires for step in values(steps))
while stream:
for node in stream.popleft():
node = symbol_by_name(node)
if node.name not in self.steps:
steps[node.name] = node
stream.append(node.requires)
def _firstpass(self, steps):
for step in values(steps):
step.requires = [symbol_by_name(dep) for dep in step.requires]
stream = deque(step.requires for step in values(steps))
while stream:
for node in stream.popleft():
node = symbol_by_name(node)
if node.name not in self.steps:
steps[node.name] = node
stream.append(node.requires)
def symbol_by_name(name):
""" Get symbol by qualified name.
"""
try:
return utils.symbol_by_name(name)
except:
pass
if ':' in name:
name = name.replace(':', '.')
attrs = name.split('.')
base_module = utils.symbol_by_name(attrs.pop(0))
return reduce(getattr, attrs, base_module)
def symbol_by_name(name):
""" Get symbol by qualified name.
"""
try:
return utils.symbol_by_name(name)
except:
pass
if ':' in name:
name = name.replace(':', '.')
attrs = name.split('.')
base_module = utils.symbol_by_name(attrs.pop(0))
return reduce(getattr, attrs, base_module)
def _import_function(func_name):
""" Import class by full name, check type and return.
"""
sym = symbol_by_name(func_name)
if hasattr(sym, '__call__'):
return sym
raise TypeError("Symbol '{}' is not a function".format(func_name))
def compare_data(config):
"""
Core task that runs every 1-60 seconds
"""
logger = compare_data.get_logger()
now = time()
logger.info('Compare data started {0}'.format(now))
results = []
for alias, comparator_path in config.COMPARATOR_ALIASES.iteritems():
# inicialize comparator
try:
comparator_cls = symbol_by_name(comparator_path)
comparator = comparator_cls(config)
except Exception, e:
logger.warning('Initialize comparator {0} was failed {1}, next processing will be skipped.'.format(comparator_path, e))
if comparator:
# process compare
try:
compare_results = comparator.compare()
results.append(compare_results)
logger.info("Results {0} from {1} comparator".format(compare_results, alias))
except Exception, e:
logger.error('Process comparator({0}) compare was failed {1}'.format(comparator_path, e))
def db(self):
"""only alias for db
"""
try:
self.db_backend_cls = symbol_by_name(BACKEND_ALIASES[self.db_backend_name])
except Exception, e:
raise e
def is_database_scheduler(scheduler):
"""Return true if Celery is configured to use the db scheduler."""
if not scheduler:
return False
from kombu.utils import symbol_by_name
from .schedulers import DatabaseScheduler
return issubclass(symbol_by_name(scheduler), DatabaseScheduler)
def instantiate(name, *args, **kwargs):
"""Instantiate class by name.
See :func:`symbol_by_name`.
"""
return symbol_by_name(name)(*args, **kwargs)
def _import_function(func_name):
""" Import class by full name, check type and return.
"""
sym = symbol_by_name(func_name)
if hasattr(sym, '__call__'):
return sym
raise TypeError("Symbol '{}' is not a function".format(func_name))
def _import_model(model_name):
""" Import class by full name, check type and return.
"""
sym = symbol_by_name(model_name)
if inspect.isclass(sym) and issubclass(sym, Model):
return sym
raise TypeError(
"Symbol '{}' is not a Django model".format(model_name))
def _import_serializer(serializer_name):
""" Import class by full name, check type and return.
"""
sym = symbol_by_name(serializer_name)
if inspect.isclass(sym) and issubclass(sym, ModelSerializer):
return sym
raise TypeError(
"Symbol '{}' is not a DRF serializer".format(serializer_name))
def _import_model(model_name):
""" Import class by full name, check type and return.
"""
sym = symbol_by_name(model_name)
if inspect.isclass(sym) and issubclass(sym, Model):
return sym
raise TypeError(
"Symbol '{}' is not a Django model".format(model_name))
def _import_serializer(serializer_name):
""" Import class by full name, check type and return.
"""
sym = symbol_by_name(serializer_name)
if inspect.isclass(sym) and issubclass(sym, ModelSerializer):
return sym
raise TypeError(
"Symbol '{}' is not a DRF serializer".format(serializer_name))
def is_database_scheduler(scheduler):
"""Return true if Celery is configured to use the db scheduler."""
if not scheduler:
return False
from kombu.utils import symbol_by_name
from .schedulers import DatabaseScheduler
return (
scheduler == 'django'
or issubclass(symbol_by_name(scheduler), DatabaseScheduler)
)
def install(self):
# Need to add project directory to path
sys.path.append(os.getcwd())
self._settings = symbol_by_name('django.conf:settings')
self.app.loader.now = self.now
signals.import_modules.connect(self.on_import_modules)
signals.worker_init.connect(self.on_worker_init)
return self
def _firstpass(self, steps):
stream = deque(step.requires for step in steps.itervalues())
while stream:
for node in stream.popleft():
node = symbol_by_name(node)
if node.name not in self.steps:
steps[node.name] = node
stream.append(node.requires)
# Make sure we have all the steps
assert [steps[req.name] for step in steps.values()
for req in step.requires]
def resolve_transport(transport=None):
if isinstance(transport, string_t):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
raise KeyError('No such transport: {0}'.format(transport))
else:
if isinstance(transport, Callable):
transport = transport()
return symbol_by_name(transport)
return transport
def resolve_transport(transport=None):
if isinstance(transport, string_t):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
raise KeyError('No such transport: {0}'.format(transport))
else:
if isinstance(transport, Callable):
transport = transport()
return symbol_by_name(transport)
return transport
def resolve_transport(transport=None):
if isinstance(transport, basestring):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
raise KeyError('No such transport: %s' % transport)
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def resolve_transport(transport=None):
if isinstance(transport, basestring):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
raise KeyError('No such transport: %s' % transport)
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def spawn(self, cls, id, kwargs={}):
"""Add actor to the registry and start the actor's main method."""
try:
actor = symbol_by_name(cls)(
connection=self.connection, id=id, **kwargs)
if actor.id in self.registry:
warn('Actor id %r already exists', actor.id)
self._start_actor_consumer(actor)
debug('Actor registered: %s', cls)
return actor.id
except Exception as exc:
error('Cannot start actor: %r', exc, exc_info=True)
def spawn(self, cls, id, kwargs={}):
"""Add actor to the registry and start the actor's main method."""
try:
actor = symbol_by_name(cls)(connection=self.connection,
id=id,
**kwargs)
if actor.id in self.registry:
warn('Actor id %r already exists', actor.id)
self._start_actor_consumer(actor)
debug('Actor registered: %s', cls)
return actor.id
except Exception as exc:
error('Cannot start actor: %r', exc, exc_info=True)
def resolve_transport(transport=None):
if isinstance(transport, basestring):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
from kombu.utils.text import fmatch_best
alt = fmatch_best(transport, TRANSPORT_ALIASES)
if alt:
raise KeyError('No such transport: %s. Did you mean %s?' %
(transport, alt))
raise KeyError('No such transport: %s' % transport)
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def resolve_transport(transport=None):
if isinstance(transport, string_t):
try:
transport = TRANSPORT_ALIASES[transport]
except KeyError:
if '.' not in transport and ':' not in transport:
from kombu.utils.text import fmatch_best
alt = fmatch_best(transport, TRANSPORT_ALIASES)
if alt:
raise KeyError(
'No such transport: {0}. Did you mean {1}?'.format(
transport, alt))
raise KeyError('No such transport: {0}'.format(transport))
else:
if callable(transport):
transport = transport()
return symbol_by_name(transport)
return transport
def get_base_task_class(base_task_name):
""" Provide base task for actual tasks
Load custom base task if overrides are in config or return default base task.
:param base_task_name: name of default base task in this module
:return: base celery task class
"""
base_task = globals().get(base_task_name)
custom_task_name = config.OVERRIDE_BASE_TASKS.get(base_task_name)
if not custom_task_name:
return base_task
sym = symbol_by_name(custom_task_name)
if inspect.isclass(sym) and issubclass(sym, base_task):
return sym
raise TypeError(
"Symbol '{}' has not a base ".format(custom_task_name,
base_task.__name__))
def test_imp_reraises_ValueError(self):
imp = Mock()
imp.side_effect = ValueError()
with self.assertRaises(ValueError):
utils.symbol_by_name('kombu.Connection', imp=imp)
def test_no_default(self):
with self.assertRaises(ImportError):
utils.symbol_by_name('xyz.ryx.qedoa.weq:foz')
def _now(self):
try:
return symbol_by_name('django.utils.timezone:now')
except (AttributeError, ImportError): # pre django-1.4
return datetime.now
def load_step(self, step):
step = symbol_by_name(step)
return step.name, step
def load_step(self, step):
step = symbol_by_name(step)
return step.name, step
def _now(self):
try:
return symbol_by_name('django.utils.timezone:now')
except (AttributeError, ImportError): # pre django-1.4
return datetime.now
def test_returns_default(self):
default = object()
self.assertIs(
utils.symbol_by_name('xyz.ryx.qedoa.weq:foz', default=default),
default,
)
def test_instance_returns_instance(self):
instance = object()
self.assertIs(utils.symbol_by_name(instance), instance)
def is_database_scheduler(scheduler):
if not scheduler:
return False
from kombu.utils import symbol_by_name
from .schedulers import DatabaseScheduler
return issubclass(symbol_by_name(scheduler), DatabaseScheduler)
def factory(self):
result = symbol_by_name(self.factory_name)
return result
class UnitLogging(symbol_by_name(Celery.log_cls)):
def __init__(self, *args, **kwargs):
super(UnitLogging, self).__init__(*args, **kwargs)
self.already_setup = True
def Queue(self):
return symbol_by_name(self.queue_model)
def mixin_template(template, conf):
cls = symbol_by_name(templates[template])
conf.update(dict(
(k, v) for k, v in items(vars(cls))
if k.isupper() and not k.startswith('_')
))
def get_implementation(cls):
return symbol_by_name(cls, ALIASES)
def _mail_admins(self):
return symbol_by_name('django.core.mail:mail_admins')
def test_no_default(self):
with self.assertRaises(ImportError):
utils.symbol_by_name('xyz.ryx.qedoa.weq:foz')
def is_database_scheduler(scheduler):
if not scheduler:
return False
from kombu.utils import symbol_by_name
from .schedulers import DatabaseScheduler
return issubclass(symbol_by_name(scheduler), DatabaseScheduler)
def test_imp_reraises_ValueError(self):
imp = Mock()
imp.side_effect = ValueError()
with self.assertRaises(ValueError):
utils.symbol_by_name('kombu.Connection', imp=imp)
def Queue(self):
return symbol_by_name(self.queue_model)
def test_instance_returns_instance(self):
instance = object()
self.assertIs(utils.symbol_by_name(instance), instance)
def _mail_admins(self):
return symbol_by_name('django.core.mail:mail_admins')
def test_returns_default(self):
default = object()
self.assertIs(
utils.symbol_by_name('xyz.ryx.qedoa.weq:foz', default=default),
default,
)
def __init__(self, name, id, async_start_result=None, **kwargs):
kwargs.update({'id': id})
self._actor = symbol_by_name(name)(**kwargs)
self.id = self._actor.id
self.async_start_result = async_start_result
def mixin_template(template, conf):
cls = symbol_by_name(templates[template])
conf.update(
dict((k, v) for k, v in items(vars(cls))
if k.isupper() and not k.startswith('_')))
