def test_package(self): from kombu.entity import Exchange self.assertIs( utils.symbol_by_name('.entity:Exchange', package='kombu'), Exchange, ) self.assertTrue(utils.symbol_by_name(':Consumer', package='kombu'))
def __init__(self, app): self.app = app self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None) self._db = import_module('django.db') self._cache = import_module('django.core.cache') self._settings = symbol_by_name('django.conf:settings') self._mail_admins = symbol_by_name('django.core.mail:mail_admins') # Current time and date try: self._now = symbol_by_name('django.utils.timezone:now') except ImportError: # pre django-1.4 self._now = datetime.now # noqa # Database-related exceptions. DatabaseError = symbol_by_name('django.db:DatabaseError') try: import MySQLdb as mysql _my_database_errors = (mysql.DatabaseError, mysql.InterfaceError, mysql.OperationalError) except ImportError: _my_database_errors = () # noqa try: import psycopg2 as pg _pg_database_errors = (pg.DatabaseError, pg.InterfaceError, pg.OperationalError) except ImportError: _pg_database_errors = () # noqa try: import sqlite3 _lite_database_errors = (sqlite3.DatabaseError, sqlite3.InterfaceError, sqlite3.OperationalError) except ImportError: _lite_database_errors = () # noqa try: import cx_Oracle as oracle _oracle_database_errors = (oracle.DatabaseError, oracle.InterfaceError, oracle.OperationalError) except ImportError: _oracle_database_errors = () # noqa try: self._close_old_connections = symbol_by_name( 'django.db:close_old_connections', ) except ImportError: self._close_old_connections = None self.database_errors = ( (DatabaseError, ) + _my_database_errors + _pg_database_errors + _lite_database_errors + _oracle_database_errors )
def __init__(self, app): self.app = app self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None) self._db = import_module('django.db') self._cache = import_module('django.core.cache') self._settings = symbol_by_name('django.conf:settings') try: self.interface_errors = ( symbol_by_name('django.db.utils.InterfaceError'), ) except (ImportError, AttributeError): self._interface_errors = () # Database-related exceptions. DatabaseError = symbol_by_name('django.db:DatabaseError') try: import MySQLdb as mysql _my_database_errors = (mysql.DatabaseError, mysql.InterfaceError, mysql.OperationalError) except ImportError: _my_database_errors = () # noqa try: import psycopg2 as pg _pg_database_errors = (pg.DatabaseError, pg.InterfaceError, pg.OperationalError) except ImportError: _pg_database_errors = () # noqa try: import sqlite3 _lite_database_errors = (sqlite3.DatabaseError, sqlite3.InterfaceError, sqlite3.OperationalError) except ImportError: _lite_database_errors = () # noqa try: import cx_Oracle as oracle _oracle_database_errors = (oracle.DatabaseError, oracle.InterfaceError, oracle.OperationalError) except ImportError: _oracle_database_errors = () # noqa try: self._close_old_connections = symbol_by_name( 'django.db:close_old_connections', ) except (ImportError, AttributeError): self._close_old_connections = None self.database_errors = ( (DatabaseError,) + _my_database_errors + _pg_database_errors + _lite_database_errors + _oracle_database_errors )
def _firstpass(self, steps): for step in values(steps): step.requires = [symbol_by_name(dep) for dep in step.requires] stream = deque(step.requires for step in values(steps)) while stream: for node in stream.popleft(): node = symbol_by_name(node) if node.name not in self.steps: steps[node.name] = node stream.append(node.requires)
def symbol_by_name(name): """ Get symbol by qualified name. """ try: return utils.symbol_by_name(name) except: pass if ':' in name: name = name.replace(':', '.') attrs = name.split('.') base_module = utils.symbol_by_name(attrs.pop(0)) return reduce(getattr, attrs, base_module)
def _import_function(func_name): """ Import class by full name, check type and return. """ sym = symbol_by_name(func_name) if hasattr(sym, '__call__'): return sym raise TypeError("Symbol '{}' is not a function".format(func_name))
def compare_data(config): """ Core task that runs every 1-60 seconds """ logger = compare_data.get_logger() now = time() logger.info('Compare data started {0}'.format(now)) results = [] for alias, comparator_path in config.COMPARATOR_ALIASES.iteritems(): # inicialize comparator try: comparator_cls = symbol_by_name(comparator_path) comparator = comparator_cls(config) except Exception, e: logger.warning('Initialize comparator {0} was failed {1}, next processing will be skipped.'.format(comparator_path, e)) if comparator: # process compare try: compare_results = comparator.compare() results.append(compare_results) logger.info("Results {0} from {1} comparator".format(compare_results, alias)) except Exception, e: logger.error('Process comparator({0}) compare was failed {1}'.format(comparator_path, e))
def db(self): """only alias for db """ try: self.db_backend_cls = symbol_by_name(BACKEND_ALIASES[self.db_backend_name]) except Exception, e: raise e
def is_database_scheduler(scheduler): """Return true if Celery is configured to use the db scheduler.""" if not scheduler: return False from kombu.utils import symbol_by_name from .schedulers import DatabaseScheduler return issubclass(symbol_by_name(scheduler), DatabaseScheduler)
def instantiate(name, *args, **kwargs): """Instantiate class by name. See :func:`symbol_by_name`. """ return symbol_by_name(name)(*args, **kwargs)
def _import_model(model_name): """ Import class by full name, check type and return. """ sym = symbol_by_name(model_name) if inspect.isclass(sym) and issubclass(sym, Model): return sym raise TypeError( "Symbol '{}' is not a Django model".format(model_name))
def _import_serializer(serializer_name): """ Import class by full name, check type and return. """ sym = symbol_by_name(serializer_name) if inspect.isclass(sym) and issubclass(sym, ModelSerializer): return sym raise TypeError( "Symbol '{}' is not a DRF serializer".format(serializer_name))
def is_database_scheduler(scheduler): """Return true if Celery is configured to use the db scheduler.""" if not scheduler: return False from kombu.utils import symbol_by_name from .schedulers import DatabaseScheduler return ( scheduler == 'django' or issubclass(symbol_by_name(scheduler), DatabaseScheduler) )
def install(self): # Need to add project directory to path sys.path.append(os.getcwd()) self._settings = symbol_by_name('django.conf:settings') self.app.loader.now = self.now signals.import_modules.connect(self.on_import_modules) signals.worker_init.connect(self.on_worker_init) return self
def _firstpass(self, steps): stream = deque(step.requires for step in steps.itervalues()) while stream: for node in stream.popleft(): node = symbol_by_name(node) if node.name not in self.steps: steps[node.name] = node stream.append(node.requires) # Make sure we have all the steps assert [steps[req.name] for step in steps.values() for req in step.requires]
def resolve_transport(transport=None): if isinstance(transport, string_t): try: transport = TRANSPORT_ALIASES[transport] except KeyError: if '.' not in transport and ':' not in transport: raise KeyError('No such transport: {0}'.format(transport)) else: if isinstance(transport, Callable): transport = transport() return symbol_by_name(transport) return transport
def resolve_transport(transport=None): if isinstance(transport, basestring): try: transport = TRANSPORT_ALIASES[transport] except KeyError: if '.' not in transport and ':' not in transport: raise KeyError('No such transport: %s' % transport) else: if callable(transport): transport = transport() return symbol_by_name(transport) return transport
def spawn(self, cls, id, kwargs={}): """Add actor to the registry and start the actor's main method.""" try: actor = symbol_by_name(cls)( connection=self.connection, id=id, **kwargs) if actor.id in self.registry: warn('Actor id %r already exists', actor.id) self._start_actor_consumer(actor) debug('Actor registered: %s', cls) return actor.id except Exception as exc: error('Cannot start actor: %r', exc, exc_info=True)
def spawn(self, cls, id, kwargs={}): """Add actor to the registry and start the actor's main method.""" try: actor = symbol_by_name(cls)(connection=self.connection, id=id, **kwargs) if actor.id in self.registry: warn('Actor id %r already exists', actor.id) self._start_actor_consumer(actor) debug('Actor registered: %s', cls) return actor.id except Exception as exc: error('Cannot start actor: %r', exc, exc_info=True)
def resolve_transport(transport=None): if isinstance(transport, basestring): try: transport = TRANSPORT_ALIASES[transport] except KeyError: if '.' not in transport and ':' not in transport: from kombu.utils.text import fmatch_best alt = fmatch_best(transport, TRANSPORT_ALIASES) if alt: raise KeyError('No such transport: %s. Did you mean %s?' % (transport, alt)) raise KeyError('No such transport: %s' % transport) else: if callable(transport): transport = transport() return symbol_by_name(transport) return transport
def resolve_transport(transport=None): if isinstance(transport, string_t): try: transport = TRANSPORT_ALIASES[transport] except KeyError: if '.' not in transport and ':' not in transport: from kombu.utils.text import fmatch_best alt = fmatch_best(transport, TRANSPORT_ALIASES) if alt: raise KeyError( 'No such transport: {0}. Did you mean {1}?'.format( transport, alt)) raise KeyError('No such transport: {0}'.format(transport)) else: if callable(transport): transport = transport() return symbol_by_name(transport) return transport
def get_base_task_class(base_task_name): """ Provide base task for actual tasks Load custom base task if overrides are in config or return default base task. :param base_task_name: name of default base task in this module :return: base celery task class """ base_task = globals().get(base_task_name) custom_task_name = config.OVERRIDE_BASE_TASKS.get(base_task_name) if not custom_task_name: return base_task sym = symbol_by_name(custom_task_name) if inspect.isclass(sym) and issubclass(sym, base_task): return sym raise TypeError( "Symbol '{}' has not a base ".format(custom_task_name, base_task.__name__))
def test_imp_reraises_ValueError(self): imp = Mock() imp.side_effect = ValueError() with self.assertRaises(ValueError): utils.symbol_by_name('kombu.Connection', imp=imp)
def test_no_default(self): with self.assertRaises(ImportError): utils.symbol_by_name('xyz.ryx.qedoa.weq:foz')
def _now(self): try: return symbol_by_name('django.utils.timezone:now') except (AttributeError, ImportError): # pre django-1.4 return datetime.now
def load_step(self, step): step = symbol_by_name(step) return step.name, step
def test_returns_default(self): default = object() self.assertIs( utils.symbol_by_name('xyz.ryx.qedoa.weq:foz', default=default), default, )
def test_instance_returns_instance(self): instance = object() self.assertIs(utils.symbol_by_name(instance), instance)
def is_database_scheduler(scheduler): if not scheduler: return False from kombu.utils import symbol_by_name from .schedulers import DatabaseScheduler return issubclass(symbol_by_name(scheduler), DatabaseScheduler)
def factory(self): result = symbol_by_name(self.factory_name) return result
class UnitLogging(symbol_by_name(Celery.log_cls)): def __init__(self, *args, **kwargs): super(UnitLogging, self).__init__(*args, **kwargs) self.already_setup = True
def Queue(self): return symbol_by_name(self.queue_model)
def mixin_template(template, conf): cls = symbol_by_name(templates[template]) conf.update(dict( (k, v) for k, v in items(vars(cls)) if k.isupper() and not k.startswith('_') ))
def get_implementation(cls): return symbol_by_name(cls, ALIASES)
def _mail_admins(self): return symbol_by_name('django.core.mail:mail_admins')
def __init__(self, name, id, async_start_result=None, **kwargs): kwargs.update({'id': id}) self._actor = symbol_by_name(name)(**kwargs) self.id = self._actor.id self.async_start_result = async_start_result
def mixin_template(template, conf): cls = symbol_by_name(templates[template]) conf.update( dict((k, v) for k, v in items(vars(cls)) if k.isupper() and not k.startswith('_')))