def find_obj(self, model_class, uuid): """ Find instance of ``model_class`` in local database or through single object discovery (if absent in local database). :param model_class: model class object :param uuid: object identifier :return: model class instance """ model_qualname = utils.qualname(model_class) LOG.debug('Trying to find %s with ID %s in cloud %s', model_qualname, uuid, self.cloud.name) if uuid is None: return None object_id = model.ObjectId(uuid, self.cloud.name) try: with model.Session() as session: if session.is_missing(model_class, object_id): LOG.debug('Object %s with ID %s is stored as missing', model_qualname, object_id) return None return session.retrieve(model_class, object_id) except model.NotFound: LOG.debug('Object %s with ID %s not found in local database', model_qualname, object_id) try: discoverer_class = self.cloud.discoverers.get(model_class) if discoverer_class is None: LOG.warning('Can\'t find discoverer class for %s', model_qualname) raise DiscovererNotFound(model_class) LOG.debug('Trying to discover %s with ID %s using %s', model_qualname, object_id, utils.qualname(discoverer_class)) discoverer = discoverer_class(self.config, self.cloud) return discoverer.discover_one(uuid) except NotFound: LOG.warning('Object %s with uuid %s not found in cloud %s', model_class.get_class_qualname(), uuid, self.cloud.name) with model.Session() as session: session.store_missing(model_class, model.ObjectId(uuid, self.cloud.name)) except model.ValidationError as e: LOG.warning('Invalid %s with uuid %s in cloud %s: %s', model_class.get_class_qualname(), uuid, self.cloud.name, e) return None
def find_obj(self, model_class, uuid): """ Find instance of ``model_class`` in local database or through single object discovery (if absent in local database). :param model_class: model class object :param uuid: object identifier :return: model class instance """ model_qualname = utils.qualname(model_class) LOG.debug('Trying to find %s with ID %s in cloud %s', model_qualname, uuid, self.cloud.name) if uuid is None: return None object_id = model.ObjectId(uuid, self.cloud.name) try: with model.Session() as session: if session.is_missing(model_class, object_id): LOG.debug('Object %s with ID %s is stored as missing', model_qualname, object_id) return None return session.retrieve(model_class, object_id) except model.NotFound: LOG.debug('Object %s with ID %s not found in local database', model_qualname, object_id) try: discoverer_class = self.cloud.discoverers.get(model_class) if discoverer_class is None: LOG.warning('Can\'t find discoverer class for %s', model_qualname) raise DiscovererNotFound(model_class) LOG.debug('Trying to discover %s with ID %s using %s', model_qualname, object_id, utils.qualname(discoverer_class)) discoverer = discoverer_class(self.config, self.cloud) return discoverer.discover_one(uuid) except NotFound: LOG.warning('Object %s with uuid %s not found in cloud %s', model_class.get_class_qualname(), uuid, self.cloud.name) with model.Session() as session: session.store_missing( model_class, model.ObjectId(uuid, self.cloud.name)) except model.ValidationError as e: LOG.warning('Invalid %s with uuid %s in cloud %s: %s', model_class.get_class_qualname(), uuid, self.cloud.name, e) return None
def list(self, cls, cloud=None): """ Returns list of all objects of class ``cls`` stored in the database. If cloud argument is not None, then list is filtered by cloud. :param cls: model class :param cloud: config.Cloud instance or None :return: list of model instances """ if cloud is None: cloud_name = None query = self._make_sql(cls, 'type', list=True) else: cloud_name = cloud.name query = self._make_sql(cls, 'type', 'cloud', list=True) result = [] for obj in self.session.values(): if isinstance(obj, cls) and \ (cloud is None or cloud_name == obj.primary_key.cloud): result.append(obj) schema = cls.get_schema() for row in self.tx.query(query, type=utils.qualname(cls), cloud=cloud_name): uuid, cloud_name = row[:2] key = (cls, ObjectId(uuid, cloud_name)) if key in self.session or not row[2]: continue loaded, _ = schema.load(self._merge_obj(row[2:])) obj = cls.create(loaded, schema=schema, mark_dirty=False) self.session[key] = obj result.append(obj) return result
def __str__(self): try: if len(self.args) > 1: if isinstance(self.args[0], basestring): # We suspect that anything can happen here, like __repr__ # or __str__ raising arbitrary exceptions. # We want to suppress them and deliver to the user as much # original information as we can and don't clutter it with # exception that happend due to conversion of exception to # string. return self.args[0] % self.args[1:] elif len(self.args) == 1: if isinstance(self.args[0], basestring): return self.args[0] else: return 'ValidationError' except Exception: # pylint: disable=broad-except pass # If we got here, then either exception was raised or first argument is # not a string args_repr = [] for arg in self.args: try: args_repr.append(repr(arg)) except Exception: # pylint: disable=broad-except args_repr.append('<%s id:%d>' % (utils.qualname(type(arg)), id(arg))) if len(args_repr) == 1: return '(' + args_repr[0] + ',)' else: return '(' + ', '.join(args_repr) + ')'
def _store_none(self, cls, pk): uuid = pk.id cloud_name = pk.cloud type_name = utils.qualname(cls) self.tx.execute('INSERT OR REPLACE INTO objects ' 'VALUES (:uuid, :cloud, :type_name, NULL)', uuid=uuid, cloud=cloud_name, type_name=type_name)
def load_from_cloud(cfg, cloud, model_class, data): discoverer_class = cloud.discoverers.get(model_class) if discoverer_class is None: LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class)) raise DiscovererNotFound(model_class) discoverer = discoverer_class(cfg, cloud) return discoverer.load_from_cloud(data)
def __str__(self): try: if len(self.args) > 1: if isinstance(self.args[0], str): # We suspect that anything can happen here, like __repr__ # or __str__ raising arbitrary exceptions. # We want to suppress them and deliver to the user as much # original information as we can and don't clutter it with # exception that happend due to conversion of exception to # string. return self.args[0] % self.args[1:] elif len(self.args) == 1: if isinstance(self.args[0], str): return self.args[0] else: return 'ValidationError' except Exception: # pylint: disable=broad-except pass # If we got here, then either exception was raised or first argument is # not a string args_repr = [] for arg in self.args: try: args_repr.append(repr(arg)) except Exception: # pylint: disable=broad-except args_repr.append('<%s id:%d>' % (utils.qualname(type(arg)), id(arg))) if len(args_repr) == 1: return '(' + args_repr[0] + ',)' else: return '(' + ', '.join(args_repr) + ')'
def __init__(self, config, migration, obj, name_suffix=None, requires=None, **kwargs): name = '{0}_{1}'.format(utils.qualname(self.__class__), taskflow_utils.object_name(obj)) if name_suffix is not None: name += '_' + name_suffix if requires is None: requires = [] else: requires = list(requires) requires.extend(reflection.get_callable_args(self.migrate)) super(MigrationTask, self).__init__(name=name, requires=requires, **kwargs) self.src_cloud = config.clouds[migration.source] self.dst_cloud = config.clouds[migration.destination] self.config = config self.migration = migration self.created_object = None
def __repr__(self): schema = self.get_schema() obj_fields = sorted(schema.fields.keys()) cls = self.__class__ return '<{cls} {fields}>'.format( cls=utils.qualname(cls), fields=' '.join('{0}:{1}'.format(f, getattr(self, f)) for f in obj_fields))
def discover_all(cfg, cloud): """ Discovers all objects using discoverers specified for the cloud. :param cfg: config.Configuration instance :param cloud: config.Cloud instance """ LOG.info('Start discovery process for cloud %s', cloud.name) for discoverer_class in cloud.discoverers.values(): LOG.debug('Starting discovering %s using %s', utils.qualname(discoverer_class.discovered_class), utils.qualname(discoverer_class)) discoverer = discoverer_class(cfg, cloud) discoverer.discover_all() LOG.debug('Finished discovering %s using %s', utils.qualname(discoverer_class.discovered_class), utils.qualname(discoverer_class)) LOG.info('Finished discovery process for cloud %s', cloud.name)
def execute(self, *args, **kwargs): LOG.info('Starting to migrate %s with id %s', utils.qualname(self.source_obj.get_class()), self.source_obj.primary_key) return [{ 'type': self.source_obj.get_class_qualname(), 'object': self.source_obj.dump(), }]
def to_dict(self, cls): """ Serialize ObjectId to dictionary representation. """ return { 'id': self.id, 'cloud': self.cloud, 'type': utils.qualname(cls), }
def migrate(self, source_obj, dst_object, *args, **kwargs): LOG.debug('Remebering migration: %s -> %s', source_obj, dst_object) with model.Session() as session: source_obj.link_to(dst_object) session.store(dst_object) session.store(source_obj) LOG.info('Finished migrating %s with id %s to %s', utils.qualname(source_obj.get_class()), source_obj.primary_key, dst_object.primary_key)
def store_missing(self, cls, object_id): """ Stores information that object is missing in cloud :param cls: model class object :param object_id: model.ObjectId instance """ LOG.debug('Storing missing: %s %s', utils.qualname(cls), object_id) key = (cls, object_id) self.session[key] = None
def __init__(self, obj, config, migration, **kwargs): super(MigrationTask, self).__init__( name='{0}_{1}'.format(utils.qualname(self.__class__), taskflow_utils.object_name(obj)), requires=reflection.get_callable_args(self.migrate), **kwargs) self.src_object = obj self.config = config self.migration = migration self.created_object = None
def _update_row(self, obj, table): pk = obj.primary_key uuid = pk.id cloud_name = pk.cloud type_name = utils.qualname(obj.get_class()) sql_statement = \ 'INSERT OR REPLACE INTO {table} ' \ 'VALUES (:uuid, :cloud, :type_name, :data)'.format(table=table) self.tx.execute(sql_statement, uuid=uuid, cloud=cloud_name, type_name=type_name, data=local_db.Json(obj.dump(table))) obj.clear_dirty(table) assert not obj.is_dirty(table)
def get_discoverer(cfg, cloud, model_class): """ Returns discoverer for given model class and cloud :param cfg: cloudferry configuration object :param cloud: cloud configuration object :param model_class: model.Model derived class :return: discoverer object """ discoverer_class = cloud.discoverers.get(model_class) if discoverer_class is None: LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class)) raise DiscovererNotFound(model_class) return discoverer_class(cfg, cloud)
def exists(self, cls, object_id): """ Returns True if object exists in database, False otherwise :param cls: model class :param object_id: model.ObjectId instance :return: True or False """ key = (cls, object_id) if key in self.session: return self.session[key] is not None result = self.tx.query_one('SELECT EXISTS(SELECT 1 FROM objects ' 'WHERE uuid=:uuid AND cloud=:cloud ' 'AND type=:type LIMIT 1)', uuid=object_id.id, cloud=object_id.cloud, type=utils.qualname(cls)) return bool(result[0])
def is_missing(self, cls, object_id): """ Check if object couldn't be found in cloud (e.g. was deleted) :param cls: model class :param object_id: model.ObjectId instance :return: True or False """ key = (cls, object_id) if key in self.session: return self.session[key] is None result = self.tx.query_one('SELECT json FROM objects WHERE uuid=:uuid ' 'AND cloud=:cloud AND type=:type_name', uuid=object_id.id, cloud=object_id.cloud, type_name=utils.qualname(cls)) if not result: raise NotFound(cls, object_id) return result[0] is None
def _delete_rows(self, cls, cloud_name, object_id, table): predicates = [] kwargs = {} if cls is not None: predicates.append('type=:type_name') kwargs['type_name'] = utils.qualname(cls) if object_id is not None: predicates.append('uuid=:uuid') kwargs['uuid'] = object_id.id if cloud_name is None: cloud_name = object_id.cloud else: assert cloud_name == object_id.cloud if cloud_name is not None: predicates.append('cloud=:cloud') kwargs['cloud'] = cloud_name statement = 'DELETE FROM {table} WHERE '.format(table=table) if predicates: statement += ' AND '.join(predicates) else: statement += ' 1' self.tx.execute(statement, **kwargs)
def retrieve(self, cls, object_id): """ Loads object from database using class and object_id. If no such object were found, this method will throw ``model.NotFound`` exception. :param cls: model class :param object_id: model.ObjectId instance :return: model instance """ key = (cls, object_id) if key in self.session: return self.session[key] query = self._make_sql(cls, 'uuid', 'cloud', 'type') result = self.tx.query_one(query, uuid=object_id.id, cloud=object_id.cloud, type=utils.qualname(cls)) if not result or not result[0]: raise NotFound(cls, object_id) schema = cls.get_schema() loaded, _ = schema.load(self._merge_obj(result)) obj = cls.create(loaded, schema=schema, mark_dirty=False) self.session[key] = obj return obj
def __init__(self, discoverer_class, *args): super(DiscovererNotFound, self).__init__('Discoverer for % not found.', utils.qualname(discoverer_class), *args)
def __init__(self, discoverer_class, *args): super(DiscovererNotFound, self).__init__( 'Discoverer for % not found.', utils.qualname(discoverer_class), *args)
def __str__(self): return '{0} object with id {1} not found.'.format( utils.qualname(self.cls), self.object_id)
def get_class_qualname(self): """ Return fully qualified name of class (with module name, etc...) """ return utils.qualname(self._model)
def get_class_qualname(): return utils.qualname(Destructor)
def revert(self, *args, **kwargs): LOG.error('Failed to migrate %s with id %s', utils.qualname(self.source_obj.get_class()), self.source_obj.primary_key)