Beispiel #1
0
    def find_obj(self, model_class, uuid):
        """
        Find instance of ``model_class`` in local database or through single
        object discovery (if absent in local database).
        :param model_class: model class object
        :param uuid: object identifier
        :return: model class instance
        """
        model_qualname = utils.qualname(model_class)
        LOG.debug('Trying to find %s with ID %s in cloud %s', model_qualname,
                  uuid, self.cloud.name)
        if uuid is None:
            return None
        object_id = model.ObjectId(uuid, self.cloud.name)
        try:
            with model.Session() as session:
                if session.is_missing(model_class, object_id):
                    LOG.debug('Object %s with ID %s is stored as missing',
                              model_qualname, object_id)
                    return None
                return session.retrieve(model_class, object_id)
        except model.NotFound:
            LOG.debug('Object %s with ID %s not found in local database',
                      model_qualname, object_id)

        try:
            discoverer_class = self.cloud.discoverers.get(model_class)
            if discoverer_class is None:
                LOG.warning('Can\'t find discoverer class for %s',
                            model_qualname)
                raise DiscovererNotFound(model_class)
            LOG.debug('Trying to discover %s with ID %s using %s',
                      model_qualname, object_id,
                      utils.qualname(discoverer_class))
            discoverer = discoverer_class(self.config, self.cloud)
            return discoverer.discover_one(uuid)
        except NotFound:
            LOG.warning('Object %s with uuid %s not found in cloud %s',
                        model_class.get_class_qualname(), uuid,
                        self.cloud.name)
            with model.Session() as session:
                session.store_missing(model_class,
                                      model.ObjectId(uuid, self.cloud.name))
        except model.ValidationError as e:
            LOG.warning('Invalid %s with uuid %s in cloud %s: %s',
                        model_class.get_class_qualname(), uuid,
                        self.cloud.name, e)
            return None
Beispiel #2
0
    def find_obj(self, model_class, uuid):
        """
        Find instance of ``model_class`` in local database or through single
        object discovery (if absent in local database).
        :param model_class: model class object
        :param uuid: object identifier
        :return: model class instance
        """
        model_qualname = utils.qualname(model_class)
        LOG.debug('Trying to find %s with ID %s in cloud %s',
                  model_qualname, uuid, self.cloud.name)
        if uuid is None:
            return None
        object_id = model.ObjectId(uuid, self.cloud.name)
        try:
            with model.Session() as session:
                if session.is_missing(model_class, object_id):
                    LOG.debug('Object %s with ID %s is stored as missing',
                              model_qualname, object_id)
                    return None
                return session.retrieve(model_class, object_id)
        except model.NotFound:
            LOG.debug('Object %s with ID %s not found in local database',
                      model_qualname, object_id)

        try:
            discoverer_class = self.cloud.discoverers.get(model_class)
            if discoverer_class is None:
                LOG.warning('Can\'t find discoverer class for %s',
                            model_qualname)
                raise DiscovererNotFound(model_class)
            LOG.debug('Trying to discover %s with ID %s using %s',
                      model_qualname, object_id,
                      utils.qualname(discoverer_class))
            discoverer = discoverer_class(self.config, self.cloud)
            return discoverer.discover_one(uuid)
        except NotFound:
            LOG.warning('Object %s with uuid %s not found in cloud %s',
                        model_class.get_class_qualname(), uuid,
                        self.cloud.name)
            with model.Session() as session:
                session.store_missing(
                    model_class, model.ObjectId(uuid, self.cloud.name))
        except model.ValidationError as e:
            LOG.warning('Invalid %s with uuid %s in cloud %s: %s',
                        model_class.get_class_qualname(), uuid,
                        self.cloud.name, e)
            return None
Beispiel #3
0
    def list(self, cls, cloud=None):
        """
        Returns list of all objects of class ``cls`` stored in the database. If
        cloud argument is not None, then list is filtered by cloud.
        :param cls: model class
        :param cloud: config.Cloud instance or None
        :return: list of model instances
        """
        if cloud is None:
            cloud_name = None
            query = self._make_sql(cls, 'type', list=True)
        else:
            cloud_name = cloud.name
            query = self._make_sql(cls, 'type', 'cloud', list=True)
        result = []
        for obj in self.session.values():
            if isinstance(obj, cls) and \
                    (cloud is None or cloud_name == obj.primary_key.cloud):
                result.append(obj)

        schema = cls.get_schema()
        for row in self.tx.query(query,
                                 type=utils.qualname(cls),
                                 cloud=cloud_name):
            uuid, cloud_name = row[:2]
            key = (cls, ObjectId(uuid, cloud_name))
            if key in self.session or not row[2]:
                continue
            loaded, _ = schema.load(self._merge_obj(row[2:]))
            obj = cls.create(loaded, schema=schema, mark_dirty=False)
            self.session[key] = obj
            result.append(obj)
        return result
Beispiel #4
0
    def __str__(self):
        try:
            if len(self.args) > 1:
                if isinstance(self.args[0], basestring):
                    # We suspect that anything can happen here, like __repr__
                    # or __str__ raising arbitrary exceptions.
                    # We want to suppress them and deliver to the user as much
                    # original information as we can and don't clutter it with
                    # exception that happend due to conversion of exception to
                    # string.
                    return self.args[0] % self.args[1:]
            elif len(self.args) == 1:
                if isinstance(self.args[0], basestring):
                    return self.args[0]
            else:
                return 'ValidationError'
        except Exception:  # pylint: disable=broad-except
            pass

        # If we got here, then either exception was raised or first argument is
        # not a string
        args_repr = []
        for arg in self.args:
            try:
                args_repr.append(repr(arg))
            except Exception:  # pylint: disable=broad-except
                args_repr.append('<%s id:%d>' % (utils.qualname(type(arg)),
                                                 id(arg)))
        if len(args_repr) == 1:
            return '(' + args_repr[0] + ',)'
        else:
            return '(' + ', '.join(args_repr) + ')'
Beispiel #5
0
 def _store_none(self, cls, pk):
     uuid = pk.id
     cloud_name = pk.cloud
     type_name = utils.qualname(cls)
     self.tx.execute('INSERT OR REPLACE INTO objects '
                     'VALUES (:uuid, :cloud, :type_name, NULL)',
                     uuid=uuid, cloud=cloud_name, type_name=type_name)
Beispiel #6
0
def load_from_cloud(cfg, cloud, model_class, data):
    discoverer_class = cloud.discoverers.get(model_class)
    if discoverer_class is None:
        LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class))
        raise DiscovererNotFound(model_class)
    discoverer = discoverer_class(cfg, cloud)
    return discoverer.load_from_cloud(data)
Beispiel #7
0
    def __str__(self):
        try:
            if len(self.args) > 1:
                if isinstance(self.args[0], str):
                    # We suspect that anything can happen here, like __repr__
                    # or __str__ raising arbitrary exceptions.
                    # We want to suppress them and deliver to the user as much
                    # original information as we can and don't clutter it with
                    # exception that happend due to conversion of exception to
                    # string.
                    return self.args[0] % self.args[1:]
            elif len(self.args) == 1:
                if isinstance(self.args[0], str):
                    return self.args[0]
            else:
                return 'ValidationError'
        except Exception:  # pylint: disable=broad-except
            pass

        # If we got here, then either exception was raised or first argument is
        # not a string
        args_repr = []
        for arg in self.args:
            try:
                args_repr.append(repr(arg))
            except Exception:  # pylint: disable=broad-except
                args_repr.append('<%s id:%d>' %
                                 (utils.qualname(type(arg)), id(arg)))
        if len(args_repr) == 1:
            return '(' + args_repr[0] + ',)'
        else:
            return '(' + ', '.join(args_repr) + ')'
Beispiel #8
0
def load_from_cloud(cfg, cloud, model_class, data):
    discoverer_class = cloud.discoverers.get(model_class)
    if discoverer_class is None:
        LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class))
        raise DiscovererNotFound(model_class)
    discoverer = discoverer_class(cfg, cloud)
    return discoverer.load_from_cloud(data)
Beispiel #9
0
    def list(self, cls, cloud=None):
        """
        Returns list of all objects of class ``cls`` stored in the database. If
        cloud argument is not None, then list is filtered by cloud.
        :param cls: model class
        :param cloud: config.Cloud instance or None
        :return: list of model instances
        """
        if cloud is None:
            cloud_name = None
            query = self._make_sql(cls, 'type', list=True)
        else:
            cloud_name = cloud.name
            query = self._make_sql(cls, 'type', 'cloud', list=True)
        result = []
        for obj in self.session.values():
            if isinstance(obj, cls) and \
                    (cloud is None or cloud_name == obj.primary_key.cloud):
                result.append(obj)

        schema = cls.get_schema()
        for row in self.tx.query(query, type=utils.qualname(cls),
                                 cloud=cloud_name):
            uuid, cloud_name = row[:2]
            key = (cls, ObjectId(uuid, cloud_name))
            if key in self.session or not row[2]:
                continue
            loaded, _ = schema.load(self._merge_obj(row[2:]))
            obj = cls.create(loaded, schema=schema, mark_dirty=False)
            self.session[key] = obj
            result.append(obj)
        return result
Beispiel #10
0
 def _store_none(self, cls, pk):
     uuid = pk.id
     cloud_name = pk.cloud
     type_name = utils.qualname(cls)
     self.tx.execute('INSERT OR REPLACE INTO objects '
                     'VALUES (:uuid, :cloud, :type_name, NULL)',
                     uuid=uuid, cloud=cloud_name, type_name=type_name)
Beispiel #11
0
    def __init__(self,
                 config,
                 migration,
                 obj,
                 name_suffix=None,
                 requires=None,
                 **kwargs):
        name = '{0}_{1}'.format(utils.qualname(self.__class__),
                                taskflow_utils.object_name(obj))
        if name_suffix is not None:
            name += '_' + name_suffix
        if requires is None:
            requires = []
        else:
            requires = list(requires)
        requires.extend(reflection.get_callable_args(self.migrate))

        super(MigrationTask, self).__init__(name=name,
                                            requires=requires,
                                            **kwargs)
        self.src_cloud = config.clouds[migration.source]
        self.dst_cloud = config.clouds[migration.destination]
        self.config = config
        self.migration = migration
        self.created_object = None
Beispiel #12
0
 def __repr__(self):
     schema = self.get_schema()
     obj_fields = sorted(schema.fields.keys())
     cls = self.__class__
     return '<{cls} {fields}>'.format(
         cls=utils.qualname(cls),
         fields=' '.join('{0}:{1}'.format(f, getattr(self, f))
                         for f in obj_fields))
Beispiel #13
0
def discover_all(cfg, cloud):
    """
    Discovers all objects using discoverers specified for the cloud.
    :param cfg: config.Configuration instance
    :param cloud: config.Cloud instance
    """
    LOG.info('Start discovery process for cloud %s', cloud.name)
    for discoverer_class in cloud.discoverers.values():
        LOG.debug('Starting discovering %s using %s',
                  utils.qualname(discoverer_class.discovered_class),
                  utils.qualname(discoverer_class))
        discoverer = discoverer_class(cfg, cloud)
        discoverer.discover_all()
        LOG.debug('Finished discovering %s using %s',
                  utils.qualname(discoverer_class.discovered_class),
                  utils.qualname(discoverer_class))
    LOG.info('Finished discovery process for cloud %s', cloud.name)
Beispiel #14
0
 def __repr__(self):
     schema = self.get_schema()
     obj_fields = sorted(schema.fields.keys())
     cls = self.__class__
     return '<{cls} {fields}>'.format(
         cls=utils.qualname(cls),
         fields=' '.join('{0}:{1}'.format(f, getattr(self, f))
                         for f in obj_fields))
Beispiel #15
0
def discover_all(cfg, cloud):
    """
    Discovers all objects using discoverers specified for the cloud.
    :param cfg: config.Configuration instance
    :param cloud: config.Cloud instance
    """
    LOG.info('Start discovery process for cloud %s', cloud.name)
    for discoverer_class in cloud.discoverers.values():
        LOG.debug('Starting discovering %s using %s',
                  utils.qualname(discoverer_class.discovered_class),
                  utils.qualname(discoverer_class))
        discoverer = discoverer_class(cfg, cloud)
        discoverer.discover_all()
        LOG.debug('Finished discovering %s using %s',
                  utils.qualname(discoverer_class.discovered_class),
                  utils.qualname(discoverer_class))
    LOG.info('Finished discovery process for cloud %s', cloud.name)
Beispiel #16
0
 def execute(self, *args, **kwargs):
     LOG.info('Starting to migrate %s with id %s',
              utils.qualname(self.source_obj.get_class()),
              self.source_obj.primary_key)
     return [{
         'type': self.source_obj.get_class_qualname(),
         'object': self.source_obj.dump(),
     }]
Beispiel #17
0
 def to_dict(self, cls):
     """
     Serialize ObjectId to dictionary representation.
     """
     return {
         'id': self.id,
         'cloud': self.cloud,
         'type': utils.qualname(cls),
     }
Beispiel #18
0
 def migrate(self, source_obj, dst_object, *args, **kwargs):
     LOG.debug('Remebering migration: %s -> %s', source_obj, dst_object)
     with model.Session() as session:
         source_obj.link_to(dst_object)
         session.store(dst_object)
         session.store(source_obj)
     LOG.info('Finished migrating %s with id %s to %s',
              utils.qualname(source_obj.get_class()),
              source_obj.primary_key, dst_object.primary_key)
Beispiel #19
0
 def store_missing(self, cls, object_id):
     """
     Stores information that object is missing in cloud
     :param cls: model class object
     :param object_id: model.ObjectId instance
     """
     LOG.debug('Storing missing: %s %s', utils.qualname(cls), object_id)
     key = (cls, object_id)
     self.session[key] = None
Beispiel #20
0
 def store_missing(self, cls, object_id):
     """
     Stores information that object is missing in cloud
     :param cls: model class object
     :param object_id: model.ObjectId instance
     """
     LOG.debug('Storing missing: %s %s', utils.qualname(cls), object_id)
     key = (cls, object_id)
     self.session[key] = None
Beispiel #21
0
 def to_dict(self, cls):
     """
     Serialize ObjectId to dictionary representation.
     """
     return {
         'id': self.id,
         'cloud': self.cloud,
         'type': utils.qualname(cls),
     }
Beispiel #22
0
 def __init__(self, obj, config, migration, **kwargs):
     super(MigrationTask, self).__init__(
         name='{0}_{1}'.format(utils.qualname(self.__class__),
                               taskflow_utils.object_name(obj)),
         requires=reflection.get_callable_args(self.migrate),
         **kwargs)
     self.src_object = obj
     self.config = config
     self.migration = migration
     self.created_object = None
Beispiel #23
0
 def __init__(self, obj, config, migration, **kwargs):
     super(MigrationTask, self).__init__(
         name='{0}_{1}'.format(utils.qualname(self.__class__),
                               taskflow_utils.object_name(obj)),
         requires=reflection.get_callable_args(self.migrate),
         **kwargs)
     self.src_object = obj
     self.config = config
     self.migration = migration
     self.created_object = None
Beispiel #24
0
 def _update_row(self, obj, table):
     pk = obj.primary_key
     uuid = pk.id
     cloud_name = pk.cloud
     type_name = utils.qualname(obj.get_class())
     sql_statement = \
         'INSERT OR REPLACE INTO {table} ' \
         'VALUES (:uuid, :cloud, :type_name, :data)'.format(table=table)
     self.tx.execute(sql_statement,
                     uuid=uuid, cloud=cloud_name, type_name=type_name,
                     data=local_db.Json(obj.dump(table)))
     obj.clear_dirty(table)
     assert not obj.is_dirty(table)
Beispiel #25
0
def get_discoverer(cfg, cloud, model_class):
    """
    Returns discoverer for given model class and cloud
    :param cfg: cloudferry configuration object
    :param cloud: cloud configuration object
    :param model_class: model.Model derived class
    :return: discoverer object
    """
    discoverer_class = cloud.discoverers.get(model_class)
    if discoverer_class is None:
        LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class))
        raise DiscovererNotFound(model_class)
    return discoverer_class(cfg, cloud)
Beispiel #26
0
 def _update_row(self, obj, table):
     pk = obj.primary_key
     uuid = pk.id
     cloud_name = pk.cloud
     type_name = utils.qualname(obj.get_class())
     sql_statement = \
         'INSERT OR REPLACE INTO {table} ' \
         'VALUES (:uuid, :cloud, :type_name, :data)'.format(table=table)
     self.tx.execute(sql_statement,
                     uuid=uuid, cloud=cloud_name, type_name=type_name,
                     data=local_db.Json(obj.dump(table)))
     obj.clear_dirty(table)
     assert not obj.is_dirty(table)
Beispiel #27
0
 def exists(self, cls, object_id):
     """
     Returns True if object exists in database, False otherwise
     :param cls: model class
     :param object_id: model.ObjectId instance
     :return: True or False
     """
     key = (cls, object_id)
     if key in self.session:
         return self.session[key] is not None
     result = self.tx.query_one('SELECT EXISTS(SELECT 1 FROM objects '
                                'WHERE uuid=:uuid AND cloud=:cloud '
                                'AND type=:type LIMIT 1)',
                                uuid=object_id.id,
                                cloud=object_id.cloud,
                                type=utils.qualname(cls))
     return bool(result[0])
Beispiel #28
0
 def exists(self, cls, object_id):
     """
     Returns True if object exists in database, False otherwise
     :param cls: model class
     :param object_id: model.ObjectId instance
     :return: True or False
     """
     key = (cls, object_id)
     if key in self.session:
         return self.session[key] is not None
     result = self.tx.query_one('SELECT EXISTS(SELECT 1 FROM objects '
                                'WHERE uuid=:uuid AND cloud=:cloud '
                                'AND type=:type LIMIT 1)',
                                uuid=object_id.id,
                                cloud=object_id.cloud,
                                type=utils.qualname(cls))
     return bool(result[0])
Beispiel #29
0
 def is_missing(self, cls, object_id):
     """
     Check if object couldn't be found in cloud (e.g. was deleted)
     :param cls: model class
     :param object_id: model.ObjectId instance
     :return: True or False
     """
     key = (cls, object_id)
     if key in self.session:
         return self.session[key] is None
     result = self.tx.query_one('SELECT json FROM objects WHERE uuid=:uuid '
                                'AND cloud=:cloud AND type=:type_name',
                                uuid=object_id.id,
                                cloud=object_id.cloud,
                                type_name=utils.qualname(cls))
     if not result:
         raise NotFound(cls, object_id)
     return result[0] is None
Beispiel #30
0
 def is_missing(self, cls, object_id):
     """
     Check if object couldn't be found in cloud (e.g. was deleted)
     :param cls: model class
     :param object_id: model.ObjectId instance
     :return: True or False
     """
     key = (cls, object_id)
     if key in self.session:
         return self.session[key] is None
     result = self.tx.query_one('SELECT json FROM objects WHERE uuid=:uuid '
                                'AND cloud=:cloud AND type=:type_name',
                                uuid=object_id.id,
                                cloud=object_id.cloud,
                                type_name=utils.qualname(cls))
     if not result:
         raise NotFound(cls, object_id)
     return result[0] is None
Beispiel #31
0
 def _delete_rows(self, cls, cloud_name, object_id, table):
     predicates = []
     kwargs = {}
     if cls is not None:
         predicates.append('type=:type_name')
         kwargs['type_name'] = utils.qualname(cls)
     if object_id is not None:
         predicates.append('uuid=:uuid')
         kwargs['uuid'] = object_id.id
         if cloud_name is None:
             cloud_name = object_id.cloud
         else:
             assert cloud_name == object_id.cloud
     if cloud_name is not None:
         predicates.append('cloud=:cloud')
         kwargs['cloud'] = cloud_name
     statement = 'DELETE FROM {table} WHERE '.format(table=table)
     if predicates:
         statement += ' AND '.join(predicates)
     else:
         statement += ' 1'
     self.tx.execute(statement, **kwargs)
Beispiel #32
0
 def _delete_rows(self, cls, cloud_name, object_id, table):
     predicates = []
     kwargs = {}
     if cls is not None:
         predicates.append('type=:type_name')
         kwargs['type_name'] = utils.qualname(cls)
     if object_id is not None:
         predicates.append('uuid=:uuid')
         kwargs['uuid'] = object_id.id
         if cloud_name is None:
             cloud_name = object_id.cloud
         else:
             assert cloud_name == object_id.cloud
     if cloud_name is not None:
         predicates.append('cloud=:cloud')
         kwargs['cloud'] = cloud_name
     statement = 'DELETE FROM {table} WHERE '.format(table=table)
     if predicates:
         statement += ' AND '.join(predicates)
     else:
         statement += ' 1'
     self.tx.execute(statement, **kwargs)
Beispiel #33
0
    def retrieve(self, cls, object_id):
        """
        Loads object from database using class and object_id. If no such object
        were found, this method will throw ``model.NotFound`` exception.
        :param cls: model class
        :param object_id: model.ObjectId instance
        :return: model instance
        """
        key = (cls, object_id)
        if key in self.session:
            return self.session[key]

        query = self._make_sql(cls, 'uuid', 'cloud', 'type')
        result = self.tx.query_one(query, uuid=object_id.id,
                                   cloud=object_id.cloud,
                                   type=utils.qualname(cls))
        if not result or not result[0]:
            raise NotFound(cls, object_id)
        schema = cls.get_schema()
        loaded, _ = schema.load(self._merge_obj(result))
        obj = cls.create(loaded, schema=schema, mark_dirty=False)
        self.session[key] = obj
        return obj
Beispiel #34
0
    def retrieve(self, cls, object_id):
        """
        Loads object from database using class and object_id. If no such object
        were found, this method will throw ``model.NotFound`` exception.
        :param cls: model class
        :param object_id: model.ObjectId instance
        :return: model instance
        """
        key = (cls, object_id)
        if key in self.session:
            return self.session[key]

        query = self._make_sql(cls, 'uuid', 'cloud', 'type')
        result = self.tx.query_one(query, uuid=object_id.id,
                                   cloud=object_id.cloud,
                                   type=utils.qualname(cls))
        if not result or not result[0]:
            raise NotFound(cls, object_id)
        schema = cls.get_schema()
        loaded, _ = schema.load(self._merge_obj(result))
        obj = cls.create(loaded, schema=schema, mark_dirty=False)
        self.session[key] = obj
        return obj
Beispiel #35
0
 def __init__(self, discoverer_class, *args):
     super(DiscovererNotFound,
           self).__init__('Discoverer for % not found.',
                          utils.qualname(discoverer_class), *args)
Beispiel #36
0
 def __init__(self, discoverer_class, *args):
     super(DiscovererNotFound, self).__init__(
         'Discoverer for % not found.',
         utils.qualname(discoverer_class), *args)
Beispiel #37
0
 def __str__(self):
     return '{0} object with id {1} not found.'.format(
         utils.qualname(self.cls), self.object_id)
Beispiel #38
0
 def __str__(self):
     return '{0} object with id {1} not found.'.format(
         utils.qualname(self.cls), self.object_id)
Beispiel #39
0
 def get_class_qualname(self):
     """
     Return fully qualified name of class (with module name, etc...)
     """
     return utils.qualname(self._model)
Beispiel #40
0
 def get_class_qualname():
     return utils.qualname(Destructor)
Beispiel #41
0
 def get_class_qualname(self):
     """
     Return fully qualified name of class (with module name, etc...)
     """
     return utils.qualname(self._model)
Beispiel #42
0
 def revert(self, *args, **kwargs):
     LOG.error('Failed to migrate %s with id %s',
               utils.qualname(self.source_obj.get_class()),
               self.source_obj.primary_key)