def find_obj(self, model_class, uuid):
"""
Find instance of ``model_class`` in local database or through single
object discovery (if absent in local database).
:param model_class: model class object
:param uuid: object identifier
:return: model class instance
"""
model_qualname = utils.qualname(model_class)
LOG.debug('Trying to find %s with ID %s in cloud %s', model_qualname,
uuid, self.cloud.name)
if uuid is None:
return None
object_id = model.ObjectId(uuid, self.cloud.name)
try:
with model.Session() as session:
if session.is_missing(model_class, object_id):
LOG.debug('Object %s with ID %s is stored as missing',
model_qualname, object_id)
return None
return session.retrieve(model_class, object_id)
except model.NotFound:
LOG.debug('Object %s with ID %s not found in local database',
model_qualname, object_id)
try:
discoverer_class = self.cloud.discoverers.get(model_class)
if discoverer_class is None:
LOG.warning('Can\'t find discoverer class for %s',
model_qualname)
raise DiscovererNotFound(model_class)
LOG.debug('Trying to discover %s with ID %s using %s',
model_qualname, object_id,
utils.qualname(discoverer_class))
discoverer = discoverer_class(self.config, self.cloud)
return discoverer.discover_one(uuid)
except NotFound:
LOG.warning('Object %s with uuid %s not found in cloud %s',
model_class.get_class_qualname(), uuid,
self.cloud.name)
with model.Session() as session:
session.store_missing(model_class,
model.ObjectId(uuid, self.cloud.name))
except model.ValidationError as e:
LOG.warning('Invalid %s with uuid %s in cloud %s: %s',
model_class.get_class_qualname(), uuid,
self.cloud.name, e)
return None
def find_obj(self, model_class, uuid):
"""
Find instance of ``model_class`` in local database or through single
object discovery (if absent in local database).
:param model_class: model class object
:param uuid: object identifier
:return: model class instance
"""
model_qualname = utils.qualname(model_class)
LOG.debug('Trying to find %s with ID %s in cloud %s',
model_qualname, uuid, self.cloud.name)
if uuid is None:
return None
object_id = model.ObjectId(uuid, self.cloud.name)
try:
with model.Session() as session:
if session.is_missing(model_class, object_id):
LOG.debug('Object %s with ID %s is stored as missing',
model_qualname, object_id)
return None
return session.retrieve(model_class, object_id)
except model.NotFound:
LOG.debug('Object %s with ID %s not found in local database',
model_qualname, object_id)
try:
discoverer_class = self.cloud.discoverers.get(model_class)
if discoverer_class is None:
LOG.warning('Can\'t find discoverer class for %s',
model_qualname)
raise DiscovererNotFound(model_class)
LOG.debug('Trying to discover %s with ID %s using %s',
model_qualname, object_id,
utils.qualname(discoverer_class))
discoverer = discoverer_class(self.config, self.cloud)
return discoverer.discover_one(uuid)
except NotFound:
LOG.warning('Object %s with uuid %s not found in cloud %s',
model_class.get_class_qualname(), uuid,
self.cloud.name)
with model.Session() as session:
session.store_missing(
model_class, model.ObjectId(uuid, self.cloud.name))
except model.ValidationError as e:
LOG.warning('Invalid %s with uuid %s in cloud %s: %s',
model_class.get_class_qualname(), uuid,
self.cloud.name, e)
return None
def list(self, cls, cloud=None):
"""
Returns list of all objects of class ``cls`` stored in the database. If
cloud argument is not None, then list is filtered by cloud.
:param cls: model class
:param cloud: config.Cloud instance or None
:return: list of model instances
"""
if cloud is None:
cloud_name = None
query = self._make_sql(cls, 'type', list=True)
else:
cloud_name = cloud.name
query = self._make_sql(cls, 'type', 'cloud', list=True)
result = []
for obj in self.session.values():
if isinstance(obj, cls) and \
(cloud is None or cloud_name == obj.primary_key.cloud):
result.append(obj)
schema = cls.get_schema()
for row in self.tx.query(query,
type=utils.qualname(cls),
cloud=cloud_name):
uuid, cloud_name = row[:2]
key = (cls, ObjectId(uuid, cloud_name))
if key in self.session or not row[2]:
continue
loaded, _ = schema.load(self._merge_obj(row[2:]))
obj = cls.create(loaded, schema=schema, mark_dirty=False)
self.session[key] = obj
result.append(obj)
return result
def __str__(self):
try:
if len(self.args) > 1:
if isinstance(self.args[0], basestring):
# We suspect that anything can happen here, like __repr__
# or __str__ raising arbitrary exceptions.
# We want to suppress them and deliver to the user as much
# original information as we can and don't clutter it with
# exception that happend due to conversion of exception to
# string.
return self.args[0] % self.args[1:]
elif len(self.args) == 1:
if isinstance(self.args[0], basestring):
return self.args[0]
else:
return 'ValidationError'
except Exception: # pylint: disable=broad-except
pass
# If we got here, then either exception was raised or first argument is
# not a string
args_repr = []
for arg in self.args:
try:
args_repr.append(repr(arg))
except Exception: # pylint: disable=broad-except
args_repr.append('<%s id:%d>' % (utils.qualname(type(arg)),
id(arg)))
if len(args_repr) == 1:
return '(' + args_repr[0] + ',)'
else:
return '(' + ', '.join(args_repr) + ')'
def _store_none(self, cls, pk):
uuid = pk.id
cloud_name = pk.cloud
type_name = utils.qualname(cls)
self.tx.execute('INSERT OR REPLACE INTO objects '
'VALUES (:uuid, :cloud, :type_name, NULL)',
uuid=uuid, cloud=cloud_name, type_name=type_name)
def load_from_cloud(cfg, cloud, model_class, data):
discoverer_class = cloud.discoverers.get(model_class)
if discoverer_class is None:
LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class))
raise DiscovererNotFound(model_class)
discoverer = discoverer_class(cfg, cloud)
return discoverer.load_from_cloud(data)
def __str__(self):
try:
if len(self.args) > 1:
if isinstance(self.args[0], str):
# We suspect that anything can happen here, like __repr__
# or __str__ raising arbitrary exceptions.
# We want to suppress them and deliver to the user as much
# original information as we can and don't clutter it with
# exception that happend due to conversion of exception to
# string.
return self.args[0] % self.args[1:]
elif len(self.args) == 1:
if isinstance(self.args[0], str):
return self.args[0]
else:
return 'ValidationError'
except Exception: # pylint: disable=broad-except
pass
# If we got here, then either exception was raised or first argument is
# not a string
args_repr = []
for arg in self.args:
try:
args_repr.append(repr(arg))
except Exception: # pylint: disable=broad-except
args_repr.append('<%s id:%d>' %
(utils.qualname(type(arg)), id(arg)))
if len(args_repr) == 1:
return '(' + args_repr[0] + ',)'
else:
return '(' + ', '.join(args_repr) + ')'
def load_from_cloud(cfg, cloud, model_class, data):
discoverer_class = cloud.discoverers.get(model_class)
if discoverer_class is None:
LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class))
raise DiscovererNotFound(model_class)
discoverer = discoverer_class(cfg, cloud)
return discoverer.load_from_cloud(data)
def list(self, cls, cloud=None):
"""
Returns list of all objects of class ``cls`` stored in the database. If
cloud argument is not None, then list is filtered by cloud.
:param cls: model class
:param cloud: config.Cloud instance or None
:return: list of model instances
"""
if cloud is None:
cloud_name = None
query = self._make_sql(cls, 'type', list=True)
else:
cloud_name = cloud.name
query = self._make_sql(cls, 'type', 'cloud', list=True)
result = []
for obj in self.session.values():
if isinstance(obj, cls) and \
(cloud is None or cloud_name == obj.primary_key.cloud):
result.append(obj)
schema = cls.get_schema()
for row in self.tx.query(query, type=utils.qualname(cls),
cloud=cloud_name):
uuid, cloud_name = row[:2]
key = (cls, ObjectId(uuid, cloud_name))
if key in self.session or not row[2]:
continue
loaded, _ = schema.load(self._merge_obj(row[2:]))
obj = cls.create(loaded, schema=schema, mark_dirty=False)
self.session[key] = obj
result.append(obj)
return result
def _store_none(self, cls, pk):
uuid = pk.id
cloud_name = pk.cloud
type_name = utils.qualname(cls)
self.tx.execute('INSERT OR REPLACE INTO objects '
'VALUES (:uuid, :cloud, :type_name, NULL)',
uuid=uuid, cloud=cloud_name, type_name=type_name)
def __init__(self,
config,
migration,
obj,
name_suffix=None,
requires=None,
**kwargs):
name = '{0}_{1}'.format(utils.qualname(self.__class__),
taskflow_utils.object_name(obj))
if name_suffix is not None:
name += '_' + name_suffix
if requires is None:
requires = []
else:
requires = list(requires)
requires.extend(reflection.get_callable_args(self.migrate))
super(MigrationTask, self).__init__(name=name,
requires=requires,
**kwargs)
self.src_cloud = config.clouds[migration.source]
self.dst_cloud = config.clouds[migration.destination]
self.config = config
self.migration = migration
self.created_object = None
def __repr__(self):
schema = self.get_schema()
obj_fields = sorted(schema.fields.keys())
cls = self.__class__
return '<{cls} {fields}>'.format(
cls=utils.qualname(cls),
fields=' '.join('{0}:{1}'.format(f, getattr(self, f))
for f in obj_fields))
def discover_all(cfg, cloud):
"""
Discovers all objects using discoverers specified for the cloud.
:param cfg: config.Configuration instance
:param cloud: config.Cloud instance
"""
LOG.info('Start discovery process for cloud %s', cloud.name)
for discoverer_class in cloud.discoverers.values():
LOG.debug('Starting discovering %s using %s',
utils.qualname(discoverer_class.discovered_class),
utils.qualname(discoverer_class))
discoverer = discoverer_class(cfg, cloud)
discoverer.discover_all()
LOG.debug('Finished discovering %s using %s',
utils.qualname(discoverer_class.discovered_class),
utils.qualname(discoverer_class))
LOG.info('Finished discovery process for cloud %s', cloud.name)
def __repr__(self):
schema = self.get_schema()
obj_fields = sorted(schema.fields.keys())
cls = self.__class__
return '<{cls} {fields}>'.format(
cls=utils.qualname(cls),
fields=' '.join('{0}:{1}'.format(f, getattr(self, f))
for f in obj_fields))
def discover_all(cfg, cloud):
"""
Discovers all objects using discoverers specified for the cloud.
:param cfg: config.Configuration instance
:param cloud: config.Cloud instance
"""
LOG.info('Start discovery process for cloud %s', cloud.name)
for discoverer_class in cloud.discoverers.values():
LOG.debug('Starting discovering %s using %s',
utils.qualname(discoverer_class.discovered_class),
utils.qualname(discoverer_class))
discoverer = discoverer_class(cfg, cloud)
discoverer.discover_all()
LOG.debug('Finished discovering %s using %s',
utils.qualname(discoverer_class.discovered_class),
utils.qualname(discoverer_class))
LOG.info('Finished discovery process for cloud %s', cloud.name)
def execute(self, *args, **kwargs):
LOG.info('Starting to migrate %s with id %s',
utils.qualname(self.source_obj.get_class()),
self.source_obj.primary_key)
return [{
'type': self.source_obj.get_class_qualname(),
'object': self.source_obj.dump(),
}]
def to_dict(self, cls):
"""
Serialize ObjectId to dictionary representation.
"""
return {
'id': self.id,
'cloud': self.cloud,
'type': utils.qualname(cls),
}
def migrate(self, source_obj, dst_object, *args, **kwargs):
LOG.debug('Remebering migration: %s -> %s', source_obj, dst_object)
with model.Session() as session:
source_obj.link_to(dst_object)
session.store(dst_object)
session.store(source_obj)
LOG.info('Finished migrating %s with id %s to %s',
utils.qualname(source_obj.get_class()),
source_obj.primary_key, dst_object.primary_key)
def store_missing(self, cls, object_id):
"""
Stores information that object is missing in cloud
:param cls: model class object
:param object_id: model.ObjectId instance
"""
LOG.debug('Storing missing: %s %s', utils.qualname(cls), object_id)
key = (cls, object_id)
self.session[key] = None
def store_missing(self, cls, object_id):
"""
Stores information that object is missing in cloud
:param cls: model class object
:param object_id: model.ObjectId instance
"""
LOG.debug('Storing missing: %s %s', utils.qualname(cls), object_id)
key = (cls, object_id)
self.session[key] = None
def to_dict(self, cls):
"""
Serialize ObjectId to dictionary representation.
"""
return {
'id': self.id,
'cloud': self.cloud,
'type': utils.qualname(cls),
}
def __init__(self, obj, config, migration, **kwargs):
super(MigrationTask, self).__init__(
name='{0}_{1}'.format(utils.qualname(self.__class__),
taskflow_utils.object_name(obj)),
requires=reflection.get_callable_args(self.migrate),
**kwargs)
self.src_object = obj
self.config = config
self.migration = migration
self.created_object = None
def __init__(self, obj, config, migration, **kwargs):
super(MigrationTask, self).__init__(
name='{0}_{1}'.format(utils.qualname(self.__class__),
taskflow_utils.object_name(obj)),
requires=reflection.get_callable_args(self.migrate),
**kwargs)
self.src_object = obj
self.config = config
self.migration = migration
self.created_object = None
def _update_row(self, obj, table):
pk = obj.primary_key
uuid = pk.id
cloud_name = pk.cloud
type_name = utils.qualname(obj.get_class())
sql_statement = \
'INSERT OR REPLACE INTO {table} ' \
'VALUES (:uuid, :cloud, :type_name, :data)'.format(table=table)
self.tx.execute(sql_statement,
uuid=uuid, cloud=cloud_name, type_name=type_name,
data=local_db.Json(obj.dump(table)))
obj.clear_dirty(table)
assert not obj.is_dirty(table)
def get_discoverer(cfg, cloud, model_class):
"""
Returns discoverer for given model class and cloud
:param cfg: cloudferry configuration object
:param cloud: cloud configuration object
:param model_class: model.Model derived class
:return: discoverer object
"""
discoverer_class = cloud.discoverers.get(model_class)
if discoverer_class is None:
LOG.error('Can\'t find discoverer for %s', utils.qualname(model_class))
raise DiscovererNotFound(model_class)
return discoverer_class(cfg, cloud)
def _update_row(self, obj, table):
pk = obj.primary_key
uuid = pk.id
cloud_name = pk.cloud
type_name = utils.qualname(obj.get_class())
sql_statement = \
'INSERT OR REPLACE INTO {table} ' \
'VALUES (:uuid, :cloud, :type_name, :data)'.format(table=table)
self.tx.execute(sql_statement,
uuid=uuid, cloud=cloud_name, type_name=type_name,
data=local_db.Json(obj.dump(table)))
obj.clear_dirty(table)
assert not obj.is_dirty(table)
def exists(self, cls, object_id):
"""
Returns True if object exists in database, False otherwise
:param cls: model class
:param object_id: model.ObjectId instance
:return: True or False
"""
key = (cls, object_id)
if key in self.session:
return self.session[key] is not None
result = self.tx.query_one('SELECT EXISTS(SELECT 1 FROM objects '
'WHERE uuid=:uuid AND cloud=:cloud '
'AND type=:type LIMIT 1)',
uuid=object_id.id,
cloud=object_id.cloud,
type=utils.qualname(cls))
return bool(result[0])
def exists(self, cls, object_id):
"""
Returns True if object exists in database, False otherwise
:param cls: model class
:param object_id: model.ObjectId instance
:return: True or False
"""
key = (cls, object_id)
if key in self.session:
return self.session[key] is not None
result = self.tx.query_one('SELECT EXISTS(SELECT 1 FROM objects '
'WHERE uuid=:uuid AND cloud=:cloud '
'AND type=:type LIMIT 1)',
uuid=object_id.id,
cloud=object_id.cloud,
type=utils.qualname(cls))
return bool(result[0])
def is_missing(self, cls, object_id):
"""
Check if object couldn't be found in cloud (e.g. was deleted)
:param cls: model class
:param object_id: model.ObjectId instance
:return: True or False
"""
key = (cls, object_id)
if key in self.session:
return self.session[key] is None
result = self.tx.query_one('SELECT json FROM objects WHERE uuid=:uuid '
'AND cloud=:cloud AND type=:type_name',
uuid=object_id.id,
cloud=object_id.cloud,
type_name=utils.qualname(cls))
if not result:
raise NotFound(cls, object_id)
return result[0] is None
def is_missing(self, cls, object_id):
"""
Check if object couldn't be found in cloud (e.g. was deleted)
:param cls: model class
:param object_id: model.ObjectId instance
:return: True or False
"""
key = (cls, object_id)
if key in self.session:
return self.session[key] is None
result = self.tx.query_one('SELECT json FROM objects WHERE uuid=:uuid '
'AND cloud=:cloud AND type=:type_name',
uuid=object_id.id,
cloud=object_id.cloud,
type_name=utils.qualname(cls))
if not result:
raise NotFound(cls, object_id)
return result[0] is None
def _delete_rows(self, cls, cloud_name, object_id, table):
predicates = []
kwargs = {}
if cls is not None:
predicates.append('type=:type_name')
kwargs['type_name'] = utils.qualname(cls)
if object_id is not None:
predicates.append('uuid=:uuid')
kwargs['uuid'] = object_id.id
if cloud_name is None:
cloud_name = object_id.cloud
else:
assert cloud_name == object_id.cloud
if cloud_name is not None:
predicates.append('cloud=:cloud')
kwargs['cloud'] = cloud_name
statement = 'DELETE FROM {table} WHERE '.format(table=table)
if predicates:
statement += ' AND '.join(predicates)
else:
statement += ' 1'
self.tx.execute(statement, **kwargs)
def _delete_rows(self, cls, cloud_name, object_id, table):
predicates = []
kwargs = {}
if cls is not None:
predicates.append('type=:type_name')
kwargs['type_name'] = utils.qualname(cls)
if object_id is not None:
predicates.append('uuid=:uuid')
kwargs['uuid'] = object_id.id
if cloud_name is None:
cloud_name = object_id.cloud
else:
assert cloud_name == object_id.cloud
if cloud_name is not None:
predicates.append('cloud=:cloud')
kwargs['cloud'] = cloud_name
statement = 'DELETE FROM {table} WHERE '.format(table=table)
if predicates:
statement += ' AND '.join(predicates)
else:
statement += ' 1'
self.tx.execute(statement, **kwargs)
def retrieve(self, cls, object_id):
"""
Loads object from database using class and object_id. If no such object
were found, this method will throw ``model.NotFound`` exception.
:param cls: model class
:param object_id: model.ObjectId instance
:return: model instance
"""
key = (cls, object_id)
if key in self.session:
return self.session[key]
query = self._make_sql(cls, 'uuid', 'cloud', 'type')
result = self.tx.query_one(query, uuid=object_id.id,
cloud=object_id.cloud,
type=utils.qualname(cls))
if not result or not result[0]:
raise NotFound(cls, object_id)
schema = cls.get_schema()
loaded, _ = schema.load(self._merge_obj(result))
obj = cls.create(loaded, schema=schema, mark_dirty=False)
self.session[key] = obj
return obj
def retrieve(self, cls, object_id):
"""
Loads object from database using class and object_id. If no such object
were found, this method will throw ``model.NotFound`` exception.
:param cls: model class
:param object_id: model.ObjectId instance
:return: model instance
"""
key = (cls, object_id)
if key in self.session:
return self.session[key]
query = self._make_sql(cls, 'uuid', 'cloud', 'type')
result = self.tx.query_one(query, uuid=object_id.id,
cloud=object_id.cloud,
type=utils.qualname(cls))
if not result or not result[0]:
raise NotFound(cls, object_id)
schema = cls.get_schema()
loaded, _ = schema.load(self._merge_obj(result))
obj = cls.create(loaded, schema=schema, mark_dirty=False)
self.session[key] = obj
return obj
def __init__(self, discoverer_class, *args):
super(DiscovererNotFound,
self).__init__('Discoverer for % not found.',
utils.qualname(discoverer_class), *args)
def __init__(self, discoverer_class, *args):
super(DiscovererNotFound, self).__init__(
'Discoverer for % not found.',
utils.qualname(discoverer_class), *args)
def __str__(self):
return '{0} object with id {1} not found.'.format(
utils.qualname(self.cls), self.object_id)
def __str__(self):
return '{0} object with id {1} not found.'.format(
utils.qualname(self.cls), self.object_id)
def get_class_qualname(self):
"""
Return fully qualified name of class (with module name, etc...)
"""
return utils.qualname(self._model)
def get_class_qualname():
return utils.qualname(Destructor)
def get_class_qualname(self):
"""
Return fully qualified name of class (with module name, etc...)
"""
return utils.qualname(self._model)
def revert(self, *args, **kwargs):
LOG.error('Failed to migrate %s with id %s',
utils.qualname(self.source_obj.get_class()),
self.source_obj.primary_key)