def load_foreign_relations(object_type):
"""
This method will return a mapping of all relations towards a certain hybrid object type.
The resulting mapping will be stored in volatile storage so it can be fetched faster
"""
relation_key = 'ovs_relations_{0}'.format(object_type.__name__.lower())
volatile = VolatileFactory.get_client()
relation_info = volatile.get(relation_key)
if relation_info is None:
Toolbox.log_cache_hit('relations', False)
relation_info = {}
hybrid_structure = HybridRunner.get_hybrids()
for class_descriptor in hybrid_structure.values(): # Extended objects
cls = Descriptor().load(class_descriptor).get_object()
for relation in cls._relations:
if relation.foreign_type is None:
remote_class = cls
else:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
remote_class = Descriptor().load(hybrid_structure[identifier]).get_object()
else:
remote_class = relation.foreign_type
itemname = remote_class.__name__
if itemname == object_type.__name__:
relation_info[relation.foreign_key] = {'class': Descriptor(cls).descriptor,
'key': relation.name,
'list': not relation.onetoone}
volatile.set(relation_key, relation_info)
else:
Toolbox.log_cache_hit('relations', True)
return relation_info
def load_foreign_relations(object_type):
"""
This method will return a mapping of all relations towards a certain hybrid object type.
The resulting mapping will be stored in volatile storage so it can be fetched faster
"""
relation_key = 'ovs_relations_{0}'.format(object_type.__name__.lower())
volatile = VolatileFactory.get_client()
relation_info = volatile.get(relation_key)
if relation_info is None:
Toolbox.log_cache_hit('relations', False)
relation_info = {}
hybrid_structure = HybridRunner.get_hybrids()
for class_descriptor in hybrid_structure.values(): # Extended objects
cls = Descriptor().load(class_descriptor).get_object()
for relation in cls._relations:
if relation.foreign_type is None:
remote_class = cls
else:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
remote_class = Descriptor().load(hybrid_structure[identifier]).get_object()
else:
remote_class = relation.foreign_type
itemname = remote_class.__name__
if itemname == object_type.__name__:
relation_info[relation.foreign_key] = {'class': Descriptor(cls).descriptor,
'key': relation.name,
'list': not relation.onetoone}
volatile.set(relation_key, relation_info)
else:
Toolbox.log_cache_hit('relations', True)
return relation_info
def get_relation_set(remote_class, remote_key, own_class, own_key,
own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
"""
# Example:
# * remote_class = vDisk
# * remote_key = vmachine
# * own_class = vMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
volatile = VolatileFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList({},
'{0}_{1}_{2}'.format(own_name, own_guid,
remote_key),
load=False)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
# Check whether the requested information is available in cache
reverse_index = volatile.get(reverse_key)
if reverse_index is not None and own_key in reverse_index:
Toolbox.log_cache_hit('datalist', True)
datalist.data = reverse_index[own_key]
datalist.from_cache = True
return datalist
Toolbox.log_cache_hit('datalist', False)
mutex = VolatileMutex('reverseindex')
remote_name = remote_class.__name__.lower()
blueprint_object = remote_class() # vDisk object
foreign_guids = {}
remote_namespace = blueprint_object._namespace
remote_keys = DataList.get_pks(remote_namespace, remote_name)
handled_flows = []
for guid in remote_keys:
try:
instance = remote_class(guid)
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
foreign_namespace = blueprint_object._namespace
else:
classname = relation.foreign_type.__name__.lower()
foreign_namespace = relation.foreign_type()._namespace
if classname not in foreign_guids:
foreign_guids[classname] = DataList.get_pks(
foreign_namespace, classname)
flow = '{0}_{1}'.format(classname, relation.foreign_key)
if flow not in handled_flows:
handled_flows.append(flow)
try:
mutex.acquire(60)
for foreign_guid in foreign_guids[classname]:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(
classname, foreign_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
volatile.set(reverse_key, reverse_index)
finally:
mutex.release()
key = getattr(instance, '{0}_guid'.format(relation.name))
if key is not None:
try:
mutex.acquire(60)
reverse_index = volatile.get(
'ovs_reverseindex_{0}_{1}'.format(
classname, key))
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
if guid not in reverse_index[relation.foreign_key]:
reverse_index[relation.foreign_key].append(
guid)
volatile.set(
'ovs_reverseindex_{0}_{1}'.format(
classname, key), reverse_index)
finally:
mutex.release()
except ObjectNotFoundException:
pass
try:
mutex.acquire(60)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if own_key not in reverse_index:
reverse_index[own_key] = []
volatile.set(reverse_key, reverse_index)
datalist.data = reverse_index[own_key]
datalist.from_cache = False
finally:
mutex.release()
return datalist
def _load(self):
"""
Tries to load the result for the given key from the volatile cache, or executes the query
if not yet available. Afterwards (if a key is given), the result will be (re)cached
"""
self.data = self._volatile.get(
self._key) if self._key is not None else None
if self.data is None:
# The query should be a dictionary:
# {'object': Disk, # Object on which the query should be executed
# 'data' : DataList.select.XYZ, # The requested result
# 'query' : <query>} # The actual query
# Where <query> is a query(group) dictionary:
# {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR
# 'items': <items>} # The items in the group
# Where the <items> is any combination of one or more <filter> or <query>
# A <filter> tuple example:
# (<field>, DataList.operator.GHI, <value>) # For example EQUALS
# The field is any property you would also find on the given object. In case of
# properties, you can dot as far as you like. This means you can combine AND and OR
# in any possible combination
Toolbox.log_cache_hit('datalist', False)
hybrid_structure = HybridRunner.get_hybrids()
items = self._query['query']['items']
query_type = self._query['query']['type']
query_data = self._query['data']
query_object = self._query['object']
query_object_id = Descriptor(query_object).descriptor['identifier']
if query_object_id in hybrid_structure and query_object_id != hybrid_structure[
query_object_id]['identifier']:
query_object = Descriptor().load(
hybrid_structure[query_object_id]).get_object()
invalidations = {query_object.__name__.lower(): ['__all']}
DataList._build_invalidations(invalidations, query_object, items)
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
mutex = VolatileMutex('listcache_{0}'.format(class_name))
try:
mutex.acquire(60)
cache_list = Toolbox.try_get(key, {})
current_fields = cache_list.get(self._key, [])
current_fields = list(
set(current_fields + ['__all'] +
invalidations[class_name]))
cache_list[self._key] = current_fields
self._volatile.set(key, cache_list)
self._persistent.set(key, cache_list)
finally:
mutex.release()
self.from_cache = False
namespace = query_object()._namespace
name = query_object.__name__.lower()
guids = DataList.get_pks(namespace, name)
if query_data == DataList.select.COUNT:
self.data = 0
else:
self.data = []
for guid in guids:
try:
instance = query_object(guid)
if query_type == DataList.where_operator.AND:
include = self._exec_and(instance, items)
elif query_type == DataList.where_operator.OR:
include = self._exec_or(instance, items)
else:
raise NotImplementedError(
'The given operator is not yet implemented.')
if include:
if query_data == DataList.select.COUNT:
self.data += 1
elif query_data == DataList.select.GUIDS:
self.data.append(guid)
else:
raise NotImplementedError(
'The given selector type is not implemented')
except ObjectNotFoundException:
pass
if self._post_query_hook is not None:
self._post_query_hook(self)
if self._key is not None and len(guids) > 0 and self._can_cache:
invalidated = False
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
cache_list = Toolbox.try_get(key, {})
if self._key not in cache_list:
invalidated = True
# If the key under which the list should be saved was already invalidated since the invalidations
# were saved, the returned list is most likely outdated. This is OK for this result, but the list
# won't get cached
if invalidated is False:
self._volatile.set(
self._key, self.data, 300 +
randint(0, 300)) # Cache between 5 and 10 minutes
else:
Toolbox.log_cache_hit('datalist', True)
self.from_cache = True
return self
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None):
"""
Loads an object with a given guid. If no guid is given, a new object
is generated with a new guid.
* guid: The guid indicating which object should be loaded
* datastoreWins: Optional boolean indicating save conflict resolve management.
** True: when saving, external modified fields will not be saved
** False: when saving, all changed data will be saved, regardless of external updates
** None: in case changed field were also changed externally, an error will be raised
"""
# Initialize super class
super(DataObject, self).__init__()
# Initialize internal fields
self._frozen = False
self._datastore_wins = datastore_wins
self._guid = None # Guid identifier of the object
self._original = {} # Original data copy
self._metadata = {} # Some metadata, mainly used for unit testing
self._data = {} # Internal data storage
self._objects = {} # Internal objects storage
# Initialize public fields
self.dirty = False
self.volatile = volatile
# Worker fields/objects
self._classname = self.__class__.__name__.lower()
self._namespace = 'ovs_data' # Namespace of the object
self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname))
self._mutex_reverseindex = VolatileMutex('reverseindex')
# Rebuild _relation types
hybrid_structure = HybridRunner.get_hybrids()
for relation in self._relations:
if relation.foreign_type is not None:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object()
# Init guid
self._new = False
if guid is None:
self._guid = str(uuid.uuid4())
self._new = True
else:
guid = str(guid).lower()
if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None:
self._guid = str(guid)
else:
raise ValueError('The given guid is invalid: {0}'.format(guid))
# Build base keys
self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid)
# Worker mutexes
self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid))
# Load data from cache or persistent backend where appropriate
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._metadata['cache'] = None
if self._new:
self._data = {}
else:
self._data = self._volatile.get(self._key)
if self._data is None:
Toolbox.log_cache_hit('object_load', False)
self._metadata['cache'] = False
try:
self._data = self._persistent.get(self._key)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
else:
Toolbox.log_cache_hit('object_load', True)
self._metadata['cache'] = True
# Set default values on new fields
for prop in self._properties:
if prop.name not in self._data:
self._data[prop.name] = prop.default
self._add_property(prop)
# Load relations
for relation in self._relations:
if relation.name not in self._data:
if relation.foreign_type is None:
cls = self.__class__
else:
cls = relation.foreign_type
self._data[relation.name] = Descriptor(cls).descriptor
self._add_relation_property(relation)
# Add wrapped properties
for dynamic in self._dynamics:
self._add_dynamic_property(dynamic)
# Load foreign keys
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
self._objects[key] = {'info': info,
'data': None}
self._add_list_property(key, info['list'])
# Store original data
self._original = copy.deepcopy(self._data)
if _hook is not None and hasattr(_hook, '__call__'):
_hook()
if not self._new:
# Re-cache the object, if required
if self._metadata['cache'] is False:
# The data wasn't loaded from the cache, so caching is required now
try:
self._mutex_version.acquire(30)
this_version = self._data['_version']
store_version = self._persistent.get(self._key)['_version']
if this_version == store_version:
self._volatile.set(self._key, self._data)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
finally:
self._mutex_version.release()
# Freeze property creation
self._frozen = True
# Optionally, initialize some fields
if data is not None:
for prop in self._properties:
if prop.name in data:
setattr(self, prop.name, data[prop.name])
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
"""
# Example:
# * remote_class = vDisk
# * remote_key = vmachine
# * own_class = vMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
volatile = VolatileFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
# Check whether the requested information is available in cache
reverse_index = volatile.get(reverse_key)
if reverse_index is not None and own_key in reverse_index:
Toolbox.log_cache_hit('datalist', True)
datalist.data = reverse_index[own_key]
datalist.from_cache = True
return datalist
Toolbox.log_cache_hit('datalist', False)
mutex = VolatileMutex('reverseindex')
remote_name = remote_class.__name__.lower()
blueprint_object = remote_class() # vDisk object
foreign_guids = {}
remote_namespace = blueprint_object._namespace
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
foreign_namespace = blueprint_object._namespace
else:
classname = relation.foreign_type.__name__.lower()
foreign_namespace = relation.foreign_type()._namespace
if classname not in foreign_guids:
foreign_guids[classname] = DataList.get_pks(foreign_namespace, classname)
try:
mutex.acquire(60)
for foreign_guid in foreign_guids[classname]:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, foreign_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
volatile.set(reverse_key, reverse_index)
finally:
mutex.release()
remote_keys = DataList.get_pks(remote_namespace, remote_name)
for guid in remote_keys:
try:
instance = remote_class(guid)
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
else:
classname = relation.foreign_type.__name__.lower()
key = getattr(instance, '{0}_guid'.format(relation.name))
if key is not None:
try:
mutex.acquire(60)
reverse_index = volatile.get('ovs_reverseindex_{0}_{1}'.format(classname, key))
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
if guid not in reverse_index[relation.foreign_key]:
if instance.updated_on_datastore():
raise ConcurrencyException()
reverse_index[relation.foreign_key].append(guid)
volatile.set('ovs_reverseindex_{0}_{1}'.format(classname, key), reverse_index)
finally:
mutex.release()
except ObjectNotFoundException:
pass
except ConcurrencyException:
pass
try:
mutex.acquire(60)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if own_key not in reverse_index:
reverse_index[own_key] = []
volatile.set(reverse_key, reverse_index)
datalist.data = reverse_index[own_key]
datalist.from_cache = False
finally:
mutex.release()
return datalist
def _load(self):
"""
Tries to load the result for the given key from the volatile cache, or executes the query
if not yet available. Afterwards (if a key is given), the result will be (re)cached
"""
self.data = self._volatile.get(self._key) if self._key is not None else None
if self.data is None:
# The query should be a dictionary:
# {'object': Disk, # Object on which the query should be executed
# 'data' : DataList.select.XYZ, # The requested result
# 'query' : <query>} # The actual query
# Where <query> is a query(group) dictionary:
# {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR
# 'items': <items>} # The items in the group
# Where the <items> is any combination of one or more <filter> or <query>
# A <filter> tuple example:
# (<field>, DataList.operator.GHI, <value>) # For example EQUALS
# The field is any property you would also find on the given object. In case of
# properties, you can dot as far as you like. This means you can combine AND and OR
# in any possible combination
Toolbox.log_cache_hit('datalist', False)
hybrid_structure = HybridRunner.get_hybrids()
items = self._query['query']['items']
query_type = self._query['query']['type']
query_data = self._query['data']
query_object = self._query['object']
query_object_id = Descriptor(query_object).descriptor['identifier']
if query_object_id in hybrid_structure and query_object_id != hybrid_structure[query_object_id]['identifier']:
query_object = Descriptor().load(hybrid_structure[query_object_id]).get_object()
invalidations = {query_object.__name__.lower(): ['__all']}
DataList._build_invalidations(invalidations, query_object, items)
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
mutex = VolatileMutex('listcache_{0}'.format(class_name))
try:
mutex.acquire(60)
cache_list = Toolbox.try_get(key, {})
current_fields = cache_list.get(self._key, [])
current_fields = list(set(current_fields + ['__all'] + invalidations[class_name]))
cache_list[self._key] = current_fields
self._volatile.set(key, cache_list)
self._persistent.set(key, cache_list)
finally:
mutex.release()
self.from_cache = False
namespace = query_object()._namespace
name = query_object.__name__.lower()
guids = DataList.get_pks(namespace, name)
if query_data == DataList.select.COUNT:
self.data = 0
else:
self.data = []
for guid in guids:
try:
instance = query_object(guid)
if query_type == DataList.where_operator.AND:
include = self._exec_and(instance, items)
elif query_type == DataList.where_operator.OR:
include = self._exec_or(instance, items)
else:
raise NotImplementedError('The given operator is not yet implemented.')
if include:
if query_data == DataList.select.COUNT:
self.data += 1
elif query_data == DataList.select.GUIDS:
self.data.append(guid)
else:
raise NotImplementedError('The given selector type is not implemented')
except ObjectNotFoundException:
pass
if 'post_query' in DataList.test_hooks:
DataList.test_hooks['post_query'](self)
if self._key is not None and len(guids) > 0 and self._can_cache:
invalidated = False
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
cache_list = Toolbox.try_get(key, {})
if self._key not in cache_list:
invalidated = True
# If the key under which the list should be saved was already invalidated since the invalidations
# were saved, the returned list is most likely outdated. This is OK for this result, but the list
# won't get cached
if invalidated is False:
self._volatile.set(self._key, self.data, 300 + randint(0, 300)) # Cache between 5 and 10 minutes
else:
Toolbox.log_cache_hit('datalist', True)
self.from_cache = True
return self
