def load_foreign_relations(object_type): """ This method will return a mapping of all relations towards a certain hybrid object type. The resulting mapping will be stored in volatile storage so it can be fetched faster """ relation_key = 'ovs_relations_{0}'.format(object_type.__name__.lower()) volatile = VolatileFactory.get_client() relation_info = volatile.get(relation_key) if relation_info is None: Toolbox.log_cache_hit('relations', False) relation_info = {} hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): # Extended objects cls = Descriptor().load(class_descriptor).get_object() for relation in cls._relations: if relation.foreign_type is None: remote_class = cls else: identifier = Descriptor(relation.foreign_type).descriptor['identifier'] if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']: remote_class = Descriptor().load(hybrid_structure[identifier]).get_object() else: remote_class = relation.foreign_type itemname = remote_class.__name__ if itemname == object_type.__name__: relation_info[relation.foreign_key] = {'class': Descriptor(cls).descriptor, 'key': relation.name, 'list': not relation.onetoone} volatile.set(relation_key, relation_info) else: Toolbox.log_cache_hit('relations', True) return relation_info
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid): """ This method will get a DataList for a relation. On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will update all relations in the mean time. """ # Example: # * remote_class = vDisk # * remote_key = vmachine # * own_class = vMachine # * own_key = vdisks # Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects) # * own_guid = this vMachine object's guid volatile = VolatileFactory.get_client() own_name = own_class.__name__.lower() datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) # Check whether the requested information is available in cache reverse_index = volatile.get(reverse_key) if reverse_index is not None and own_key in reverse_index: Toolbox.log_cache_hit('datalist', True) datalist.data = reverse_index[own_key] datalist.from_cache = True return datalist Toolbox.log_cache_hit('datalist', False) mutex = VolatileMutex('reverseindex') remote_name = remote_class.__name__.lower() blueprint_object = remote_class() # vDisk object foreign_guids = {} remote_namespace = blueprint_object._namespace remote_keys = DataList.get_pks(remote_namespace, remote_name) handled_flows = [] for guid in remote_keys: try: instance = remote_class(guid) for relation in blueprint_object._relations: # E.g. vmachine or vpool relation if relation.foreign_type is None: classname = remote_name foreign_namespace = blueprint_object._namespace else: classname = relation.foreign_type.__name__.lower() foreign_namespace = relation.foreign_type()._namespace if classname not in foreign_guids: foreign_guids[classname] = DataList.get_pks( foreign_namespace, classname) flow = '{0}_{1}'.format(classname, relation.foreign_key) if flow not in handled_flows: handled_flows.append(flow) try: mutex.acquire(60) for foreign_guid in foreign_guids[classname]: reverse_key = 'ovs_reverseindex_{0}_{1}'.format( classname, foreign_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] volatile.set(reverse_key, reverse_index) finally: mutex.release() key = getattr(instance, '{0}_guid'.format(relation.name)) if key is not None: try: mutex.acquire(60) reverse_index = volatile.get( 'ovs_reverseindex_{0}_{1}'.format( classname, key)) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] if guid not in reverse_index[relation.foreign_key]: reverse_index[relation.foreign_key].append( guid) volatile.set( 'ovs_reverseindex_{0}_{1}'.format( classname, key), reverse_index) finally: mutex.release() except ObjectNotFoundException: pass try: mutex.acquire(60) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if own_key not in reverse_index: reverse_index[own_key] = [] volatile.set(reverse_key, reverse_index) datalist.data = reverse_index[own_key] datalist.from_cache = False finally: mutex.release() return datalist
def _load(self): """ Tries to load the result for the given key from the volatile cache, or executes the query if not yet available. Afterwards (if a key is given), the result will be (re)cached """ self.data = self._volatile.get( self._key) if self._key is not None else None if self.data is None: # The query should be a dictionary: # {'object': Disk, # Object on which the query should be executed # 'data' : DataList.select.XYZ, # The requested result # 'query' : <query>} # The actual query # Where <query> is a query(group) dictionary: # {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR # 'items': <items>} # The items in the group # Where the <items> is any combination of one or more <filter> or <query> # A <filter> tuple example: # (<field>, DataList.operator.GHI, <value>) # For example EQUALS # The field is any property you would also find on the given object. In case of # properties, you can dot as far as you like. This means you can combine AND and OR # in any possible combination Toolbox.log_cache_hit('datalist', False) hybrid_structure = HybridRunner.get_hybrids() items = self._query['query']['items'] query_type = self._query['query']['type'] query_data = self._query['data'] query_object = self._query['object'] query_object_id = Descriptor(query_object).descriptor['identifier'] if query_object_id in hybrid_structure and query_object_id != hybrid_structure[ query_object_id]['identifier']: query_object = Descriptor().load( hybrid_structure[query_object_id]).get_object() invalidations = {query_object.__name__.lower(): ['__all']} DataList._build_invalidations(invalidations, query_object, items) for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) mutex = VolatileMutex('listcache_{0}'.format(class_name)) try: mutex.acquire(60) cache_list = Toolbox.try_get(key, {}) current_fields = cache_list.get(self._key, []) current_fields = list( set(current_fields + ['__all'] + invalidations[class_name])) cache_list[self._key] = current_fields self._volatile.set(key, cache_list) self._persistent.set(key, cache_list) finally: mutex.release() self.from_cache = False namespace = query_object()._namespace name = query_object.__name__.lower() guids = DataList.get_pks(namespace, name) if query_data == DataList.select.COUNT: self.data = 0 else: self.data = [] for guid in guids: try: instance = query_object(guid) if query_type == DataList.where_operator.AND: include = self._exec_and(instance, items) elif query_type == DataList.where_operator.OR: include = self._exec_or(instance, items) else: raise NotImplementedError( 'The given operator is not yet implemented.') if include: if query_data == DataList.select.COUNT: self.data += 1 elif query_data == DataList.select.GUIDS: self.data.append(guid) else: raise NotImplementedError( 'The given selector type is not implemented') except ObjectNotFoundException: pass if self._post_query_hook is not None: self._post_query_hook(self) if self._key is not None and len(guids) > 0 and self._can_cache: invalidated = False for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) cache_list = Toolbox.try_get(key, {}) if self._key not in cache_list: invalidated = True # If the key under which the list should be saved was already invalidated since the invalidations # were saved, the returned list is most likely outdated. This is OK for this result, but the list # won't get cached if invalidated is False: self._volatile.set( self._key, self.data, 300 + randint(0, 300)) # Cache between 5 and 10 minutes else: Toolbox.log_cache_hit('datalist', True) self.from_cache = True return self
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None): """ Loads an object with a given guid. If no guid is given, a new object is generated with a new guid. * guid: The guid indicating which object should be loaded * datastoreWins: Optional boolean indicating save conflict resolve management. ** True: when saving, external modified fields will not be saved ** False: when saving, all changed data will be saved, regardless of external updates ** None: in case changed field were also changed externally, an error will be raised """ # Initialize super class super(DataObject, self).__init__() # Initialize internal fields self._frozen = False self._datastore_wins = datastore_wins self._guid = None # Guid identifier of the object self._original = {} # Original data copy self._metadata = {} # Some metadata, mainly used for unit testing self._data = {} # Internal data storage self._objects = {} # Internal objects storage # Initialize public fields self.dirty = False self.volatile = volatile # Worker fields/objects self._classname = self.__class__.__name__.lower() self._namespace = 'ovs_data' # Namespace of the object self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname)) self._mutex_reverseindex = VolatileMutex('reverseindex') # Rebuild _relation types hybrid_structure = HybridRunner.get_hybrids() for relation in self._relations: if relation.foreign_type is not None: identifier = Descriptor(relation.foreign_type).descriptor['identifier'] if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']: relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object() # Init guid self._new = False if guid is None: self._guid = str(uuid.uuid4()) self._new = True else: guid = str(guid).lower() if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None: self._guid = str(guid) else: raise ValueError('The given guid is invalid: {0}'.format(guid)) # Build base keys self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid) # Worker mutexes self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid)) # Load data from cache or persistent backend where appropriate self._volatile = VolatileFactory.get_client() self._persistent = PersistentFactory.get_client() self._metadata['cache'] = None if self._new: self._data = {} else: self._data = self._volatile.get(self._key) if self._data is None: Toolbox.log_cache_hit('object_load', False) self._metadata['cache'] = False try: self._data = self._persistent.get(self._key) except KeyNotFoundException: raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format( self.__class__.__name__, self._guid )) else: Toolbox.log_cache_hit('object_load', True) self._metadata['cache'] = True # Set default values on new fields for prop in self._properties: if prop.name not in self._data: self._data[prop.name] = prop.default self._add_property(prop) # Load relations for relation in self._relations: if relation.name not in self._data: if relation.foreign_type is None: cls = self.__class__ else: cls = relation.foreign_type self._data[relation.name] = Descriptor(cls).descriptor self._add_relation_property(relation) # Add wrapped properties for dynamic in self._dynamics: self._add_dynamic_property(dynamic) # Load foreign keys relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): self._objects[key] = {'info': info, 'data': None} self._add_list_property(key, info['list']) # Store original data self._original = copy.deepcopy(self._data) if _hook is not None and hasattr(_hook, '__call__'): _hook() if not self._new: # Re-cache the object, if required if self._metadata['cache'] is False: # The data wasn't loaded from the cache, so caching is required now try: self._mutex_version.acquire(30) this_version = self._data['_version'] store_version = self._persistent.get(self._key)['_version'] if this_version == store_version: self._volatile.set(self._key, self._data) except KeyNotFoundException: raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format( self.__class__.__name__, self._guid )) finally: self._mutex_version.release() # Freeze property creation self._frozen = True # Optionally, initialize some fields if data is not None: for prop in self._properties: if prop.name in data: setattr(self, prop.name, data[prop.name])
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid): """ This method will get a DataList for a relation. On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will update all relations in the mean time. """ # Example: # * remote_class = vDisk # * remote_key = vmachine # * own_class = vMachine # * own_key = vdisks # Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects) # * own_guid = this vMachine object's guid volatile = VolatileFactory.get_client() own_name = own_class.__name__.lower() datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) # Check whether the requested information is available in cache reverse_index = volatile.get(reverse_key) if reverse_index is not None and own_key in reverse_index: Toolbox.log_cache_hit('datalist', True) datalist.data = reverse_index[own_key] datalist.from_cache = True return datalist Toolbox.log_cache_hit('datalist', False) mutex = VolatileMutex('reverseindex') remote_name = remote_class.__name__.lower() blueprint_object = remote_class() # vDisk object foreign_guids = {} remote_namespace = blueprint_object._namespace for relation in blueprint_object._relations: # E.g. vmachine or vpool relation if relation.foreign_type is None: classname = remote_name foreign_namespace = blueprint_object._namespace else: classname = relation.foreign_type.__name__.lower() foreign_namespace = relation.foreign_type()._namespace if classname not in foreign_guids: foreign_guids[classname] = DataList.get_pks(foreign_namespace, classname) try: mutex.acquire(60) for foreign_guid in foreign_guids[classname]: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, foreign_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] volatile.set(reverse_key, reverse_index) finally: mutex.release() remote_keys = DataList.get_pks(remote_namespace, remote_name) for guid in remote_keys: try: instance = remote_class(guid) for relation in blueprint_object._relations: # E.g. vmachine or vpool relation if relation.foreign_type is None: classname = remote_name else: classname = relation.foreign_type.__name__.lower() key = getattr(instance, '{0}_guid'.format(relation.name)) if key is not None: try: mutex.acquire(60) reverse_index = volatile.get('ovs_reverseindex_{0}_{1}'.format(classname, key)) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] if guid not in reverse_index[relation.foreign_key]: if instance.updated_on_datastore(): raise ConcurrencyException() reverse_index[relation.foreign_key].append(guid) volatile.set('ovs_reverseindex_{0}_{1}'.format(classname, key), reverse_index) finally: mutex.release() except ObjectNotFoundException: pass except ConcurrencyException: pass try: mutex.acquire(60) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if own_key not in reverse_index: reverse_index[own_key] = [] volatile.set(reverse_key, reverse_index) datalist.data = reverse_index[own_key] datalist.from_cache = False finally: mutex.release() return datalist
def _load(self): """ Tries to load the result for the given key from the volatile cache, or executes the query if not yet available. Afterwards (if a key is given), the result will be (re)cached """ self.data = self._volatile.get(self._key) if self._key is not None else None if self.data is None: # The query should be a dictionary: # {'object': Disk, # Object on which the query should be executed # 'data' : DataList.select.XYZ, # The requested result # 'query' : <query>} # The actual query # Where <query> is a query(group) dictionary: # {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR # 'items': <items>} # The items in the group # Where the <items> is any combination of one or more <filter> or <query> # A <filter> tuple example: # (<field>, DataList.operator.GHI, <value>) # For example EQUALS # The field is any property you would also find on the given object. In case of # properties, you can dot as far as you like. This means you can combine AND and OR # in any possible combination Toolbox.log_cache_hit('datalist', False) hybrid_structure = HybridRunner.get_hybrids() items = self._query['query']['items'] query_type = self._query['query']['type'] query_data = self._query['data'] query_object = self._query['object'] query_object_id = Descriptor(query_object).descriptor['identifier'] if query_object_id in hybrid_structure and query_object_id != hybrid_structure[query_object_id]['identifier']: query_object = Descriptor().load(hybrid_structure[query_object_id]).get_object() invalidations = {query_object.__name__.lower(): ['__all']} DataList._build_invalidations(invalidations, query_object, items) for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) mutex = VolatileMutex('listcache_{0}'.format(class_name)) try: mutex.acquire(60) cache_list = Toolbox.try_get(key, {}) current_fields = cache_list.get(self._key, []) current_fields = list(set(current_fields + ['__all'] + invalidations[class_name])) cache_list[self._key] = current_fields self._volatile.set(key, cache_list) self._persistent.set(key, cache_list) finally: mutex.release() self.from_cache = False namespace = query_object()._namespace name = query_object.__name__.lower() guids = DataList.get_pks(namespace, name) if query_data == DataList.select.COUNT: self.data = 0 else: self.data = [] for guid in guids: try: instance = query_object(guid) if query_type == DataList.where_operator.AND: include = self._exec_and(instance, items) elif query_type == DataList.where_operator.OR: include = self._exec_or(instance, items) else: raise NotImplementedError('The given operator is not yet implemented.') if include: if query_data == DataList.select.COUNT: self.data += 1 elif query_data == DataList.select.GUIDS: self.data.append(guid) else: raise NotImplementedError('The given selector type is not implemented') except ObjectNotFoundException: pass if 'post_query' in DataList.test_hooks: DataList.test_hooks['post_query'](self) if self._key is not None and len(guids) > 0 and self._can_cache: invalidated = False for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) cache_list = Toolbox.try_get(key, {}) if self._key not in cache_list: invalidated = True # If the key under which the list should be saved was already invalidated since the invalidations # were saved, the returned list is most likely outdated. This is OK for this result, but the list # won't get cached if invalidated is False: self._volatile.set(self._key, self.data, 300 + randint(0, 300)) # Cache between 5 and 10 minutes else: Toolbox.log_cache_hit('datalist', True) self.from_cache = True return self