def test_pk_stretching(self): """ Validates whether the primary key lists scale correctly. * X entries will be added (e.g. 10000) * X/2 random entries will be deleted (5000) * X/2 entries will be added again (5000) * X entries will be removed (10000) No entries should be remaining """ print '' print 'starting test' amount_of_objects = 10000 # Must be an even number! machine = TestMachine() runtimes = [] # First fill start = time.time() keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(list(keys)), 0, 'There should be no primary keys yet ({0})'.format( len(list(keys)))) guids = [] mstart = time.time() for i in xrange(0, amount_of_objects): guid = str(uuid.uuid4()) guids.append(guid) DataList.add_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format( len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - start) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress( '* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'. format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() print '' # First delete amount_of_objects /= 2 shuffle(guids) # Make the test a bit more realistic guids_copy = guids[:] dstart = time.time() mstart = time.time() for i in xrange(0, amount_of_objects): guid = guids_copy[i] guids.remove(guid) DataList.delete_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format( len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - dstart) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress( '* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'. format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(list(keys)), amount_of_objects, 'There should be {0} primary keys ({1})'.format( amount_of_objects, len(list(keys)))) print '' # Second round sstart = time.time() mstart = time.time() for i in xrange(0, amount_of_objects): guid = str(uuid.uuid4()) guids.append(guid) DataList.add_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format( len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - sstart) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress( '* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'. format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() print '' # Second delete amount_of_objects *= 2 shuffle(guids) # Make the test a bit more realistic guids_copy = guids[:] dstart = time.time() mstart = time.time() for i in xrange(0, amount_of_objects): guid = guids_copy[i] guids.remove(guid) DataList.delete_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format( len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - dstart) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress( '* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'. format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual( len(guids), 0, 'All guids should be removed. {0} left'.format(len(guids))) self.assertEqual( len(list(keys)), 0, 'There should be no primary keys ({0})'.format(len(list(keys)))) seconds_passed = (time.time() - start) runtimes.sort() print '\ncompleted in {0} seconds (avg: {1} ops, min: {2} ops, max: {3} ops)'.format( round(seconds_passed, 2), round((amount_of_objects * 3) / seconds_passed, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def save(self, recursive=False, skip=None): """ Save the object to the persistent backend and clear cache, making use of the specified conflict resolve settings. It will also invalidate certain caches if required. For example lists pointing towards this object """ invalid_fields = [] for prop in self._properties: if prop.mandatory is True and self._data[prop.name] is None: invalid_fields.append(prop.name) for relation in self._relations: if relation.mandatory is True and self._data[relation.name]['guid'] is None: invalid_fields.append(relation.name) if len(invalid_fields) > 0: raise MissingMandatoryFieldsException('Missing fields on {0}: {1}'.format(self._name, ', '.join(invalid_fields))) if recursive: # Save objects that point to us (e.g. disk.vmachine - if this is disk) for relation in self._relations: if relation.name != skip: # disks will be skipped item = getattr(self, relation.name) if item is not None: item.save(recursive=True, skip=relation.foreign_key) # Save object we point at (e.g. machine.disks - if this is machine) relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): if key != skip: # machine will be skipped if info['list'] is True: for item in getattr(self, key).iterloaded(): item.save(recursive=True, skip=info['key']) else: item = getattr(self, key) if item is not None: item.save(recursive=True, skip=info['key']) try: data = self._persistent.get(self._key) except KeyNotFoundException: if self._new: data = {} else: raise ObjectNotFoundException('{0} with guid \'{1}\' was deleted'.format( self.__class__.__name__, self._guid )) changed_fields = [] data_conflicts = [] for attribute in self._data.keys(): if self._data[attribute] != self._original[attribute]: # We changed this value changed_fields.append(attribute) if attribute in data and self._original[attribute] != data[attribute]: # Some other process also wrote to the database if self._datastore_wins is None: # In case we didn't set a policy, we raise the conflicts data_conflicts.append(attribute) elif self._datastore_wins is False: # If the datastore should not win, we just overwrite the data data[attribute] = self._data[attribute] # If the datastore should win, we discard/ignore our change else: # Normal scenario, saving data data[attribute] = self._data[attribute] elif attribute not in data: data[attribute] = self._data[attribute] if data_conflicts: raise ConcurrencyException('Got field conflicts while saving {0}. Conflicts: {1}'.format( self._name, ', '.join(data_conflicts) )) # Refresh internal data structure self._data = copy.deepcopy(data) # First, update reverse index try: self._mutex_reverseindex.acquire(60) for relation in self._relations: key = relation.name original_guid = self._original[key]['guid'] new_guid = self._data[key]['guid'] if original_guid != new_guid: if relation.foreign_type is None: classname = self.__class__.__name__.lower() else: classname = relation.foreign_type.__name__.lower() if original_guid is not None: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid in entries: entries.remove(self.guid) reverse_index[relation.foreign_key] = entries self._volatile.set(reverse_key, reverse_index) if new_guid is not None: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, new_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid not in entries: entries.append(self.guid) reverse_index[relation.foreign_key] = entries self._volatile.set(reverse_key, reverse_index) else: reverse_index[relation.foreign_key] = [self.guid] self._volatile.set(reverse_key, reverse_index) else: reverse_index = {relation.foreign_key: [self.guid]} self._volatile.set(reverse_key, reverse_index) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(self._name, self.guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is None: reverse_index = {} relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, _ in relations.iteritems(): reverse_index[key] = [] self._volatile.set(reverse_key, reverse_index) finally: self._mutex_reverseindex.release() # Second, invalidate property lists try: self._mutex_listcache.acquire(60) cache_key = '{0}_{1}'.format(DataList.cachelink, self._name) cache_list = Toolbox.try_get(cache_key, {}) change = False for list_key in cache_list.keys(): fields = cache_list[list_key] if ('__all' in fields and self._new) or list(set(fields) & set(changed_fields)): change = True self._volatile.delete(list_key) del cache_list[list_key] if change is True: self._volatile.set(cache_key, cache_list) self._persistent.set(cache_key, cache_list) finally: self._mutex_listcache.release() # Save the data self._persistent.set(self._key, self._data) DataList.add_pk(self._namespace, self._name, self._guid) # Invalidate the cache self._volatile.delete(self._key) self._original = copy.deepcopy(self._data) self.dirty = False self._new = False
def test_pk_stretching(self): """ Validates whether the primary key lists scale correctly. * X entries will be added (e.g. 10000) * X/2 random entries will be deleted (5000) * X/2 entries will be added again (5000) * X entries will be removed (10000) No entries should be remaining """ print '' print 'starting test' amount_of_objects = 10000 # Must be an even number! machine = TestMachine() runtimes = [] # First fill start = time.time() keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(list(keys)), 0, 'There should be no primary keys yet ({0})'.format(len(list(keys)))) guids = [] mstart = time.time() for i in xrange(0, amount_of_objects): guid = str(uuid.uuid4()) guids.append(guid) DataList.add_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - start) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() print '' # First delete amount_of_objects /= 2 shuffle(guids) # Make the test a bit more realistic guids_copy = guids[:] dstart = time.time() mstart = time.time() for i in xrange(0, amount_of_objects): guid = guids_copy[i] guids.remove(guid) DataList.delete_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - dstart) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(list(keys)), amount_of_objects, 'There should be {0} primary keys ({1})'.format(amount_of_objects, len(list(keys)))) print '' # Second round sstart = time.time() mstart = time.time() for i in xrange(0, amount_of_objects): guid = str(uuid.uuid4()) guids.append(guid) DataList.add_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - sstart) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() print '' # Second delete amount_of_objects *= 2 shuffle(guids) # Make the test a bit more realistic guids_copy = guids[:] dstart = time.time() mstart = time.time() for i in xrange(0, amount_of_objects): guid = guids_copy[i] guids.remove(guid) DataList.delete_pk(machine._namespace, machine._name, guid) keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys)))) if i % 100 == 99: avgitemspersec = (i + 1) / (time.time() - dstart) itemspersec = 100 / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2))) mstart = time.time() keys = DataList._get_pks(machine._namespace, machine._name) self.assertEqual(len(guids), 0, 'All guids should be removed. {0} left'.format(len(guids))) self.assertEqual(len(list(keys)), 0, 'There should be no primary keys ({0})'.format(len(list(keys)))) seconds_passed = (time.time() - start) runtimes.sort() print '\ncompleted in {0} seconds (avg: {1} ops, min: {2} ops, max: {3} ops)'.format(round(seconds_passed, 2), round((amount_of_objects * 3) / seconds_passed, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def save(self, recursive=False, skip=None): """ Save the object to the persistent backend and clear cache, making use of the specified conflict resolve settings. It will also invalidate certain caches if required. For example lists pointing towards this object """ invalid_fields = [] for prop in self._properties: if prop.mandatory is True and self._data[prop.name] is None: invalid_fields.append(prop.name) for relation in self._relations: if relation.mandatory is True and self._data[ relation.name]['guid'] is None: invalid_fields.append(relation.name) if len(invalid_fields) > 0: raise MissingMandatoryFieldsException( 'Missing fields on {0}: {1}'.format(self._name, ', '.join(invalid_fields))) if recursive: # Save objects that point to us (e.g. disk.vmachine - if this is disk) for relation in self._relations: if relation.name != skip: # disks will be skipped item = getattr(self, relation.name) if item is not None: item.save(recursive=True, skip=relation.foreign_key) # Save object we point at (e.g. machine.disks - if this is machine) relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): if key != skip: # machine will be skipped if info['list'] is True: for item in getattr(self, key).iterloaded(): item.save(recursive=True, skip=info['key']) else: item = getattr(self, key) if item is not None: item.save(recursive=True, skip=info['key']) try: data = self._persistent.get(self._key) except KeyNotFoundException: if self._new: data = {} else: raise ObjectNotFoundException( '{0} with guid \'{1}\' was deleted'.format( self.__class__.__name__, self._guid)) changed_fields = [] data_conflicts = [] for attribute in self._data.keys(): if self._data[attribute] != self._original[attribute]: # We changed this value changed_fields.append(attribute) if attribute in data and self._original[attribute] != data[ attribute]: # Some other process also wrote to the database if self._datastore_wins is None: # In case we didn't set a policy, we raise the conflicts data_conflicts.append(attribute) elif self._datastore_wins is False: # If the datastore should not win, we just overwrite the data data[attribute] = self._data[attribute] # If the datastore should win, we discard/ignore our change else: # Normal scenario, saving data data[attribute] = self._data[attribute] elif attribute not in data: data[attribute] = self._data[attribute] if data_conflicts: raise ConcurrencyException( 'Got field conflicts while saving {0}. Conflicts: {1}'.format( self._name, ', '.join(data_conflicts))) # Refresh internal data structure self._data = copy.deepcopy(data) # First, update reverse index try: self._mutex_reverseindex.acquire(60) for relation in self._relations: key = relation.name original_guid = self._original[key]['guid'] new_guid = self._data[key]['guid'] if original_guid != new_guid: if relation.foreign_type is None: classname = self.__class__.__name__.lower() else: classname = relation.foreign_type.__name__.lower() if original_guid is not None: reverse_key = 'ovs_reverseindex_{0}_{1}'.format( classname, original_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid in entries: entries.remove(self.guid) reverse_index[ relation.foreign_key] = entries self._volatile.set(reverse_key, reverse_index) if new_guid is not None: reverse_key = 'ovs_reverseindex_{0}_{1}'.format( classname, new_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid not in entries: entries.append(self.guid) reverse_index[ relation.foreign_key] = entries self._volatile.set(reverse_key, reverse_index) else: reverse_index[relation.foreign_key] = [ self.guid ] self._volatile.set(reverse_key, reverse_index) else: reverse_index = {relation.foreign_key: [self.guid]} self._volatile.set(reverse_key, reverse_index) reverse_key = 'ovs_reverseindex_{0}_{1}'.format( self._name, self.guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is None: reverse_index = {} relations = RelationMapper.load_foreign_relations( self.__class__) if relations is not None: for key, _ in relations.iteritems(): reverse_index[key] = [] self._volatile.set(reverse_key, reverse_index) finally: self._mutex_reverseindex.release() # Second, invalidate property lists try: self._mutex_listcache.acquire(60) cache_key = '{0}_{1}'.format(DataList.cachelink, self._name) cache_list = Toolbox.try_get(cache_key, {}) change = False for list_key in cache_list.keys(): fields = cache_list[list_key] if ('__all' in fields and self._new) or list(set(fields) & set(changed_fields)): change = True self._volatile.delete(list_key) del cache_list[list_key] if change is True: self._volatile.set(cache_key, cache_list) self._persistent.set(cache_key, cache_list) finally: self._mutex_listcache.release() # Save the data self._persistent.set(self._key, self._data) DataList.add_pk(self._namespace, self._name, self._guid) # Invalidate the cache self._volatile.delete(self._key) self._original = copy.deepcopy(self._data) self.dirty = False self._new = False