def test_pk_stretching(self):
"""
Validates whether the primary key lists scale correctly.
* X entries will be added (e.g. 10000)
* X/2 random entries will be deleted (5000)
* X/2 entries will be added again (5000)
* X entries will be removed (10000)
No entries should be remaining
"""
print ''
print 'starting test'
amount_of_objects = 10000 # Must be an even number!
machine = TestMachine()
runtimes = []
# First fill
start = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)),
0, 'There should be no primary keys yet ({0})'.format(
len(list(keys))))
guids = []
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - start)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# First delete
amount_of_objects /= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), amount_of_objects,
'There should be {0} primary keys ({1})'.format(
amount_of_objects, len(list(keys))))
print ''
# Second round
sstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - sstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# Second delete
amount_of_objects *= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(guids), 0,
'All guids should be removed. {0} left'.format(len(guids)))
self.assertEqual(
len(list(keys)), 0,
'There should be no primary keys ({0})'.format(len(list(keys))))
seconds_passed = (time.time() - start)
runtimes.sort()
print '\ncompleted in {0} seconds (avg: {1} ops, min: {2} ops, max: {3} ops)'.format(
round(seconds_passed, 2),
round((amount_of_objects * 3) / seconds_passed, 2),
round(runtimes[1], 2), round(runtimes[-2], 2))
def save(self, recursive=False, skip=None):
"""
Save the object to the persistent backend and clear cache, making use
of the specified conflict resolve settings.
It will also invalidate certain caches if required. For example lists pointing towards this
object
"""
invalid_fields = []
for prop in self._properties:
if prop.mandatory is True and self._data[prop.name] is None:
invalid_fields.append(prop.name)
for relation in self._relations:
if relation.mandatory is True and self._data[relation.name]['guid'] is None:
invalid_fields.append(relation.name)
if len(invalid_fields) > 0:
raise MissingMandatoryFieldsException('Missing fields on {0}: {1}'.format(self._name, ', '.join(invalid_fields)))
if recursive:
# Save objects that point to us (e.g. disk.vmachine - if this is disk)
for relation in self._relations:
if relation.name != skip: # disks will be skipped
item = getattr(self, relation.name)
if item is not None:
item.save(recursive=True, skip=relation.foreign_key)
# Save object we point at (e.g. machine.disks - if this is machine)
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
if key != skip: # machine will be skipped
if info['list'] is True:
for item in getattr(self, key).iterloaded():
item.save(recursive=True, skip=info['key'])
else:
item = getattr(self, key)
if item is not None:
item.save(recursive=True, skip=info['key'])
try:
data = self._persistent.get(self._key)
except KeyNotFoundException:
if self._new:
data = {}
else:
raise ObjectNotFoundException('{0} with guid \'{1}\' was deleted'.format(
self.__class__.__name__, self._guid
))
changed_fields = []
data_conflicts = []
for attribute in self._data.keys():
if self._data[attribute] != self._original[attribute]:
# We changed this value
changed_fields.append(attribute)
if attribute in data and self._original[attribute] != data[attribute]:
# Some other process also wrote to the database
if self._datastore_wins is None:
# In case we didn't set a policy, we raise the conflicts
data_conflicts.append(attribute)
elif self._datastore_wins is False:
# If the datastore should not win, we just overwrite the data
data[attribute] = self._data[attribute]
# If the datastore should win, we discard/ignore our change
else:
# Normal scenario, saving data
data[attribute] = self._data[attribute]
elif attribute not in data:
data[attribute] = self._data[attribute]
if data_conflicts:
raise ConcurrencyException('Got field conflicts while saving {0}. Conflicts: {1}'.format(
self._name, ', '.join(data_conflicts)
))
# Refresh internal data structure
self._data = copy.deepcopy(data)
# First, update reverse index
try:
self._mutex_reverseindex.acquire(60)
for relation in self._relations:
key = relation.name
original_guid = self._original[key]['guid']
new_guid = self._data[key]['guid']
if original_guid != new_guid:
if relation.foreign_type is None:
classname = self.__class__.__name__.lower()
else:
classname = relation.foreign_type.__name__.lower()
if original_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid in entries:
entries.remove(self.guid)
reverse_index[relation.foreign_key] = entries
self._volatile.set(reverse_key, reverse_index)
if new_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, new_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid not in entries:
entries.append(self.guid)
reverse_index[relation.foreign_key] = entries
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index[relation.foreign_key] = [self.guid]
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index = {relation.foreign_key: [self.guid]}
self._volatile.set(reverse_key, reverse_index)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(self._name, self.guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, _ in relations.iteritems():
reverse_index[key] = []
self._volatile.set(reverse_key, reverse_index)
finally:
self._mutex_reverseindex.release()
# Second, invalidate property lists
try:
self._mutex_listcache.acquire(60)
cache_key = '{0}_{1}'.format(DataList.cachelink, self._name)
cache_list = Toolbox.try_get(cache_key, {})
change = False
for list_key in cache_list.keys():
fields = cache_list[list_key]
if ('__all' in fields and self._new) or list(set(fields) & set(changed_fields)):
change = True
self._volatile.delete(list_key)
del cache_list[list_key]
if change is True:
self._volatile.set(cache_key, cache_list)
self._persistent.set(cache_key, cache_list)
finally:
self._mutex_listcache.release()
# Save the data
self._persistent.set(self._key, self._data)
DataList.add_pk(self._namespace, self._name, self._guid)
# Invalidate the cache
self._volatile.delete(self._key)
self._original = copy.deepcopy(self._data)
self.dirty = False
self._new = False
def test_pk_stretching(self):
"""
Validates whether the primary key lists scale correctly.
* X entries will be added (e.g. 10000)
* X/2 random entries will be deleted (5000)
* X/2 entries will be added again (5000)
* X entries will be removed (10000)
No entries should be remaining
"""
print ''
print 'starting test'
amount_of_objects = 10000 # Must be an even number!
machine = TestMachine()
runtimes = []
# First fill
start = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), 0, 'There should be no primary keys yet ({0})'.format(len(list(keys))))
guids = []
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - start)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# First delete
amount_of_objects /= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), amount_of_objects, 'There should be {0} primary keys ({1})'.format(amount_of_objects, len(list(keys))))
print ''
# Second round
sstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - sstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# Second delete
amount_of_objects *= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(guids), 0, 'All guids should be removed. {0} left'.format(len(guids)))
self.assertEqual(len(list(keys)), 0, 'There should be no primary keys ({0})'.format(len(list(keys))))
seconds_passed = (time.time() - start)
runtimes.sort()
print '\ncompleted in {0} seconds (avg: {1} ops, min: {2} ops, max: {3} ops)'.format(round(seconds_passed, 2), round((amount_of_objects * 3) / seconds_passed, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def save(self, recursive=False, skip=None):
"""
Save the object to the persistent backend and clear cache, making use
of the specified conflict resolve settings.
It will also invalidate certain caches if required. For example lists pointing towards this
object
"""
invalid_fields = []
for prop in self._properties:
if prop.mandatory is True and self._data[prop.name] is None:
invalid_fields.append(prop.name)
for relation in self._relations:
if relation.mandatory is True and self._data[
relation.name]['guid'] is None:
invalid_fields.append(relation.name)
if len(invalid_fields) > 0:
raise MissingMandatoryFieldsException(
'Missing fields on {0}: {1}'.format(self._name,
', '.join(invalid_fields)))
if recursive:
# Save objects that point to us (e.g. disk.vmachine - if this is disk)
for relation in self._relations:
if relation.name != skip: # disks will be skipped
item = getattr(self, relation.name)
if item is not None:
item.save(recursive=True, skip=relation.foreign_key)
# Save object we point at (e.g. machine.disks - if this is machine)
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
if key != skip: # machine will be skipped
if info['list'] is True:
for item in getattr(self, key).iterloaded():
item.save(recursive=True, skip=info['key'])
else:
item = getattr(self, key)
if item is not None:
item.save(recursive=True, skip=info['key'])
try:
data = self._persistent.get(self._key)
except KeyNotFoundException:
if self._new:
data = {}
else:
raise ObjectNotFoundException(
'{0} with guid \'{1}\' was deleted'.format(
self.__class__.__name__, self._guid))
changed_fields = []
data_conflicts = []
for attribute in self._data.keys():
if self._data[attribute] != self._original[attribute]:
# We changed this value
changed_fields.append(attribute)
if attribute in data and self._original[attribute] != data[
attribute]:
# Some other process also wrote to the database
if self._datastore_wins is None:
# In case we didn't set a policy, we raise the conflicts
data_conflicts.append(attribute)
elif self._datastore_wins is False:
# If the datastore should not win, we just overwrite the data
data[attribute] = self._data[attribute]
# If the datastore should win, we discard/ignore our change
else:
# Normal scenario, saving data
data[attribute] = self._data[attribute]
elif attribute not in data:
data[attribute] = self._data[attribute]
if data_conflicts:
raise ConcurrencyException(
'Got field conflicts while saving {0}. Conflicts: {1}'.format(
self._name, ', '.join(data_conflicts)))
# Refresh internal data structure
self._data = copy.deepcopy(data)
# First, update reverse index
try:
self._mutex_reverseindex.acquire(60)
for relation in self._relations:
key = relation.name
original_guid = self._original[key]['guid']
new_guid = self._data[key]['guid']
if original_guid != new_guid:
if relation.foreign_type is None:
classname = self.__class__.__name__.lower()
else:
classname = relation.foreign_type.__name__.lower()
if original_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(
classname, original_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid in entries:
entries.remove(self.guid)
reverse_index[
relation.foreign_key] = entries
self._volatile.set(reverse_key,
reverse_index)
if new_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(
classname, new_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid not in entries:
entries.append(self.guid)
reverse_index[
relation.foreign_key] = entries
self._volatile.set(reverse_key,
reverse_index)
else:
reverse_index[relation.foreign_key] = [
self.guid
]
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index = {relation.foreign_key: [self.guid]}
self._volatile.set(reverse_key, reverse_index)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(
self._name, self.guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
relations = RelationMapper.load_foreign_relations(
self.__class__)
if relations is not None:
for key, _ in relations.iteritems():
reverse_index[key] = []
self._volatile.set(reverse_key, reverse_index)
finally:
self._mutex_reverseindex.release()
# Second, invalidate property lists
try:
self._mutex_listcache.acquire(60)
cache_key = '{0}_{1}'.format(DataList.cachelink, self._name)
cache_list = Toolbox.try_get(cache_key, {})
change = False
for list_key in cache_list.keys():
fields = cache_list[list_key]
if ('__all' in fields and
self._new) or list(set(fields) & set(changed_fields)):
change = True
self._volatile.delete(list_key)
del cache_list[list_key]
if change is True:
self._volatile.set(cache_key, cache_list)
self._persistent.set(cache_key, cache_list)
finally:
self._mutex_listcache.release()
# Save the data
self._persistent.set(self._key, self._data)
DataList.add_pk(self._namespace, self._name, self._guid)
# Invalidate the cache
self._volatile.delete(self._key)
self._original = copy.deepcopy(self._data)
self.dirty = False
self._new = False
