def _clean_all(self):
"""
Cleans all disks and machines
"""
machine = TestMachine()
prefix = '{0}_{1}_'.format(DataObject.NAMESPACE, machine._classname)
keys = self.persistent.prefix(prefix)
for key in keys:
try:
guid = key.replace(prefix, '')
machine = TestMachine(guid)
for disk in machine.disks:
disk.delete()
machine.delete()
except (ObjectNotFoundException, ValueError):
pass
for key in self.persistent.prefix('ovs_reverseindex_{0}'.format(machine._classname)):
self.persistent.delete(key)
disk = TestDisk()
prefix = '{0}_{1}_'.format(DataObject.NAMESPACE, disk._classname)
keys = self.persistent.prefix(prefix)
for key in keys:
try:
guid = key.replace(prefix, '')
disk = TestDisk(guid)
disk.delete()
except (ObjectNotFoundException, ValueError):
pass
for key in self.persistent.prefix('ovs_reverseindex_{0}'.format(disk._classname)):
self.persistent.delete(key)
def test_mandatory_fields(self):
"""
Validates whether mandatory properties and relations work
"""
machine = TestMachine()
machine.extended = 'extended'
machine.name = 'machine'
machine.save()
disk = TestDisk()
# Modify relation to mandatory
[_ for _ in disk._relations if _.name == 'machine'][0].mandatory = True
# Continue test
disk.name = None
with self.assertRaises(MissingMandatoryFieldsException) as exception:
disk.save()
self.assertIn('name', exception.exception.message, 'Field name should be in exception message: {0}'.format(exception.exception.message))
self.assertIn('machine', exception.exception.message, 'Field machine should be in exception message: {0}'.format(exception.exception.message))
disk.name = 'disk'
disk.machine = machine
disk.save()
disk.description = 'test'
disk.storage = machine
disk.save()
# Restore relation
[_ for _ in disk._relations if _.name == 'machine'][0].mandatory = False
def test_datalistactions(self):
"""
Validates all actions that can be executed agains DataLists
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk1 = TestDisk()
disk1.name = 'disk1'
disk1.machine = machine
disk1.save()
disk2 = TestDisk()
disk2.name = 'disk2'
disk2.machine = machine
disk2.save()
disk3 = TestDisk()
disk3.name = 'disk3'
disk3.machine = machine
disk3.save()
self.assertEqual(machine.disks.count(disk1), 1, 'Disk should be available only once')
self.assertGreaterEqual(machine.disks.index(disk1), 0, 'We should retreive an index')
machine.disks.sort()
guid = machine.disks[0].guid
machine.disks.reverse()
self.assertEqual(machine.disks[-1].guid, guid, 'Reverse and sort should work')
machine.disks.sort()
self.assertEqual(machine.disks[0].guid, guid, 'And the guid should be first again')
def test_relationcache(self):
"""
Validates whether the relational properties are cached correctly, and whether
they are invalidated when required
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk1 = TestDisk()
disk1.name = 'disk1'
disk1.save()
disk2 = TestDisk()
disk2.name = 'disk2'
disk2.save()
disk3 = TestDisk()
disk3.name = 'disk3'
disk3.save()
self.assertEqual(len(machine.disks), 0, 'There should be no disks on the machine')
disk1.machine = machine
disk1.save()
self.assertEqual(len(machine.disks), 1, 'There should be 1 disks on the machine')
disk2.machine = machine
disk2.save()
self.assertEqual(len(machine.disks), 2, 'There should be 2 disks on the machine')
disk3.machine = machine
disk3.save()
self.assertEqual(len(machine.disks), 3, 'There should be 3 disks on the machine')
machine.disks[0].name = 'disk1_'
machine.disks[1].name = 'disk2_'
machine.disks[2].name = 'disk3_'
disk1.machine = None
disk1.save()
disk2.machine = None
disk2.save()
self.assertEqual(len(machine.disks), 1, 'There should be 1 disks on the machine')
def test_invalidqueries(self):
"""
Validates invalid queries
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk = TestDisk()
disk.name = 'disk'
disk.machine = machine
disk.save()
setattr(DataList.select, 'SOMETHING', 'SOMETHING')
with self.assertRaises(NotImplementedError):
DataList({'object': TestDisk,
'data': DataList.select.SOMETHING,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
setattr(DataList.where_operator, 'SOMETHING', 'SOMETHING')
with self.assertRaises(NotImplementedError):
DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.SOMETHING,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
setattr(DataList.operator, 'SOMETHING', 'SOMETHING')
with self.assertRaises(NotImplementedError):
DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.SOMETHING, 'machine')]}}) # noqa
def test_copy(self):
"""
Validates whether the copy function works correct
"""
machine = TestMachine()
machine.name = 'testmachine1'
machine.save()
disk1 = TestDisk()
disk1.name = 'test1'
disk1.size = 100
disk1.order = 1
disk1.type = 'ONE'
disk1.machine = machine
disk1.save()
disk2 = TestDisk()
disk2.copy(disk1)
self.assertEqual(disk2.name, 'test1', 'Properties should be copied')
self.assertEqual(disk2.size, 100, 'Properties should be copied')
self.assertEqual(disk2.order, 1, 'Properties should be copied')
self.assertEqual(disk2.type, 'ONE', 'Properties should be copied')
self.assertEqual(disk2.machine, None, 'Relations should not be copied')
disk3 = TestDisk()
disk3.copy(disk1, include_relations=True)
self.assertEqual(disk3.machine.name, 'testmachine1', 'Relations should be copied')
disk4 = TestDisk()
disk4.copy(disk1, include=['name'])
self.assertEqual(disk4.name, 'test1', 'Name should be copied')
self.assertEqual(disk4.size, 0, 'Size should not be copied')
self.assertEqual(disk4.machine, None, 'Relations should not be copied')
disk5 = TestDisk()
disk5.copy(disk1, exclude=['name'])
self.assertEqual(disk5.name, None, 'Name should not be copied')
self.assertEqual(disk5.size, 100, 'Size should be copied')
self.assertEqual(disk5.machine, None, 'Relations should not be copied')
def test_saveorder(self):
"""
Validates whether the order of saving related objects doesn't matter
"""
machine1 = TestMachine()
machine1.name = 'machine'
disk1_1 = TestDisk()
disk1_1.name = 'disk1'
disk1_1.machine = machine1
disk1_1.save()
disk1_2 = TestDisk()
disk1_2.name = 'disk2'
disk1_2.machine = machine1
disk1_2.save()
machine1.save()
self.assertEqual(len(machine1.disks), 2, 'There should be two disks. {0}'.format(len(machine1.disks)))
machine2 = TestMachine()
machine2.name = 'machine'
machine2.save()
disk2_1 = TestDisk()
disk2_1.name = 'disk1'
disk2_1.machine = machine2
disk2_1.save()
disk2_2 = TestDisk()
disk2_2.name = 'disk2'
disk2_2.machine = machine2
disk2_2.save()
self.assertEqual(len(machine2.disks), 2, 'There should be two disks. {0}'.format(len(machine2.disks)))
def test_guid_query(self):
"""
Validates whether queries can use the _guid fields
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk = TestDisk()
disk.name = 'test'
disk.machine = machine
disk.save()
data = DataList({'object': TestDisk,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': [('machine_guid', DataList.operator.EQUALS, machine.guid)]}}).data
disks = DataObjectList(data, TestDisk)
self.assertEqual(len(disks), 1, 'There should be one disk ({0})'.format(len(disks)))
def test_invalidonetoone(self):
"""
Validates that if a one-to-one is used as a one-to-many an exception will be raised
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
self.assertIsNone(machine.one, 'There should not be any disk(s)')
disk1 = TestDisk()
disk1.name = 'disk1'
disk1.one = machine
disk1.save()
self.assertEqual(machine.one, disk1, 'The correct disk should be returned')
disk2 = TestDisk()
disk2.name = 'disk2'
disk2.one = machine
disk2.save()
with self.assertRaises(InvalidRelationException):
_ = machine.one
def test_outdated_listobjects(self):
"""
Validates whether elements in a (cached) list are reloaded if they are changed externally
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk1 = TestDisk()
disk1.name = 'disk1'
disk1.machine = machine
disk1.save()
disk2 = TestDisk()
disk2.name = 'disk2'
disk2.machine = machine
disk2.save()
self.assertListEqual(['disk1', 'disk2'], sorted([disk.name for disk in machine.disks]), 'Names should be disk1 and disk2')
disk2.name = 'disk_'
self.assertListEqual(['disk1', 'disk2'], sorted([disk.name for disk in machine.disks]), 'Names should still be disk1 and disk2')
disk2.save()
self.assertListEqual(['disk1', 'disk_'], sorted([disk.name for disk in machine.disks]), 'Names should be disk1 and disk_')
def test_listcache(self):
"""
Validates whether lists are cached and invalidated correctly
"""
keys = ['list_cache', None]
for key in keys:
disk0 = TestDisk()
disk0.name = 'disk 0'
disk0.save()
list_cache = DataList(key=key,
query={'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key))
self.assertEqual(list_cache.data, 0, 'List should find no entries (mode: {0})'.format(key))
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk1 = TestDisk()
disk1.name = 'disk 1'
disk1.machine = machine
disk1.save()
list_cache = DataList(key=key,
query={'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key))
self.assertEqual(list_cache.data, 1, 'List should find one entry (mode: {0})'.format(key))
list_cache = DataList(key=key,
query={'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
self.assertTrue(list_cache.from_cache, 'List should be loaded from cache (mode: {0})'.format(key))
disk2 = TestDisk()
disk2.machine = machine
disk2.name = 'disk 2'
disk2.save()
list_cache = DataList(key=key,
query={'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key))
self.assertEqual(list_cache.data, 2, 'List should find two entries (mode: {0})'.format(key))
machine.name = 'x'
machine.save()
list_cache = DataList(key=key,
query={'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa
self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key))
self.assertEqual(list_cache.data, 0, 'List should have no matches (mode: {0})'.format(key))
def test_serialization(self):
"""
Validates whether serialization works as expected
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk = TestDisk()
disk.name = 'disk'
disk.machine = machine
disk.save()
dictionary = disk.serialize()
self.assertIn('name', dictionary, 'Serialized object should have correct properties')
self.assertEqual(dictionary['name'], 'disk', 'Serialized object should have correct name')
self.assertIn('machine_guid', dictionary, 'Serialized object should have correct depth')
self.assertEqual(dictionary['machine_guid'], machine.guid,
'Serialized object should have correct properties')
dictionary = disk.serialize(depth=1)
self.assertIn('machine', dictionary, 'Serialized object should have correct depth')
self.assertEqual(dictionary['machine']['name'], 'machine',
'Serialized object should have correct properties at all depths')
def _clean_all(self):
"""
Cleans all disks and machines
"""
machine = TestMachine()
prefix = '{0}_{1}_'.format(DataObject.NAMESPACE, machine._classname)
keys = self.persistent.prefix(prefix)
for key in keys:
try:
guid = key.replace(prefix, '')
machine = TestMachine(guid)
for disk in machine.disks:
disk.delete()
machine.delete()
except (ObjectNotFoundException, ValueError):
pass
for prefix in ['ovs_reverseindex_{0}', 'ovs_unique_{0}', 'ovs_index_{0}']:
for key in self.persistent.prefix(prefix.format(machine._classname)):
self.persistent.delete(key)
disk = TestDisk()
prefix = '{0}_{1}_'.format(DataObject.NAMESPACE, disk._classname)
keys = self.persistent.prefix(prefix)
for key in keys:
try:
guid = key.replace(prefix, '')
disk = TestDisk(guid)
disk.delete()
except (ObjectNotFoundException, ValueError):
pass
for prefix in ['ovs_reverseindex_{0}', 'ovs_unique_{0}', 'ovs_index_{0}']:
for key in self.persistent.prefix(prefix.format(disk._classname)):
self.persistent.delete(key)
def test_recursive(self):
"""
Validates the recursive save
"""
machine = TestMachine()
machine.name = 'original'
machine.save()
disks = []
for i in xrange(0, 10):
disk = TestDisk()
disk.name = 'test_{0}'.format(i)
if i % 2:
disk.machine = machine
else:
disk.machine = machine
self.assertEqual(disk.machine.name, 'original', 'child should be set')
disk.machine = None
self.assertIsNone(disk.machine, 'child should be cleared')
disks.append(disk)
disk.save()
counter = 1
for disk in machine.disks:
disk.size = counter
counter += 1
machine.save(recursive=True)
disk = TestDisk(machine.disks[0].guid)
self.assertEqual(disk.size, 1, 'lists should be saved recursively')
disk.machine.name = 'mtest'
disk.save(recursive=True)
machine2 = TestMachine(machine.guid)
self.assertEqual(machine2.disks[1].size, 2, 'lists should be saved recursively')
self.assertEqual(machine2.name, 'mtest', 'properties should be saved recursively')
def _clean_all(self):
"""
Cleans all disks and machines
"""
machine = TestMachine()
keys = DataList.get_pks(machine._namespace, machine._classname)
for guid in keys:
try:
machine = TestMachine(guid)
for disk in machine.disks:
disk.delete()
machine.delete()
except (ObjectNotFoundException, ValueError):
pass
disk = TestDisk()
keys = DataList.get_pks(disk._namespace, disk._classname)
for guid in keys:
try:
disk = TestDisk(guid)
disk.delete()
except (ObjectNotFoundException, ValueError):
pass
def test_extended_filter(self):
"""
Validates whether base and extended hybrids behave the same in lists
"""
machine1 = TestMachine()
machine1.name = 'basic'
machine1.save()
machine2 = TestEMachine()
machine2.name = 'extended'
machine2.save()
data = DataList({'object': TestMachine,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
datalist = DataObjectList(data, TestMachine)
self.assertEqual(len(datalist), 2, 'There should be two machines if searched for TestMachine ({0})'.format(len(datalist)))
data = DataList({'object': TestEMachine,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
datalist = DataObjectList(data, TestMachine)
self.assertEqual(len(datalist), 2, 'There should be two machines if searched for TestEMachine ({0})'.format(len(datalist)))
def test_1_to_1(self):
"""
Validates whether 1-to-1 relations work correct
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
self.assertIsNone(machine.one, 'The machine should not have a reverse disk relation')
self.assertIsNone(machine.one_guid, 'The machine should have an empty disk _guid property')
disk = TestDisk()
disk.name = 'test'
disk.one = machine
disk.save()
self.assertIsNotNone(machine.one, 'The machine should have a reverse disk relation')
self.assertEqual(machine.one.name, 'test', 'The reverse 1-to-1 relation should work')
self.assertEqual(disk.one.name, 'machine', 'The normal 1-to-1 relation should work')
self.assertEqual(machine.one_guid, disk.guid, 'The reverse disk should be the correct one')
with self.assertRaises(RuntimeError):
machine.one = disk
def test_relation_inheritance(self):
"""
Validates whether relations on inherited hybrids behave OK
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk = TestDisk()
disk.name = 'disk'
disk.machine = machine # Validates relation acceptance (accepts TestEMachine)
disk.save()
machine.the_disk = disk # Validates whether _relations is build correctly
machine.save()
disk2 = TestDisk(disk.guid)
self.assertEqual(Descriptor(disk2.machine.__class__), Descriptor(TestEMachine), 'The machine should be a TestEMachine')
def _clean_all(self):
"""
Cleans all disks and machines
"""
machine = TestMachine()
keys = DataList.get_pks(machine._namespace, machine._classname)
for guid in keys:
try:
machine = TestMachine(guid)
for disk in machine.disks:
disk.delete()
machine.delete()
except (ObjectNotFoundException, ValueError):
pass
disk = TestDisk()
keys = DataList.get_pks(disk._namespace, disk._classname)
for guid in keys:
try:
disk = TestDisk(guid)
disk.delete()
except (ObjectNotFoundException, ValueError):
pass
def test_delete_abandoning(self):
"""
Validates the abandoning behavior of the delete method
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
disk_1 = TestDisk()
disk_1.name = 'disk 1'
disk_1.machine = machine
disk_1.save()
disk_2 = TestDisk()
disk_2.name = 'disk 2'
disk_2.machine = machine
disk_2.save()
self.assertRaises(LinkedObjectException, machine.delete)
disk_3 = TestDisk(disk_1.guid)
self.assertIsNotNone(disk_3.machine, 'The machine should still be linked')
_ = machine.disks # Make sure we loaded the list
disk_2.delete()
machine.delete(abandon=True) # Should not raise due to disk_2 being deleted
disk_4 = TestDisk(disk_1.guid)
self.assertIsNone(disk_4.machine, 'The machine should be unlinked')
def test_pk_stretching(self):
"""
Validates whether the primary key lists scale correctly.
* X entries will be added (e.g. 10000)
* X/2 random entries will be deleted (5000)
* X/2 entries will be added again (5000)
* X entries will be removed (10000)
No entries should be remaining
"""
print ''
print 'starting test'
amount_of_objects = 10000 # Must be an even number!
machine = TestMachine()
runtimes = []
# First fill
start = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)),
0, 'There should be no primary keys yet ({0})'.format(
len(list(keys))))
guids = []
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - start)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# First delete
amount_of_objects /= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), amount_of_objects,
'There should be {0} primary keys ({1})'.format(
amount_of_objects, len(list(keys))))
print ''
# Second round
sstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - sstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# Second delete
amount_of_objects *= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(list(keys)), len(guids),
'There should be {0} primary keys instead of {1}'.format(
len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
'* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.
format(i + 1, int(amount_of_objects),
round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(
len(guids), 0,
'All guids should be removed. {0} left'.format(len(guids)))
self.assertEqual(
len(list(keys)), 0,
'There should be no primary keys ({0})'.format(len(list(keys))))
seconds_passed = (time.time() - start)
runtimes.sort()
print '\ncompleted in {0} seconds (avg: {1} ops, min: {2} ops, max: {3} ops)'.format(
round(seconds_passed, 2),
round((amount_of_objects * 3) / seconds_passed, 2),
round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ""
print "cleaning up"
self._clean_all()
print "start test"
tstart = time.time()
if getattr(LotsOfObjects, "amount_of_machines", None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, "amount_of_disks", None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
if load_data:
print "start loading data"
start = time.time()
mguids = []
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = "machine_{0}".format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = "disk_{0}_{1}".format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
"* machine {0}/{1} (run: {2} dps, avg: {3} dps)".format(
i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)
)
)
runtimes.sort()
print "\nloading done ({0}s). min: {1} dps, max: {2} dps".format(
round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)
)
test_queries = True
if test_queries:
print "start queries"
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
self.assertEqual(
len(machine.disks),
LotsOfObjects.amount_of_disks,
"Not all disks were retreived ({0})".format(len(machine.disks)),
)
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
"* machine {0}/{1} (run: {2} dps, avg: {3} dps)".format(
i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)
)
)
runtimes.sort()
print "\ncompleted ({0}s). min: {1} dps, max: {2} dps".format(
round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)
)
print "start full query on disk property"
start = time.time()
amount = DataList(
{
"object": TestDisk,
"data": DataList.select.COUNT,
"query": {
"type": DataList.where_operator.AND,
"items": [
("size", DataList.operator.GT, 100),
("size", DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100),
],
},
}
).data
self.assertEqual(
amount,
(LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines,
"Incorrect amount of disks. Found {0} instead of {1}".format(
amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)
),
)
seconds_passed = time.time() - start
print "completed ({0}s) in {1} seconds (avg: {2} dps)".format(
round(time.time() - tstart, 2),
round(seconds_passed, 2),
round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2),
)
clean_data = True
if clean_data:
print "cleaning up"
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress(
"* machine {0}/{1} (run: {2} dps, avg: {3} dps)".format(
i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)
)
)
runtimes.sort()
print "\ncompleted ({0}s). min: {1} dps, max: {2} dps".format(
round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)
)
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
if load_data:
print 'start loading data'
start = time.time()
mguids = []
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
test_queries = True
if test_queries:
print 'start queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
self.assertEqual(len(machine.disks), LotsOfObjects.amount_of_disks, 'Not all disks were retreived ({0})'.format(len(machine.disks)))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
print 'start full query on disk property'
start = time.time()
amount = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}}).data
self.assertEqual(amount, (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print 'cleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
if load_data:
print 'start loading data'
start = time.time()
mguids = []
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
test_queries = True
if test_queries:
print 'start queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
self.assertEqual(len(machine.disks), LotsOfObjects.amount_of_disks, 'Not all disks were retreived ({0})'.format(len(machine.disks)))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
print 'start full query on disk property'
start = time.time()
amount = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}}).data
self.assertEqual(amount, (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print 'cleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
mguids = []
if load_data:
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (
time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() -
mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress(
'* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(
i + 1, int(LotsOfObjects.amount_of_machines),
round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(
round(time.time() - tstart, 2), round(runtimes[1], 2),
round(runtimes[-2], 2))
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(
machine.disks
) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(
len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (
time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() -
mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress(
'* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(
i + 1, int(LotsOfObjects.amount_of_machines),
round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(
round(time.time() - tstart, 2), round(runtimes[1], 2),
round(runtimes[-2], 2))
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(
TestDisk, {
'type':
DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT,
(LotsOfObjects.amount_of_disks - 1) * 100)]
})
amount = len(dlist)
assert amount == (
LotsOfObjects.amount_of_disks - 3
) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(
amount,
int((LotsOfObjects.amount_of_disks - 3) *
LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nloading disks (all)'
start = time.time()
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(
TestDisk, {
'type':
DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT,
(LotsOfObjects.amount_of_disks - 1) * 100)]
})
amount = len(dlist)
assert amount == (
LotsOfObjects.amount_of_disks - 3
) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(
amount,
int((LotsOfObjects.amount_of_disks - 3) *
LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart property sort'
dlist = DataList(TestDisk, {
'type': DataList.where_operator.AND,
'items': []
})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {
'type': DataList.where_operator.AND,
'items': []
})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (
time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() -
mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress(
'* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(
i + 1, int(LotsOfObjects.amount_of_machines),
round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(
round(time.time() - tstart, 2), round(runtimes[1], 2),
round(runtimes[-2], 2))
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.factory = None
PersistentFactory.get_client().clean()
VolatileFactory.get_client().clean()
admin_group = Group()
admin_group.name = 'administrators'
admin_group.description = 'Administrators'
admin_group.save()
viewers_group = Group()
viewers_group.name = 'viewers'
viewers_group.description = 'Viewers'
viewers_group.save()
# Create users
admin = User()
admin.username = '******'
admin.password = hashlib.sha256('admin').hexdigest()
admin.is_active = True
admin.group = admin_group
admin.save()
admin_npg = User()
admin_npg.username = '******'
admin_npg.password = hashlib.sha256('admin_npg').hexdigest()
admin_npg.is_active = True
admin_npg.group = admin_group
admin_npg.save()
admin_na = User()
admin_na.username = '******'
admin_na.password = hashlib.sha256('admin_na').hexdigest()
admin_na.is_active = False
admin_na.group = admin_group
admin_na.save()
user = User()
user.username = '******'
user.password = hashlib.sha256('user').hexdigest()
user.is_active = True
user.group = viewers_group
user.save()
sort_combinations = [('bb', 'aa'), ('aa', 'cc'), ('bb', 'dd'), ('aa', 'bb')] # No logical ordering
for name, description in sort_combinations:
machine = TestMachine()
machine.name = name
machine.description = description
machine.save()
# Create internal OAuth 2 clients
admin_client = Client()
admin_client.ovs_type = 'INTERNAL'
admin_client.grant_type = 'PASSWORD'
admin_client.user = admin
admin_client.save()
admin_na_client = Client()
admin_na_client.ovs_type = 'INTERNAL'
admin_na_client.grant_type = 'PASSWORD'
admin_na_client.user = admin_na
admin_na_client.save()
user_client = Client()
user_client.ovs_type = 'INTERNAL'
user_client.grant_type = 'PASSWORD'
user_client.user = user
user_client.save()
# Create roles
read_role = Role()
read_role.code = 'read'
read_role.name = 'Read'
read_role.description = 'Can read objects'
read_role.save()
write_role = Role()
write_role.code = 'write'
write_role.name = 'Write'
write_role.description = 'Can write objects'
write_role.save()
manage_role = Role()
manage_role.code = 'manage'
manage_role.name = 'Manage'
manage_role.description = 'Can manage the system'
manage_role.save()
# Attach groups to roles
mapping = [
(admin_group, [read_role, write_role, manage_role]),
(viewers_group, [read_role])
]
for setting in mapping:
for role in setting[1]:
rolegroup = RoleGroup()
rolegroup.group = setting[0]
rolegroup.role = role
rolegroup.save()
for user in setting[0].users:
for role in setting[1]:
for client in user.clients:
roleclient = RoleClient()
roleclient.client = client
roleclient.role = role
roleclient.save()
from django.conf import settings
settings.VERSION = (1, 2, 3)
from django.test import RequestFactory
cls.factory = RequestFactory()
fakesleep.monkey_patch()
def test_versioning(self):
"""
Validates whether the versioning system works
"""
machine = TestMachine()
machine.name = 'machine0'
machine.save()
self.assertEqual(machine._data['_version'], 1, 'Version should be 1, is {0}'.format(machine._data['_version']))
machine.save()
self.assertEqual(machine._data['_version'], 2, 'Version should be 2, is {0}'.format(machine._data['_version']))
machine_x = TestMachine(machine.guid)
machine_x.name = 'machine1'
machine_x.save()
self.assertTrue(machine.updated_on_datastore(), 'Machine is updated on datastore')
machine.name = 'machine2'
machine.save()
self.assertEqual(machine._data['_version'], 4, 'Version should be 4, is {0}'.format(machine._data['_version']))
self.assertFalse(machine.updated_on_datastore(), 'Machine is not updated on datastore')
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'preparing'
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
else:
LotsOfObjects.amount_of_machines = int(LotsOfObjects.amount_of_machines)
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
else:
LotsOfObjects.amount_of_disks = int(LotsOfObjects.amount_of_disks)
if getattr(LotsOfObjects, 'repetition_scan', None) is None:
LotsOfObjects.repetition_scan = 32
else:
LotsOfObjects.repetition_scan = int(LotsOfObjects.repetition_scan)
total_amount_of_disks = LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks
mguids = []
uuids = [str(uuid.uuid4()) for _ in xrange(total_amount_of_disks)]
counter = 0
repetition = []
for i in xrange(LotsOfObjects.repetition_scan):
repetition.append([])
dguids = []
print 'start test'
tstart = time.time()
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(LotsOfObjects.amount_of_machines):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(LotsOfObjects.amount_of_disks):
current_uuid = uuids[counter]
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.description = 'disk_{0}'.format(i)
disk.size = ii * 100
disk.machine = machine
disk.something = current_uuid
disk.save()
dguids.append(disk.guid)
random.choice(repetition).append(current_uuid)
counter += 1
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\nloading done ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(LotsOfObjects.amount_of_machines):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nloading disks (all)'
start = time.time()
for i in xrange(LotsOfObjects.amount_of_machines):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nindexed single query'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('something', DataList.operator.EQUALS, repetition[0][0])]})
start = time.time()
assert len(dlist) == 1, 'One disk should be found'
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.3f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart property sort'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'size'))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'predictable'))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nrepetition scan'
start = time.time()
times = []
for i in xrange(LotsOfObjects.repetition_scan):
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('something', DataList.operator.IN, repetition[i])]})
run_start = time.time()
assert len(repetition[i]) == len(dlist), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(repetition[i]))
times.append(time.time() - run_start)
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (run avg: {2:.3f}s, avg: {3:.2f} dps)'.format(time.time() - tstart, seconds_passed, sum(times) / float(LotsOfObjects.repetition_scan), LotsOfObjects.repetition_scan * total_amount_of_disks / seconds_passed)
print '\nguid index query'
start = time.time()
guids = dguids[len(dguids)/2:]
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('guid', DataList.operator.IN, guids)]})
assert len(dlist) == len(guids), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(guids))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
def test_queries(self):
"""
Validates whether executing queries returns the expected results
"""
machine = TestMachine()
machine.name = 'machine'
machine.save()
for i in xrange(0, 20):
disk = TestDisk()
disk.name = 'test_{0}'.format(i)
disk.size = i
if i < 10:
disk.machine = machine
else:
disk.storage = machine
disk.save()
self.assertEqual(len(machine.disks), 10, 'query should find added machines')
# pylint: disable=line-too-long
list_1 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.EQUALS, 1)]}}).data # noqa
self.assertEqual(list_1, 1, 'list should contain int 1')
list_2 = DataList({'object': TestDisk,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.EQUALS, 1)]}}).data # noqa
found_object = Descriptor(TestDisk, list_2[0]).get_object(True)
self.assertEqual(found_object.name, 'test_1', 'list should contain correct machine')
list_3 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 3),
('size', DataList.operator.LT, 6)]}}).data # noqa
self.assertEqual(list_3, 2, 'list should contain int 2') # disk 4 and 5
list_4 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.OR,
'items': [('size', DataList.operator.LT, 3),
('size', DataList.operator.GT, 6)]}}).data # noqa
# at least disk 0, 1, 2, 7, 8, 9, 10-19
self.assertGreaterEqual(list_4, 16, 'list should contain >= 16')
list_5 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.guid', DataList.operator.EQUALS, machine.guid), # noqa
{'type': DataList.where_operator.OR,
'items': [('size', DataList.operator.LT, 3),
('size', DataList.operator.GT, 6)]}]}}).data # noqa
self.assertEqual(list_5, 6, 'list should contain int 6') # disk 0, 1, 2, 7, 8, 9
list_6 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.LT, 3),
('size', DataList.operator.GT, 6)]}}).data # noqa
self.assertEqual(list_6, 0, 'list should contain int 0') # no disks
list_7 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.OR,
'items': [('machine.guid', DataList.operator.EQUALS, '123'), # noqa
('used_size', DataList.operator.EQUALS, -1),
{'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 3),
('size', DataList.operator.LT, 6)]}]}}).data # noqa
self.assertEqual(list_7, 2, 'list should contain int 2') # disk 4 and 5
list_8 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('machine.name', DataList.operator.EQUALS, 'machine'), # noqa
('name', DataList.operator.EQUALS, 'test_3')]}}).data # noqa
self.assertEqual(list_8, 1, 'list should contain int 1') # disk 3
list_9 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 3),
{'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.LT, 6)]}]}}).data # noqa
self.assertEqual(list_9, 2, 'list should contain int 2') # disk 4 and 5
list_10 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.OR,
'items': [('size', DataList.operator.LT, 3),
{'type': DataList.where_operator.OR,
'items': [('size', DataList.operator.GT, 6)]}]}}).data # noqa
# at least disk 0, 1, 2, 7, 8, 9, 10-19
self.assertGreaterEqual(list_10, 16, 'list should contain >= 16')
list_11 = DataList({'object': TestDisk,
'data': DataList.select.COUNT,
'query': {'type': DataList.where_operator.AND,
'items': [('storage.name', DataList.operator.EQUALS, 'machine')]}}).data # noqa
self.assertEqual(list_11, 10, 'list should contain int 10') # disk 10-19
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
mguids = []
if load_data:
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nloading disks (all)'
start = time.time()
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart property sort'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
DalHelper.setup(fake_sleep=True)
admin_group = Group()
admin_group.name = 'administrators'
admin_group.description = 'Administrators'
admin_group.save()
viewers_group = Group()
viewers_group.name = 'viewers'
viewers_group.description = 'Viewers'
viewers_group.save()
# Create users
admin = User()
admin.username = '******'
admin.password = hashlib.sha256('admin').hexdigest()
admin.is_active = True
admin.group = admin_group
admin.save()
admin_npg = User()
admin_npg.username = '******'
admin_npg.password = hashlib.sha256('admin_npg').hexdigest()
admin_npg.is_active = True
admin_npg.group = admin_group
admin_npg.save()
admin_na = User()
admin_na.username = '******'
admin_na.password = hashlib.sha256('admin_na').hexdigest()
admin_na.is_active = False
admin_na.group = admin_group
admin_na.save()
user = User()
user.username = '******'
user.password = hashlib.sha256('user').hexdigest()
user.is_active = True
user.group = viewers_group
user.save()
sort_combinations = [('bb', 'aa'), ('aa', 'cc'), ('bb', 'dd'),
('aa', 'bb')] # No logical ordering
for name, description in sort_combinations:
machine = TestMachine()
machine.name = name
machine.description = description
machine.save()
# Create internal OAuth 2 clients
admin_client = Client()
admin_client.ovs_type = 'INTERNAL'
admin_client.grant_type = 'PASSWORD'
admin_client.user = admin
admin_client.save()
admin_na_client = Client()
admin_na_client.ovs_type = 'INTERNAL'
admin_na_client.grant_type = 'PASSWORD'
admin_na_client.user = admin_na
admin_na_client.save()
user_client = Client()
user_client.ovs_type = 'INTERNAL'
user_client.grant_type = 'PASSWORD'
user_client.user = user
user_client.save()
# Create roles
read_role = Role()
read_role.code = 'read'
read_role.name = 'Read'
read_role.description = 'Can read objects'
read_role.save()
write_role = Role()
write_role.code = 'write'
write_role.name = 'Write'
write_role.description = 'Can write objects'
write_role.save()
manage_role = Role()
manage_role.code = 'manage'
manage_role.name = 'Manage'
manage_role.description = 'Can manage the system'
manage_role.save()
# Attach groups to roles
mapping = [(admin_group, [read_role, write_role, manage_role]),
(viewers_group, [read_role])]
for setting in mapping:
for role in setting[1]:
rolegroup = RoleGroup()
rolegroup.group = setting[0]
rolegroup.role = role
rolegroup.save()
for user in setting[0].users:
for role in setting[1]:
for client in user.clients:
roleclient = RoleClient()
roleclient.client = client
roleclient.role = role
roleclient.save()
from django.conf import settings
settings.VERSION = (1, 2, 3)
from django.test import RequestFactory
cls.factory = RequestFactory()
