def test_datalistactions(self): """ Validates all actions that can be executed agains DataLists """ machine = TestMachine() machine.name = 'machine' machine.save() disk1 = TestDisk() disk1.name = 'disk1' disk1.machine = machine disk1.save() disk2 = TestDisk() disk2.name = 'disk2' disk2.machine = machine disk2.save() disk3 = TestDisk() disk3.name = 'disk3' disk3.machine = machine disk3.save() self.assertEqual(machine.disks.count(disk1), 1, 'Disk should be available only once') self.assertGreaterEqual(machine.disks.index(disk1), 0, 'We should retreive an index') machine.disks.sort() guid = machine.disks[0].guid machine.disks.reverse() self.assertEqual(machine.disks[-1].guid, guid, 'Reverse and sort should work') machine.disks.sort() self.assertEqual(machine.disks[0].guid, guid, 'And the guid should be first again')
def test_ownrelations(self): """ Validates whether relations to the object itself are working """ pdisk = TestDisk() pdisk.name = 'parent' pdisk.save() cdisk1 = TestDisk() cdisk1.name = 'child 1' cdisk1.size = 100 cdisk1.parent = pdisk cdisk1.save() cdisk2 = TestDisk() cdisk2.name = 'child 2' cdisk2.size = 100 cdisk2.parent = pdisk cdisk2.save() self.assertEqual(len(pdisk.children), 2, 'There should be 2 children ({0})'.format(len(pdisk.children))) self.assertEqual(cdisk1.parent.name, 'parent', 'Parent should be loaded correctly') data = DataList({'object': TestDisk, 'data': DataList.select.GUIDS, 'query': {'type': DataList.where_operator.AND, 'items': [('parent.name', DataList.operator.EQUALS, 'parent')]}}).data datalist = DataObjectList(data, TestDisk) self.assertEqual(len(datalist), 2, 'There should be two items ({0})'.format(len(datalist))) cdisk2.parent = None cdisk2.save() data = DataList({'object': TestDisk, 'data': DataList.select.GUIDS, 'query': {'type': DataList.where_operator.AND, 'items': [('parent.name', DataList.operator.EQUALS, 'parent')]}}).data datalist = DataObjectList(data, TestDisk) self.assertEqual(len(datalist), 1, 'There should be one item ({0})'.format(len(datalist)))
def test_saveorder(self): """ Validates whether the order of saving related objects doesn't matter """ machine1 = TestMachine() machine1.name = 'machine' disk1_1 = TestDisk() disk1_1.name = 'disk1' disk1_1.machine = machine1 disk1_1.save() disk1_2 = TestDisk() disk1_2.name = 'disk2' disk1_2.machine = machine1 disk1_2.save() machine1.save() self.assertEqual(len(machine1.disks), 2, 'There should be two disks. {0}'.format(len(machine1.disks))) machine2 = TestMachine() machine2.name = 'machine' machine2.save() disk2_1 = TestDisk() disk2_1.name = 'disk1' disk2_1.machine = machine2 disk2_1.save() disk2_2 = TestDisk() disk2_2.name = 'disk2' disk2_2.machine = machine2 disk2_2.save() self.assertEqual(len(machine2.disks), 2, 'There should be two disks. {0}'.format(len(machine2.disks)))
def test_mandatory_fields(self): """ Validates whether mandatory properties and relations work """ machine = TestMachine() machine.extended = 'extended' machine.name = 'machine' machine.save() disk = TestDisk() # Modify relation to mandatory [_ for _ in disk._relations if _.name == 'machine'][0].mandatory = True # Continue test disk.name = None with self.assertRaises(MissingMandatoryFieldsException) as exception: disk.save() self.assertIn('name', exception.exception.message, 'Field name should be in exception message: {0}'.format(exception.exception.message)) self.assertIn('machine', exception.exception.message, 'Field machine should be in exception message: {0}'.format(exception.exception.message)) disk.name = 'disk' disk.machine = machine disk.save() disk.description = 'test' disk.storage = machine disk.save() # Restore relation [_ for _ in disk._relations if _.name == 'machine'][0].mandatory = False
def test_relationcache(self): """ Validates whether the relational properties are cached correctly, and whether they are invalidated when required """ machine = TestMachine() machine.name = 'machine' machine.save() disk1 = TestDisk() disk1.name = 'disk1' disk1.save() disk2 = TestDisk() disk2.name = 'disk2' disk2.save() disk3 = TestDisk() disk3.name = 'disk3' disk3.save() self.assertEqual(len(machine.disks), 0, 'There should be no disks on the machine') disk1.machine = machine disk1.save() self.assertEqual(len(machine.disks), 1, 'There should be 1 disks on the machine') disk2.machine = machine disk2.save() self.assertEqual(len(machine.disks), 2, 'There should be 2 disks on the machine') disk3.machine = machine disk3.save() self.assertEqual(len(machine.disks), 3, 'There should be 3 disks on the machine') machine.disks[0].name = 'disk1_' machine.disks[1].name = 'disk2_' machine.disks[2].name = 'disk3_' disk1.machine = None disk1.save() disk2.machine = None disk2.save() self.assertEqual(len(machine.disks), 1, 'There should be 1 disks on the machine')
def test_nofilterquery(self): """ Validates whether empty queries return the full resultset """ disk1 = TestDisk() disk1.name = 'disk 1' disk1.save() disk2 = TestDisk() disk2.name = 'disk 2' disk2.save() amount = DataList(key='some_list', query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': []}}).data self.assertEqual(amount, 2, 'There should be two disks ({0})'.format(amount)) disk3 = TestDisk() disk3.name = 'disk 3' disk3.save() amount = DataList(key='some_list', query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': []}}).data self.assertEqual(amount, 3, 'There should be three disks ({0})'.format(amount))
def test_listcache(self): """ Validates whether lists are cached and invalidated correctly """ keys = ['list_cache', None] for key in keys: disk0 = TestDisk() disk0.name = 'disk 0' disk0.save() list_cache = DataList(key=key, query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key)) self.assertEqual(list_cache.data, 0, 'List should find no entries (mode: {0})'.format(key)) machine = TestMachine() machine.name = 'machine' machine.save() disk1 = TestDisk() disk1.name = 'disk 1' disk1.machine = machine disk1.save() list_cache = DataList(key=key, query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key)) self.assertEqual(list_cache.data, 1, 'List should find one entry (mode: {0})'.format(key)) list_cache = DataList(key=key, query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa self.assertTrue(list_cache.from_cache, 'List should be loaded from cache (mode: {0})'.format(key)) disk2 = TestDisk() disk2.machine = machine disk2.name = 'disk 2' disk2.save() list_cache = DataList(key=key, query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key)) self.assertEqual(list_cache.data, 2, 'List should find two entries (mode: {0})'.format(key)) machine.name = 'x' machine.save() list_cache = DataList(key=key, query={'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa self.assertFalse(list_cache.from_cache, 'List should not be loaded from cache (mode: {0})'.format(key)) self.assertEqual(list_cache.data, 0, 'List should have no matches (mode: {0})'.format(key))
def test_discard(self): """ Validates the behavior regarding pending changes discard """ disk = TestDisk() disk.name = 'one' disk.save() disk.name = 'two' # Discarding an object should rollback all changes disk.discard() self.assertEqual(disk.name, 'one', 'Data should be discarded')
def test_datastoreraises(self): """ Validates the "datastore_wins" behavior in the usecase where it's supposed to raise """ disk = TestDisk() disk.name = 'initial' disk.save() disk2 = TestDisk(disk.guid, datastore_wins=None) disk.name = 'one' disk.save() disk2.name = 'two' # with datastore_wins set to None, concurrency conflicts are raised self.assertRaises(ConcurrencyException, disk2.save)
def test_datastoreloses(self): """ Validates the "datastore_wins" behavior in the usecase where it loses """ disk = TestDisk() disk.name = 'initial' disk.save() disk2 = TestDisk(disk.guid, datastore_wins=False) disk.name = 'one' disk.save() disk2.name = 'two' disk2.save() # With datastore_wins set to False, the datastore loses concurrency conflicts self.assertEqual(disk2.name, 'two', 'Data should not be overwritten')
def test_invalidqueries(self): """ Validates invalid queries """ machine = TestMachine() machine.name = 'machine' machine.save() disk = TestDisk() disk.name = 'disk' disk.machine = machine disk.save() setattr(DataList.select, 'SOMETHING', 'SOMETHING') with self.assertRaises(NotImplementedError): DataList({'object': TestDisk, 'data': DataList.select.SOMETHING, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa setattr(DataList.where_operator, 'SOMETHING', 'SOMETHING') with self.assertRaises(NotImplementedError): DataList({'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.SOMETHING, 'items': [('machine.name', DataList.operator.EQUALS, 'machine')]}}) # noqa setattr(DataList.operator, 'SOMETHING', 'SOMETHING') with self.assertRaises(NotImplementedError): DataList({'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('machine.name', DataList.operator.SOMETHING, 'machine')]}}) # noqa
def test_dol_advanced(self): """ Validates the DataObjectList advanced functions (indexer, sort) """ sizes = [7, 2, 0, 4, 6, 1, 5, 9, 3, 8] guids = [] for i in xrange(0, 10): disk = TestDisk() disk.name = 'disk_{0}'.format(i) disk.size = sizes[i] disk.save() guids.append(disk.guid) data = DataList({'object': TestDisk, 'data': DataList.select.GUIDS, 'query': {'type': DataList.where_operator.AND, 'items': []}}).data disks = DataObjectList(data, TestDisk) disks.sort() guids.sort() self.assertEqual(disks[0].guid, guids[0], 'Disks should be sorted on guid') self.assertEqual(disks[4].guid, guids[4], 'Disks should be sorted on guid') disks.sort(cmp=lambda a, b: a.size - b.size) self.assertEqual(disks[0].size, 0, 'Disks should be sorted on size') self.assertEqual(disks[4].size, 4, 'Disks should be sorted on size') disks.sort(key=lambda a: a.name) self.assertEqual(disks[0].name, 'disk_0', 'Disks should be sorted on name') self.assertEqual(disks[4].name, 'disk_4', 'Disks should be sorted on name') filtered = disks[1:4] self.assertEqual(filtered[0].name, 'disk_1', 'Disks should be properly sliced') self.assertEqual(filtered[2].name, 'disk_3', 'Disks should be properly sliced')
def test_recursive(self): """ Validates the recursive save """ machine = TestMachine() machine.name = 'original' machine.save() disks = [] for i in xrange(0, 10): disk = TestDisk() disk.name = 'test_{0}'.format(i) if i % 2: disk.machine = machine else: disk.machine = machine self.assertEqual(disk.machine.name, 'original', 'child should be set') disk.machine = None self.assertIsNone(disk.machine, 'child should be cleared') disks.append(disk) disk.save() counter = 1 for disk in machine.disks: disk.size = counter counter += 1 machine.save(recursive=True) disk = TestDisk(machine.disks[0].guid) self.assertEqual(disk.size, 1, 'lists should be saved recursively') disk.machine.name = 'mtest' disk.save(recursive=True) machine2 = TestMachine(machine.guid) self.assertEqual(machine2.disks[1].size, 2, 'lists should be saved recursively') self.assertEqual(machine2.name, 'mtest', 'properties should be saved recursively')
def test_copy(self): """ Validates whether the copy function works correct """ machine = TestMachine() machine.name = 'testmachine1' machine.save() disk1 = TestDisk() disk1.name = 'test1' disk1.size = 100 disk1.order = 1 disk1.type = 'ONE' disk1.machine = machine disk1.save() disk2 = TestDisk() disk2.copy(disk1) self.assertEqual(disk2.name, 'test1', 'Properties should be copied') self.assertEqual(disk2.size, 100, 'Properties should be copied') self.assertEqual(disk2.order, 1, 'Properties should be copied') self.assertEqual(disk2.type, 'ONE', 'Properties should be copied') self.assertEqual(disk2.machine, None, 'Relations should not be copied') disk3 = TestDisk() disk3.copy(disk1, include_relations=True) self.assertEqual(disk3.machine.name, 'testmachine1', 'Relations should be copied') disk4 = TestDisk() disk4.copy(disk1, include=['name']) self.assertEqual(disk4.name, 'test1', 'Name should be copied') self.assertEqual(disk4.size, 0, 'Size should not be copied') self.assertEqual(disk4.machine, None, 'Relations should not be copied') disk5 = TestDisk() disk5.copy(disk1, exclude=['name']) self.assertEqual(disk5.name, None, 'Name should not be copied') self.assertEqual(disk5.size, 100, 'Size should be copied') self.assertEqual(disk5.machine, None, 'Relations should not be copied')
def inject_new(datalist_object): """ Creates a new object """ _ = datalist_object disk_x = TestDisk() disk_x.name = 'test' disk_x.save()
def test_silentdatarefresh(self): """ Validates whether the default scenario (datastore_wins=False) will execute silent data refresh """ disk = TestDisk() disk.name = 'initial' disk.save() disk2 = TestDisk(disk.guid, datastore_wins=False) disk.name = 'one' disk.save() disk2.name = 'two' disk2.save() disk.save() # This should not overwrite anything but instead refresh data # With datastore_wins set to False, the datastore loses concurrency conflicts self.assertEqual(disk2.name, 'two', 'Data should not be overwritten') self.assertEqual(disk.name, 'two', 'Data should be refreshed')
def test_save_deleted(self): """ Validates whether saving a previously deleted object raises """ disk = TestDisk() disk.name = 'disk' disk.save() disk.delete() self.assertRaises(ObjectNotFoundException, disk.save, 'Cannot resave a deleted object')
def test_primarykeyvalidation(self): """ Validates whether the passed in key (guid) of an object is validated """ self.assertRaises(ValueError, TestDisk, 'foo', None) disk = TestDisk() # Should not raise disk.name = 'disk' disk.save() _ = TestDisk(disk.guid) # Should not raise
def test_updateproperty(self): """ Validates the behavior regarding updating properties """ disk = TestDisk() disk.name = 'test' disk.description = 'desc' # A property should be writable self.assertIs(disk.name, 'test', 'Property should be updated') self.assertIs(disk.description, 'desc', 'Property should be updated')
def test_clearedcache(self): """ Validates the correct behavior when the volatile cache is cleared """ disk = TestDisk() disk.name = 'somedisk' disk.save() VolatileFactory.store.delete(disk._key) disk2 = TestDisk(disk.guid) self.assertEqual(disk2.name, 'somedisk', 'Disk should be fetched from persistent store')
def test_invalidonetoone(self): """ Validates that if a one-to-one is used as a one-to-many an exception will be raised """ machine = TestMachine() machine.name = 'machine' machine.save() self.assertIsNone(machine.one, 'There should not be any disk(s)') disk1 = TestDisk() disk1.name = 'disk1' disk1.one = machine disk1.save() self.assertEqual(machine.one, disk1, 'The correct disk should be returned') disk2 = TestDisk() disk2.name = 'disk2' disk2.one = machine disk2.save() with self.assertRaises(InvalidRelationException): _ = machine.one
def test_typesafety(self): """ Validates typesafety checking on object properties """ disk = TestDisk() disk.name = 'test' disk.name = u'test' disk.name = None disk.size = 100 disk.size = 100.5 disk.order = 100 with self.assertRaises(TypeError): disk.order = 100.5 with self.assertRaises(TypeError): disk.__dict__['wrong_type_data'] = None disk.wrong_type_data = 'string' _ = disk.wrong_type with self.assertRaises(TypeError): disk.type = 'THREE' disk.type = 'ONE'
def test_outdated_listobjects(self): """ Validates whether elements in a (cached) list are reloaded if they are changed externally """ machine = TestMachine() machine.name = 'machine' machine.save() disk1 = TestDisk() disk1.name = 'disk1' disk1.machine = machine disk1.save() disk2 = TestDisk() disk2.name = 'disk2' disk2.machine = machine disk2.save() self.assertListEqual(['disk1', 'disk2'], sorted([disk.name for disk in machine.disks]), 'Names should be disk1 and disk2') disk2.name = 'disk_' self.assertListEqual(['disk1', 'disk2'], sorted([disk.name for disk in machine.disks]), 'Names should still be disk1 and disk2') disk2.save() self.assertListEqual(['disk1', 'disk_'], sorted([disk.name for disk in machine.disks]), 'Names should be disk1 and disk_')
def test_datapersistent(self): """ Validates whether data is persisted correctly """ disk = TestDisk() guid = disk.guid disk.name = 'test' disk.save() # Retreiving an object should return the data as when it was saved disk2 = TestDisk(guid) self.assertEqual(disk.name, disk2.name, 'Data should be persistent')
def test_newobject_delete(self): """ Validates the behavior on object deletions """ disk = TestDisk() disk.name = 'disk' disk.save() # An object should always have a guid guid = disk.guid self.assertIsNotNone(guid, 'Guid should not be None') # After deleting, the object should not be retreivable disk.delete() self.assertRaises(Exception, TestDisk, guid, None)
def test_querydynamic(self): """ Validates whether a query that queried dynamic properties is never cached """ def get_disks(): return DataList({'object': TestDisk, 'data': DataList.select.GUIDS, 'query': {'type': DataList.where_operator.AND, 'items': [('used_size', DataList.operator.NOT_EQUALS, -1)]}}) disk1 = TestDisk() disk1.name = 'disk 1' disk1.size = 100 disk1.save() disk2 = TestDisk() disk2.name = 'disk 2' disk2.size = 100 disk2.save() query_result = get_disks() self.assertEqual(len(query_result.data), 2, 'There should be 2 disks ({0})'.format(len(query_result.data))) self.assertFalse(query_result.from_cache, 'Disk should not be loaded from cache') query_result = get_disks() self.assertFalse(query_result.from_cache, 'Disk should not be loaded from cache')
def test_primarykeys(self): """ Validates whether the primary keys are kept in sync """ disk = TestDisk() disk.name = 'disk' keys = DataList.get_pks(disk._namespace, disk._name) self.assertEqual(len(keys), 0, 'There should be no primary keys ({0})'.format(len(keys))) disk.save() keys = DataList.get_pks(disk._namespace, disk._name) self.assertEqual(len(keys), 1, 'There should be one primary key ({0})'.format(len(keys))) disk.delete() keys = DataList.get_pks(disk._namespace, disk._name) self.assertEqual(len(keys), 0, 'There should be no primary keys ({0})'.format(len(keys)))
def test_delete_abandoning(self): """ Validates the abandoning behavior of the delete method """ machine = TestMachine() machine.name = 'machine' machine.save() disk_1 = TestDisk() disk_1.name = 'disk 1' disk_1.machine = machine disk_1.save() disk_2 = TestDisk() disk_2.name = 'disk 2' disk_2.machine = machine disk_2.save() self.assertRaises(LinkedObjectException, machine.delete) disk_3 = TestDisk(disk_1.guid) self.assertIsNotNone(disk_3.machine, 'The machine should still be linked') _ = machine.disks # Make sure we loaded the list disk_2.delete() machine.delete(abandon=True) # Should not raise due to disk_2 being deleted disk_4 = TestDisk(disk_1.guid) self.assertIsNone(disk_4.machine, 'The machine should be unlinked')
def test_relation_inheritance(self): """ Validates whether relations on inherited hybrids behave OK """ machine = TestMachine() machine.name = 'machine' machine.save() disk = TestDisk() disk.name = 'disk' disk.machine = machine # Validates relation acceptance (accepts TestEMachine) disk.save() machine.the_disk = disk # Validates whether _relations is build correctly machine.save() disk2 = TestDisk(disk.guid) self.assertEqual(Descriptor(disk2.machine.__class__), Descriptor(TestEMachine), 'The machine should be a TestEMachine')
def test_reduceddatalist(self): """ Validates the reduced list """ disk = TestDisk() disk.name = 'test' disk.save() data = DataList({'object': TestDisk, 'data': DataList.select.GUIDS, 'query': {'type': DataList.where_operator.AND, 'items': []}}).data datalist = DataObjectList(data, TestDisk) self.assertEqual(len(datalist), 1, 'There should be only one item ({0})'.format(len(datalist))) item = datalist.reduced[0] with self.assertRaises(AttributeError): print item.name self.assertEqual(item.guid, disk.guid, 'The guid should be available')
def test_lotsofobjects(self): """ A test creating, linking and querying a lot of objects """ print '' print 'cleaning up' self._clean_all() print 'start test' tstart = time.time() if getattr(LotsOfObjects, 'amount_of_machines', None) is None: LotsOfObjects.amount_of_machines = 50 if getattr(LotsOfObjects, 'amount_of_disks', None) is None: LotsOfObjects.amount_of_disks = 5 load_data = True mguids = [] if load_data: print '\nstart loading data' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine() machine.name = 'machine_{0}'.format(i) machine.save() mguids.append(machine.guid) for ii in xrange(0, int(LotsOfObjects.amount_of_disks)): disk = TestDisk() disk.name = 'disk_{0}_{1}'.format(i, ii) disk.size = ii * 100 disk.machine = machine disk.save() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / ( time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress( '* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format( i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format( round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) test_queries = True if test_queries: print '\nstart queries' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) assert len( machine.disks ) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format( len(machine.disks)) avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / ( time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress( '* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format( i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format( round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) print '\nstart full query on disk property' start = time.time() dlist = DataList( TestDisk, { 'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)] }) amount = len(dlist) assert amount == ( LotsOfObjects.amount_of_disks - 3 ) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format( amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nloading disks (all)' start = time.time() for i in xrange(0, int(LotsOfObjects.amount_of_machines)): machine = TestMachine(mguids[i]) _ = [_ for _ in machine.disks] seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart full query on disk property (using cached objects)' dlist._volatile.delete(dlist._key) start = time.time() dlist = DataList( TestDisk, { 'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)] }) amount = len(dlist) assert amount == ( LotsOfObjects.amount_of_disks - 3 ) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format( amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart property sort' dlist = DataList(TestDisk, { 'type': DataList.where_operator.AND, 'items': [] }) start = time.time() dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size')) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart dynamic sort' dlist._volatile.delete(dlist._key) dlist = DataList(TestDisk, { 'type': DataList.where_operator.AND, 'items': [] }) start = time.time() dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable')) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) clean_data = True if clean_data: print '\ncleaning up' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) for disk in machine.disks: disk.delete() machine.delete() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / ( time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress( '* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format( i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format( round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self): """ A test creating, linking and querying a lot of objects """ print '' print 'cleaning up' self._clean_all() print 'preparing' if getattr(LotsOfObjects, 'amount_of_machines', None) is None: LotsOfObjects.amount_of_machines = 50 else: LotsOfObjects.amount_of_machines = int(LotsOfObjects.amount_of_machines) if getattr(LotsOfObjects, 'amount_of_disks', None) is None: LotsOfObjects.amount_of_disks = 5 else: LotsOfObjects.amount_of_disks = int(LotsOfObjects.amount_of_disks) if getattr(LotsOfObjects, 'repetition_scan', None) is None: LotsOfObjects.repetition_scan = 32 else: LotsOfObjects.repetition_scan = int(LotsOfObjects.repetition_scan) total_amount_of_disks = LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks mguids = [] uuids = [str(uuid.uuid4()) for _ in xrange(total_amount_of_disks)] counter = 0 repetition = [] for i in xrange(LotsOfObjects.repetition_scan): repetition.append([]) dguids = [] print 'start test' tstart = time.time() print '\nstart loading data' start = time.time() runtimes = [] for i in xrange(LotsOfObjects.amount_of_machines): mstart = time.time() machine = TestMachine() machine.name = 'machine_{0}'.format(i) machine.save() mguids.append(machine.guid) for ii in xrange(LotsOfObjects.amount_of_disks): current_uuid = uuids[counter] disk = TestDisk() disk.name = 'disk_{0}_{1}'.format(i, ii) disk.description = 'disk_{0}'.format(i) disk.size = ii * 100 disk.machine = machine disk.something = current_uuid disk.save() dguids.append(disk.guid) random.choice(repetition).append(current_uuid) counter += 1 avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec)) runtimes.sort() print '\nloading done ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2]) test_queries = True if test_queries: print '\nstart queries' start = time.time() runtimes = [] for i in xrange(LotsOfObjects.amount_of_machines): mstart = time.time() machine = TestMachine(mguids[i]) assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks)) avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec)) runtimes.sort() print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2]) print '\nstart full query on disk property' start = time.time() dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}) amount = len(dlist) assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines)) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nloading disks (all)' start = time.time() for i in xrange(LotsOfObjects.amount_of_machines): machine = TestMachine(mguids[i]) _ = [_ for _ in machine.disks] seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nstart full query on disk property (using cached objects)' dlist._volatile.delete(dlist._key) start = time.time() dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}) amount = len(dlist) assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines)) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nindexed single query' dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('something', DataList.operator.EQUALS, repetition[0][0])]}) start = time.time() assert len(dlist) == 1, 'One disk should be found' seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.3f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nstart property sort' dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': []}) start = time.time() dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'size')) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nstart dynamic sort' dlist._volatile.delete(dlist._key) dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': []}) start = time.time() dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'predictable')) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nrepetition scan' start = time.time() times = [] for i in xrange(LotsOfObjects.repetition_scan): dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('something', DataList.operator.IN, repetition[i])]}) run_start = time.time() assert len(repetition[i]) == len(dlist), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(repetition[i])) times.append(time.time() - run_start) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (run avg: {2:.3f}s, avg: {3:.2f} dps)'.format(time.time() - tstart, seconds_passed, sum(times) / float(LotsOfObjects.repetition_scan), LotsOfObjects.repetition_scan * total_amount_of_disks / seconds_passed) print '\nguid index query' start = time.time() guids = dguids[len(dguids)/2:] dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('guid', DataList.operator.IN, guids)]}) assert len(dlist) == len(guids), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(guids)) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) clean_data = True if clean_data: print '\ncleaning up' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) for disk in machine.disks: disk.delete() machine.delete() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec)) runtimes.sort() print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
def test_lotsofobjects(self): """ A test creating, linking and querying a lot of objects """ print '' print 'cleaning up' self._clean_all() print 'start test' tstart = time.time() if getattr(LotsOfObjects, 'amount_of_machines', None) is None: LotsOfObjects.amount_of_machines = 50 if getattr(LotsOfObjects, 'amount_of_disks', None) is None: LotsOfObjects.amount_of_disks = 5 load_data = True if load_data: print 'start loading data' start = time.time() mguids = [] runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine() machine.name = 'machine_{0}'.format(i) machine.save() mguids.append(machine.guid) for ii in xrange(0, int(LotsOfObjects.amount_of_disks)): disk = TestDisk() disk.name = 'disk_{0}_{1}'.format(i, ii) disk.size = ii * 100 disk.machine = machine disk.save() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) test_queries = True if test_queries: print 'start queries' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) self.assertEqual(len(machine.disks), LotsOfObjects.amount_of_disks, 'Not all disks were retreived ({0})'.format(len(machine.disks))) avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) print 'start full query on disk property' start = time.time() amount = DataList({'object': TestDisk, 'data': DataList.select.COUNT, 'query': {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}}).data self.assertEqual(amount, (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) clean_data = True if clean_data: print 'cleaning up' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) for disk in machine.disks: disk.delete() machine.delete() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) self._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))