def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'preparing'
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
else:
LotsOfObjects.amount_of_machines = int(LotsOfObjects.amount_of_machines)
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
else:
LotsOfObjects.amount_of_disks = int(LotsOfObjects.amount_of_disks)
if getattr(LotsOfObjects, 'repetition_scan', None) is None:
LotsOfObjects.repetition_scan = 32
else:
LotsOfObjects.repetition_scan = int(LotsOfObjects.repetition_scan)
total_amount_of_disks = LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks
mguids = []
uuids = [str(uuid.uuid4()) for _ in xrange(total_amount_of_disks)]
counter = 0
repetition = []
for i in xrange(LotsOfObjects.repetition_scan):
repetition.append([])
dguids = []
print 'start test'
tstart = time.time()
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(LotsOfObjects.amount_of_machines):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(LotsOfObjects.amount_of_disks):
current_uuid = uuids[counter]
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.description = 'disk_{0}'.format(i)
disk.size = ii * 100
disk.machine = machine
disk.something = current_uuid
disk.save()
dguids.append(disk.guid)
random.choice(repetition).append(current_uuid)
counter += 1
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\nloading done ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(LotsOfObjects.amount_of_machines):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nloading disks (all)'
start = time.time()
for i in xrange(LotsOfObjects.amount_of_machines):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nindexed single query'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('something', DataList.operator.EQUALS, repetition[0][0])]})
start = time.time()
assert len(dlist) == 1, 'One disk should be found'
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.3f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart property sort'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'size'))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'predictable'))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nrepetition scan'
start = time.time()
times = []
for i in xrange(LotsOfObjects.repetition_scan):
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('something', DataList.operator.IN, repetition[i])]})
run_start = time.time()
assert len(repetition[i]) == len(dlist), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(repetition[i]))
times.append(time.time() - run_start)
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (run avg: {2:.3f}s, avg: {3:.2f} dps)'.format(time.time() - tstart, seconds_passed, sum(times) / float(LotsOfObjects.repetition_scan), LotsOfObjects.repetition_scan * total_amount_of_disks / seconds_passed)
print '\nguid index query'
start = time.time()
guids = dguids[len(dguids)/2:]
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('guid', DataList.operator.IN, guids)]})
assert len(dlist) == len(guids), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(guids))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
