def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
mguids = []
if load_data:
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nloading disks (all)'
start = time.time()
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart property sort'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
mguids = []
if load_data:
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (
time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() -
mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress(
'* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(
i + 1, int(LotsOfObjects.amount_of_machines),
round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(
round(time.time() - tstart, 2), round(runtimes[1], 2),
round(runtimes[-2], 2))
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(
machine.disks
) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(
len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (
time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() -
mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress(
'* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(
i + 1, int(LotsOfObjects.amount_of_machines),
round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(
round(time.time() - tstart, 2), round(runtimes[1], 2),
round(runtimes[-2], 2))
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(
TestDisk, {
'type':
DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT,
(LotsOfObjects.amount_of_disks - 1) * 100)]
})
amount = len(dlist)
assert amount == (
LotsOfObjects.amount_of_disks - 3
) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(
amount,
int((LotsOfObjects.amount_of_disks - 3) *
LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nloading disks (all)'
start = time.time()
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(
TestDisk, {
'type':
DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT,
(LotsOfObjects.amount_of_disks - 1) * 100)]
})
amount = len(dlist)
assert amount == (
LotsOfObjects.amount_of_disks - 3
) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(
amount,
int((LotsOfObjects.amount_of_disks - 3) *
LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart property sort'
dlist = DataList(TestDisk, {
'type': DataList.where_operator.AND,
'items': []
})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {
'type': DataList.where_operator.AND,
'items': []
})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(
round(time.time() - tstart, 2), round(seconds_passed, 2),
round(
LotsOfObjects.amount_of_machines *
LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (
time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() -
mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress(
'* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(
i + 1, int(LotsOfObjects.amount_of_machines),
round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(
round(time.time() - tstart, 2), round(runtimes[1], 2),
round(runtimes[-2], 2))
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'preparing'
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
else:
LotsOfObjects.amount_of_machines = int(LotsOfObjects.amount_of_machines)
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
else:
LotsOfObjects.amount_of_disks = int(LotsOfObjects.amount_of_disks)
if getattr(LotsOfObjects, 'repetition_scan', None) is None:
LotsOfObjects.repetition_scan = 32
else:
LotsOfObjects.repetition_scan = int(LotsOfObjects.repetition_scan)
total_amount_of_disks = LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks
mguids = []
uuids = [str(uuid.uuid4()) for _ in xrange(total_amount_of_disks)]
counter = 0
repetition = []
for i in xrange(LotsOfObjects.repetition_scan):
repetition.append([])
dguids = []
print 'start test'
tstart = time.time()
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(LotsOfObjects.amount_of_machines):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(LotsOfObjects.amount_of_disks):
current_uuid = uuids[counter]
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.description = 'disk_{0}'.format(i)
disk.size = ii * 100
disk.machine = machine
disk.something = current_uuid
disk.save()
dguids.append(disk.guid)
random.choice(repetition).append(current_uuid)
counter += 1
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\nloading done ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(LotsOfObjects.amount_of_machines):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nloading disks (all)'
start = time.time()
for i in xrange(LotsOfObjects.amount_of_machines):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nindexed single query'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('something', DataList.operator.EQUALS, repetition[0][0])]})
start = time.time()
assert len(dlist) == 1, 'One disk should be found'
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.3f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart property sort'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'size'))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'predictable'))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
print '\nrepetition scan'
start = time.time()
times = []
for i in xrange(LotsOfObjects.repetition_scan):
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('something', DataList.operator.IN, repetition[i])]})
run_start = time.time()
assert len(repetition[i]) == len(dlist), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(repetition[i]))
times.append(time.time() - run_start)
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (run avg: {2:.3f}s, avg: {3:.2f} dps)'.format(time.time() - tstart, seconds_passed, sum(times) / float(LotsOfObjects.repetition_scan), LotsOfObjects.repetition_scan * total_amount_of_disks / seconds_passed)
print '\nguid index query'
start = time.time()
guids = dguids[len(dguids)/2:]
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('guid', DataList.operator.IN, guids)]})
assert len(dlist) == len(guids), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(guids))
seconds_passed = time.time() - start
print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed)
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec))
runtimes.sort()
print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
def new_function(*args, **kwargs):
"""
Wrapped function
This function will process the api request an apply:
- Paging (only return a subset of all results)
- Sorting (sort on properties)
- Filtering (filter on properties)
Request arguments for paging:
- page: The page number for which the items should be displayed (string/int)
- page_size: The size of the pages (string/int)
Request arguments for sorting:
- sort: Comma separated list of the properties to sort on. Prefix with '-' to use descending order (eg name,-description) (string)
Request arguments for filtering: identical to DataList query params
- query: The query to perform. See DataList execute_query method for more info
"""
request = _find_request(args)
timings = {}
# 1. Pre-loading request data
start = time.time()
sort = request.QUERY_PARAMS.get('sort')
query = request.QUERY_PARAMS.get('query')
if query is not None:
try:
query = json.loads(query)
DataList.validate_query(query)
except ValueError as ex:
raise ValueError('Query is not valid: \'{0}\''.format(
str(ex)))
if sort is None and default_sort is not None:
sort = default_sort
sort = None if sort is None else [
s for s in reversed(sort.split(','))
]
page = request.QUERY_PARAMS.get('page')
page = int(page) if page is not None and (
isinstance(page, int) or page.isdigit()) else None
page_size = request.QUERY_PARAMS.get('page_size')
page_size = int(page_size) if page_size is not None and (
isinstance(page_size, int) or page_size.isdigit()) else None
contents = request.QUERY_PARAMS.get('contents')
contents = None if contents is None else contents.split(',')
timings['preload'] = [time.time() - start, 'Data preloading']
# 2. Construct hints for decorated function (so it can provide full objects if required)
start = time.time()
if 'hints' not in kwargs:
kwargs['hints'] = {}
kwargs['hints']['full'] = sort is not None or contents is not None
timings['hinting'] = [time.time() - start, 'Request hinting']
# 3. Fetch data
start = time.time()
data_list = f(*args, **kwargs)
guid_list = isinstance(data_list,
list) and len(data_list) > 0 and isinstance(
data_list[0], basestring)
timings['fetch'] = [time.time() - start, 'Fetching data']
# 4. Filtering data
if query is not None:
start = time.time()
if guid_list is True:
guids = data_list
guid_list = False # The list will be converted to a datalist
else:
guids = data_list.guids
# Use the guids from the result list as a base to query inside the functions results and apply the query
data_list = DataList(object_type, query=query, guids=guids)
# Trigger the query
_ = data_list.guids
timings['querying'] = [time.time() - start, 'Querying data']
# 5. Sorting
if sort is not None:
start = time.time()
if guid_list is True:
data_list = DataList(object_type, guids=data_list)
guid_list = False # The list is converted to objects
for sort_item in sort:
desc = sort_item[0] == '-'
field = sort_item[1 if desc else 0:]
data_list.sort(
key=lambda e: DalToolbox.extract_key(e, field),
reverse=desc)
timings['sort'] = [time.time() - start, 'Sorting data']
# 6. Paging
start = time.time()
total_items = len(data_list)
page_metadata = {
'total_items': total_items,
'current_page': 1,
'max_page': 1,
'page_size': page_size,
'start_number': min(1, total_items),
'end_number': total_items
}
if page is not None:
max_page = int(math.ceil(total_items / (page_size * 1.0)))
if page > max_page:
page = max_page
if page == 0:
start_number = -1
end_number = 0
else:
start_number = (
page - 1
) * page_size # Index - e.g. 0 for page 1, 10 for page 2
end_number = start_number + page_size # Index - e.g. 10 for page 1, 20 for page 2
data_list = data_list[start_number:end_number]
page_metadata.update({
'current_page': max(1, page),
'max_page': max(1, max_page),
'start_number': start_number + 1,
'end_number': min(total_items, end_number)
})
else:
page_metadata['page_size'] = total_items
timings['paging'] = [time.time() - start, 'Selecting current page']
# 7. Serializing
start = time.time()
if contents is not None:
if guid_list is True:
data_list = DataList(object_type, guids=data_list)
data = FullSerializer(object_type,
contents=contents,
instance=data_list,
many=True).data
else:
if guid_list is False:
data_list = data_list.guids # 'data_list' is a ovs.dal.datalist.DataList which has the guids stored
data = data_list
timings['serializing'] = [time.time() - start, 'Serializing']
# Add timings about dynamics
if contents is not None and len(data_list) > 0:
object_timings = {}
for obj in data_list:
dynamic_timings = obj.get_timings()
for timing in dynamic_timings:
key = 'dynamic_{0}'.format(timing)
if key not in object_timings:
object_timings[key] = []
object_timings[key].append([
dynamic_timings[timing],
'Load \'{0}\''.format(timing)
])
for key in object_timings:
times = [entry[0] for entry in object_timings[key]]
timings[key] = [sum(times), object_timings[key][0][1]]
timings['{0}_avg'.format(key)] = [
sum(times) / len(times),
'{0} (avg)'.format(object_timings[key][0][1])
]
timings['{0}_min'.format(key)] = [
min(times),
'{0} (min)'.format(object_timings[key][0][1])
]
timings['{0}_max'.format(key)] = [
max(times),
'{0} (max)'.format(object_timings[key][0][1])
]
result = {
'data': data,
'_paging': page_metadata,
'_contents': contents,
'_sorting': [s for s in reversed(sort)] if sort else sort
}
# 8. Building response
return OVSResponse(result,
status=status.HTTP_200_OK,
timings=timings)
