def test_lotsofobjects(self): """ A test creating, linking and querying a lot of objects """ print '' print 'cleaning up' self._clean_all() print 'start test' tstart = time.time() if getattr(LotsOfObjects, 'amount_of_machines', None) is None: LotsOfObjects.amount_of_machines = 50 if getattr(LotsOfObjects, 'amount_of_disks', None) is None: LotsOfObjects.amount_of_disks = 5 load_data = True mguids = [] if load_data: print '\nstart loading data' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine() machine.name = 'machine_{0}'.format(i) machine.save() mguids.append(machine.guid) for ii in xrange(0, int(LotsOfObjects.amount_of_disks)): disk = TestDisk() disk.name = 'disk_{0}_{1}'.format(i, ii) disk.size = ii * 100 disk.machine = machine disk.save() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) test_queries = True if test_queries: print '\nstart queries' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks)) avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) print '\nstart full query on disk property' start = time.time() dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}) amount = len(dlist) assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nloading disks (all)' start = time.time() for i in xrange(0, int(LotsOfObjects.amount_of_machines)): machine = TestMachine(mguids[i]) _ = [_ for _ in machine.disks] seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart full query on disk property (using cached objects)' dlist._volatile.delete(dlist._key) start = time.time() dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}) amount = len(dlist) assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart property sort' dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': []}) start = time.time() dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size')) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart dynamic sort' dlist._volatile.delete(dlist._key) dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': []}) start = time.time() dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable')) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) clean_data = True if clean_data: print '\ncleaning up' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) for disk in machine.disks: disk.delete() machine.delete() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self): """ A test creating, linking and querying a lot of objects """ print '' print 'cleaning up' self._clean_all() print 'start test' tstart = time.time() if getattr(LotsOfObjects, 'amount_of_machines', None) is None: LotsOfObjects.amount_of_machines = 50 if getattr(LotsOfObjects, 'amount_of_disks', None) is None: LotsOfObjects.amount_of_disks = 5 load_data = True mguids = [] if load_data: print '\nstart loading data' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine() machine.name = 'machine_{0}'.format(i) machine.save() mguids.append(machine.guid) for ii in xrange(0, int(LotsOfObjects.amount_of_disks)): disk = TestDisk() disk.name = 'disk_{0}_{1}'.format(i, ii) disk.size = ii * 100 disk.machine = machine disk.save() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / ( time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress( '* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format( i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format( round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) test_queries = True if test_queries: print '\nstart queries' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) assert len( machine.disks ) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format( len(machine.disks)) avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / ( time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress( '* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format( i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format( round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2)) print '\nstart full query on disk property' start = time.time() dlist = DataList( TestDisk, { 'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)] }) amount = len(dlist) assert amount == ( LotsOfObjects.amount_of_disks - 3 ) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format( amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nloading disks (all)' start = time.time() for i in xrange(0, int(LotsOfObjects.amount_of_machines)): machine = TestMachine(mguids[i]) _ = [_ for _ in machine.disks] seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart full query on disk property (using cached objects)' dlist._volatile.delete(dlist._key) start = time.time() dlist = DataList( TestDisk, { 'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)] }) amount = len(dlist) assert amount == ( LotsOfObjects.amount_of_disks - 3 ) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format( amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines)) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart property sort' dlist = DataList(TestDisk, { 'type': DataList.where_operator.AND, 'items': [] }) start = time.time() dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size')) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) print '\nstart dynamic sort' dlist._volatile.delete(dlist._key) dlist = DataList(TestDisk, { 'type': DataList.where_operator.AND, 'items': [] }) start = time.time() dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable')) seconds_passed = (time.time() - start) print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format( round(time.time() - tstart, 2), round(seconds_passed, 2), round( LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2)) clean_data = True if clean_data: print '\ncleaning up' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) for disk in machine.disks: disk.delete() machine.delete() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / ( time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress( '* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format( i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2))) runtimes.sort() print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format( round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_lotsofobjects(self): """ A test creating, linking and querying a lot of objects """ print '' print 'cleaning up' self._clean_all() print 'preparing' if getattr(LotsOfObjects, 'amount_of_machines', None) is None: LotsOfObjects.amount_of_machines = 50 else: LotsOfObjects.amount_of_machines = int(LotsOfObjects.amount_of_machines) if getattr(LotsOfObjects, 'amount_of_disks', None) is None: LotsOfObjects.amount_of_disks = 5 else: LotsOfObjects.amount_of_disks = int(LotsOfObjects.amount_of_disks) if getattr(LotsOfObjects, 'repetition_scan', None) is None: LotsOfObjects.repetition_scan = 32 else: LotsOfObjects.repetition_scan = int(LotsOfObjects.repetition_scan) total_amount_of_disks = LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks mguids = [] uuids = [str(uuid.uuid4()) for _ in xrange(total_amount_of_disks)] counter = 0 repetition = [] for i in xrange(LotsOfObjects.repetition_scan): repetition.append([]) dguids = [] print 'start test' tstart = time.time() print '\nstart loading data' start = time.time() runtimes = [] for i in xrange(LotsOfObjects.amount_of_machines): mstart = time.time() machine = TestMachine() machine.name = 'machine_{0}'.format(i) machine.save() mguids.append(machine.guid) for ii in xrange(LotsOfObjects.amount_of_disks): current_uuid = uuids[counter] disk = TestDisk() disk.name = 'disk_{0}_{1}'.format(i, ii) disk.description = 'disk_{0}'.format(i) disk.size = ii * 100 disk.machine = machine disk.something = current_uuid disk.save() dguids.append(disk.guid) random.choice(repetition).append(current_uuid) counter += 1 avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec)) runtimes.sort() print '\nloading done ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2]) test_queries = True if test_queries: print '\nstart queries' start = time.time() runtimes = [] for i in xrange(LotsOfObjects.amount_of_machines): mstart = time.time() machine = TestMachine(mguids[i]) assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks)) avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec)) runtimes.sort() print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2]) print '\nstart full query on disk property' start = time.time() dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}) amount = len(dlist) assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines)) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nloading disks (all)' start = time.time() for i in xrange(LotsOfObjects.amount_of_machines): machine = TestMachine(mguids[i]) _ = [_ for _ in machine.disks] seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nstart full query on disk property (using cached objects)' dlist._volatile.delete(dlist._key) start = time.time() dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('size', DataList.operator.GT, 100), ('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]}) amount = len(dlist) assert amount == (max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((max(0, LotsOfObjects.amount_of_disks - 3)) * LotsOfObjects.amount_of_machines)) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nindexed single query' dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('something', DataList.operator.EQUALS, repetition[0][0])]}) start = time.time() assert len(dlist) == 1, 'One disk should be found' seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.3f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nstart property sort' dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': []}) start = time.time() dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'size')) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nstart dynamic sort' dlist._volatile.delete(dlist._key) dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': []}) start = time.time() dlist.sort(key=lambda a: DalToolbox.extract_key(a, 'predictable')) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) print '\nrepetition scan' start = time.time() times = [] for i in xrange(LotsOfObjects.repetition_scan): dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('something', DataList.operator.IN, repetition[i])]}) run_start = time.time() assert len(repetition[i]) == len(dlist), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(repetition[i])) times.append(time.time() - run_start) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (run avg: {2:.3f}s, avg: {3:.2f} dps)'.format(time.time() - tstart, seconds_passed, sum(times) / float(LotsOfObjects.repetition_scan), LotsOfObjects.repetition_scan * total_amount_of_disks / seconds_passed) print '\nguid index query' start = time.time() guids = dguids[len(dguids)/2:] dlist = DataList(TestDisk, {'type': DataList.where_operator.AND, 'items': [('guid', DataList.operator.IN, guids)]}) assert len(dlist) == len(guids), 'Incorrect amount of found disks. Found {0} instead of {1}'.format(len(dlist), len(guids)) seconds_passed = time.time() - start print 'completed ({0:.2f}s) in {1:.2f} seconds (avg: {2:.2f} dps)'.format(time.time() - tstart, seconds_passed, total_amount_of_disks / seconds_passed) clean_data = True if clean_data: print '\ncleaning up' start = time.time() runtimes = [] for i in xrange(0, int(LotsOfObjects.amount_of_machines)): mstart = time.time() machine = TestMachine(mguids[i]) for disk in machine.disks: disk.delete() machine.delete() avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start) itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart) runtimes.append(itemspersec) LotsOfObjects._print_progress('* machine {0}/{1} (run: {2:.2f} dps, avg: {3:.2f} dps)'.format(i + 1, LotsOfObjects.amount_of_machines, itemspersec, avgitemspersec)) runtimes.sort() print '\ncompleted ({0:.2f}s). min: {1:.2f} dps, max: {2:.2f} dps'.format(time.time() - tstart, runtimes[1], runtimes[-2])
def new_function(*args, **kwargs): """ Wrapped function This function will process the api request an apply: - Paging (only return a subset of all results) - Sorting (sort on properties) - Filtering (filter on properties) Request arguments for paging: - page: The page number for which the items should be displayed (string/int) - page_size: The size of the pages (string/int) Request arguments for sorting: - sort: Comma separated list of the properties to sort on. Prefix with '-' to use descending order (eg name,-description) (string) Request arguments for filtering: identical to DataList query params - query: The query to perform. See DataList execute_query method for more info """ request = _find_request(args) timings = {} # 1. Pre-loading request data start = time.time() sort = request.QUERY_PARAMS.get('sort') query = request.QUERY_PARAMS.get('query') if query is not None: try: query = json.loads(query) DataList.validate_query(query) except ValueError as ex: raise ValueError('Query is not valid: \'{0}\''.format( str(ex))) if sort is None and default_sort is not None: sort = default_sort sort = None if sort is None else [ s for s in reversed(sort.split(',')) ] page = request.QUERY_PARAMS.get('page') page = int(page) if page is not None and ( isinstance(page, int) or page.isdigit()) else None page_size = request.QUERY_PARAMS.get('page_size') page_size = int(page_size) if page_size is not None and ( isinstance(page_size, int) or page_size.isdigit()) else None contents = request.QUERY_PARAMS.get('contents') contents = None if contents is None else contents.split(',') timings['preload'] = [time.time() - start, 'Data preloading'] # 2. Construct hints for decorated function (so it can provide full objects if required) start = time.time() if 'hints' not in kwargs: kwargs['hints'] = {} kwargs['hints']['full'] = sort is not None or contents is not None timings['hinting'] = [time.time() - start, 'Request hinting'] # 3. Fetch data start = time.time() data_list = f(*args, **kwargs) guid_list = isinstance(data_list, list) and len(data_list) > 0 and isinstance( data_list[0], basestring) timings['fetch'] = [time.time() - start, 'Fetching data'] # 4. Filtering data if query is not None: start = time.time() if guid_list is True: guids = data_list guid_list = False # The list will be converted to a datalist else: guids = data_list.guids # Use the guids from the result list as a base to query inside the functions results and apply the query data_list = DataList(object_type, query=query, guids=guids) # Trigger the query _ = data_list.guids timings['querying'] = [time.time() - start, 'Querying data'] # 5. Sorting if sort is not None: start = time.time() if guid_list is True: data_list = DataList(object_type, guids=data_list) guid_list = False # The list is converted to objects for sort_item in sort: desc = sort_item[0] == '-' field = sort_item[1 if desc else 0:] data_list.sort( key=lambda e: DalToolbox.extract_key(e, field), reverse=desc) timings['sort'] = [time.time() - start, 'Sorting data'] # 6. Paging start = time.time() total_items = len(data_list) page_metadata = { 'total_items': total_items, 'current_page': 1, 'max_page': 1, 'page_size': page_size, 'start_number': min(1, total_items), 'end_number': total_items } if page is not None: max_page = int(math.ceil(total_items / (page_size * 1.0))) if page > max_page: page = max_page if page == 0: start_number = -1 end_number = 0 else: start_number = ( page - 1 ) * page_size # Index - e.g. 0 for page 1, 10 for page 2 end_number = start_number + page_size # Index - e.g. 10 for page 1, 20 for page 2 data_list = data_list[start_number:end_number] page_metadata.update({ 'current_page': max(1, page), 'max_page': max(1, max_page), 'start_number': start_number + 1, 'end_number': min(total_items, end_number) }) else: page_metadata['page_size'] = total_items timings['paging'] = [time.time() - start, 'Selecting current page'] # 7. Serializing start = time.time() if contents is not None: if guid_list is True: data_list = DataList(object_type, guids=data_list) data = FullSerializer(object_type, contents=contents, instance=data_list, many=True).data else: if guid_list is False: data_list = data_list.guids # 'data_list' is a ovs.dal.datalist.DataList which has the guids stored data = data_list timings['serializing'] = [time.time() - start, 'Serializing'] # Add timings about dynamics if contents is not None and len(data_list) > 0: object_timings = {} for obj in data_list: dynamic_timings = obj.get_timings() for timing in dynamic_timings: key = 'dynamic_{0}'.format(timing) if key not in object_timings: object_timings[key] = [] object_timings[key].append([ dynamic_timings[timing], 'Load \'{0}\''.format(timing) ]) for key in object_timings: times = [entry[0] for entry in object_timings[key]] timings[key] = [sum(times), object_timings[key][0][1]] timings['{0}_avg'.format(key)] = [ sum(times) / len(times), '{0} (avg)'.format(object_timings[key][0][1]) ] timings['{0}_min'.format(key)] = [ min(times), '{0} (min)'.format(object_timings[key][0][1]) ] timings['{0}_max'.format(key)] = [ max(times), '{0} (max)'.format(object_timings[key][0][1]) ] result = { 'data': data, '_paging': page_metadata, '_contents': contents, '_sorting': [s for s in reversed(sort)] if sort else sort } # 8. Building response return OVSResponse(result, status=status.HTTP_200_OK, timings=timings)