def testGetPagedResultsForDirectionPrevious(self): """Tests getting previous page.""" entities_on_page1, _, bottom_cursor1 = dashboard_util.GetPagedResults( Entity.query(), [(Entity.time, 'desc')], direction='next', page_size=1) _, top_cursor2, _ = dashboard_util.GetPagedResults( Entity.query(), [(Entity.time, 'desc')], cursor=bottom_cursor1, direction='next', page_size=1) back_to_page1_entities, _, _ = dashboard_util.GetPagedResults( Entity.query(), [(Entity.time, 'desc')], cursor=top_cursor2, direction='previous', page_size=1) self.assertListEqual(entities_on_page1, back_to_page1_entities)
def testGetPagedResultsForDirectionNext(self): """Tests getting next page.""" entities, _, _ = dashboard_util.GetPagedResults( Entity.query(), [(Entity.time, 'desc')], direction='next', page_size=1) self.assertEqual(len(entities), 1) self.assertEqual(entities, [self.entities[2]])
def _GetDisabledTestsQueryResults(cursor, direction, page_size): """Gets queried results of disabled tests. Args: cursor (None or str): The cursor provides a cursor in the current query results, allowing you to retrieve the next set based on the offset. direction (str): Either previous or next. page_size (int): Number of entities to show per page. Returns: A tuple of (disabled_tests, prev_cursor, cursor). disabled_tests ([LuciTest]): List of disabled_tests to be displayed at the current page. prev_cursor (str): The urlsafe encoding of the cursor, which is at the top position of entities of the current page. cursor (str): The urlsafe encoding of the cursor, which is at the bottom position of entities of the current page. """ disabled_tests_query = LuciTest.query(LuciTest.disabled == True) # pylint: disable=singleton-comparison return dashboard_util.GetPagedResults( disabled_tests_query, order_properties=[ (LuciTest.last_updated_time, dashboard_util.DESC), (LuciTest.normalized_test_name, dashboard_util.ASC), ], cursor=cursor, direction=direction, page_size=page_size)
def _GetFlakeQueryResults(luci_project, cursor, direction, page_size): """Gets queried results of flakes. Args: luci_project (str): Luci project of the flakes. cursor (None or str): The cursor provides a cursor in the current query results, allowing you to retrieve the next set based on the offset. direction (str): Either previous or next. page_size (int): Number of entities to show per page. Returns: A tuple of (flakes, prev_cursor, cursor). flakes (list): List of flakes to be displayed at the current page. prev_cursor (str): The urlsafe encoding of the cursor, which is at the top position of entities of the current page. cursor (str): The urlsafe encoding of the cursor, which is at the bottom position of entities of the current page. """ flake_query = Flake.query( ndb.AND(Flake.luci_project == luci_project, Flake.archived == False)) # pylint: disable=singleton-comparison # Orders flakes by flake_score_last_week. flake_query = flake_query.filter(Flake.flake_score_last_week > 0) first_sort_property = Flake.flake_score_last_week return dashboard_util.GetPagedResults( flake_query, order_properties=[ (first_sort_property, dashboard_util.DESC), (Flake.last_occurred_time, dashboard_util.DESC), (Flake.normalized_step_name, dashboard_util.ASC), (Flake.test_label_name, dashboard_util.ASC), ], cursor=cursor, direction=direction, page_size=page_size)
def testFetchPreviousPages(self): query = _GetFlakeAnalysisFilterQuery(MasterFlakeAnalysis.query()) result2, _, _ = dashboard_util.GetPagedResults( query, [(MasterFlakeAnalysis.request_time, dashboard_util.DESC)], cursor=self.cursor, direction='previous') self.assertEqual(len(result2), 1) self.assertEqual(self.results, result2)
def HandleGet(self): """Shows crash analysis results in an HTML page.""" start_date, end_date = dashboard_util.GetStartAndEndDates( self.request.get('start_date'), self.request.get('end_date')) query = self.Filter(start_date, end_date) try: page_size = int(self.request.get('n')) except (ValueError, TypeError): page_size = _PAGE_SIZE crash_analyses, top_cusor, bottom_cursor = dashboard_util.GetPagedResults( query, [(self.crash_analysis_cls.requested_time, dashboard_util.DESC)], cursor=self.request.get('cursor'), direction=self.request.get('direction', 'next'), page_size=page_size) # TODO(katesonia): An optimization is to index analysis.status. crash_analyses = [analysis for analysis in crash_analyses if analysis.completed] data = { 'start_date': time_util.FormatDatetime(start_date), 'end_date': time_util.FormatDatetime(end_date), 'found_suspects': self.request.get('found_suspects', '-1'), 'has_regression_range': self.request.get('has_regression_range', '-1'), 'suspected_cls_triage_status': self.request.get( 'suspected_cls_triage_status', '-1'), 'regression_range_triage_status': self.request.get( 'regression_range_triage_status', '-1'), 'client': self.client, 'crashes': self.CrashDataToDisplay(crash_analyses), 'signature': self.request.get('signature'), 'top_cursor': top_cusor, 'bottom_cursor': bottom_cursor, } return { 'template': self.template, 'data': data }
def _MockCursor(self): results, prev_cursor, cursor = dashboard_util.GetPagedResults( MasterFlakeAnalysis.query(), [(MasterFlakeAnalysis.request_time, dashboard_util.DESC)], page_size=1) return results, prev_cursor, cursor
def GetFlakesByFilter(flake_filter, luci_project, cursor, direction, page_size=None): # pragma: no cover. """Gets flakes by the given filter, then sorts them by the flake score. Args: flake_filter (str): It could be a test name, or a tag-based filter in the following forms: * tag::value * tag1::value1@tag2::value2 * tag1::value1@-tag2:value2 luci_project (str): The Luci project that the flakes are for. cursor (None or str): The cursor provides a cursor in the current query results, allowing you to retrieve the next set based on the offset. direction (str): Either previous or next. page_size (int): Limit of results required in one page. Returns: (flakes, prev_cursor, cursor, grouping_search, error_message) flakes (list): A list of Flakes filtered by tags. prev_cursor (str): The urlsafe encoding of the cursor, which is at the top position of entities of the current page. cursor (str): The urlsafe encoding of the cursor, which is at the bottom position of entities of the current page. grouping_search (bool): Whether it is a group searching. error_message (str): An error message if there is one; otherwise None. """ logging.info('Searching filter: %s', flake_filter) flakes = [] error_message = None grouping_search = True filters = [f.strip() for f in flake_filter.split('@') if f.strip()] # The resulted flakes are those: # * Match all of positive filters # * Not match any of negative filters positive_filters = [] negative_filters = [] invalid_filters = [] for f in filters: parts = [p.strip() for p in f.split(TAG_DELIMITER)] if len(parts) != 2 or not parts[1]: invalid_filters.append(f) continue if parts[0] == _TEST_FILTER_NAME: # Search for a specific test. grouping_search = False flakes = Flake.query( Flake.normalized_test_name == Flake.NormalizeTestName(parts[1]) ).filter(Flake.luci_project == luci_project).fetch() return flakes, '', '', grouping_search, error_message negative = False if parts[0][0] == '-': parts[0] = parts[0][1:] negative = True if parts[0] not in SUPPORTED_TAGS: invalid_filters.append(f) continue if negative: negative_filters.append(TAG_DELIMITER.join(parts)) else: positive_filters.append(TAG_DELIMITER.join(parts)) if invalid_filters: error_message = 'Unsupported tag filters: %s' % ', '.join( invalid_filters) return flakes, '', '', grouping_search, error_message if not positive_filters: # At least one positive filter should be given. error_message = 'At least one positive filter required' return flakes, '', '', grouping_search, error_message logging.info('Positive filters: %r', positive_filters) logging.info('Negative filters: %r', negative_filters) query = Flake.query( ndb.AND(Flake.luci_project == luci_project, Flake.archived == False)) # pylint: disable=singleton-comparison for tag in positive_filters: query = query.filter(Flake.tags == tag) query = query.filter(Flake.flake_score_last_week > 0) minimum_flake_count_in_page = max( 1, page_size / 2) if page_size else DEFAULT_PAGE_SIZE / 2 while True: results, prev_cursor, cursor = dashboard_util.GetPagedResults( query, order_properties=[ (Flake.flake_score_last_week, dashboard_util.DESC), (Flake.last_occurred_time, dashboard_util.DESC), (Flake.normalized_step_name, dashboard_util.ASC), (Flake.test_label_name, dashboard_util.ASC), ], cursor=cursor, direction=direction, page_size=page_size or DEFAULT_PAGE_SIZE) for result in results: if negative_filters and any(t in result.tags for t in negative_filters): continue flakes.append(result) if ((direction == dashboard_util.PREVIOUS and prev_cursor == '') or cursor == '' or len(flakes) >= minimum_flake_count_in_page): # No more results or gets enough flakes on a page. # Ideally we expect the page shows the same amount of flakes as the # page_size suggests, but in the case with negative_filters, the number of # flakes left after filtering out negative_filters is unknown. # Uses minimum_flake_count_in_page to cap the flake count in one page from # 0.5 page_size to 1.5 page_size. break return flakes, prev_cursor, cursor, grouping_search, error_message
def HandleGet(self): status_code = int( self.request.get('result_status', result_status.UNSPECIFIED)) step_name = self.request.get('step_name').strip() test_name = self.request.get('test_name').strip() triage = self.request.get('triage') == '1' # Only allow querying by start/end dates for admins during triage to avoid # overcomplicating the UI for other users. start_date, end_date = self._GetStartAndEndDates(triage) master_flake_analysis_query = _GetFlakeAnalysisFilterQuery( MasterFlakeAnalysis.query(), step_name, test_name, start_date, end_date, status_code) # If filters by step_name and/or test_name, don't do paging. if step_name or test_name: analyses = master_flake_analysis_query.order( -MasterFlakeAnalysis.request_time).fetch() prev_cursor = '' cursor = '' else: analyses, prev_cursor, cursor = dashboard_util.GetPagedResults( master_flake_analysis_query, MasterFlakeAnalysis.request_time, self.request.get('cursor'), self.request.get('direction').strip(), page_size=PAGE_SIZE) data = { 'master_flake_analyses': [], 'result_status_filter': status_code, 'step_name_filter': step_name, 'test_name_filter': test_name, 'prev_cursor': prev_cursor, 'cursor': cursor, } if triage: # pragma: no cover data['triage'] = triage data['start_date'] = start_date data['end_date'] = end_date for master_flake_analysis in analyses: data['master_flake_analyses'].append({ 'build_analysis_status': master_flake_analysis.status_description, 'build_number': master_flake_analysis.build_number, 'builder_name': master_flake_analysis.builder_name, 'confidence_in_suspected_build': (master_flake_analysis.confidence_in_suspected_build), 'culprit': (master_flake_analysis.culprit.ToDict() if master_flake_analysis.culprit else {}), 'key': master_flake_analysis.key.urlsafe(), 'master_name': master_flake_analysis.master_name, 'request_time': time_util.FormatDatetime(master_flake_analysis.request_time), 'result_status': result_status.RESULT_STATUS_TO_DESCRIPTION.get( master_flake_analysis.result_status), 'step_name': master_flake_analysis.step_name, 'suspected_build': master_flake_analysis.suspected_flake_build_number, 'test_name': master_flake_analysis.test_name, 'try_job_status': analysis_status.STATUS_TO_DESCRIPTION.get( master_flake_analysis.try_job_status), }) return {'template': 'flake/dashboard.html', 'data': data}
def HandleGet(self): status_code = int( self.request.get('result_status', result_status.UNSPECIFIED)) step_name = self.request.get('step_name').strip() test_name = self.request.get('test_name').strip() triage = self.request.get('triage') == '1' # Only allow querying by start/end dates for admins during triage to avoid # overcomplicating the UI for other users. start_date, end_date = self._GetStartAndEndDates(triage) master_flake_analysis_query = _GetFlakeAnalysisFilterQuery( MasterFlakeAnalysis.query(), step_name, test_name, start_date, end_date, status_code) # If filters by step_name and/or test_name, don't do paging. if step_name or test_name: analyses = master_flake_analysis_query.order( -MasterFlakeAnalysis.request_time).fetch() prev_cursor = '' cursor = '' else: analyses, prev_cursor, cursor = dashboard_util.GetPagedResults( master_flake_analysis_query, [(MasterFlakeAnalysis.request_time, dashboard_util.DESC)], self.request.get('cursor'), self.request.get('direction').strip(), page_size=PAGE_SIZE) data = { 'master_flake_analyses': [], 'result_status_filter': status_code, 'step_name_filter': step_name, 'test_name_filter': test_name, 'prev_cursor': prev_cursor, 'cursor': cursor, } if triage: # pragma: no cover data['triage'] = triage data['start_date'] = start_date data['end_date'] = end_date for analysis in analyses: culprit = None if analysis.culprit_urlsafe_key: culprit_key = ndb.Key(urlsafe=analysis.culprit_urlsafe_key) # TODO(crbug.com/799308): Remove this hack when bug is fixed. assert culprit_key.pairs()[0] assert culprit_key.pairs()[0][0] # Name of the model. assert culprit_key.pairs()[0][1] # Id of the model. culprit = ndb.Key(culprit_key.pairs()[0][0], culprit_key.pairs()[0][1]).get() status = analysis.status if analysis.heuristic_analysis_status == analysis_status.ERROR: status = analysis_status.ERROR timestamp = ( time_util.ConvertToTimestamp(analysis.request_time) if analysis.request_time else 'None') data['master_flake_analyses'].append({ 'master_name': analysis.original_master_name or analysis.master_name, 'builder_name': analysis.original_builder_name or analysis.builder_name, 'build_number': analysis.original_build_number or analysis.build_number, 'step_name': analysis.original_step_name or analysis.step_name, 'test_name': analysis.original_test_name or analysis.test_name, 'bug_id': analysis.bug_id, 'confidence_in_culprit': analysis.confidence_in_culprit, 'culprit': culprit.to_dict() if culprit else {}, 'key': analysis.key.urlsafe(), 'request_utc_timestamp': timestamp, 'result_status': result_status.RESULT_STATUS_TO_DESCRIPTION.get( analysis.result_status), 'suspected_build': analysis.suspected_flake_build_number, 'status': analysis_status.STATUS_TO_DESCRIPTION.get(status), }) return {'template': 'flake/dashboard.html', 'data': data}