def _GetFlakeQueryResults(luci_project, cursor, direction, page_size): """Gets queried results of flakes. Args: luci_project (str): Luci project of the flakes. cursor (None or str): The cursor provides a cursor in the current query results, allowing you to retrieve the next set based on the offset. direction (str): Either previous or next. page_size (int): Number of entities to show per page. Returns: A tuple of (flakes, prev_cursor, cursor). flakes (list): List of flakes to be displayed at the current page. prev_cursor (str): The urlsafe encoding of the cursor, which is at the top position of entities of the current page. cursor (str): The urlsafe encoding of the cursor, which is at the bottom position of entities of the current page. """ flake_query = Flake.query( ndb.AND(Flake.luci_project == luci_project, Flake.archived == False)) # pylint: disable=singleton-comparison # Orders flakes by flake_score_last_week. flake_query = flake_query.filter(Flake.flake_score_last_week > 0) first_sort_property = Flake.flake_score_last_week return dashboard_util.GetPagedResults( flake_query, order_properties=[ (first_sort_property, dashboard_util.DESC), (Flake.last_occurred_time, dashboard_util.DESC), (Flake.normalized_step_name, dashboard_util.ASC), (Flake.test_label_name, dashboard_util.ASC), ], cursor=cursor, direction=direction, page_size=page_size)
def _ArchiveFlakesForClosedIssue(flake_issue): """Archives flakes with closed issue. Flakes with closed issue should be archived since they are fixed or not actionable. """ flakes = Flake.query(Flake.flake_issue_key == flake_issue.key).fetch() for flake in flakes: flake.archived = True ndb.put_multi(flakes)
def testCreate(self): luci_project = 'chromium' normalized_step_name = 'normalized_step' normalized_test_name = 'normalized_test' test_label_name = 'test_label' flake = Flake.Create(luci_project=luci_project, normalized_step_name=normalized_step_name, normalized_test_name=normalized_test_name, test_label_name=test_label_name) flake.put() fetched_flakes = Flake.query().fetch() self.assertEqual(1, len(fetched_flakes)) self.assertEqual(flake, fetched_flakes[0])
def HandleGet(self): total = self.request.get('total').strip() component = self.request.get('component').strip() luci_project = self.request.get( 'luci_project').strip() or DEFAULT_LUCI_PROJECT if not component and not total: return self.CreateError( 'A component is required to show its flake report, or add total=1 to ' 'show total numbers.', return_code=404) if component: component_reports = _QueryComponentReports(component, luci_project) if not component_reports: return self.CreateError( 'Didn\'t find reports for project {}, component {}.'. format(luci_project, component), return_code=404) report_json = _GenerateComponentReportJson(component_reports) top_flakes, _, _, _, _ = flake_detection_utils.GetFlakesByFilter( ComponentFlakinessReport.GenerateTag('component', component), luci_project, cursor=None, direction='next', page_size=_DEFAULT_TOP_FLAKE_NUM) else: total_reports = _QueryTotalReports(luci_project) report_json = _GenerateComponentReportJson(total_reports) top_flakes = Flake.query().order( -Flake.flake_score_last_week).fetch(_DEFAULT_TOP_FLAKE_NUM) flakes_data = flake_detection_utils.GenerateFlakesData(top_flakes) data = { 'report_json': report_json, 'component': component if component else 'All', 'top_flakes': flakes_data, 'total': total, 'luci_project': (luci_project if luci_project != DEFAULT_LUCI_PROJECT else ''), } return {'template': 'flake/report/component_report.html', 'data': data}
def _UpdateCountsForNewFlake(start_date): """Updates counts for new, re-occurred or rare flakes. Args: start_date(datetime): Earliest time to check. """ more = True cursor = None while more: ndb.get_context().clear_cache() flakes, cursor, more = Flake.query().filter( Flake.last_occurred_time > start_date).filter( Flake.flake_score_last_week == 0).fetch_page( 100, start_cursor=cursor) for flake in flakes: _UpdateFlakeCountsAndScore(flake, start_date) ndb.put_multi(flakes)
def _UpdateCountsForOldFlake(start_date): """Updates counts for old flakes - flakes with score greater than 0. a. if the flake has 1+ occurrences within the time range, updates counts. b. if the flake didn't occurred within the time range, resets counts. Args: start_date(datetime): Earliest time to check. """ more = True cursor = None while more: ndb.get_context().clear_cache() flakes, cursor, more = Flake.query().filter( Flake.flake_score_last_week > 0).fetch_page(100, start_cursor=cursor) for flake in flakes: _UpdateFlakeCountsAndScore(flake, start_date) ndb.put_multi(flakes)
def _GetFlakesByBug(monorail_project, bug_id): """Gets flakes link to the same bug. Gets flakes directly link to the bug and also flakes link to bugs that are merged into this bug. """ flake_issue = FlakeIssue.Get(monorail_project, bug_id) assert flake_issue, 'Requested FlakeIssue {} not found.'.format(bug_id) all_issue_keys = [flake_issue.key] issue_leaves = FlakeIssue.query( FlakeIssue.merge_destination_key == flake_issue.key).fetch( keys_only=True) all_issue_keys.extend(issue_leaves) flakes = [] for issue_key in all_issue_keys: flakes_to_issue = Flake.query( Flake.flake_issue_key == issue_key).fetch() flakes.extend(flakes_to_issue) flakes.sort(key=lambda flake: flake.flake_score_last_week, reverse=True) return flakes
def _AddFlakesToCounters(counters, flake_info_dict, report_time, save_test_report): """Queries all flakes that have happened after report_time and adds their info to counters. """ query = Flake.query() query = query.filter(Flake.last_occurred_time >= report_time) cursor = None more = True while more: flakes, cursor, more = query.fetch_page(500, start_cursor=cursor) for flake in flakes: luci_project = flake.luci_project if luci_project not in counters: counters[luci_project] = _NewTally( TotalFlakinessReport.MakeId(report_time, luci_project)) _AddFlakeToTally(counters[luci_project], flake, report_time) component = flake.GetComponent() test = flake.normalized_test_name flake_info_dict[flake.key] = { 'luci_project': luci_project, 'component': component, 'test': test } if component not in counters[luci_project]: counters[luci_project][component] = _NewTally(component) _AddFlakeToTally(counters[luci_project][component], flake, report_time) if save_test_report: # pragma: no branch. if test not in counters[luci_project][component]: counters[luci_project][component][test] = _NewTally(test) _AddFlakeToTally(counters[luci_project][component][test], flake, report_time)
def GetFlakesByFilter(flake_filter, luci_project, cursor, direction, page_size=None): # pragma: no cover. """Gets flakes by the given filter, then sorts them by the flake score. Args: flake_filter (str): It could be a test name, or a tag-based filter in the following forms: * tag::value * tag1::value1@tag2::value2 * tag1::value1@-tag2:value2 luci_project (str): The Luci project that the flakes are for. cursor (None or str): The cursor provides a cursor in the current query results, allowing you to retrieve the next set based on the offset. direction (str): Either previous or next. page_size (int): Limit of results required in one page. Returns: (flakes, prev_cursor, cursor, grouping_search, error_message) flakes (list): A list of Flakes filtered by tags. prev_cursor (str): The urlsafe encoding of the cursor, which is at the top position of entities of the current page. cursor (str): The urlsafe encoding of the cursor, which is at the bottom position of entities of the current page. grouping_search (bool): Whether it is a group searching. error_message (str): An error message if there is one; otherwise None. """ logging.info('Searching filter: %s', flake_filter) flakes = [] error_message = None grouping_search = True filters = [f.strip() for f in flake_filter.split('@') if f.strip()] # The resulted flakes are those: # * Match all of positive filters # * Not match any of negative filters positive_filters = [] negative_filters = [] invalid_filters = [] for f in filters: parts = [p.strip() for p in f.split(TAG_DELIMITER)] if len(parts) != 2 or not parts[1]: invalid_filters.append(f) continue if parts[0] == _TEST_FILTER_NAME: # Search for a specific test. grouping_search = False flakes = Flake.query( Flake.normalized_test_name == Flake.NormalizeTestName(parts[1]) ).filter(Flake.luci_project == luci_project).fetch() return flakes, '', '', grouping_search, error_message negative = False if parts[0][0] == '-': parts[0] = parts[0][1:] negative = True if parts[0] not in SUPPORTED_TAGS: invalid_filters.append(f) continue if negative: negative_filters.append(TAG_DELIMITER.join(parts)) else: positive_filters.append(TAG_DELIMITER.join(parts)) if invalid_filters: error_message = 'Unsupported tag filters: %s' % ', '.join( invalid_filters) return flakes, '', '', grouping_search, error_message if not positive_filters: # At least one positive filter should be given. error_message = 'At least one positive filter required' return flakes, '', '', grouping_search, error_message logging.info('Positive filters: %r', positive_filters) logging.info('Negative filters: %r', negative_filters) query = Flake.query( ndb.AND(Flake.luci_project == luci_project, Flake.archived == False)) # pylint: disable=singleton-comparison for tag in positive_filters: query = query.filter(Flake.tags == tag) query = query.filter(Flake.flake_score_last_week > 0) minimum_flake_count_in_page = max( 1, page_size / 2) if page_size else DEFAULT_PAGE_SIZE / 2 while True: results, prev_cursor, cursor = dashboard_util.GetPagedResults( query, order_properties=[ (Flake.flake_score_last_week, dashboard_util.DESC), (Flake.last_occurred_time, dashboard_util.DESC), (Flake.normalized_step_name, dashboard_util.ASC), (Flake.test_label_name, dashboard_util.ASC), ], cursor=cursor, direction=direction, page_size=page_size or DEFAULT_PAGE_SIZE) for result in results: if negative_filters and any(t in result.tags for t in negative_filters): continue flakes.append(result) if ((direction == dashboard_util.PREVIOUS and prev_cursor == '') or cursor == '' or len(flakes) >= minimum_flake_count_in_page): # No more results or gets enough flakes on a page. # Ideally we expect the page shows the same amount of flakes as the # page_size suggests, but in the case with negative_filters, the number of # flakes left after filtering out negative_filters is unknown. # Uses minimum_flake_count_in_page to cap the flake count in one page from # 0.5 page_size to 1.5 page_size. break return flakes, prev_cursor, cursor, grouping_search, error_message