def testGetStepMetadataCached(self, mock_fn, *_): mock_fn.side_effect = [None, {'canonical_step_name': 'step_name'}] # Returns the invalid step_metadata but not cache it. self.assertEqual( None, step_util.GetStepMetadata(123, 'step_name on a platform')) self.assertTrue(mock_fn.call_count == 1) # Returns the valid step_metadata and cache it. self.assertEqual({'canonical_step_name': 'step_name'}, step_util.GetStepMetadata(123, 'step_name on a platform')) self.assertTrue(mock_fn.call_count == 2) self.assertEqual({'canonical_step_name': 'step_name'}, step_util.GetStepMetadata(123, 'step_name on a platform')) self.assertTrue(mock_fn.call_count == 2)
def GetTestLocation(build_id, step_name, test_name, normalized_test_name): """Returns a TestLocation for the given test. Currently only supports webkit_layout_tests and Gtests. Args: build_id (int): Build id of the build. step_name (str): The name of the step. test_name (str): The name of the test. normalized_step_name (str): The normalized version of the step name. """ if 'webkit_layout_tests' in step_name: # For Webkit layout tests, assume that the normalized test name is # the directory name. return TestLocation( file_path=_NormalizePath('third_party/blink/web_tests/%s' % normalized_test_name)) if not test_name_util.GTEST_REGEX.match(normalized_test_name): return None # For Gtest, we read the test location from the output.json step_metadata = step_util.GetStepMetadata(build_id, step_name, partial_match=True) task_ids = step_metadata.get('swarm_task_ids') for task_id in task_ids: test_path = swarmed_test_util.GetTestLocation(task_id, test_name) if test_path: return TestLocation(file_path=_NormalizePath(test_path.file), line_number=test_path.line) return None
def RunImpl(self, build_key): """Triggers flake analyses for flaky tests found by CI failure analysis.""" master_name, builder_name, build_number = build_key.GetParts() flake_settings = waterfall_config.GetCheckFlakeSettings() throttled = flake_settings.get('throttle_flake_analyses', True) analysis = WfAnalysis.Get(master_name, builder_name, build_number) if not analysis or not analysis.flaky_tests: return analysis_counts = defaultdict(lambda: defaultdict(int)) for step, flaky_tests in analysis.flaky_tests.iteritems(): logging.info('%s/%s/%s/%s has %s flaky tests.', master_name, builder_name, build_number, step, len(flaky_tests)) for test_name in flaky_tests: # TODO(crbug.com/904050): Deprecate FlakeAnalysisRequest in favor of # Flake. flake = flake_util.GetFlake(_LUCI_PROJECT, step, test_name, master_name, builder_name, build_number) request = FlakeAnalysisRequest.Create(test_name, False, None) request.AddBuildStep(master_name, builder_name, build_number, step, time_util.GetUTCNow()) request.flake_key = flake.key scheduled = flake_analysis_service.ScheduleAnalysisForFlake( request, '*****@*****.**', False, triggering_sources.FINDIT_PIPELINE) if scheduled: # pragma: no branch analysis_counts[step]['analyzed'] += 1 logging.info( 'A flake analysis has been triggered for %s/%s', step, test_name) if throttled and len(flaky_tests) > 1: logging.info( 'Throttling is enabled, skipping %d tests.', len(flaky_tests) - 1) analysis_counts[step]['throttled'] = len( flaky_tests) - 1 break # If we're throttled, stop after the first. else: analysis_counts[step]['error'] += 1 for step, step_counts in analysis_counts.iteritems(): # Collects metrics. step_metadata = step_util.GetStepMetadata(master_name, builder_name, build_number, step) canonical_step_name = step_metadata.get( 'canonical_step_name') or 'Unknown' isolate_target_name = step_metadata.get( 'isolate_target_name') or 'Unknown' for operation, count in step_counts.iteritems(): monitoring.OnFlakeIdentified(canonical_step_name, isolate_target_name, operation, count)
def _GetTestLocation(flake_occurrence): """Returns a TestLocation for the given FlakeOccurrence instance.""" step_metadata = step_util.GetStepMetadata( flake_occurrence.build_configuration.legacy_master_name, flake_occurrence.build_configuration.luci_builder, flake_occurrence.build_configuration.legacy_build_number, flake_occurrence.step_ui_name) task_ids = step_metadata.get('swarm_task_ids') for task_id in task_ids: test_path = swarmed_test_util.GetTestLocation( task_id, flake_occurrence.test_name) if test_path: return TestLocation(file_path=_NormalizePath(test_path.file), line_number=test_path.line) return None
def testGetStepMetadataPartialMatch(self, mock_step_log): step_util.GetStepMetadata(123, 'step', True) self.assertIn(True, mock_step_log.call_args[0]) step_util.GetStepMetadata(123, 'step', False) self.assertIn(False, mock_step_log.call_args[0])
def testGetStepMetadata(self, cases, mock_step_log): mock_step_log.return_value = cases['step_log_return'] step_metadata = step_util.GetStepMetadata(123, 'step') self.assertEqual(cases['expected_step_metadata'], step_metadata)
def FindMatchingWaterfallStep(build_step, test_name): """Finds the matching Waterfall step and checks whether it is supported. Only Swarmed and gtest-based steps are supported at the moment. Args: build_step (BuildStep): A build step on Waterfall or Commit Queue. It will be updated with the matching Waterfall step and whether it is Swarmed and supported. test_name (str): The name of the test. """ build_step.swarmed = False build_step.supported = False http_client = FinditHttpClient() if build_step.on_cq: wf_master_name, wf_builder_name, wf_build_number, wf_step_name, metadata = ( _GetMatchingWaterfallBuildStep(build_step, http_client)) build_step.wf_master_name = wf_master_name build_step.wf_builder_name = wf_builder_name build_step.wf_build_number = wf_build_number build_step.wf_step_name = wf_step_name if not build_step.has_matching_waterfall_step: return else: build_step.wf_master_name = build_step.master_name build_step.wf_builder_name = build_step.builder_name build_step.wf_build_number = build_step.build_number build_step.wf_step_name = build_step.step_name metadata = build_step.step_metadata or step_util.GetStepMetadata( build_step.master_name, build_step.builder_name, build_step.build_number, build_step.step_name) if not metadata: logging.error('Couldn\'t get step_metadata') return build_step.step_metadata = metadata # Query Swarming for isolated data. build_step.swarmed = True if metadata.get('swarm_task_ids') else False if build_step.swarmed: need_to_continue = False for task_id in metadata['swarm_task_ids']: output = swarmed_test_util.GetTestResultForSwarmingTask( task_id, http_client) if output: # Guess from the format. test_result_object = test_results_util.GetTestResultObject( output, partial_result=True) if not test_result_object: build_step.supported = False elif not step_util.IsStepSupportedByFindit( test_result_object, metadata.get('canonical_step_name') or build_step.step_name, build_step.wf_master_name): build_step.supported = False else: test_exists = test_result_object.DoesTestExist(test_name) if test_exists: build_step.supported = True elif test_result_object.contains_all_tests: # There is no such test for sure. build_step.supported = False else: # Test is not in this task, but cannot determine if it's in other # tasks. need_to_continue = True if not need_to_continue: break
def _GetMatchingWaterfallBuildStep(cq_build_step, http_client): """Returns the matching Waterfall build step of the given CQ one. Args: cq_build_step (BuildStep): A build step on Commit Queue. http_client (RetryHttpClient): A http client to send http requests. Returns: (master_name, builder_name, build_number, step_name, step_metadata) or None """ no_matching_result = (None, None, None, None, None) # 0. Get step_metadata. step_metadata = cq_build_step.step_metadata or step_util.GetStepMetadata( cq_build_step.master_name, cq_build_step.builder_name, cq_build_step.build_number, cq_build_step.step_name) if not step_metadata: logging.error('Couldn\'t get step_metadata') return no_matching_result # 1. Map a cq trybot to the matching waterfall buildbot: # get master_name and builder_name. wf_master_name = step_metadata.get('waterfall_mastername') wf_builder_name = step_metadata.get('waterfall_buildername') if not wf_master_name or not wf_builder_name: # Either waterfall_mastername or waterfall_buildername doesn't exist. logging.info('%s/%s has no matching Waterfall buildbot', cq_build_step.master_name, cq_build_step.builder_name) return no_matching_result # No matching Waterfall buildbot. # 2. Get "name" of the CQ trybot step. # Name of the step in the tags of a Swarming task. # Can't use step name, as cq one is with "(with patch)" while waterfall one # without. name = step_metadata.get('canonical_step_name') # The OS in which the test runs on. The same test binary might run on two # different OS platforms. os_name = step_metadata.get('dimensions', {}).get('os') if not name or not os_name: logging.error('Couldn\'t find name/os') return no_matching_result # No name of the step. # TODO: cache and throttle QPS to the same master. # 3. Retrieve latest completed build cycle on the buildbot. builds = buildbot.GetRecentCompletedBuilds(wf_master_name, wf_builder_name, page_size=1) if not builds: logging.error('Couldn\'t find latest builds.') return no_matching_result # No name of the step. # 4. Check whether there is matching step. tasks = swarming.ListSwarmingTasksDataByTags(http_client, wf_master_name, wf_builder_name, builds[0], additional_tag_filters={ 'name': name, 'os': os_name }) if tasks: # One matching buildbot is found. wf_step_name = tasks[0].tags['stepname'][0] if 'stepname' in tasks[ 0].tags else '' logging.info('%s/%s/%s is mapped to %s/%s/%s', cq_build_step.master_name, cq_build_step.builder_name, cq_build_step.step_name, wf_master_name, wf_builder_name, wf_step_name) return (wf_master_name, wf_builder_name, builds[0], wf_step_name, step_metadata) return no_matching_result
def ScheduleAnalysisIfNeeded( normalized_test, original_test, flake_key, bug_id=None, allow_new_analysis=False, force=False, manually_triggered=False, user_email=None, triggering_source=triggering_sources.FINDIT_PIPELINE, queue_name=constants.DEFAULT_QUEUE): """Schedules an analysis if needed and returns the MasterFlakeAnalysis. When the build failure was already analyzed and a new analysis is scheduled, the returned WfAnalysis will still have the result of last completed analysis. Args: normalized_test (TestInfo): Info of the normalized flaky test after mapping a CQ trybot step to a Waterfall buildbot step, striping prefix "PRE_" from a gtest, etc. original_test (TestInfo): Info of the original flaky test. flake_key (ndb.Key): The key to the Flake responsible for triggering this analysis. bug_id (int): The monorail bug id to update when analysis is done. allow_new_analysis (bool): Indicate whether a new analysis is allowed. force (bool): Indicate whether to force a rerun of current analysis. manually_triggered (bool): True if the analysis was requested manually, such as by a Chromium sheriff. user_email (str): The email of the user requesting the analysis. triggering_source (int): From where this analysis was triggered, such as through Findit pipeline, UI, or through Findit API. queue_name (str): The App Engine queue to run the analysis. Returns: A MasterFlakeAnalysis instance. None if no analysis was scheduled and the user has no permission to. """ need_new_analysis, analysis = _NeedANewAnalysis( normalized_test, original_test, flake_key, bug_id=bug_id, allow_new_analysis=allow_new_analysis, force=force, user_email=user_email, triggering_source=triggering_source) if need_new_analysis: # _NeedANewAnalysis just created master_flake_analysis. Use the latest # version number and pass that along to the other pipelines for updating # results and data. logging.info( 'A new master flake analysis was successfully saved for %s (%s) and ' 'will be captured in version %s', repr(normalized_test), repr(original_test), analysis.version_number) step_metadata = (step_util.GetStepMetadata( normalized_test.master_name, normalized_test.builder_name, normalized_test.build_number, normalized_test.step_name) or step_util.GetStepMetadata( original_test.master_name, original_test.builder_name, original_test.build_number, original_test.step_name)) logging.info('Initializing flake analysis pipeline for key: %s', analysis.key) starting_build_info = build_util.GetBuildInfo( normalized_test.master_name, normalized_test.builder_name, normalized_test.build_number) original_build_info = build_util.GetBuildInfo( original_test.master_name, original_test.builder_name, original_test.build_number) assert starting_build_info, ( 'Failed to get starting build for flake analysis') starting_commit_position = starting_build_info.commit_position assert starting_commit_position is not None, ( 'Cannot analyze flake without a starting commit position') assert original_build_info, 'Failed to get original build info' # Get the dimensions of the bot for when try jobs are needed to compile. dimensions = try_job_service.GetDimensionsFromBuildInfo( starting_build_info) analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( culprit_commit_id=None, next_commit_id=CommitID( commit_position=starting_commit_position, revision=starting_build_info.chromium_revision)), commit_position_range=IntRange(lower=None, upper=starting_commit_position), dimensions=ListOfBasestring.FromSerializable(dimensions), manually_triggered=manually_triggered, retries=0, rerun=force, step_metadata=StepMetadata.FromSerializable(step_metadata)) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.target = appengine_util.GetTargetNameForModule( constants.WATERFALL_BACKEND) pipeline_job.start(queue_name=queue_name) analysis.pipeline_status_path = pipeline_job.pipeline_status_path analysis.root_pipeline_id = pipeline_job.root_pipeline_id analysis.build_id = starting_build_info.buildbucket_id analysis.original_build_id = original_build_info.buildbucket_id analysis.put() analysis.LogInfo(( 'A flake analysis was scheduled using commit-based pipelines with ' 'path {}').format(pipeline_job.pipeline_status_path)) else: logging.info('A flake analysis not necessary for build %s, %s, %s, %s', normalized_test.master_name, normalized_test.builder_name, normalized_test.build_number, normalized_test.step_name) return analysis