def testAnalyzeFlakePipelineStartTaskAfterDelay(self, mocked_delay, _): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') # Random date in the past, for coverage. analysis.request_time = datetime.datetime(2015, 1, 1, 1, 1, 1) analysis.Save() start_commit_position = 1000 start_revision = 'r1000' delay = 60 step_metadata = StepMetadata( canonical_step_name='s', dimensions=None, full_step_name='s', patched=False, swarm_task_ids=None, waterfall_buildername='b', waterfall_mastername='w', isolate_target_name='s') mocked_delay.return_value = delay analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=CommitID( commit_position=start_commit_position, revision=start_revision), culprit_commit_id=None), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable(['os:testOS']), manually_triggered=False, rerun=False, retries=0, step_metadata=step_metadata) expected_retried_analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=CommitID( commit_position=start_commit_position, revision=start_revision), culprit_commit_id=None), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable(['os:testOS']), manually_triggered=False, rerun=False, retries=1, step_metadata=step_metadata) self.MockAsynchronousPipeline(DelayPipeline, delay, delay) self.MockGeneratorPipeline(RecursiveAnalyzeFlakePipeline, expected_retried_analyze_flake_input, None) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.start() self.execute_queued_tasks()
def testAnalyzeFlakePipelineAnalysisFinishedNoFindings(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.Save() analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=None, culprit_commit_id=None), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable([]), manually_triggered=False, rerun=False, retries=0, step_metadata=None) expected_report_event_input = ReportEventInput( analysis_urlsafe_key=analysis.key.urlsafe()) self.MockGeneratorPipeline(ReportAnalysisEventPipeline, expected_report_event_input, None) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.start() self.execute_queued_tasks() self.assertIsNone(analysis.culprit_urlsafe_key) self.assertEqual(analysis_status.COMPLETED, analysis.status) self.assertEqual(result_status.NOT_FOUND_UNTRIAGED, analysis.result_status)
def testRerunAnalysisWithAnalyzeFlakePipeline( self, mocked_analysis, mocked_pipeline, mocked_need_analysis, mocked_build_info, mock_dimensions, *_): buildbucket_id = 'id' mock_dimensions.return_value = ['os:Mac', 'cpu:x86'] start_commit_position = 1000 start_build_info = BuildInfo('m', 'b 1', 123) start_build_info.commit_position = start_commit_position start_build_info.chromium_revision = 'r1000' start_build_info.buildbucket_id = buildbucket_id mocked_build_info.return_value = start_build_info mocked_analysis.pipeline_status_path.return_value = 'status' mocked_analysis.key.urlsafe.return_value = 'urlsafe_key' mocked_need_analysis.return_value = (True, mocked_analysis) test = TestInfo('m', 'b 1', 123, 's', 't') manually_triggered = False flake = Flake.Create('chromium', 's', 't', 'l') analysis = initialize_flake_pipeline.ScheduleAnalysisIfNeeded( test, test, flake.key, bug_id=None, allow_new_analysis=True, manually_triggered=manually_triggered, force=True, queue_name=constants.DEFAULT_QUEUE) self.assertIsNotNone(analysis) self.assertEqual(buildbucket_id, analysis.build_id) self.assertEqual(buildbucket_id, analysis.original_build_id) analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key='urlsafe_key', analyze_commit_position_parameters=NextCommitPositionOutput( culprit_commit_id=None, next_commit_id=CommitID( commit_position=start_commit_position, revision=start_build_info.chromium_revision)), commit_position_range=IntRange(lower=None, upper=start_commit_position), dimensions=ListOfBasestring.FromSerializable( ['os:Mac', 'cpu:x86', 'pool:luci.chromium.findit']), manually_triggered=manually_triggered, rerun=True, retries=0, step_metadata=StepMetadata.FromSerializable({})) mocked_pipeline.assert_has_calls([ mock.call(analyze_flake_input), mock.call().start(queue_name=constants.DEFAULT_QUEUE) ])
def testOnFinalizedNoError(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.Save() analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=CommitID(commit_position=1000, revision='rev'), culprit_commit_id=None), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable(['os:testOS']), manually_triggered=False, rerun=False, retries=0, step_metadata=None) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.OnFinalized(analyze_flake_input) self.assertEqual(analysis_status.COMPLETED, analysis.status)
def testRecursiveAnalyzeFlakePipeline(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.Save() analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=CommitID(commit_position=1000, revision='rev'), culprit_commit_id=None), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable([]), manually_triggered=False, rerun=False, retries=0, step_metadata=None) self.MockGeneratorPipeline(AnalyzeFlakePipeline, analyze_flake_input, None) pipeline_job = RecursiveAnalyzeFlakePipeline(analyze_flake_input) pipeline_job.start() self.execute_queued_tasks()
def ScheduleAnalysisIfNeeded( normalized_test, original_test, flake_key, bug_id=None, allow_new_analysis=False, force=False, manually_triggered=False, user_email=None, triggering_source=triggering_sources.FINDIT_PIPELINE, queue_name=constants.DEFAULT_QUEUE): """Schedules an analysis if needed and returns the MasterFlakeAnalysis. When the build failure was already analyzed and a new analysis is scheduled, the returned WfAnalysis will still have the result of last completed analysis. Args: normalized_test (TestInfo): Info of the normalized flaky test after mapping a CQ trybot step to a Waterfall buildbot step, striping prefix "PRE_" from a gtest, etc. original_test (TestInfo): Info of the original flaky test. flake_key (ndb.Key): The key to the Flake responsible for triggering this analysis. bug_id (int): The monorail bug id to update when analysis is done. allow_new_analysis (bool): Indicate whether a new analysis is allowed. force (bool): Indicate whether to force a rerun of current analysis. manually_triggered (bool): True if the analysis was requested manually, such as by a Chromium sheriff. user_email (str): The email of the user requesting the analysis. triggering_source (int): From where this analysis was triggered, such as through Findit pipeline, UI, or through Findit API. queue_name (str): The App Engine queue to run the analysis. Returns: A MasterFlakeAnalysis instance. None if no analysis was scheduled and the user has no permission to. """ need_new_analysis, analysis = _NeedANewAnalysis( normalized_test, original_test, flake_key, bug_id=bug_id, allow_new_analysis=allow_new_analysis, force=force, user_email=user_email, triggering_source=triggering_source) if need_new_analysis: # _NeedANewAnalysis just created master_flake_analysis. Use the latest # version number and pass that along to the other pipelines for updating # results and data. logging.info( 'A new master flake analysis was successfully saved for %s (%s) and ' 'will be captured in version %s', repr(normalized_test), repr(original_test), analysis.version_number) step_metadata = (step_util.LegacyGetStepMetadata( normalized_test.master_name, normalized_test.builder_name, normalized_test.build_number, normalized_test.step_name) or step_util.LegacyGetStepMetadata( original_test.master_name, original_test.builder_name, original_test.build_number, original_test.step_name)) logging.info('Initializing flake analysis pipeline for key: %s', analysis.key) starting_build_info = build_util.GetBuildInfo( normalized_test.master_name, normalized_test.builder_name, normalized_test.build_number) original_build_info = build_util.GetBuildInfo( original_test.master_name, original_test.builder_name, original_test.build_number) assert starting_build_info, ( 'Failed to get starting build for flake analysis') starting_commit_position = starting_build_info.commit_position assert starting_commit_position is not None, ( 'Cannot analyze flake without a starting commit position') assert original_build_info, 'Failed to get original build info' # Get the dimensions of the bot for when try jobs are needed to compile. dimensions = try_job_service.GetDimensionsFromBuildInfo( starting_build_info) analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( culprit_commit_id=None, next_commit_id=CommitID( commit_position=starting_commit_position, revision=starting_build_info.chromium_revision)), commit_position_range=IntRange(lower=None, upper=starting_commit_position), dimensions=ListOfBasestring.FromSerializable(dimensions), manually_triggered=manually_triggered, retries=0, rerun=force, step_metadata=StepMetadata.FromSerializable(step_metadata)) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.target = appengine_util.GetTargetNameForModule( constants.WATERFALL_BACKEND) pipeline_job.start(queue_name=queue_name) analysis.pipeline_status_path = pipeline_job.pipeline_status_path analysis.root_pipeline_id = pipeline_job.root_pipeline_id analysis.build_id = starting_build_info.buildbucket_id analysis.original_build_id = original_build_info.buildbucket_id analysis.put() analysis.LogInfo(( 'A flake analysis was scheduled using commit-based pipelines with ' 'path {}').format(pipeline_job.pipeline_status_path)) else: logging.info('A flake analysis not necessary for build %s, %s, %s, %s', normalized_test.master_name, normalized_test.builder_name, normalized_test.build_number, normalized_test.step_name) return analysis
def testAnalyzeFlakePipelineCanStartAnalysisImmediately(self, _): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.Save() start_commit_position = 1000 start_revision = 'r1000' isolate_sha = 'sha1' next_commit_position = 999 pass_rate = 0.5 build_url = 'url' try_job_url = None get_sha_output = GetIsolateShaOutput( isolate_sha=isolate_sha, build_url=build_url, try_job_url=try_job_url) step_metadata = StepMetadata( canonical_step_name='s', dimensions=None, full_step_name='s', patched=False, swarm_task_ids=None, waterfall_buildername='b', waterfall_mastername='w', isolate_target_name='s') expected_flakiness = Flakiness( build_url=build_url, commit_position=start_commit_position, revision=start_revision, pass_rate=pass_rate) analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=CommitID( commit_position=start_commit_position, revision=start_revision), culprit_commit_id=None), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable(['os:testOS']), manually_triggered=False, rerun=False, retries=0, step_metadata=step_metadata) expected_isolate_sha_input = GetIsolateShaForCommitPositionParameters( analysis_urlsafe_key=analysis.key.urlsafe(), commit_position=start_commit_position, dimensions=ListOfBasestring.FromSerializable(['os:testOS']), revision=start_revision, step_metadata=step_metadata, upper_bound_build_number=analysis.build_number) expected_pass_rate_input = DetermineApproximatePassRateInput( builder_name=analysis.builder_name, commit_position=start_commit_position, flakiness_thus_far=None, get_isolate_sha_output=get_sha_output, master_name=analysis.master_name, previous_swarming_task_output=None, reference_build_number=analysis.build_number, revision=start_revision, step_name=analysis.step_name, test_name=analysis.test_name) expected_update_data_points_input = UpdateFlakeAnalysisDataPointsInput( analysis_urlsafe_key=analysis.key.urlsafe(), flakiness=expected_flakiness) expected_next_commit_position_input = NextCommitPositionInput( analysis_urlsafe_key=analysis.key.urlsafe(), commit_position_range=IntRange(lower=None, upper=None), step_metadata=step_metadata) next_commit_id = CommitID( commit_position=next_commit_position, revision='r999') expected_next_commit_position_output = NextCommitPositionOutput( next_commit_id=next_commit_id, culprit_commit_id=None) expected_recursive_analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=expected_next_commit_position_output, commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable(['os:testOS']), manually_triggered=False, rerun=False, retries=0, step_metadata=step_metadata) self.MockGeneratorPipeline(GetIsolateShaForCommitPositionPipeline, expected_isolate_sha_input, get_sha_output) self.MockGeneratorPipeline(DetermineApproximatePassRatePipeline, expected_pass_rate_input, expected_flakiness) self.MockSynchronousPipeline(UpdateFlakeAnalysisDataPointsPipeline, expected_update_data_points_input, None) self.MockSynchronousPipeline(NextCommitPositionPipeline, expected_next_commit_position_input, expected_next_commit_position_output) self.MockGeneratorPipeline(RecursiveAnalyzeFlakePipeline, expected_recursive_analyze_flake_input, None) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.start() self.execute_queued_tasks()
def testAnalyzeFlakePipelineAnalysisFinishedWithCulprit( self, mocked_confidence, mocked_culprit, _): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' culprit_commit_position = 999 analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.data_points = [ DataPoint.Create(commit_position=culprit_commit_position) ] analysis.original_master_name = master_name analysis.original_builder_name = builder_name analysis.original_build_number = build_number analysis.original_step_name = step_name analysis.original_test_name = test_name analysis.Save() culprit_revision = 'r999' confidence_score = 0.85 culprit = FlakeCulprit.Create('chromium', culprit_revision, culprit_commit_position) culprit.put() mocked_confidence.return_value = confidence_score mocked_culprit.return_value = culprit analyze_flake_input = AnalyzeFlakeInput( analysis_urlsafe_key=analysis.key.urlsafe(), analyze_commit_position_parameters=NextCommitPositionOutput( next_commit_id=None, culprit_commit_id=CommitID( commit_position=culprit_commit_position, revision=culprit_revision)), commit_position_range=IntRange(lower=None, upper=None), dimensions=ListOfBasestring.FromSerializable([]), manually_triggered=False, rerun=False, retries=0, step_metadata=None) expected_recent_flakiness_input = AnalyzeRecentFlakinessInput( analysis_urlsafe_key=analysis.key.urlsafe()) expected_auto_action_input = _PerformAutoActionsInput( analysis_urlsafe_key=analysis.key.urlsafe()) expected_report_event_input = ReportEventInput( analysis_urlsafe_key=analysis.key.urlsafe()) self.MockGeneratorPipeline(AnalyzeRecentFlakinessPipeline, expected_recent_flakiness_input, None) self.MockGeneratorPipeline(_PerformAutoActionsPipeline, expected_auto_action_input, None) self.MockGeneratorPipeline(ReportAnalysisEventPipeline, expected_report_event_input, None) pipeline_job = AnalyzeFlakePipeline(analyze_flake_input) pipeline_job.start() self.execute_queued_tasks() self.assertEqual(analysis_status.COMPLETED, analysis.status) self.assertEqual(result_status.FOUND_UNTRIAGED, analysis.result_status)