def testCreateAndSubmitRevertOverLimit(self, under_limit, can_revert, revert_fn, commit_fn): build_key = 'mock_build_key' repo = 'chromium' rev = 'rev1' commit_position = 100 pipeline_id = 'foo' culprit = FlakeCulprit.Create(repo, rev, commit_position) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.put() pipeline_input = CreateAndSubmitRevertInput( analysis_urlsafe_key=analysis.key.urlsafe(), build_key=build_key) culprit_util.CreateAndSubmitRevert(pipeline_input, pipeline_id) under_limit.assert_called_once() can_revert.assert_not_called() revert_fn.assert_not_called() commit_fn.assert_not_called() self.assertFalse(analysis.has_created_autorevert) self.assertFalse(analysis.has_submitted_autorevert)
def testCreateAndSubmitRevertCreateFailed(self, under_limit, can_revert, revert_fn, commit_fn): build_key = 'mock_build_key' repo = 'chromium' rev = 'rev1' commit_position = 100 pipeline_id = 'foo' culprit = FlakeCulprit.Create(repo, rev, commit_position) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.put() revert_expected = CreateRevertCLParameters( cl_key=culprit.key.urlsafe(), build_key=build_key, failure_type=failure_type.FLAKY_TEST) pipeline_input = CreateAndSubmitRevertInput( analysis_urlsafe_key=analysis.key.urlsafe(), build_key=build_key) culprit_util.CreateAndSubmitRevert(pipeline_input, pipeline_id) under_limit.assert_called_once() can_revert.assert_called_once_with(analysis) revert_fn.assert_called_once_with(revert_expected, pipeline_id) commit_fn.assert_not_called() self.assertFalse(analysis.has_created_autorevert) self.assertFalse(analysis.has_submitted_autorevert)
def testGetFlakeAnalysesAsDicts(self): analysis1 = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't1') analysis1.confidence_in_culprit = 0.3 analysis1.put() analysis2 = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't2') analysis2.confidence_in_culprit = 0.8 analysis2.put() culprit = FlakeCulprit.Create('repo', 'revision', 1000) culprit.flake_analysis_urlsafe_keys = [ analysis1.key.urlsafe(), analysis2.key.urlsafe() ] expected_result = [{ 'master_name': 'm', 'builder_name': 'b', 'step_name': 's', 'test_name': 't1', 'key': analysis1.key.urlsafe(), 'confidence_in_culprit': 0.3, }, { 'master_name': 'm', 'builder_name': 'b', 'step_name': 's', 'test_name': 't2', 'key': analysis2.key.urlsafe(), 'confidence_in_culprit': 0.8, }] self.assertEqual(expected_result, flake_culprit._GetFlakeAnalysesAsDicts(culprit))
def UpdateCulprit(analysis_urlsafe_key, revision, commit_position, repo_name='chromium'): """Sets culprit information. Args: analysis_urlafe_key (str): The urlsafe-key to the MasterFlakeAnalysis to update culprit information for. revision (str): The culprit's chromium revision. commit_position (int): The culprit's commit position. repo_name (str): The name of the repo the culprit is in. """ culprit = (FlakeCulprit.Get(repo_name, revision) or FlakeCulprit.Create(repo_name, revision, commit_position)) needs_updating = False if culprit.url is None: change_log = _GIT_REPO.GetChangeLog(revision) if change_log: culprit.url = change_log.code_review_url or change_log.commit_url needs_updating = True else: logging.error('Unable to retrieve change logs for %s', revision) if analysis_urlsafe_key not in culprit.flake_analysis_urlsafe_keys: culprit.flake_analysis_urlsafe_keys.append(analysis_urlsafe_key) needs_updating = True if needs_updating: culprit.put() return culprit
def testSaveFlakeCulpritsForSuspectedRevisionsExistingCulprit( self, mocked_commit_position, mocked_fn): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' suspected_revision = 'r1' suspect_commit_position = 995 suspected_revisions = [suspected_revision] mocked_commit_position.return_value = suspect_commit_position analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash=suspected_revision) ] suspect = FlakeCulprit.Create('chromium', suspected_revision, suspect_commit_position) suspect.url = 'url' suspect.put() analysis.suspect_urlsafe_keys = [suspect.key.urlsafe()] analysis.Save() mocked_fn.return_value = None heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions( analysis.key.urlsafe(), suspected_revisions) analysis = MasterFlakeAnalysis.GetVersion( master_name, builder_name, build_number, step_name, test_name) self.assertIn(suspect.key.urlsafe(), analysis.suspect_urlsafe_keys)
def testGenerateDuplicateComment(self): commit_position = 12345 culprit = FlakeCulprit.Create('c', str(commit_position), commit_position, 'http://') culprit.put() comment = issue_generator.GenerateDuplicateComment(culprit) self.assertIn(str(commit_position), comment) self.assertIn(culprit.key.urlsafe(), comment)
def testCanRevertForAnalysisNotNewlyAddedTest(self): culprit = FlakeCulprit.Create('chromium', 'r13', 100) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 100, 's', 't') analysis.data_points = [DataPoint.Create(commit_position=99, pass_rate=1.0)] analysis.Update( confidence_in_culprit=1.0, culprit_urlsafe_key=culprit.key.urlsafe()) self.assertFalse(culprit_util.CanRevertForAnalysis(analysis))
def testMergeOrSplitFlakeIssueByCulpritIssueAlreadyMerged( self, mocked_get_issue, mocked_merge_issues, _): # Culprit's flake issue 12344 was already merged into 12346. # Incoming flake issue's id is 12345 and is expected to be merged as well. project = 'chromium' merged_bug_id = 12344 open_bug_id = 12345 destination_bug_id = 12346 revision = 'r1000' commit_position = 1000 flake_issue = FlakeIssue.Create(project, open_bug_id) flake_issue.put() destination_issue = FlakeIssue.Create(project, destination_bug_id) destination_issue.put() culprit_flake_issue = FlakeIssue.Create(project, merged_bug_id) culprit_flake_issue.status = 'Merged' culprit_flake_issue.merge_destination_key = destination_issue.key culprit_flake_issue.put() flake_culprit = FlakeCulprit.Create(project, revision, commit_position) flake_culprit.flake_issue_key = culprit_flake_issue.key flake_culprit.put() flake_monorail_issue = Issue({ 'status': 'Available', 'projectId': 'chromium', 'id': str(open_bug_id) }) destination_monorail_issue = Issue({ 'status': 'Available', 'projectId': 'chromium', 'id': str(destination_bug_id) }) mocked_get_issue.side_effect = [ destination_monorail_issue, flake_monorail_issue, ] (duplicate, destination) = flake_analysis_actions.MergeOrSplitFlakeIssueByCulprit( flake_issue.key, flake_culprit.key) mocked_merge_issues.assert_called_once_with( flake_monorail_issue, destination_monorail_issue, mock.ANY) flake_issue = flake_issue.key.get() self.assertEqual(flake_issue.key, duplicate) self.assertEqual(destination_issue.key, destination) self.assertEqual(destination_issue.key, flake_issue.merge_destination_key)
def testCreate(self): repo_name = 'chromium' revision = 'a1b2c3d4' commit_position = 12345 url = 'url' culprit = FlakeCulprit.Create(repo_name, revision, commit_position, url) self.assertEqual(repo_name, culprit.repo_name) self.assertEqual(revision, culprit.revision) self.assertEqual(commit_position, culprit.commit_position) self.assertEqual(url, culprit.url)
def testCanRevertForAnalysis(self, mock_time_fn): culprit = FlakeCulprit.Create('chromium', 'r13', 100) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 100, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=99, pass_rate=-1.0) ] analysis.Update( confidence_in_culprit=1.0, culprit_urlsafe_key=culprit.key.urlsafe()) self.assertTrue(culprit_util.CanRevertForAnalysis(analysis)) mock_time_fn.assert_called_with('r13')
def testGetNextCommitIdFromHeuristicResultsNoDataPoints(self, _): suspect = FlakeCulprit.Create('repo', 'revision', 1000) suspect.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.suspect_urlsafe_keys.append(suspect.key.urlsafe()) analysis.put() next_commit_id = CommitID(commit_position=999, revision='r999') self.assertEqual( next_commit_id, next_commit_position_utils.GetNextCommitIdFromHeuristicResults( analysis.key.urlsafe()))
def testMergeOrSplitFlakeIssueByCulpritMergeIntoManuallyCreated( self, mocked_get_issue, mocked_merge_issues, _): project = 'chromium' duplicate_bug_id = 12344 manually_created_bug_id = 12345 revision = 'r1000' commit_position = 1000 flake_issue = FlakeIssue.Create(project, manually_created_bug_id) flake_issue.status = 'Assigned' flake_issue.put() culprit_flake_issue = FlakeIssue.Create(project, duplicate_bug_id) culprit_flake_issue.put() flake_culprit = FlakeCulprit.Create(project, revision, commit_position) flake_culprit.flake_issue_key = culprit_flake_issue.key flake_culprit.put() # Even though the flake issue associated with the culprit was identified # first, the incoming flake issue was manually created. Merge into the # manually created one. flake_monorail_issue = Issue({ 'status': 'Available', 'projectId': 'chromium', 'id': str(manually_created_bug_id) }) culprit_monorail_issue = Issue({ 'status': 'Available', 'projectId': 'chromium', 'id': str(duplicate_bug_id) }) mocked_get_issue.side_effect = [ culprit_monorail_issue, flake_monorail_issue ] (duplicate, destination) = flake_analysis_actions.MergeOrSplitFlakeIssueByCulprit( flake_issue.key, flake_culprit.key) mocked_merge_issues.assert_called_once_with(culprit_monorail_issue, flake_monorail_issue, mock.ANY) flake_culprit = flake_culprit.key.get() flake_issue = flake_issue.key.get() culprit_flake_issue = culprit_flake_issue.key.get() self.assertEqual(culprit_flake_issue.key, duplicate) self.assertEqual(flake_issue.key, destination) self.assertEqual(flake_issue.flake_culprit_key, flake_culprit.key) self.assertEqual(flake_issue.key, culprit_flake_issue.merge_destination_key)
def testSendNotificationForCulprit(self, mocked_culprit_info, mocked_post_message, _): mocked_culprit_info.return_value = { 'review_server_host': 'host', 'review_change_id': 'change_id' } mocked_post_message.return_value = True culprit = FlakeCulprit.Create('repo', 'revision', 12345) culprit.put() self.assertTrue(culprit_util.NotifyCulprit(culprit, None)) culprit = ndb.Key(urlsafe=culprit.key.urlsafe()).get() self.assertEqual(culprit.cr_notification_status, analysis_status.COMPLETED)
def testSendNotificationForCulpritNoChangeId( self, mocked_culprit_info, mocked_post_message, mocked_get_code_review): mocked_culprit_info.return_value = { 'review_server_host': 'host', } mocked_post_message.return_value = False mocked_get_code_review.return_value = False culprit = FlakeCulprit.Create('repo', 'revision', 12345) culprit.put() self.assertFalse(culprit_util.NotifyCulprit(culprit, None)) culprit = ndb.Key(urlsafe=culprit.key.urlsafe()).get() self.assertEqual(culprit.cr_notification_status, analysis_status.ERROR)
def testPrepareCulpritForSendingNotificationAlreadyRunning(self): repo_name = 'repo' revision = 'r1' url = 'code.review.url' commit_position = 1000 culprit = FlakeCulprit.Create(repo_name, revision, commit_position, url) culprit.cr_notification_status = analysis_status.RUNNING culprit.put() self.assertFalse( culprit_util.PrepareCulpritForSendingNotification( culprit.key.urlsafe()))
def testGetAutoAssignOwnerNonChromiumAccount(self, mock_author): revision = 'r' culprit = FlakeCulprit.Create('c', revision, 123, 'http://') culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.Save() author = mock.MagicMock() author.email = 'author@something_else.com' mock_author.return_value = author self.assertIsNone(issue_generator._GetAutoAssignOwner(analysis))
def testGetNextCommitIdFromHeuristicResultsAlreadyFlaky(self): suspect = FlakeCulprit.Create('repo', 'revision', 1000) suspect.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.suspect_urlsafe_keys.append(suspect.key.urlsafe()) analysis.data_points = [ DataPoint.Create(commit_position=999, pass_rate=0.5) ] analysis.put() self.assertIsNone( next_commit_position_utils.GetNextCommitIdFromHeuristicResults( analysis.key.urlsafe()))
def testMergeOrSplitFlakeIssueByCulpritIssueClosed(self, mocked_get_issue, mocked_merge_issues, *_): project = 'chromium' closed_bug_id = 12344 open_bug_id = 12345 revision = 'r1000' commit_position = 1000 flake_issue = FlakeIssue.Create(project, open_bug_id) flake_issue.put() culprit_flake_issue = FlakeIssue.Create(project, closed_bug_id) culprit_flake_issue.status = 'Fixed' culprit_flake_issue.last_updated_time_in_monorail = datetime( 2019, 1, 1) culprit_flake_issue.put() flake_culprit = FlakeCulprit.Create(project, revision, commit_position) flake_culprit.flake_issue_key = culprit_flake_issue.key flake_culprit.put() # Even though the flake issue associated with the culprit was identified # first, it has been closed. FlakeCulprit should have its flake issue # updated to the incoming one. flake_monorail_issue = Issue({ 'status': 'Available', 'projectId': 'chromium', 'id': str(open_bug_id) }) culprit_monorail_issue = Issue({ 'status': 'Fixed', 'projectId': 'chromium', 'id': str(closed_bug_id) }) mocked_get_issue.side_effect = [ culprit_monorail_issue, flake_monorail_issue ] (duplicate, destination) = flake_analysis_actions.MergeOrSplitFlakeIssueByCulprit( flake_issue.key, flake_culprit.key) mocked_merge_issues.assert_not_called() flake_culprit = flake_culprit.key.get() self.assertIsNone(duplicate) self.assertIsNone(destination) self.assertIsNone(flake_issue.merge_destination_key) self.assertEqual(flake_issue.key, flake_culprit.flake_issue_key)
def testShouldTakeAutoAction(self): culprit_commit_position = 1000 culprit = FlakeCulprit.Create('chromium', 'r', culprit_commit_position) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.data_points = [ DataPoint.Create(commit_position=culprit_commit_position, pass_rate=0.5), DataPoint.Create(commit_position=culprit_commit_position - 1, pass_rate=1.0) ] self.assertTrue( flake_analysis_util.ShouldTakeAutoAction(analysis, False))
def testShouldSendNotificationAlreadyProcessed(self, _): repo_name = 'repo' revision = 'r1' url = 'code.review.url' commit_position = 1000 analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') culprit = FlakeCulprit.Create(repo_name, revision, commit_position, url) culprit.cr_notification_status = analysis_status.COMPLETED culprit.put() analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.put() self.assertFalse(culprit_util.ShouldNotifyCulprit(analysis))
def testPrepareCulpritForSendingNotification(self): repo_name = 'repo' revision = 'r1' url = 'code.review.url' commit_position = 1000 culprit = FlakeCulprit.Create(repo_name, revision, commit_position, url) culprit.put() self.assertTrue( culprit_util.PrepareCulpritForSendingNotification( culprit.key.urlsafe())) culprit = ndb.Key(urlsafe=culprit.key.urlsafe()).get() self.assertEqual(analysis_status.RUNNING, culprit.cr_notification_status)
def testAttachCulpritFlakeIssueToFlakeNoCulpritFlakeIssue(self): project = 'chromium' step_name = 's' test_name = 't' label = 'l' revision = 'r1000' commit_position = 1000 flake = Flake.Create(project, step_name, test_name, label) flake.put() culprit = FlakeCulprit.Create(project, revision, commit_position) culprit.put() self.assertIsNone( flake_analysis_actions._AttachCulpritFlakeIssueToFlake( flake, culprit.key))
def testGetCulpritError(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.put() culprit = FlakeCulprit.Create('chromium', 'r1', 1000) culprit.flake_analysis_urlsafe_keys.append(analysis.key.urlsafe()) culprit.cr_notification_status = analysis_status.COMPLETED culprit.cr_notification_time = datetime(2017, 07, 19, 10, 03, 00) response = self.test_app.get( '/p/chromium/flake-portal/analysis/culprit', params={ 'key': culprit.key.urlsafe(), 'format': 'json', }, status=404) self.assertEqual('Culprit not found!', response.json_body.get('error_message'))
def testOnCulpritIdentified(self, mocked_update_monorail, mocked_update_issues, mocked_merge): project = 'chromium' master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' label = 'l' bug_id = 12345 merged_bug_id = 12344 revision = 'r1000' commit_position = 1000 merged_issue = FlakeIssue.Create(project, merged_bug_id) merged_issue.put() issue = FlakeIssue.Create(project, bug_id) issue.merge_destination_key = merged_issue.key issue.put() flake = Flake.Create(project, step_name, test_name, label) flake.flake_issue_key = issue.key flake.put() culprit = FlakeCulprit.Create(project, revision, commit_position) culprit.flake_issue_key = merged_issue.key culprit.put() mocked_merge.return_value = (issue.key, merged_issue.key) analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.flake_key = flake.key analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.confidence_in_culprit = 0.9 analysis.put() flake_analysis_actions.OnCulpritIdentified(analysis.key.urlsafe()) mocked_merge.assert_called_once_with(issue.key, culprit.key) mocked_update_issues.assert_called_once_with( issue.key, issue.merge_destination_key) mocked_update_monorail.assert_called_once_with(analysis.key.urlsafe())
def testNotifyCulpritPipelineShoudNotify(self, mocked_notify, *_): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.Save() culprit = FlakeCulprit.Create('repo', 'r1000', 1000) culprit.flake_analysis_urlsafe_keys.append(analysis.key.urlsafe()) culprit.put() analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.put() notify_culprit_input = NotifyCulpritInput( analysis_urlsafe_key=analysis.key.urlsafe()) pipeline_job = NotifyCulpritPipeline(notify_culprit_input) pipeline_job.start() self.execute_queued_tasks() self.assertTrue(mocked_notify.called) self.assertTrue(analysis.has_commented_on_cl)
def testNotifyCulpritPipelineShoudNotNotify(self, _): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.Save() culprit = FlakeCulprit.Create('repo', 'r1000', 1000) culprit.flake_analysis_urlsafe_keys.append(analysis.key.urlsafe()) culprit.put() analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.put() notify_culprit_input = NotifyCulpritInput( analysis_urlsafe_key=analysis.key.urlsafe()) pipeline_job = NotifyCulpritPipeline(notify_culprit_input) pipeline_job.start() self.execute_queued_tasks() pipeline_job = pipelines.pipeline.Pipeline.from_id(pipeline_job.pipeline_id) self.assertFalse(pipeline_job.outputs.default.value)
def testNextCommitPositionPipelineWithHeuristicResults( self, mock_heuristic_result, mock_run_heuristic, mock_next_commit): master_name = 'm' builder_name = 'b' build_number = 105 step_name = 's' test_name = 't' start_commit_position = 1000 suspect_commit_position = 95 expected_next_commit_id = CommitID(commit_position=94, revision='r94') suspect = FlakeCulprit.Create('repo', 'revision', suspect_commit_position) suspect.commit_position = suspect_commit_position suspect.put() analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.suspect_urlsafe_keys.append(suspect.key.urlsafe()) analysis.put() mock_run_heuristic.return_value = False mock_heuristic_result.return_value = expected_next_commit_id calculated_next_commit_id = CommitID(commit_position=999, revision='r999') mock_next_commit.return_value = (calculated_next_commit_id, None) next_commit_position_input = NextCommitPositionInput( analysis_urlsafe_key=analysis.key.urlsafe(), commit_position_range=IntRange(lower=None, upper=start_commit_position), step_metadata=None) pipeline_job = NextCommitPositionPipeline(next_commit_position_input) pipeline_job.start() self.execute_queued_tasks() pipeline_job = pipelines.pipeline.Pipeline.from_id(pipeline_job.pipeline_id) next_commit_position_output = pipeline_job.outputs.default.value self.assertFalse(pipeline_job.was_aborted) self.assertIsNone(next_commit_position_output['culprit_commit_id']) self.assertEqual(expected_next_commit_id.ToSerializable(), next_commit_position_output['next_commit_id']) mock_heuristic_result.assert_called_once_with(analysis.key.urlsafe())
def testMergeOrSplitFlakeIssueByCulprit(self): project = 'chromium' bug_id = 12345 revision = 'r1000' commit_position = 1000 issue = FlakeIssue.Create(project, bug_id) issue.put() culprit = FlakeCulprit.Create(project, revision, commit_position) culprit.put() flake_analysis_actions.MergeOrSplitFlakeIssueByCulprit( issue.key, culprit.key) issue = issue.key.get() culprit = culprit.key.get() self.assertEqual(culprit.key, issue.flake_culprit_key) self.assertEqual(issue.key, culprit.flake_issue_key)
def testAbort(self, abort_fn): build_key = 'mock_build_key' repo = 'chromium' rev = 'rev1' commit_position = 100 culprit = FlakeCulprit.Create(repo, rev, commit_position) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.put() pipeline_input = CreateAndSubmitRevertInput( analysis_urlsafe_key=analysis.key.urlsafe(), build_key=build_key) pipeline_job = CreateAndSubmitRevertPipeline(pipeline_input) pipeline_job.OnAbort(pipeline_input) abort_fn.assert_called_once_with(pipeline_input, pipeline_job.pipeline_id)
def testShouldSendNotification(self, *_): repo_name = 'repo' revision = 'r1' url = 'code.review.url' commit_position = 1000 analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.confidence_in_culprit = 0.6 culprit = FlakeCulprit.Create(repo_name, revision, commit_position, url) culprit.put() analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.data_points = [ DataPoint.Create(pass_rate=1.0, commit_position=commit_position - 1), DataPoint.Create(pass_rate=0.4, commit_position=commit_position), ] analysis.confidence_in_culprit = 0.6 analysis.put() self.assertTrue(culprit_util.ShouldNotifyCulprit(analysis))