def testRevertCulprit(self, mock_revert, *_):
repo_name = 'chromium'
revision = 'rev1'
build_key = 'm/b/123'
revert_cl = RevertCL()
revert_cl.revert_cl_url = 'url'
revert_cl.created_time = datetime.datetime(2018, 6, 20, 0, 0, 0)
mock_revert.return_value = (constants.CREATED_BY_FINDIT, revert_cl,
None)
culprit = WfSuspectedCL.Create(repo_name, revision, 1)
culprit.put()
pipeline_input = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.COMPILE)
self.assertEqual(
constants.CREATED_BY_FINDIT,
culprit_action.RevertCulprit(pipeline_input, 'pipeline_id'))
culprit = WfSuspectedCL.Get(repo_name, revision)
self.assertEqual(revert_cl, culprit.revert_cl)
self.assertEqual(analysis_status.COMPLETED, culprit.revert_status)
def testCreateAndSubmitRevertCreateFailed(self, under_limit, can_revert,
revert_fn, commit_fn):
build_key = 'mock_build_key'
repo = 'chromium'
rev = 'rev1'
commit_position = 100
pipeline_id = 'foo'
culprit = FlakeCulprit.Create(repo, rev, commit_position)
culprit.put()
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.culprit_urlsafe_key = culprit.key.urlsafe()
analysis.put()
revert_expected = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.FLAKY_TEST)
pipeline_input = CreateAndSubmitRevertInput(
analysis_urlsafe_key=analysis.key.urlsafe(), build_key=build_key)
culprit_util.CreateAndSubmitRevert(pipeline_input, pipeline_id)
under_limit.assert_called_once()
can_revert.assert_called_once_with(analysis)
revert_fn.assert_called_once_with(revert_expected, pipeline_id)
commit_fn.assert_not_called()
self.assertFalse(analysis.has_created_autorevert)
self.assertFalse(analysis.has_submitted_autorevert)
def CreateAndSubmitRevert(parameters, runner_id):
"""Wraps the creation and submission of autoreverts for flaky tests."""
analysis = ndb.Key(urlsafe=parameters.analysis_urlsafe_key).get()
assert analysis
culprit = ndb.Key(urlsafe=analysis.culprit_urlsafe_key).get()
assert culprit
# TODO(crbug.com/907675): Create, but not auto commit reverts for some
# culprits.
if not IsAutorevertEnabled():
analysis.LogInfo('Autorevert is disabled.')
return False
if not UnderLimitForAutorevert():
analysis.LogInfo('Not autoreverting since limit has been reached.')
return False
# Verify the conditions for revert are satisfied.
if not CanRevertForAnalysis(analysis):
analysis.LogInfo('Not reverting: CanRevertForAnalysis returned false.')
return False
# Create the revert, and check if it succeeded. If it succeeded, then
# continue on and submit it.
revert_culprit_parameters = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=parameters.build_key,
failure_type=failure_type.FLAKY_TEST)
revert_status = culprit_action.RevertCulprit(revert_culprit_parameters,
runner_id)
if revert_status != constants.CREATED_BY_FINDIT:
analysis.LogInfo(
'Not reverting: RevertCulprit wasn\'t able to create a revert.')
return False
submit_revert_paramters = SubmitRevertCLParameters(
cl_key=culprit.key.urlsafe(),
revert_status=revert_status,
failure_type=failure_type.FLAKY_TEST)
submit_revert_status = culprit_action.CommitRevert(submit_revert_paramters,
runner_id)
if submit_revert_status != constants.COMMITTED:
analysis.LogInfo(
'Not reverting: CommitRevert wasn\'t able to submit the revert')
analysis.Update(has_created_autorevert=True)
return False
analysis.Update(has_created_autorevert=True,
has_submitted_autorevert=True,
autorevert_submission_time=time_util.GetUTCNow())
return True
def testLogUnexpectedAbortingNoChange(self):
repo_name = 'chromium'
revision = 'rev1'
build_key = 'm/b/123'
culprit = WfSuspectedCL.Create(repo_name, revision, 123)
culprit.put()
pipeline_input = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.COMPILE)
CreateRevertCLPipeline(pipeline_input).OnAbort(pipeline_input)
culprit = WfSuspectedCL.Get(repo_name, revision)
self.assertIsNone(culprit.revert_status)
def test_CanCreateRevertForCulprit(self):
repo_name = 'chromium'
revision = 'rev1'
culprit = WfSuspectedCL.Create(repo_name, revision, 123)
culprit.revert_status = analysis_status.RUNNING
culprit.put()
parameters = CreateRevertCLParameters(cl_key=culprit.key.urlsafe(),
build_key='build_key')
pipeline_id = 'pipeline_id'
self.assertTrue(
culprit_action._CanCreateRevertForCulprit(parameters, pipeline_id))
def testSendNotificationToConfirmRevert(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 124
build_key = 'm/b/124'
cl_key = 'mockurlsafekey'
culprits = DictOfBasestring()
culprits['r1'] = cl_key
heuristic_cls = ListOfBasestring()
heuristic_cls.append(cl_key)
self.MockSynchronousPipeline(
CreateRevertCLPipeline,
CreateRevertCLParameters(cl_key=cl_key,
build_key=build_key,
failure_type=failure_type.COMPILE),
services_constants.CREATED_BY_FINDIT)
self.MockSynchronousPipeline(
SubmitRevertCLPipeline,
SubmitRevertCLParameters(
cl_key=cl_key,
revert_status=services_constants.CREATED_BY_FINDIT,
failure_type=failure_type.COMPILE),
services_constants.COMMITTED)
self.MockSynchronousPipeline(
SendNotificationToIrcPipeline,
SendNotificationToIrcParameters(
cl_key=cl_key,
revert_status=services_constants.CREATED_BY_FINDIT,
commit_status=services_constants.COMMITTED,
failure_type=failure_type.COMPILE), True)
self.MockSynchronousPipeline(
SendNotificationForCulpritPipeline,
SendNotificationForCulpritParameters(
cl_key=cl_key,
force_notify=True,
revert_status=services_constants.CREATED_BY_FINDIT,
failure_type=failure_type.COMPILE), True)
pipeline = wrapper_pipeline.RevertAndNotifyCompileCulpritPipeline(
CulpritActionParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
culprits=culprits,
heuristic_cls=heuristic_cls,
failure_to_culprit_map=None))
pipeline.start(queue_name=constants.DEFAULT_QUEUE)
self.execute_queued_tasks()
def testRevertCLSucceed(self, mock_fn, mock_revert, *_):
repo_name = 'chromium'
revision = 'rev1'
commit_position = 123
build_key = 'm/b/123'
cl_info = ClInfo(self.review_server_host, self.review_change_id)
cl_info.commits.append(
Commit('20001', 'rev1', ['rev0'], datetime(2017, 2, 1, 0, 0, 0)))
cl_info.owner_email = '*****@*****.**'
mock_fn.return_value = cl_info
mock_revert.return_value = '54321'
culprit = WfSuspectedCL.Create(repo_name, revision, commit_position)
culprit.builds = {
build_key: {
'status': None,
'failures': {
'step': ['test1']
}
}
}
culprit.put()
pipeline_input = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.COMPILE)
pipeline = CreateRevertCLPipeline(pipeline_input)
revert_status = pipeline.run(pipeline_input)
self.assertEquals(revert_status, constants.CREATED_BY_FINDIT)
culprit = WfSuspectedCL.Get(repo_name, revision)
self.assertEqual(culprit.revert_status, status.COMPLETED)
self.assertIsNotNone(culprit.revert_cl)
reason = textwrap.dedent("""
Findit (https://goo.gl/kROfz5) identified CL at revision %s as the
culprit for failures in the build cycles as shown on:
https://analysis.chromium.org/waterfall/culprit?key=%s\n
Sample Failed Build: %s\n
Sample Failed Step: %s""") % (commit_position, culprit.key.urlsafe(),
buildbot.CreateBuildUrl('m', 'b',
'123'), 'step')
mock_revert.assert_called_with(reason,
self.review_change_id,
'20001',
bug_id=None)
def testRevertCulpritSkipped(self, _):
repo_name = 'chromium'
revision = 'rev1'
build_key = 'm/b/123'
culprit = WfSuspectedCL.Create(repo_name, revision, 1)
culprit.put()
pipeline_input = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.COMPILE)
self.assertEqual(
constants.SKIPPED,
culprit_action.RevertCulprit(pipeline_input, 'pipeline_id'))
def RunImpl(self, pipeline_input):
culprits_should_take_actions = (
test_culprit_action.GetCulpritsShouldTakeActions(pipeline_input))
if not culprits_should_take_actions:
return
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
culprits = pipeline_input.culprits
build_failure_type = failure_type.TEST
for culprit_revision in culprits_should_take_actions:
culprit_key = culprits.get(culprit_revision)
assert culprit_key, (
'Failed to get culprit_key for culprit {} when analyzing failures'
' at build {}/{}/{}'.format(culprit_revision, master_name,
builder_name, build_number))
revert_status = constants.SKIPPED
if test_culprit_action.CanAutoCreateRevert(
culprit_key, pipeline_input): # pragma: no branch.
revert_status = yield CreateRevertCLPipeline(
CreateRevertCLParameters(cl_key=culprit_key,
build_key=build_key,
failure_type=build_failure_type))
if test_culprit_action.CanAutoCommitRevertByFindit():
submit_revert_pipeline_input = self.CreateInputObjectInstance(
SubmitRevertCLParameters,
cl_key=culprit_key,
revert_status=revert_status,
failure_type=build_failure_type)
yield SubmitRevertCLPipeline(submit_revert_pipeline_input)
# Checks if any of the culprits was also found by heuristic analysis,
# if so send notification right away.
send_notification_to_culprit_input = self.CreateInputObjectInstance(
SendNotificationForCulpritParameters,
cl_key=culprit_key,
force_notify=culprit_action.ShouldForceNotify(
culprit_key, pipeline_input),
revert_status=revert_status,
failure_type=build_failure_type)
yield SendNotificationForCulpritPipeline(
send_notification_to_culprit_input)
def RunImpl(self, pipeline_input):
if not compile_culprit_action.ShouldTakeActionsOnCulprit(
pipeline_input):
return
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
culprits = pipeline_input.culprits
culprit = culprits.values()[0]
force_notify = culprit_action.ShouldForceNotify(
culprit, pipeline_input)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
build_failure_type = failure_type.COMPILE
revert_status = constants.SKIPPED
commit_status = constants.SKIPPED
if compile_culprit_action.CanAutoCreateRevert():
revert_status = yield CreateRevertCLPipeline(
CreateRevertCLParameters(cl_key=culprit,
build_key=build_key,
failure_type=build_failure_type))
if compile_culprit_action.CanAutoCommitRevertByFindit():
submit_revert_pipeline_input = self.CreateInputObjectInstance(
SubmitRevertCLParameters,
cl_key=culprit,
revert_status=revert_status,
failure_type=build_failure_type)
commit_status = yield SubmitRevertCLPipeline(
submit_revert_pipeline_input)
send_notification_to_irc_input = self.CreateInputObjectInstance(
SendNotificationToIrcParameters,
cl_key=culprit,
revert_status=revert_status,
commit_status=commit_status,
failure_type=build_failure_type)
yield SendNotificationToIrcPipeline(send_notification_to_irc_input)
send_notification_to_culprit_input = self.CreateInputObjectInstance(
SendNotificationForCulpritParameters,
cl_key=culprit,
force_notify=force_notify,
revert_status=revert_status,
failure_type=build_failure_type)
yield SendNotificationForCulpritPipeline(
send_notification_to_culprit_input)
def testRevertSkipped(self, _):
repo_name = 'chromium'
revision = 'rev1'
build_key = 'm/b/123'
culprit = WfSuspectedCL.Create(repo_name, revision, 123)
culprit.revert_status = status.SKIPPED
culprit.put()
pipeline_input = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.COMPILE)
pipeline = CreateRevertCLPipeline(pipeline_input)
revert_status = pipeline.run(pipeline_input)
self.assertEqual(constants.SKIPPED, revert_status)
def testLogUnexpectedAbortingPipelineIdNotMatch(self):
repo_name = 'chromium'
revision = 'rev1'
build_key = 'm/b/123'
culprit = WfSuspectedCL.Create(repo_name, revision, 123)
culprit.revert_pipeline_id = 'pipeline_id'
culprit.put()
pipeline_input = CreateRevertCLParameters(
cl_key=culprit.key.urlsafe(),
build_key=build_key,
failure_type=failure_type.COMPILE)
pipeline = CreateRevertCLPipeline(pipeline_input)
pipeline.start()
pipeline.OnAbort(pipeline_input)
culprit = WfSuspectedCL.Get(repo_name, revision)
self.assertEqual(culprit.revert_pipeline_id, 'pipeline_id')
def testCreatedRevertButNotCommitted(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 124
build_key = 'm/b/124'
cl_key = 'mockurlsafekey'
culprits = DictOfBasestring()
culprits['r1'] = cl_key
heuristic_cls = ListOfBasestring()
heuristic_cls.append(cl_key)
failure_to_culprit_map = FailureToCulpritMap.FromSerializable(
{'step1': {
't1': 'r1'
}})
self.MockSynchronousPipeline(
CreateRevertCLPipeline,
CreateRevertCLParameters(cl_key=cl_key,
build_key=build_key,
failure_type=failure_type.TEST),
services_constants.CREATED_BY_FINDIT)
self.MockSynchronousPipeline(
SendNotificationForCulpritPipeline,
SendNotificationForCulpritParameters(
cl_key=cl_key,
force_notify=True,
revert_status=services_constants.CREATED_BY_FINDIT,
failure_type=failure_type.TEST), True)
pipeline = RevertAndNotifyTestCulpritPipeline(
CulpritActionParameters(
build_key=BuildKey(master_name=master_name,
builder_name=builder_name,
build_number=build_number),
culprits=culprits,
heuristic_cls=heuristic_cls,
failure_to_culprit_map=failure_to_culprit_map))
pipeline.start(queue_name=constants.DEFAULT_QUEUE)
self.execute_queued_tasks()