def testUpdateWfSuspectedCLAddSameBuild(self):
approach = analysis_approach_type.HEURISTIC
master_name = 'm'
builder_name = 'b'
build_number = 122
test_failure_type = failure_type.TEST
repo_name = 'chromium'
revision = 'r2'
commit_position = 2
failures = {'step_1': ['test1', 'test2']}
top_score = 4
suspected_cl = WfSuspectedCL.Create(repo_name, revision,
commit_position)
suspected_cl.approaches = [analysis_approach_type.HEURISTIC]
suspected_cl.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'failure_type': test_failure_type,
'failures': failures,
'status': None,
'top_score': 4
}
}
suspected_cl.failure_type = [test_failure_type]
suspected_cl.put()
suspected_cl_util.UpdateSuspectedCL(repo_name, revision,
commit_position, approach,
master_name, builder_name,
build_number, test_failure_type,
failures, top_score)
expected_builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'failure_type': test_failure_type,
'failures': failures,
'status': None,
'top_score': 4
}
}
expected_approaches = [analysis_approach_type.HEURISTIC]
suspected_cl = WfSuspectedCL.Get(repo_name, revision)
self.assertIsNotNone(suspected_cl)
self.assertEqual(expected_approaches, suspected_cl.approaches)
self.assertEqual([test_failure_type], suspected_cl.failure_type)
self.assertEqual(expected_builds, suspected_cl.builds)
def UpdateSuspectedCL(repo_name, revision, commit_position, approach,
master_name, builder_name, build_number, cl_failure_type,
failures, top_score):
suspected_cl = (
WfSuspectedCL.Get(repo_name, revision) or
WfSuspectedCL.Create(repo_name, revision, commit_position))
if not suspected_cl.identified_time: # pragma: no cover.
suspected_cl.identified_time = time_util.GetUTCNow()
suspected_cl.updated_time = time_util.GetUTCNow()
if approach not in suspected_cl.approaches:
suspected_cl.approaches.append(approach)
if cl_failure_type not in suspected_cl.failure_type:
suspected_cl.failure_type.append(cl_failure_type)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
if build_key not in suspected_cl.builds:
suspected_cl.builds[build_key] = {
'approaches': [approach],
'failure_type': cl_failure_type,
'failures': failures,
'status': _GetsStatusFromSameFailure(suspected_cl.builds, failures),
'top_score': top_score
}
else:
build = suspected_cl.builds[build_key]
if approach not in build['approaches']:
build['approaches'].append(approach)
suspected_cl.put()
def RunImpl(self, parameters):
analysis_urlsafe_key = parameters.analysis_urlsafe_key
analysis = entity_util.GetEntityFromUrlsafeKey(analysis_urlsafe_key)
assert analysis, 'Cannot retrieve analysis entry from datastore {}'.format(
analysis_urlsafe_key)
# Determine if flakiness is still persistent.
still_flaky = flake_analysis_util.FlakyAtMostRecentlyAnalyzedCommit(
analysis)
# TODO(crbug.com/905754): Call auto actions as an async taskqueue task.
flake_analysis_actions.OnCulpritIdentified(analysis_urlsafe_key)
if not still_flaky: # pragma: no cover.
analysis.LogInfo(
'No further actions taken due to latest commit being stable.')
return
# Data needed for reverts.
build_key = BaseBuildModel.CreateBuildKey(
analysis.original_master_name, analysis.original_builder_name,
analysis.original_build_number)
with pipeline.InOrder():
# Revert culprit if applicable.
yield CreateAndSubmitRevertPipeline(
self.CreateInputObjectInstance(
CreateAndSubmitRevertInput,
analysis_urlsafe_key=analysis.key.urlsafe(),
build_key=build_key))
# Update culprit code review.
yield NotifyCulpritPipeline(
self.CreateInputObjectInstance(
NotifyCulpritInput,
analysis_urlsafe_key=analysis_urlsafe_key))
def _GetAllSuspectedCLsAndCheckStatus(master_name, builder_name, build_number,
analysis):
if not analysis or not analysis.suspected_cls:
return []
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
suspected_cls = copy.deepcopy(analysis.suspected_cls)
confidences = SuspectedCLConfidence.Get()
for cl in suspected_cls:
cl['status'] = _ANALYSIS_CL_STATUS_MAP.get(analysis.result_status,
None)
cl['confidence'] = None
suspected_cl = WfSuspectedCL.Get(cl['repo_name'], cl['revision'])
if not suspected_cl:
continue
build = suspected_cl.builds.get(build_key)
if build:
cl['status'] = build['status']
cl['confidence'] = '%d%%' % (
suspected_cl_util.GetSuspectedCLConfidenceScore(
confidences, build) or 0)
return suspected_cls
def _HasBuildKeyForBuildInfoInFailureResultMap(master_name, builder_name,
build_number):
"""Checks if there is any first failed test."""
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
failure_result_map = analysis.failure_result_map
current_build_key = BaseBuildModel.CreateBuildKey(master_name,
builder_name,
build_number)
for step_keys in failure_result_map.itervalues():
for test_key in step_keys.itervalues():
if test_key == current_build_key:
return True
return False
def _NeedANewCompileTryJob(master_name, builder_name, build_number,
failure_info):
compile_failure = failure_info.failed_steps.get('compile') or None
if compile_failure:
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
analysis.failure_result_map['compile'] = BaseBuildModel.CreateBuildKey(
master_name, builder_name, compile_failure.first_failure)
analysis.put()
if (compile_failure.supported and compile_failure.first_failure
== compile_failure.current_failure):
return True
return False
def GetFirstTimeFailedSteps(master_name, builder_name, build_number):
"""Gets steps that have tests failed first time in the build."""
current_build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
assert analysis, "Cannot get WfAnalysis entity for %s" % current_build_key
first_failed_steps = []
for step, tests in (analysis.failure_result_map or {}).iteritems():
if isinstance(tests, basestring): # Non-swarming.
continue
if current_build_key in tests.values():
first_failed_steps.append(step)
return first_failed_steps
def _UpdateSuspectedCLAndAnalysis(master_name, builder_name, build_number,
cl_info, cl_status, user_name):
repo_name, revision = GetCLInfo(cl_info)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
success = (_UpdateSuspectedCL(repo_name, revision, build_key, cl_status)
and _UpdateAnalysis(master_name, builder_name, build_number,
repo_name, revision, cl_status))
if success:
_AppendTriageHistoryRecord(master_name, builder_name, build_number,
cl_info, cl_status, user_name)
return success
def RunImpl(self, pipeline_input):
culprits_should_take_actions = (
test_culprit_action.GetCulpritsShouldTakeActions(pipeline_input))
if not culprits_should_take_actions:
return
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
culprits = pipeline_input.culprits
build_failure_type = failure_type.TEST
for culprit_revision in culprits_should_take_actions:
culprit_key = culprits.get(culprit_revision)
assert culprit_key, (
'Failed to get culprit_key for culprit {} when analyzing failures'
' at build {}/{}/{}'.format(culprit_revision, master_name,
builder_name, build_number))
revert_status = constants.SKIPPED
if test_culprit_action.CanAutoCreateRevert(
culprit_key, pipeline_input): # pragma: no branch.
revert_status = yield CreateRevertCLPipeline(
CreateRevertCLParameters(cl_key=culprit_key,
build_key=build_key,
failure_type=build_failure_type))
if test_culprit_action.CanAutoCommitRevertByFindit():
submit_revert_pipeline_input = self.CreateInputObjectInstance(
SubmitRevertCLParameters,
cl_key=culprit_key,
revert_status=revert_status,
failure_type=build_failure_type)
yield SubmitRevertCLPipeline(submit_revert_pipeline_input)
# Checks if any of the culprits was also found by heuristic analysis,
# if so send notification right away.
send_notification_to_culprit_input = self.CreateInputObjectInstance(
SendNotificationForCulpritParameters,
cl_key=culprit_key,
force_notify=culprit_action.ShouldForceNotify(
culprit_key, pipeline_input),
revert_status=revert_status,
failure_type=build_failure_type)
yield SendNotificationForCulpritPipeline(
send_notification_to_culprit_input)
def RunImpl(self, pipeline_input):
if not compile_culprit_action.ShouldTakeActionsOnCulprit(
pipeline_input):
return
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
culprits = pipeline_input.culprits
culprit = culprits.values()[0]
force_notify = culprit_action.ShouldForceNotify(
culprit, pipeline_input)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
build_failure_type = failure_type.COMPILE
revert_status = constants.SKIPPED
commit_status = constants.SKIPPED
if compile_culprit_action.CanAutoCreateRevert():
revert_status = yield CreateRevertCLPipeline(
CreateRevertCLParameters(cl_key=culprit,
build_key=build_key,
failure_type=build_failure_type))
if compile_culprit_action.CanAutoCommitRevertByFindit():
submit_revert_pipeline_input = self.CreateInputObjectInstance(
SubmitRevertCLParameters,
cl_key=culprit,
revert_status=revert_status,
failure_type=build_failure_type)
commit_status = yield SubmitRevertCLPipeline(
submit_revert_pipeline_input)
send_notification_to_irc_input = self.CreateInputObjectInstance(
SendNotificationToIrcParameters,
cl_key=culprit,
revert_status=revert_status,
commit_status=commit_status,
failure_type=build_failure_type)
yield SendNotificationToIrcPipeline(send_notification_to_irc_input)
send_notification_to_culprit_input = self.CreateInputObjectInstance(
SendNotificationForCulpritParameters,
cl_key=culprit,
force_notify=force_notify,
revert_status=revert_status,
failure_type=build_failure_type)
yield SendNotificationForCulpritPipeline(
send_notification_to_culprit_input)
def testCreateWfSuspectedCL(self):
approach = analysis_approach_type.HEURISTIC
master_name = 'm'
builder_name = 'b'
build_number = 123
compile_failure_type = failure_type.COMPILE
repo_name = 'chromium'
revision = 'r1'
commit_position = 1
failures = {'compile': []}
top_score = 5
mocked_utcnow = datetime(2016, 10, 4, 0, 0, 0)
self.MockUTCNow(mocked_utcnow)
self.assertIsNone(WfSuspectedCL.Get(repo_name, revision))
suspected_cl_util.UpdateSuspectedCL(repo_name, revision,
commit_position, approach,
master_name, builder_name,
build_number, compile_failure_type,
failures, top_score)
expected_builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [approach],
'failure_type': compile_failure_type,
'failures': failures,
'status': None,
'top_score': top_score
}
}
suspected_cl = WfSuspectedCL.Get(repo_name, revision)
self.assertIsNotNone(suspected_cl)
self.assertEqual([analysis_approach_type.HEURISTIC],
suspected_cl.approaches)
self.assertEqual([compile_failure_type], suspected_cl.failure_type)
self.assertEqual(expected_builds, suspected_cl.builds)
self.assertEqual(mocked_utcnow, suspected_cl.identified_time)
self.assertEqual(mocked_utcnow, suspected_cl.updated_time)
def Create(master_name, builder_name, build_number):
key = ndb.Key(
'M',
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number),
'_DummyModel', build_number)
return _DummyModel(key=key)
def _CreateKey(master_name, builder_name,
build_number): # pragma: no cover
return ndb.Key(
'WfFailureGroup',
BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number))
def testGetsFirstFailureAtTestLevelNoFailureResultMap(self):
master_name = 'm'
builder_name = 'b'
build_number = 2
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'failed': True,
'master_name': 'm',
'builder_name': 'b',
'build_number': 2,
'chromium_revision': None,
'builds': {
2: {
'blame_list': [],
'chromium_revision': None
}
},
'failed_steps': {
'abc_test': {
'current_failure': 2,
'first_failure': 1,
'last_pass': 0,
'tests': {
'Unittest2.Subtest1': {
'current_failure': 2,
'first_failure': 2,
'last_pass': 1,
'base_test_name': 'Unittest2.Subtest1'
},
'Unittest3.Subtest2': {
'current_failure': 2,
'first_failure': 1,
'last_pass': 0,
'base_test_name': 'Unittest3.Subtest2'
}
},
'supported': True
},
'a_test': {
'current_failure': 2,
'first_failure': 1,
'last_pass': 0,
'supported': True
}
},
'failure_type': failure_type.TEST
}
expected_result = {'abc_test': ['Unittest2.Subtest1']}
expected_failure_result_map = {
'abc_test': {
'Unittest2.Subtest1':
BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number),
'Unittest3.Subtest2':
BaseBuildModel.CreateBuildKey(master_name, builder_name, 1)
}
}
result = (test_failure_analysis.GetsFirstFailureAtTestLevel(
master_name, builder_name, build_number,
TestFailureInfo.FromSerializable(failure_info), False))
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertEqual(result, expected_result)
self.assertEqual(analysis.failure_result_map,
expected_failure_result_map)
def MockedGetAllTryJobResults(master_name, builder_name, build_number,
_):
build_key = BaseBuildModel.CreateBuildKey(master_name,
builder_name,
build_number)
return SAMPLE_TRY_JOB_INFO.get(build_key, None)
def _CreateKey(master_name, builder_name, build_number,
step_name): # pragma: no cover
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
return ndb.Key('WfBuild', build_key, 'WfStep', step_name)
def _CreateKey(master_name, builder_name, build_number):
return ndb.Key(
'WfTryJob',
BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number))
def _CreateKey(master_name, builder_name,
build_number): # pragma: no cover
return ndb.Key(
'WfAnalysis',
BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number))
def GetBuildInfo(self, master_name, builder_name, build_number):
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
return self.builds.get(build_key)
def testTestLevelResultIsReturned(self, mock_fn, _):
master_name = 'm'
builder_name = 'b'
build_number = 11
master_url = 'https://build.chromium.org/p/%s' % master_name
builds = {
'builds': [{
'master_url': master_url,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': ['a', 'b on platform']
}]
}
task = WfSwarmingTask.Create(master_name, builder_name, 4,
'b on platform')
task.parameters['ref_name'] = 'b'
task.status = analysis_status.COMPLETED
task.put()
try_job = WfTryJob.Create(master_name, builder_name, 4)
try_job.status = analysis_status.COMPLETED
try_job.test_results = [{
'culprit': {
'a': {
'repo_name': 'chromium',
'revision': 'r4_2',
'commit_position': 42,
'url': None,
},
'b': {
'tests': {
'Unittest3.Subtest1': {
'repo_name': 'chromium',
'revision': 'r4_10',
'commit_position': 410,
'url': None,
},
}
}
},
}]
try_job.put()
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.status = analysis_status.COMPLETED
analysis.failure_result_map = {
'a': '/'.join([master_name, builder_name, '4']),
'b on platform': {
'Unittest1.Subtest1':
'/'.join([master_name, builder_name, '3']),
'Unittest2.Subtest1':
'/'.join([master_name, builder_name, '4']),
'Unittest3.Subtest1':
'/'.join([master_name, builder_name, '4']),
},
}
analysis.result = {
'failures': [{
'step_name':
'a',
'first_failure':
4,
'last_pass':
3,
'supported':
True,
'suspected_cls': [{
'build_number': 4,
'repo_name': 'chromium',
'revision': 'r4_2_failed',
'commit_position': None,
'url': None,
'score': 2,
'hints': {
'modified f4_2.cc (and it was in log)': 2,
},
}],
}, {
'step_name':
'b on platform',
'first_failure':
3,
'last_pass':
2,
'supported':
True,
'suspected_cls': [{
'build_number': 3,
'repo_name': 'chromium',
'revision': 'r3_1',
'commit_position': None,
'url': None,
'score': 5,
'hints': {
'added x/y/f3_1.cc (and it was in log)': 5,
},
}, {
'build_number': 4,
'repo_name': 'chromium',
'revision': 'r4_1',
'commit_position': None,
'url': None,
'score': 2,
'hints': {
'modified f4.cc (and it was in log)': 2,
},
}],
'tests': [{
'test_name':
'Unittest1.Subtest1',
'first_failure':
3,
'last_pass':
2,
'suspected_cls': [{
'build_number': 2,
'repo_name': 'chromium',
'revision': 'r2_1',
'commit_position': None,
'url': None,
'score': 5,
'hints': {
'added x/y/f99_1.cc (and it was in log)': 5,
},
}]
}, {
'test_name':
'Unittest2.Subtest1',
'first_failure':
4,
'last_pass':
2,
'suspected_cls': [{
'build_number': 2,
'repo_name': 'chromium',
'revision': 'r2_1',
'commit_position': None,
'url': None,
'score': 5,
'hints': {
'added x/y/f99_1.cc (and it was in log)': 5,
},
}]
}, {
'test_name': 'Unittest3.Subtest1',
'first_failure': 4,
'last_pass': 2,
'suspected_cls': []
}]
}, {
'step_name': 'c',
'first_failure': 4,
'last_pass': 3,
'supported': False,
'suspected_cls': [],
}]
}
analysis.put()
suspected_cl_42 = WfSuspectedCL.Create('chromium', 'r4_2', 42)
suspected_cl_42.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, 5): {
'approaches': [analysis_approach_type.TRY_JOB]
}
}
suspected_cl_42.put()
suspected_cl_21 = WfSuspectedCL.Create('chromium', 'r2_1', None)
suspected_cl_21.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, 3): {
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
},
BaseBuildModel.CreateBuildKey(master_name, builder_name, 4): {
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
},
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
}
}
suspected_cl_21.put()
suspected_cl_410 = WfSuspectedCL.Create('chromium', 'r4_10', None)
suspected_cl_410.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, 4): {
'approaches': [
analysis_approach_type.HEURISTIC,
analysis_approach_type.TRY_JOB
],
'top_score':
5
},
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
}
}
revert_cl = RevertCL()
revert_cl.revert_cl_url = 'revert_cl_url'
suspected_cl_410.revert_cl = revert_cl
suspected_cl_410.put()
def confidence_side_effect(_, build_info, first_build_info):
if (first_build_info and first_build_info.get('approaches') == [
analysis_approach_type.HEURISTIC,
analysis_approach_type.TRY_JOB
]):
return 100, analysis_approach_type.TRY_JOB
if build_info and build_info.get('top_score'):
return 90, analysis_approach_type.HEURISTIC
return 98, analysis_approach_type.TRY_JOB
mock_fn.side_effect = confidence_side_effect
expected_results = [{
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'a',
'is_sub_test':
False,
'first_known_failed_build_number':
4,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r4_2',
'commit_position': 42,
'confidence': 98,
'analysis_approach': 'TRY_JOB',
'revert_committed': False
}],
'analysis_approach':
'TRY_JOB',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'b on platform',
'is_sub_test':
True,
'test_name':
'Unittest1.Subtest1',
'first_known_failed_build_number':
3,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r2_1',
'confidence': 90,
'analysis_approach': 'HEURISTIC',
'revert_committed': False
}],
'analysis_approach':
'HEURISTIC',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'b on platform',
'is_sub_test':
True,
'test_name':
'Unittest2.Subtest1',
'first_known_failed_build_number':
4,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r2_1',
'confidence': 90,
'analysis_approach': 'HEURISTIC',
'revert_committed': False
}],
'analysis_approach':
'HEURISTIC',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'b on platform',
'is_sub_test':
True,
'test_name':
'Unittest3.Subtest1',
'first_known_failed_build_number':
4,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r4_10',
'commit_position': 410,
'analysis_approach': 'TRY_JOB',
'confidence': 100,
'revert_cl_url': 'revert_cl_url',
'revert_committed': False
}],
'analysis_approach':
'TRY_JOB',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url': master_url,
'builder_name': builder_name,
'build_number': build_number,
'step_name': 'c',
'is_sub_test': False,
'analysis_approach': 'HEURISTIC',
'is_flaky_test': False,
'has_findings': False,
'is_finished': True,
'is_supported': False,
}]
self._MockMasterIsSupported(supported=True)
response = self.call_api('AnalyzeBuildFailures', body=builds)
self.assertEqual(200, response.status_int)
self.assertItemsEqual(expected_results,
response.json_body.get('results'))
