def testGetFailedRevisionFromCompileResult(self):
self.assertIsNone(
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromCompileResult(
None))
self.assertIsNone(
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromCompileResult(
{'report': {}}))
self.assertIsNone(
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromCompileResult(
{
'report': {
'result': {
'rev1': 'passed'
}
}
}))
self.assertEqual(
'rev2',
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromCompileResult(
{
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
}
}
}))
def testFindCulpritForEachTestFailureCulpritsReturned(self):
result = {
'report': {
'culprits': {
'a_tests': {
'Test1': 'rev1'
}
}
}
}
pipeline = IdentifyTryJobCulpritPipeline()
culprit_map, failed_revisions = pipeline._FindCulpritForEachTestFailure(
result)
expected_culprit_map = {
'a_tests': {
'tests': {
'Test1': {
'revision': 'rev1'
}
}
}
}
self.assertEqual(culprit_map, expected_culprit_map)
self.assertEqual(failed_revisions, ['rev1'])
def testIdentifyCulpritForCompileTryJobNoCulprit(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
self._CreateEntities(master_name, builder_name, build_number, try_job_id)
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
None, [], failure_type.COMPILE])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.COMPILE, '1', None)
pipeline.start()
self.execute_queued_tasks()
try_job = WfTryJob.Get(master_name, builder_name, build_number)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertEqual(analysis_status.COMPLETED, try_job.status)
self.assertEqual([], try_job.compile_results)
self.assertIsNone(try_job_data.culprits)
self.assertIsNone(analysis.result_status)
self.assertIsNone(analysis.suspected_cls)
def testIdentifyCulpritForTestTryJobNoTryJobResultNoHeuristicResult(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING)
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
None, [], failure_type.TEST])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.TEST, '1', None)
pipeline.start()
self.execute_queued_tasks()
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
self.assertIsNone(analysis.result_status)
self.assertIsNone(analysis.suspected_cls)
def testIdentifyCulpritForCompileReturnNoneIfAllPassed(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'passed'
}
},
'url': 'url',
'try_job_id': try_job_id,
}
WfTryJobData.Create(try_job_id).put()
WfTryJob.Create(master_name, builder_name, build_number).put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, ['rev1'],
TryJobType.COMPILE, '1', compile_result)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertIsNone(culprit)
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
def testIdentifyCulpritForFlakyCompile(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
compile_result = {
'report': {
'result': {
'rev1': 'failed',
'rev2': 'failed'
},
'metadata': {
'sub_ranges': [
[
None,
'rev2'
]
]
}
},
'url': 'url',
'try_job_id': try_job_id,
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id)
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.result = {
'failures': [
{
'step_name': 'compile',
'suspected_cls': []
}
]
}
analysis.put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
{}, [], failure_type.COMPILE])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.COMPILE, '1', compile_result)
pipeline.start()
self.execute_queued_tasks()
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertEqual(result_status.FLAKY, analysis.result_status)
self.assertEqual([], analysis.suspected_cls)
def testIdentifyCulpritForTestTryJobReturnRevisionIfNoCulpritInfo(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
test_result = {
'report': {
'result': {
'rev3': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
}
}
}
},
'url': 'url',
'try_job_id': try_job_id
}
WfTryJobData.Create(try_job_id).put()
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.status = analysis_status.RUNNING
try_job.put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, ['rev3'], TryJobType.TEST,
'1', test_result)
expected_culprit = {
'a_test': {
'tests': {
'a_test1': {
'revision': 'rev3'
}
}
}
}
self.assertEqual(expected_culprit, culprit)
try_job_data = WfTryJobData.Get(try_job_id)
expected_culprit_data = {
'a_test': {
'a_test1': 'rev3'
}
}
self.assertEqual(expected_culprit_data, try_job_data.culprits)
def testFindCulpritForEachTestFailureRevisionNotRun(self):
result = {
'report': {
'result': {
'rev2': 'passed'
}
}
}
pipeline = IdentifyTryJobCulpritPipeline()
culprit_map, failed_revisions = pipeline._FindCulpritForEachTestFailure(
result)
self.assertEqual(culprit_map, {})
self.assertEqual(failed_revisions, [])
def testGetFailedRevisionFromResultsDict(self):
self.assertIsNone(
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromResultsDict({}))
self.assertEqual(
None,
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromResultsDict(
{'rev1': 'passed'}))
self.assertEqual(
'rev1',
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromResultsDict(
{'rev1': 'failed'}))
self.assertEqual(
'rev2',
IdentifyTryJobCulpritPipeline._GetFailedRevisionFromResultsDict(
{'rev1': 'passed', 'rev2': 'failed'}))
def testIdentifyCulpritForTestTryJobReturnNoneIfNoRevisionToCheck(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
test_result = {
'report': {
'result': {
'rev1': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
}
}
}
},
'url': 'url',
'try_job_id': try_job_id
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING)
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
{}, [], failure_type.TEST])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number, failure_type.TEST, '1',
test_result)
pipeline.start()
self.execute_queued_tasks()
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
self.assertIsNone(analysis.result_status)
self.assertIsNone(analysis.suspected_cls)
def testReturnNoneIfNoTryJob(self):
master_name = 'm'
builder_name = 'b'
build_number = 8
WfTryJob.Create(master_name, builder_name, build_number).put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
None, [], failure_type.TEST])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.TEST, None, None)
pipeline.start()
self.execute_queued_tasks()
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(try_job.test_results, [])
self.assertEqual(try_job.status, analysis_status.COMPLETED)
def testIdentifyCulpritForTestTryJobReturnNoneIfNoTryJobResult(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
WfTryJobData.Create(try_job_id).put()
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.status = analysis_status.RUNNING
try_job.put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, ['rev1', 'rev2'],
TryJobType.TEST, '1', None)
self.assertIsNone(culprit)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
def testIdentifyCulpritForCompileReturnNoneIfAllPassed(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'passed'
}
},
'url': 'url',
'try_job_id': try_job_id,
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id)
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
{}, [], failure_type.COMPILE])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.COMPILE, '1', compile_result)
pipeline.start()
self.execute_queued_tasks()
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
self.assertIsNone(analysis.result_status)
self.assertIsNone(analysis.suspected_cls)
def testIdentifyCulpritForCompileTryJobNoCulprit(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.put()
try_job_data = WfTryJobData.Create(try_job_id)
try_job_data.put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, ['rev1'],
TryJobType.COMPILE, '1', None)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(analysis_status.COMPLETED, try_job.status)
self.assertEqual([], try_job.compile_results)
self.assertIsNone(culprit)
self.assertIsNone(try_job_data.culprits)
def testIdentifyCulpritForTestTryJobNoTryJobResultWithHeuristicResult(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
suspected_cl = {
'revision': 'rev1',
'commit_position': 1,
'url': 'url_1',
'repo_name': 'chromium'
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING)
# Heuristic analysis already provided some results.
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.result_status = result_status.FOUND_UNTRIAGED
analysis.suspected_cls = [suspected_cl]
analysis.put()
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
None, [['chromium', 'rev1']],
failure_type.TEST])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.TEST, '1', None)
pipeline.start()
self.execute_queued_tasks()
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
# Ensure analysis results are not updated since no culprit from try job.
self.assertEqual(analysis.result_status, result_status.FOUND_UNTRIAGED)
self.assertEqual(analysis.suspected_cls, [suspected_cl])
def testIdentifyCulpritForTestTryJobReturnNoneIfNoRevisionToCheck(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
test_result = {
'report': {
'result': {
'rev1': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
}
}
}
},
'url': 'url',
'try_job_id': try_job_id
}
WfTryJobData.Create(try_job_id).put()
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.status = analysis_status.RUNNING
try_job.put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, [], TryJobType.TEST, '1',
test_result)
self.assertIsNone(culprit)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.culprits)
def testAnalysisIsUpdatedOnlyIfStatusOrSuspectedCLsChanged(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
repo_name = 'chromium'
revision = 'rev1'
commit_position = 1
heuristic_suspected_cl = {
'revision': revision,
'commit_position': commit_position,
'url': 'url_1',
'repo_name': repo_name
}
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.suspected_cls = [heuristic_suspected_cl]
analysis.result_status = result_status.FOUND_UNTRIAGED
analysis.put()
version = analysis.version
build_key = build_util.CreateBuildId(
master_name, builder_name, build_number)
suspected_cl = WfSuspectedCL.Create(repo_name, revision, commit_position)
suspected_cl.approaches = [analysis_approach_type.HEURISTIC]
suspected_cl.builds = {
build_key: {
'approaches': [analysis_approach_type.HEURISTIC],
'failure_type': failure_type.COMPILE,
'failures': {'compile': []},
'status': None,
'top_score': 4
}
}
suspected_cl.put()
compile_result = {
'report': {
'result': {
revision: 'failed',
},
'culprit': revision
},
'try_job_id': try_job_id,
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING,
compile_results=[compile_result])
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
{revision: heuristic_suspected_cl},
[[repo_name, revision]],
failure_type.COMPILE])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.COMPILE, '1', compile_result)
pipeline.start()
self.execute_queued_tasks()
self.assertEqual(analysis.result_status,
result_status.FOUND_UNTRIAGED)
self.assertEqual(analysis.suspected_cls, [heuristic_suspected_cl])
self.assertEqual(version, analysis.version) # No update to analysis.
expected_approaches = [
analysis_approach_type.HEURISTIC, analysis_approach_type.TRY_JOB]
expected_builds = {
build_key: {
'approaches': expected_approaches,
'failure_type': failure_type.COMPILE,
'failures': {'compile': []},
'status': None,
'top_score': 4
}
}
suspected_cl = WfSuspectedCL.Get(repo_name, revision)
self.assertEqual(expected_approaches, suspected_cl.approaches)
self.assertEqual(expected_builds, suspected_cl.builds)
def testIdentifyCulpritForCompileTryJobSuccess(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'culprit': 'rev2'
},
'try_job_id': try_job_id,
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING,
compile_results=[compile_result])
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
expected_culprit = 'rev2'
expected_suspected_cl = {
'revision': 'rev2',
'commit_position': 2,
'url': 'url_2',
'repo_name': 'chromium'
}
expected_compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'culprit': 'rev2'
},
'try_job_id': try_job_id,
'culprit': {
'compile': expected_suspected_cl
}
}
expected_analysis_suspected_cls = [{
'revision': 'rev2',
'commit_position': 2,
'url': 'url_2',
'repo_name': 'chromium',
'failures': {'compile': []},
'top_score': None
}]
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
{expected_culprit: expected_suspected_cl},
[], failure_type.COMPILE])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.COMPILE, '1', compile_result)
pipeline.start()
self.execute_queued_tasks()
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_compile_result, try_job.compile_results[-1])
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertEqual({'compile': expected_culprit}, try_job_data.culprits)
self.assertEqual(analysis.result_status,
result_status.FOUND_UNTRIAGED)
self.assertEqual(analysis.suspected_cls, expected_analysis_suspected_cls)
def testIdentifyCulpritForTestTryJobSuccess(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
test_result = {
'report': {
'result': {
'rev1': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
},
'c_test': {
'status': 'passed',
'valid': True
}
},
'rev2': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1', 'a_test2']
},
'b_test': {
'status': 'passed',
'valid': True
},
'c_test': {
'status': 'failed',
'valid': True,
'failures': []
}
}
}
},
'url': 'url',
'try_job_id': try_job_id
}
WfTryJobData.Create(try_job_id).put()
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.status = analysis_status.RUNNING
try_job.test_results = [test_result]
try_job.put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, ['rev1', 'rev2'],
TryJobType.TEST, '1', test_result)
expected_test_result = {
'report': {
'result': {
'rev1': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
},
'c_test': {
'status': 'passed',
'valid': True
}
},
'rev2': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1', 'a_test2']
},
'b_test': {
'status': 'passed',
'valid': True
},
'c_test': {
'status': 'failed',
'valid': True,
'failures': []
}
}
}
},
'url': 'url',
'try_job_id': try_job_id,
'culprit': {
'a_test': {
'tests': {
'a_test1': {
'revision': 'rev1',
'commit_position': '1',
'review_url': 'url_1'
},
'a_test2': {
'revision': 'rev2',
'commit_position': '2',
'review_url': 'url_2'
}
}
},
'b_test': {
'tests': {
'b_test1': {
'revision': 'rev1',
'commit_position': '1',
'review_url': 'url_1'
}
}
},
'c_test': {
'revision': 'rev2',
'commit_position': '2',
'review_url': 'url_2',
'tests': {}
}
}
}
self.assertEqual(expected_test_result['culprit'], culprit)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_test_result, try_job.test_results[-1])
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
expected_culprit_data = {
'a_test': {
'a_test1': 'rev1',
'a_test2': 'rev2',
},
'b_test': {
'b_test1': 'rev1',
},
'c_test': 'rev2'
}
self.assertEqual(expected_culprit_data, try_job_data.culprits)
def run(self, master_name, builder_name, build_number, failure_info,
signals, heuristic_result, build_completed, force_try_job):
"""Starts a try job if one is needed for the given failure."""
if not build_completed: # Only start try-jobs for completed builds.
return
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, signals,
heuristic_result, force_try_job)
if not need_try_job:
return
try_job_type = failure_info['failure_type']
last_pass = _GetLastPass(build_number, failure_info, try_job_type)
if last_pass is None: # pragma: no cover
logging.warning(
'Couldn"t start try job for build %s, %s, %d because'
' last_pass is not found.', master_name, builder_name,
build_number)
return
good_revision = failure_info['builds'][str(
last_pass)]['chromium_revision']
bad_revision = failure_info['builds'][str(
build_number)]['chromium_revision']
suspected_revisions = _GetSuspectsFromHeuristicResult(heuristic_result)
if try_job_type == failure_type.COMPILE:
compile_targets = try_job_util.GetFailedTargetsFromSignals(
signals, master_name, builder_name)
dimensions = waterfall_config.GetTrybotDimensions(
master_name, builder_name)
cache_name = swarming_util.GetCacheName(master_name, builder_name)
try_job_id = yield ScheduleCompileTryJobPipeline(
master_name, builder_name, build_number, good_revision,
bad_revision, try_job_type, compile_targets,
suspected_revisions, cache_name, dimensions)
else:
# If try_job_type is other type, the pipeline has returned.
# So here the try_job_type is failure_type.TEST.
# Waits and gets the swarming tasks' results.
task_results = []
for step_name, step_failure in failure_info[
'failed_steps'].iteritems():
step_has_first_time_failure = _HasFirstTimeFailure(
step_failure.get('tests', {}), build_number)
if not step_has_first_time_failure:
continue
task_result = yield ProcessSwarmingTaskResultPipeline(
master_name, builder_name, build_number, step_name)
task_results.append(task_result)
yield UpdateAnalysisWithFlakeInfoPipeline(master_name,
builder_name,
build_number,
*task_results)
parent_mastername = failure_info.get(
'parent_mastername') or master_name
parent_buildername = failure_info.get('parent_buildername') or (
builder_name)
dimensions = waterfall_config.GetTrybotDimensions(
parent_mastername, parent_buildername)
cache_name = swarming_util.GetCacheName(parent_mastername,
parent_buildername)
try_job_id = yield ScheduleTestTryJobPipeline(
master_name, builder_name, build_number, good_revision,
bad_revision, try_job_type, suspected_revisions, cache_name,
dimensions, *task_results)
try_job_result = yield MonitorTryJobPipeline(try_job_key.urlsafe(),
try_job_type, try_job_id)
yield IdentifyTryJobCulpritPipeline(master_name, builder_name,
build_number, try_job_type,
try_job_id, try_job_result)
def testIdentifyCulpritForTestTryJobReturnRevisionIfNoCulpritInfo(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
test_result = {
'report': {
'result': {
'rev3': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
}
}
},
'culprits': {
'a_test': {
'a_test1': 'rev3'
}
}
},
'url': 'url',
'try_job_id': try_job_id
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING)
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
expected_suspected_cl = {
'revision': 'rev3',
'repo_name': 'chromium'
}
expected_analysis_suspected_cls = [
{
'revision': 'rev3',
'repo_name': 'chromium',
'failures': {'a_test': ['a_test1']},
'top_score': None
}
]
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
{'rev3': expected_suspected_cl}, [],
failure_type.TEST])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number, failure_type.TEST,
'1', test_result)
pipeline.start()
self.execute_queued_tasks()
try_job_data = WfTryJobData.Get(try_job_id)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
expected_culprit_data = {
'a_test': {
'a_test1': 'rev3'
}
}
self.assertEqual(expected_culprit_data, try_job_data.culprits)
self.assertEqual(analysis.result_status,
result_status.FOUND_UNTRIAGED)
self.assertEqual(analysis.suspected_cls, expected_analysis_suspected_cls)
def testIdentifyCulpritForTestTryJobSuccess(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
test_result = {
'report': {
'result': {
'rev0': {
'a_test': {
'status': 'passed',
'valid': True,
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
}
},
'rev1': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
}
},
'rev2': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1', 'a_test2']
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
}
}
},
'culprits': {
'a_test': {
'a_test1': 'rev1',
'a_test2': 'rev2'
},
},
'flakes': {
'b_test': ['b_test1']
}
},
'url': 'url',
'try_job_id': try_job_id
}
self._CreateEntities(master_name, builder_name, build_number, try_job_id,
try_job_status=analysis_status.RUNNING,
test_results=[test_result])
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.put()
a_test1_suspected_cl = {
'revision': 'rev1',
'commit_position': 1,
'url': 'url_1',
'repo_name': 'chromium'
}
a_test2_suspected_cl = {
'revision': 'rev2',
'commit_position': 2,
'url': 'url_2',
'repo_name': 'chromium'
}
expected_test_result = {
'report': {
'result': {
'rev0': {
'a_test': {
'status': 'passed',
'valid': True,
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
}
},
'rev1': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1']
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
}
},
'rev2': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['a_test1', 'a_test2']
},
'b_test': {
'status': 'failed',
'valid': True,
'failures': ['b_test1']
}
}
},
'culprits': {
'a_test': {
'a_test1': 'rev1',
'a_test2': 'rev2'
},
},
'flakes': {
'b_test': ['b_test1']
}
},
'url': 'url',
'try_job_id': try_job_id,
'culprit': {
'a_test': {
'tests': {
'a_test1': a_test1_suspected_cl,
'a_test2': a_test2_suspected_cl
}
}
}
}
expected_culprits = {
'rev1': a_test1_suspected_cl,
'rev2': a_test2_suspected_cl
}
self.MockPipeline(RevertAndNotifyCulpritPipeline,
None,
expected_args=[master_name, builder_name, build_number,
expected_culprits, [], failure_type.TEST])
pipeline = IdentifyTryJobCulpritPipeline(
master_name, builder_name, build_number,
failure_type.TEST, '1', test_result)
pipeline.start()
self.execute_queued_tasks()
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_test_result, try_job.test_results[-1])
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
expected_culprit_data = {
'a_test': {
'a_test1': 'rev1',
'a_test2': 'rev2',
}
}
expected_cls = [
{
'revision': 'rev1',
'commit_position': 1,
'url': 'url_1',
'repo_name': 'chromium',
'failures': {
'a_test': ['a_test1'],
'b_test': ['b_test1'],
},
'top_score': None
},
{
'revision': 'rev2',
'commit_position': 2,
'url': 'url_2',
'repo_name': 'chromium',
'failures': {
'a_test': ['a_test1', 'a_test2'],
'b_test': ['b_test1'],
},
'top_score': None
}
]
self.assertEqual(expected_culprit_data, try_job_data.culprits)
self.assertEqual(analysis.result_status,
result_status.FOUND_UNTRIAGED)
self.assertEqual(analysis.suspected_cls, expected_cls)
def testIdentifyCulpritForCompileTryJobSuccess(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
},
}
try_job_data = WfTryJobData.Create(try_job_id)
try_job_data.put()
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.status = analysis_status.RUNNING
try_job.compile_results = [{
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
},
'try_job_id': try_job_id,
}]
try_job.put()
pipeline = IdentifyTryJobCulpritPipeline()
culprit = pipeline.run(
master_name, builder_name, build_number, ['rev1'],
TryJobType.COMPILE, '1', compile_result)
expected_culprit = 'rev2'
expected_compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
}
},
'try_job_id': try_job_id,
'culprit': {
'compile': {
'revision': 'rev2',
'commit_position': '2',
'review_url': 'url_2'
}
}
}
self.assertEqual(expected_compile_result['culprit'], culprit)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_compile_result, try_job.compile_results[-1])
self.assertEqual(analysis_status.COMPLETED, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertEqual({'compile': expected_culprit}, try_job_data.culprits)