def testNoCompileTryJobBecauseNoGoodRevision(self, mock_pipeline,
mock_parameter, mock_fn):
mock_parameter.return_value = RunCompileTryJobParameters(
good_revision=None)
try_job = WfTryJob.Create('m', 'b', 1)
try_job.put()
mock_fn.return_value = (True, try_job.key.urlsafe())
failure_info = {'failure_type': failure_type.COMPILE}
heuristic_result = {
'failure_info': failure_info,
'signals': {},
'heuristic_result': {}
}
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(master_name='m',
builder_name='b',
build_number=1),
heuristic_result=CompileHeuristicAnalysisOutput.FromSerializable(
heuristic_result),
build_completed=True,
force=False)
pipeline = StartCompileTryJobPipeline(start_try_job_params)
result = pipeline.RunImpl(start_try_job_params)
self.assertEqual(list(result), [])
self.assertFalse(mock_pipeline.called)
def testAnalyzeCompileFailurePipelineStartTryJob(self, mocked_pipeline,
mock_log, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 124
failure_info = {
'failed_steps': {
'compile': {
'last_pass': 122,
'current_failure': 123,
'first_failure': 123
}
}
}
self._SetupAnalysis(
master_name,
builder_name,
build_number,
status=analysis_status.RUNNING,
signals={},
failure_info=failure_info)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable({}),
build_completed=False,
force=False)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.OnAbort(pipeline_input)
heuristic_result = CompileHeuristicAnalysisOutput.FromSerializable({
'failure_info': failure_info,
'signals': {},
'heuristic_result': None
})
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
heuristic_result=heuristic_result,
build_completed=False,
force=False)
mocked_pipeline.assert_called_once_with(start_try_job_params)
mocked_pipeline.assert_has_calls(
[mock.call().start(queue_name=constants.WATERFALL_ANALYSIS_QUEUE)])
mock_log.assert_called_once_with(
'A try job pipeline for build %s, %s, %s starts after heuristic '
'analysis was aborted. Check pipeline at: %s.', master_name,
builder_name, build_number, root_pipeline.pipeline_status_path)
mock_mon.assert_called_once_with(master_name, builder_name,
analysis_status.ERROR,
analysis_approach_type.HEURISTIC)
def testBuildFailurePipelineFlow(self):
master_name = 'm'
builder_name = 'b'
build_number = 124
current_failure_info = {}
self._SetupAnalysis(master_name, builder_name, build_number)
heuristic_params = CompileHeuristicAnalysisParameters.FromSerializable({
'failure_info': current_failure_info,
'build_completed': False
})
heuristic_output = CompileHeuristicAnalysisOutput.FromSerializable({
'failure_info': None,
'signals': None,
'heuristic_result': {}
})
self.MockSynchronousPipeline(
analyze_compile_failure_pipeline.HeuristicAnalysisForCompilePipeline,
heuristic_params, heuristic_output)
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
heuristic_result=heuristic_output,
build_completed=False,
force=False)
self.MockGeneratorPipeline(
analyze_compile_failure_pipeline.StartCompileTryJobPipeline,
start_try_job_params, False)
report_event_input = report_event_pipeline.ReportEventInput(
analysis_urlsafe_key=WfAnalysis.Get(master_name, builder_name,
build_number).key.urlsafe())
self.MockGeneratorPipeline(
report_event_pipeline.ReportAnalysisEventPipeline, report_event_input,
None)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable(
current_failure_info),
build_completed=False,
force=False)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.start(queue_name=constants.DEFAULT_QUEUE)
self.execute_queued_tasks()
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertEqual(analysis_status.RUNNING, analysis.status)
def testHeuristicAnalysisForCompilePipeline(self, mock_result):
pipeline_input = CompileHeuristicAnalysisParameters.FromSerializable({
'failure_info': {},
'build_completed':
True
})
analysis_result = CompileHeuristicAnalysisOutput.FromSerializable({
'failure_info': {},
'signals': {},
'heuristic_result': {}
})
mock_result.return_value = analysis_result
pipeline = HeuristicAnalysisForCompilePipeline(pipeline_input)
result = pipeline.run(pipeline_input)
self.assertEqual(result, analysis_result.ToSerializable())
def testNotScheduleTryJobIfBuildNotCompleted(self):
heuristic_result = {
'failure_info': {},
'signals': {},
'heuristic_result': {}
}
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(master_name='m',
builder_name='b',
build_number=1),
heuristic_result=CompileHeuristicAnalysisOutput.FromSerializable(
heuristic_result),
build_completed=False,
force=False)
pipeline = StartCompileTryJobPipeline(start_try_job_params)
result = pipeline.RunImpl(start_try_job_params)
self.assertEqual(list(result), [])
def testOnAbort(self, mock_mon):
failure_info = {'failure_type': failure_type.COMPILE}
heuristic_result = {
'failure_info': failure_info,
'signals': {},
'heuristic_result': {}
}
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(master_name='m',
builder_name='b',
build_number=1),
heuristic_result=CompileHeuristicAnalysisOutput.FromSerializable(
heuristic_result),
build_completed=True,
force=False)
pipeline = StartCompileTryJobPipeline(start_try_job_params)
pipeline.OnAbort(start_try_job_params)
self.assertFalse(mock_mon.called)
def _ContinueTryJobPipeline(self, pipeline_input, failure_info, signals):
heuristic_result = {
'failure_info': failure_info,
'signals': signals,
'heuristic_result': None
}
start_compile_try_job_input = StartCompileTryJobInput(
build_key=pipeline_input.build_key,
heuristic_result=CompileHeuristicAnalysisOutput.FromSerializable(
heuristic_result),
build_completed=pipeline_input.build_completed,
force=pipeline_input.force)
try_job_pipeline = StartCompileTryJobPipeline(
start_compile_try_job_input)
try_job_pipeline.target = appengine_util.GetTargetNameForModule(
constants.WATERFALL_BACKEND)
try_job_pipeline.start(queue_name=constants.WATERFALL_ANALYSIS_QUEUE)
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
logging.info(
'A try job pipeline for build %s, %s, %s starts after heuristic '
'analysis was aborted. Check pipeline at: %s.', master_name,
builder_name, build_number, self.pipeline_status_path)
def testCompileTryJob(self, mock_fn, mock_parameter, _):
master_name = 'm'
builder_name = 'b'
build_number = 1
build_key = BuildKey(master_name=master_name,
builder_name=builder_name,
build_number=build_number)
try_job_type = failure_type.COMPILE
failure_info = {
'failure_type': try_job_type,
'builds': {
'0': {
'blame_list': ['r0', 'r1'],
'chromium_revision': 'r1'
},
'1': {
'blame_list': ['r2'],
'chromium_revision': 'r2'
}
},
'failed_steps': {
'compile': {
'first_failure': 1,
'last_pass': 0
}
}
}
good_revision = 'r1'
bad_revision = 'r2'
try_job = WfTryJob.Create('m', 'b', 1)
try_job.put()
mock_fn.return_value = True, try_job.key.urlsafe()
parameters = {
'build_key': {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number
},
'good_revision': good_revision,
'bad_revision': bad_revision,
'compile_targets': [],
'suspected_revisions': [],
'cache_name': 'cache_name',
'dimensions': [],
'urlsafe_try_job_key': 'urlsafe_try_job_key'
}
pipeline_input = RunCompileTryJobParameters.FromSerializable(
parameters)
mock_parameter.return_value = pipeline_input
expected_compile_result = {
'report': {
'culprit': None,
'last_checked_out_revision': None,
'previously_cached_revision': None,
'previously_checked_out_revision': None,
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'metadata': {}
},
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': '1',
'culprit': None
}
self.MockAsynchronousPipeline(
start_compile_try_job_pipeline.RunCompileTryJobPipeline,
pipeline_input, expected_compile_result)
identify_culprit_input = IdentifyCompileTryJobCulpritParameters(
build_key=build_key,
result=CompileTryJobResult.FromSerializable(
expected_compile_result))
self.MockGeneratorPipeline(
culprit_pipeline.IdentifyCompileTryJobCulpritPipeline,
identify_culprit_input, False)
heuristic_result = {
'failure_info': failure_info,
'signals': {},
'heuristic_result': {}
}
start_try_job_params = StartCompileTryJobInput(
build_key=build_key,
heuristic_result=CompileHeuristicAnalysisOutput.FromSerializable(
heuristic_result),
build_completed=True,
force=False)
pipeline = StartCompileTryJobPipeline(start_try_job_params)
pipeline.start()
self.execute_queued_tasks()
def HeuristicAnalysisForCompile(heuristic_params):
"""Identifies culprit CL.
Args:
heuristic_params (CompileHeuristicAnalysisParameters): A structured object
with 2 fields:
failure_info (CompileFailureInfo): An object of failure info for the
current failed build.
build_completed (bool): If the build is completed.
Returns:
A CompileHeuristicAnalysisOutput object with information about
failure_info, signals and heuristic_result.
"""
failure_info = heuristic_params.failure_info
master_name = failure_info.master_name
builder_name = failure_info.builder_name
build_number = failure_info.build_number
# 1. Detects first failed builds for failed compile step,
# updates failure_info.
failure_info = ci_failure.CheckForFirstKnownFailure(
master_name, builder_name, build_number, failure_info)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
analysis.failure_info = failure_info.ToSerializable()
analysis.put()
# Lacking chromium_revision indicates something is wrong in Findit,
assert failure_info.chromium_revision, (
'No end_revision when pulling change logs.')
# 2. Extracts failure signal.
signals = extract_compile_signal.ExtractSignalsForCompileFailure(
failure_info, FinditHttpClient())
# 3. Gets change_logs.
change_logs = git.PullChangeLogs(
ci_failure.GetGoodRevision(failure_info), failure_info.chromium_revision)
# 4. Gets deps info.
deps_info = deps.ExtractDepsInfo(failure_info, change_logs)
# 5. Analyzes the compile failure using information collected above.
heuristic_result, suspected_cls = AnalyzeCompileFailure(
failure_info, change_logs, deps_info, signals)
# Save results and other info to analysis.
build_failure_analysis.SaveAnalysisAfterHeuristicAnalysisCompletes(
master_name, builder_name, build_number, heuristic_result, suspected_cls)
# Save suspected_cls to data_store.
build_failure_analysis.SaveSuspectedCLs(suspected_cls, master_name,
builder_name, build_number,
failure_info.failure_type)
# Monitors analysis status change.
RecordCompileFailureAnalysisStateChange(master_name, builder_name,
analysis_status.COMPLETED,
analysis_approach_type.HEURISTIC)
return CompileHeuristicAnalysisOutput(
failure_info=failure_info,
signals=CompileFailureSignals.FromSerializable(signals),
heuristic_result=CompileHeuristicResult.FromSerializable(
heuristic_result))
def testHeuristicAnalysisForCompile(self, mock_result, mock_signals,
mock_failure_info, mock_mon, *_):
failure_info = {
'build_number': 213,
'master_name': 'chromium.win',
'builder_name': 'WinMSVC64 (dbg)',
'parent_mastername': None,
'parent_buildername': None,
'failed_steps': {
'compile': {
'last_pass': 212,
'current_failure': 213,
'first_failure': 213
}
},
'builds': {
'212': {
'blame_list': ['3045acb501991e37fb2416ab8816d2ff4e66735f',],
'chromium_revision': 'c7388ba52388421e91c113ed807dec16b830c45b'
},
'213': {
'blame_list': ['e282b48ad7a9715d132c649fe1aff9dde0347b1c',],
'chromium_revision': '2fefee0825b80ec3ebec5c661526818da9490180'
}
},
'failure_type': 8,
'failed': True,
'chromium_revision': '2fefee0825b80ec3ebec5c661526818da9490180',
}
signals = {
'compile': {
'failed_edges': [{
'dependencies': [
'third_party/webrtc/media/base/codec.h',
'third_party/webrtc/rtc_base/sanitizer.h',
],
'output_nodes': ['obj/third_party/webrtc/media//file.obj'],
'rule':
'CXX'
}],
'files': {
'c:/b/c/b/win/src/third_party/webrtc/media/engine/file.cc': [
76
]
},
'failed_targets': [{
'source': '../../third_party/webrtc/media/engine/target1.cc',
'target': 'obj/third_party/webrtc/media//file.obj'
}],
'failed_output_nodes': [
'obj/third_party/webrtc/media/rtc_audio_video/fon.obj'
],
'keywords': {}
}
}
mock_signals.return_value = signals
heuristic_result = {
'failures': [{
'first_failure':
213,
'supported':
True,
'suspected_cls': [{
'commit_position': 517979,
'url': 'url/0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'hints': {
'add a.cc': 5
},
'score': 5,
'build_number': 213,
'revision': '0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'repo_name': 'chromium'
}],
'step_name':
'compile',
'last_pass':
212,
'new_compile_suspected_cls': [{
'commit_position': 517979,
'url': 'url/0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'hints': {
'add a.cc': 5
},
'score': 5,
'build_number': 213,
'revision': '0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'repo_name': 'chromium'
}],
'use_ninja_dependencies':
True
}]
}
mock_result.return_value = heuristic_result, []
mock_failure_info.return_value = CompileFailureInfo.FromSerializable(
failure_info)
WfAnalysis.Create('chromium.win', 'WinMSVC64 (dbg)', 213).put()
heuristic_params = CompileHeuristicAnalysisParameters(
failure_info=CompileFailureInfo.FromSerializable(failure_info),
build_completed=True)
result = compile_failure_analysis.HeuristicAnalysisForCompile(
heuristic_params)
expected_result = {
'failure_info': failure_info,
'signals': signals,
'heuristic_result': heuristic_result
}
self.assertEqual(
result,
CompileHeuristicAnalysisOutput.FromSerializable(expected_result))
mock_mon.assert_called_once_with('chromium.win', 'WinMSVC64 (dbg)',
analysis_status.COMPLETED,
analysis_approach_type.HEURISTIC)