def testAnalyzeCompileFailurePipelineNotAbortedIfWithoutError(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 124
self._SetupAnalysis(
master_name,
builder_name,
build_number,
status=analysis_status.COMPLETED)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable({}),
build_completed=False,
force=True)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.OnAbort(pipeline_input)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertIsNotNone(analysis)
self.assertNotEqual(analysis_status.ERROR, analysis.status)
self.assertFalse(mock_mon.called)
def testAnalyzeCompileFailurePipelineAbortedIfWithError(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 124
self._SetupAnalysis(
master_name, builder_name, build_number, status=analysis_status.RUNNING)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable({}),
build_completed=False,
force=True)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.OnAbort(pipeline_input)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertIsNotNone(analysis)
self.assertEqual(analysis_status.ERROR, analysis.status)
self.assertIsNone(analysis.result_status)
self.assertTrue(analysis.aborted)
mock_mon.assert_called_once_with(master_name, builder_name,
analysis_status.ERROR,
analysis_approach_type.HEURISTIC)
def testCompileNotSupport(self):
failure_info = copy.deepcopy(_COMPILE_FAILURE_INFO)
failure_info['failed_steps']['compile']['supported'] = False
self.assertEqual({},
extract_compile_signal.ExtractSignalsForCompileFailure(
CompileFailureInfo.FromSerializable(failure_info),
None))
def testAnalyzeCompileFailurePipelineStartTryJob(self, mocked_pipeline,
mock_log, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 124
failure_info = {
'failed_steps': {
'compile': {
'last_pass': 122,
'current_failure': 123,
'first_failure': 123
}
}
}
self._SetupAnalysis(
master_name,
builder_name,
build_number,
status=analysis_status.RUNNING,
signals={},
failure_info=failure_info)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable({}),
build_completed=False,
force=False)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.OnAbort(pipeline_input)
heuristic_result = CompileHeuristicAnalysisOutput.FromSerializable({
'failure_info': failure_info,
'signals': {},
'heuristic_result': None
})
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
heuristic_result=heuristic_result,
build_completed=False,
force=False)
mocked_pipeline.assert_called_once_with(start_try_job_params)
mocked_pipeline.assert_has_calls(
[mock.call().start(queue_name=constants.WATERFALL_ANALYSIS_QUEUE)])
mock_log.assert_called_once_with(
'A try job pipeline for build %s, %s, %s starts after heuristic '
'analysis was aborted. Check pipeline at: %s.', master_name,
builder_name, build_number, root_pipeline.pipeline_status_path)
mock_mon.assert_called_once_with(master_name, builder_name,
analysis_status.ERROR,
analysis_approach_type.HEURISTIC)
def GetCompileFailureInfo(self, context, build,
first_failures_in_current_build):
"""Creates structured object expected by heuristic analysis code."""
# As per common/waterfall/failure_type.py
LEGACY_COMPILE_TYPE = 0x08
return CompileFailureInfo.FromSerializable({
'failed_steps': {
'compile': {
'supported':
True,
'last_pass':
first_failures_in_current_build['last_passed_build']
['number'],
'current_failure':
build.number,
'first_failure':
build.number,
},
},
'master_name':
build.input.properties['mastername'],
'builder_name':
build.builder.builder,
'build_number':
build.number,
'parent_mastername':
None, # These only apply to some testers.
'parent_buildername':
None,
'builds': {
build.number: {
# Construct a list of revisions since the last passing build.
'blame_list':
git.GetCommitsBetweenRevisionsInOrder(
first_failures_in_current_build['last_passed_build']
['commit_id'],
context.gitiles_id,
ascending=False),
'chromium_revision':
context.gitiles_id,
},
},
'failure_type':
LEGACY_COMPILE_TYPE,
'failed':
True,
'chromium_revision':
context.gitiles_id,
'is_luci':
True,
'buildbucket_bucket':
'luci.%s.%s' % (build.builder.project, build.builder.bucket),
'buildbucket_id':
str(build.id),
})
def testBuildFailurePipelineFlow(self):
master_name = 'm'
builder_name = 'b'
build_number = 124
current_failure_info = {}
self._SetupAnalysis(master_name, builder_name, build_number)
heuristic_params = CompileHeuristicAnalysisParameters.FromSerializable({
'failure_info': current_failure_info,
'build_completed': False
})
heuristic_output = CompileHeuristicAnalysisOutput.FromSerializable({
'failure_info': None,
'signals': None,
'heuristic_result': {}
})
self.MockSynchronousPipeline(
analyze_compile_failure_pipeline.HeuristicAnalysisForCompilePipeline,
heuristic_params, heuristic_output)
start_try_job_params = StartCompileTryJobInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
heuristic_result=heuristic_output,
build_completed=False,
force=False)
self.MockGeneratorPipeline(
analyze_compile_failure_pipeline.StartCompileTryJobPipeline,
start_try_job_params, False)
report_event_input = report_event_pipeline.ReportEventInput(
analysis_urlsafe_key=WfAnalysis.Get(master_name, builder_name,
build_number).key.urlsafe())
self.MockGeneratorPipeline(
report_event_pipeline.ReportAnalysisEventPipeline, report_event_input,
None)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable(
current_failure_info),
build_completed=False,
force=False)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.start(queue_name=constants.DEFAULT_QUEUE)
self.execute_queued_tasks()
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertEqual(analysis_status.RUNNING, analysis.status)
def testGetCompileStepSignalFromNinjaJsonOutput(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number)
signals = extract_compile_signal.ExtractSignalsForCompileFailure(
CompileFailureInfo.FromSerializable(_COMPILE_FAILURE_INFO), None)
expected_failed_edges = [{
'output_nodes': ['a/b.o'],
'rule': 'CXX',
'dependencies': ['b.h', 'b.c']
}]
self.assertEqual(expected_failed_edges, signals['compile']['failed_edges'])
def testAnalyzeCompileFailureNotSupported(self, _):
failure_info = {
'master_name': 'm',
'builder_name': 'b',
'build_number': 123,
'failed_steps': {
'compile': {
'current_failure': 123,
'first_failure': 121
}
},
'builds': {}
}
analysis_result, _ = compile_failure_analysis.AnalyzeCompileFailure(
CompileFailureInfo.FromSerializable(failure_info), None, None,
{'compile': {}})
self.assertEqual({'failures': []}, analysis_result)
def testUpdateAbortedAnalysisNoTryJob(self):
master_name = 'm'
builder_name = 'b'
build_number = 124
parameter = AnalyzeCompileFailureInput(
build_key=BuildKey(master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable({}),
build_completed=False,
force=True)
WfAnalysis.Create(master_name, builder_name, build_number).put()
analysis, run_try_job, heuristic_aborted = (
build_failure_analysis.UpdateAbortedAnalysis(parameter))
self.assertEqual(analysis_status.ERROR, analysis.status)
self.assertTrue(analysis.aborted)
self.assertFalse(run_try_job)
self.assertTrue(heuristic_aborted)
def testCompileNotInFailedSteps(self):
failure_info = {
'master_name': 'm',
'builder_name': 'b',
'build_number': 123,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'a': {
'last_pass': 122,
'current_failure': 123,
'first_failure': 123,
}
}
}
self.assertEqual({},
extract_compile_signal.ExtractSignalsForCompileFailure(
CompileFailureInfo.FromSerializable(failure_info),
None))
def testCompileStepSignalFromCachedStepLog(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'compile'
step = WfStep.Create(master_name, builder_name, build_number, step_name)
step.log_data = _NINJA_OUTPUT_JSON
step.put()
self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number)
signals = extract_compile_signal.ExtractSignalsForCompileFailure(
CompileFailureInfo.FromSerializable(_COMPILE_FAILURE_INFO), None)
expected_failed_edges = [{
'output_nodes': ['a/b.o'],
'rule': 'CXX',
'dependencies': ['b.h', 'b.c']
}]
self.assertEqual(expected_failed_edges, signals['compile']['failed_edges'])
def testOnFinalized(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 124
self._SetupAnalysis(
master_name,
builder_name,
build_number,
status=analysis_status.COMPLETED)
pipeline_input = AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable({}),
build_completed=False,
force=True)
root_pipeline = AnalyzeCompileFailurePipeline(pipeline_input)
root_pipeline.OnFinalized(pipeline_input)
mock_mon.assert_called_once_with({'type': 'compile'})
def testStartCompilePipelineForNewAnalysis(self, mock_info):
master_name = 'm'
builder_name = 'b'
build_number = 124
failure_info = {
'failed': True,
'chromium_revision': 'rev',
'failure_type': failure_type.COMPILE
}
mock_info.return_value = failure_info, True
compile_pipeline_input = (
build_failure_analysis_pipelines.AnalyzeCompileFailureInput(
build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
current_failure_info=CompileFailureInfo.FromSerializable(
failure_info),
build_completed=False,
force=False))
self.MockGeneratorPipeline(
build_failure_analysis_pipelines.AnalyzeCompileFailurePipeline,
compile_pipeline_input, None)
build_failure_analysis_pipelines.ScheduleAnalysisIfNeeded(
master_name,
builder_name,
build_number,
failed_steps=['a'],
build_completed=False,
force=False,
queue_name=constants.DEFAULT_QUEUE)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertIsNotNone(analysis)
def testAnalyzeCompileFailureNoSignal(self, mock_logging):
failure_info = {'failed_steps': {'a': {}}}
analysis_result, _ = compile_failure_analysis.AnalyzeCompileFailure(
CompileFailureInfo.FromSerializable(failure_info), None, None, {})
self.assertEqual({'failures': []}, analysis_result)
def testFailedToGetFailureLog(self, *_):
self.assertEqual(
{},
extract_compile_signal.ExtractSignalsForCompileFailure(
CompileFailureInfo.FromSerializable(_COMPILE_FAILURE_INFO), None))
def ScheduleAnalysisIfNeeded(master_name,
builder_name,
build_number,
failed_steps=None,
build_completed=False,
force=False,
queue_name=constants.DEFAULT_QUEUE):
"""Schedules an analysis if needed and returns the build analysis.
When the build failure was already analyzed and a new analysis is scheduled,
the returned WfAnalysis will still have the result of last completed analysis.
Args:
master_name (str): The master name of the failed build.
builder_name (str): The builder name of the failed build.
build_number (int): The build number of the failed build.
failed_steps (list): The names of all failed steps reported for the build.
build_completed (bool): Indicate whether the build is completed.
force (bool): If True, a fresh new analysis will be triggered even when an
old one was completed already; otherwise bail out.
queue_name (str): The task queue to be used for pipeline tasks.
Returns:
A WfAnalysis instance.
"""
if NeedANewAnalysis(master_name, builder_name, build_number, failed_steps,
build_completed, force):
failure_info, should_proceed = ci_failure.GetBuildFailureInfo(
master_name, builder_name, build_number)
if not should_proceed:
return WfAnalysis.Get(master_name, builder_name, build_number)
build_key = BuildKey(master_name=master_name,
builder_name=builder_name,
build_number=build_number)
if failure_info['failure_type'] == failure_type.COMPILE:
# Use new compile pipelines.
# TODO(crbug/869684): Use a gauge metric to track intermittent statuses.
compile_pipeline_input = AnalyzeCompileFailureInput(
build_key=build_key,
current_failure_info=CompileFailureInfo.FromSerializable(
failure_info),
build_completed=build_completed,
force=force)
pipeline_job = AnalyzeCompileFailurePipeline(
compile_pipeline_input)
else:
# TODO(crbug/869684): Use a gauge metric to track intermittent statuses.
test_pipeline_input = AnalyzeTestFailureInput(
build_key=build_key,
current_failure_info=TestFailureInfo.FromSerializable(
failure_info),
build_completed=build_completed,
force=force)
pipeline_job = AnalyzeTestFailurePipeline(test_pipeline_input)
# Explicitly run analysis in the backend module "waterfall-backend".
# Note: Just setting the target in queue.yaml does NOT work for pipeline
# when deployed to App Engine, but it does work in dev-server locally.
# A possible reason is that pipeline will pick a default target if none is
# specified explicitly, and the default target is used rather than the one
# in the queue.yaml file, but this contradicts the documentation in
# https://cloud.google.com/appengine/docs/python/taskqueue/tasks#Task.
pipeline_job.target = appengine_util.GetTargetNameForModule(
constants.WATERFALL_BACKEND)
pipeline_job.start(queue_name=queue_name)
logging.info('An analysis was scheduled for build %s, %s, %s: %s',
master_name, builder_name, build_number,
pipeline_job.pipeline_status_path)
else:
logging.info('An analysis is not needed for build %s, %s, %s',
master_name, builder_name, build_number)
return WfAnalysis.Get(master_name, builder_name, build_number)
def testHeuristicAnalysisForCompile(self, mock_result, mock_signals,
mock_failure_info, mock_mon, *_):
failure_info = {
'build_number': 213,
'master_name': 'chromium.win',
'builder_name': 'WinMSVC64 (dbg)',
'parent_mastername': None,
'parent_buildername': None,
'failed_steps': {
'compile': {
'last_pass': 212,
'current_failure': 213,
'first_failure': 213
}
},
'builds': {
'212': {
'blame_list': ['3045acb501991e37fb2416ab8816d2ff4e66735f',],
'chromium_revision': 'c7388ba52388421e91c113ed807dec16b830c45b'
},
'213': {
'blame_list': ['e282b48ad7a9715d132c649fe1aff9dde0347b1c',],
'chromium_revision': '2fefee0825b80ec3ebec5c661526818da9490180'
}
},
'failure_type': 8,
'failed': True,
'chromium_revision': '2fefee0825b80ec3ebec5c661526818da9490180',
}
signals = {
'compile': {
'failed_edges': [{
'dependencies': [
'third_party/webrtc/media/base/codec.h',
'third_party/webrtc/rtc_base/sanitizer.h',
],
'output_nodes': ['obj/third_party/webrtc/media//file.obj'],
'rule':
'CXX'
}],
'files': {
'c:/b/c/b/win/src/third_party/webrtc/media/engine/file.cc': [
76
]
},
'failed_targets': [{
'source': '../../third_party/webrtc/media/engine/target1.cc',
'target': 'obj/third_party/webrtc/media//file.obj'
}],
'failed_output_nodes': [
'obj/third_party/webrtc/media/rtc_audio_video/fon.obj'
],
'keywords': {}
}
}
mock_signals.return_value = signals
heuristic_result = {
'failures': [{
'first_failure':
213,
'supported':
True,
'suspected_cls': [{
'commit_position': 517979,
'url': 'url/0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'hints': {
'add a.cc': 5
},
'score': 5,
'build_number': 213,
'revision': '0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'repo_name': 'chromium'
}],
'step_name':
'compile',
'last_pass':
212,
'new_compile_suspected_cls': [{
'commit_position': 517979,
'url': 'url/0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'hints': {
'add a.cc': 5
},
'score': 5,
'build_number': 213,
'revision': '0366f1a82a0d2c4e0b82a3632e1dff5ee0b35690',
'repo_name': 'chromium'
}],
'use_ninja_dependencies':
True
}]
}
mock_result.return_value = heuristic_result, []
mock_failure_info.return_value = CompileFailureInfo.FromSerializable(
failure_info)
WfAnalysis.Create('chromium.win', 'WinMSVC64 (dbg)', 213).put()
heuristic_params = CompileHeuristicAnalysisParameters(
failure_info=CompileFailureInfo.FromSerializable(failure_info),
build_completed=True)
result = compile_failure_analysis.HeuristicAnalysisForCompile(
heuristic_params)
expected_result = {
'failure_info': failure_info,
'signals': signals,
'heuristic_result': heuristic_result
}
self.assertEqual(
result,
CompileHeuristicAnalysisOutput.FromSerializable(expected_result))
mock_mon.assert_called_once_with('chromium.win', 'WinMSVC64 (dbg)',
analysis_status.COMPLETED,
analysis_approach_type.HEURISTIC)
def testAnalyzeCompileFailureByDependencies(self):
failure_info = {
'master_name': 'm',
'builder_name': 'b',
'build_number': 123,
'failure_type': failure_type.COMPILE,
'failed': True,
'chromium_revision': 'r99_2',
'failed_steps': {
'compile': {
'current_failure': 99,
'first_failure': 98,
'supported': True,
},
},
'builds': {
99: {
'blame_list': ['r99_1', 'r99_2'],
},
98: {
'blame_list': ['r98_1'],
},
97: {}
}
}
deps_info = {}
failure_signals_json = {
'compile': {
'files': {},
'failed_edges': [{
'dependencies': ['src/a/b/f99_2.cc']
}]
},
}
expected_analysis_result = {
'failures': [{
'step_name':
'compile',
'supported':
True,
'first_failure':
98,
'last_pass':
None,
'suspected_cls': [{
'build_number': 99,
'repo_name': 'chromium',
'revision': 'r99_2',
'commit_position': None,
'url': None,
'score': 2,
'hints': {
('modified f99_2.cc (and it was in'
' dependencies found by ninja)'):
2,
},
}],
'new_compile_suspected_cls': [{
'build_number': 99,
'repo_name': 'chromium',
'revision': 'r99_2',
'commit_position': None,
'url': None,
'score': 2,
'hints': {
('modified f99_2.cc (and it was in'
' dependencies found by ninja)'):
2,
},
}],
'use_ninja_dependencies':
True,
}]
}
expected_suspected_cl = [{
'repo_name': 'chromium',
'revision': 'r99_2',
'commit_position': None,
'url': None,
'failures': {
'compile': []
},
'top_score': 2
}]
analysis_result, suspected_cls = (
compile_failure_analysis.AnalyzeCompileFailure(
CompileFailureInfo.FromSerializable(failure_info),
_SAMPLE_CHANGE_LOG, deps_info, failure_signals_json))
self.assertEqual(expected_analysis_result, analysis_result)
self.assertEqual(sorted(expected_suspected_cl), sorted(suspected_cls))