def testNotNeedANewWaterfallTryJobForOtherType(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {},
'builds': {
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.UNKNOWN
}
mock_fn.return_value = False
expected_try_job_key = WfTryJob.Create(master_name, builder_name,
build_number).key
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertEqual(expected_try_job_key, try_job_key)
def testBailOutForTryJobWithOutdatedTimestamp(self, mock_fn):
master_name = 'master1'
builder_name = 'builder1'
build_number = 223
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {
'compile': {
'current_failure': 221,
'first_failure': 221,
'last_pass': 220
}
},
'failure_type': failure_type.COMPILE
}
yesterday = datetime.utcnow() - timedelta(days=1)
build = WfBuild.Create(master_name, builder_name, build_number)
build.start_time = yesterday
build.put()
mock_fn.return_value = True
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertIsNone(try_job_key)
def testNotNeedANewWaterfallTryJobIfNoNewFailure(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'failed_steps': {
'a': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221,
'tests': {
'a.t2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
}
}
}
},
'failure_type': failure_type.TEST
}
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.failure_result_map = {'a': {'a.t2': 'm/b/222'}}
analysis.put()
mock_fn.return_value = False
expected_try_job_key = WfTryJob.Create(master_name, builder_name,
build_number).key
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertEqual(expected_try_job_key, try_job_key)
def testForceTryJob(self):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'failed_steps': {
'a': {
'current_failure': 223,
'first_failure': 223,
'last_pass': 222,
'tests': {
'a.t2': {
'current_failure': 223,
'first_failure': 223,
'last_pass': 222
}
}
}
},
'builds': {
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.TEST
}
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.compile_results = [['rev', 'failed']]
try_job.status = analysis_status.COMPLETED
try_job.put()
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.failure_result_map = {'a': {'a.t2': 'm/b/223'}}
analysis.put()
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None,
True)
self.assertTrue(need_try_job)
self.assertEqual(try_job_key, try_job.key)
def testNotNeedANewWaterfallTryJobIfNotFirstTimeFailure(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 223
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {
'compile': {
'current_failure': 223,
'first_failure': 221,
'last_pass': 220
}
},
'builds': {
'220': {
'blame_list': ['220-1', '220-2'],
'chromium_revision': '220-2'
},
'221': {
'blame_list': ['221-1', '221-2'],
'chromium_revision': '221-2'
},
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.COMPILE
}
WfAnalysis.Create(master_name, builder_name, build_number).put()
mock_fn.return_value = False
expected_key = WfTryJob.Create(master_name, builder_name,
build_number).key
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertEqual(expected_key, try_job_key)
def testNotNeedANewWaterfallTryJobIfBuilderIsNotSupportedYet(self):
master_name = 'master3'
builder_name = 'builder3'
build_number = 223
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {
'compile': {
'current_failure': 221,
'first_failure': 221,
'last_pass': 220
}
},
'builds': {
'220': {
'blame_list': ['220-1', '220-2'],
'chromium_revision': '220-2'
},
'221': {
'blame_list': ['221-1', '221-2'],
'chromium_revision': '221-2'
},
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.COMPILE
}
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertIsNone(try_job_key)
def testNeedANewWaterfallTryJob(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {
'compile': {
'current_failure': 223,
'first_failure': 223,
'last_pass': 222
}
},
'builds': {
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.COMPILE
}
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.failure_result_map = {'compile': 'm/b/223'}
analysis.put()
mock_fn.return_value = False
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertTrue(need_try_job)
self.assertIsNotNone(try_job_key)
def testNotNeedANewWaterfallTryJobIfOneWithResultExists(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'failed_steps': {
'compile': {
'current_failure': 223,
'first_failure': 223,
'last_pass': 220
}
},
'builds': {
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.COMPILE
}
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.compile_results = [['rev', 'failed']]
try_job.status = analysis_status.COMPLETED
try_job.put()
WfAnalysis.Create(master_name, builder_name, build_number).put()
mock_fn.return_value = False
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertEqual(try_job_key, try_job.key)
def testBailOutForTestTryJob(self, mock_fn):
master_name = 'master2'
builder_name = 'builder2'
build_number = 223
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {
'a_test': {}
},
'failure_type': failure_type.TEST
}
mock_fn.return_value = False
expected_try_job_key = WfTryJob.Create(master_name, builder_name,
build_number).key
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertEqual(expected_try_job_key, try_job_key)
def run(self, master_name, builder_name, build_number, failure_info,
signals, heuristic_result, build_completed, force_try_job):
"""Starts a try job if one is needed for the given failure."""
if not build_completed: # Only start try-jobs for completed builds.
return
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, signals,
heuristic_result, force_try_job)
if not need_try_job:
return
try_job_type = failure_info['failure_type']
last_pass = _GetLastPass(build_number, failure_info, try_job_type)
if last_pass is None: # pragma: no cover
logging.warning(
'Couldn"t start try job for build %s, %s, %d because'
' last_pass is not found.', master_name, builder_name,
build_number)
return
good_revision = failure_info['builds'][str(
last_pass)]['chromium_revision']
bad_revision = failure_info['builds'][str(
build_number)]['chromium_revision']
suspected_revisions = _GetSuspectsFromHeuristicResult(heuristic_result)
if try_job_type == failure_type.COMPILE:
compile_targets = try_job_util.GetFailedTargetsFromSignals(
signals, master_name, builder_name)
dimensions = waterfall_config.GetTrybotDimensions(
master_name, builder_name)
cache_name = swarming_util.GetCacheName(master_name, builder_name)
try_job_id = yield ScheduleCompileTryJobPipeline(
master_name, builder_name, build_number, good_revision,
bad_revision, try_job_type, compile_targets,
suspected_revisions, cache_name, dimensions)
else:
# If try_job_type is other type, the pipeline has returned.
# So here the try_job_type is failure_type.TEST.
# Waits and gets the swarming tasks' results.
task_results = []
for step_name, step_failure in failure_info[
'failed_steps'].iteritems():
step_has_first_time_failure = _HasFirstTimeFailure(
step_failure.get('tests', {}), build_number)
if not step_has_first_time_failure:
continue
task_result = yield ProcessSwarmingTaskResultPipeline(
master_name, builder_name, build_number, step_name)
task_results.append(task_result)
yield UpdateAnalysisWithFlakeInfoPipeline(master_name,
builder_name,
build_number,
*task_results)
parent_mastername = failure_info.get(
'parent_mastername') or master_name
parent_buildername = failure_info.get('parent_buildername') or (
builder_name)
dimensions = waterfall_config.GetTrybotDimensions(
parent_mastername, parent_buildername)
cache_name = swarming_util.GetCacheName(parent_mastername,
parent_buildername)
try_job_id = yield ScheduleTestTryJobPipeline(
master_name, builder_name, build_number, good_revision,
bad_revision, try_job_type, suspected_revisions, cache_name,
dimensions, *task_results)
try_job_result = yield MonitorTryJobPipeline(try_job_key.urlsafe(),
try_job_type, try_job_id)
yield IdentifyTryJobCulpritPipeline(master_name, builder_name,
build_number, try_job_type,
try_job_id, try_job_result)
def testNeedANewWaterfallTryJobIfTestFailureSwarming(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'failed_steps': {
'a': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221,
'tests': {
'a.PRE_t1': {
'current_failure': 223,
'first_failure': 223,
'last_pass': 221,
'base_test_name': 'a.t1'
},
'a.t2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
},
'a.t3': {
'current_failure': 223,
'first_failure': 223,
'last_pass': 222
}
}
},
'b': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221,
'tests': {
'b.t1': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
},
'b.t2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
}
}
}
},
'builds': {
'222': {
'blame_list': ['222-1'],
'chromium_revision': '222-1'
},
'223': {
'blame_list': ['223-1', '223-2', '223-3'],
'chromium_revision': '223-3'
}
},
'failure_type': failure_type.TEST
}
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.failure_result_map = {
'a': {
'a.PRE_t1': 'm/b/223',
'a.t2': 'm/b/222',
'a.t3': 'm/b/223'
},
'b': {
'b.t1': 'm/b/222',
'b.t2': 'm/b/222'
}
}
analysis.put()
mock_fn.return_value = False
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertTrue(need_try_job)
self.assertIsNotNone(try_job_key)