def testUpdateWfSuspectedCLAddSameBuild(self):
approach = analysis_approach_type.HEURISTIC
master_name = 'm'
builder_name = 'b'
build_number = 122
test_failure_type = failure_type.TEST
repo_name = 'chromium'
revision = 'r2'
commit_position = 2
failures = {'step_1': ['test1', 'test2']}
top_score = 4
suspected_cl = WfSuspectedCL.Create(repo_name, revision,
commit_position)
suspected_cl.approaches = [analysis_approach_type.HEURISTIC]
suspected_cl.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'failure_type': test_failure_type,
'failures': failures,
'status': None,
'top_score': 4
}
}
suspected_cl.failure_type = [test_failure_type]
suspected_cl.put()
suspected_cl_util.UpdateSuspectedCL(repo_name, revision,
commit_position, approach,
master_name, builder_name,
build_number, test_failure_type,
failures, top_score)
expected_builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'failure_type': test_failure_type,
'failures': failures,
'status': None,
'top_score': 4
}
}
expected_approaches = [analysis_approach_type.HEURISTIC]
suspected_cl = WfSuspectedCL.Get(repo_name, revision)
self.assertIsNotNone(suspected_cl)
self.assertEqual(expected_approaches, suspected_cl.approaches)
self.assertEqual([test_failure_type], suspected_cl.failure_type)
self.assertEqual(expected_builds, suspected_cl.builds)
def RunImpl(self, parameters):
analysis_urlsafe_key = parameters.analysis_urlsafe_key
analysis = entity_util.GetEntityFromUrlsafeKey(analysis_urlsafe_key)
assert analysis, 'Cannot retrieve analysis entry from datastore {}'.format(
analysis_urlsafe_key)
# Determine if flakiness is still persistent.
still_flaky = flake_analysis_util.FlakyAtMostRecentlyAnalyzedCommit(
analysis)
# TODO(crbug.com/905754): Call auto actions as an async taskqueue task.
flake_analysis_actions.OnCulpritIdentified(analysis_urlsafe_key)
if not still_flaky: # pragma: no cover.
analysis.LogInfo(
'No further actions taken due to latest commit being stable.')
return
# Data needed for reverts.
build_key = BaseBuildModel.CreateBuildKey(
analysis.original_master_name, analysis.original_builder_name,
analysis.original_build_number)
with pipeline.InOrder():
# Revert culprit if applicable.
yield CreateAndSubmitRevertPipeline(
self.CreateInputObjectInstance(
CreateAndSubmitRevertInput,
analysis_urlsafe_key=analysis.key.urlsafe(),
build_key=build_key))
# Update culprit code review.
yield NotifyCulpritPipeline(
self.CreateInputObjectInstance(
NotifyCulpritInput,
analysis_urlsafe_key=analysis_urlsafe_key))
def _GetAllSuspectedCLsAndCheckStatus(master_name, builder_name, build_number,
analysis):
if not analysis or not analysis.suspected_cls:
return []
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
suspected_cls = copy.deepcopy(analysis.suspected_cls)
confidences = SuspectedCLConfidence.Get()
for cl in suspected_cls:
cl['status'] = _ANALYSIS_CL_STATUS_MAP.get(analysis.result_status,
None)
cl['confidence'] = None
suspected_cl = WfSuspectedCL.Get(cl['repo_name'], cl['revision'])
if not suspected_cl:
continue
build = suspected_cl.builds.get(build_key)
if build:
cl['status'] = build['status']
cl['confidence'] = '%d%%' % (
suspected_cl_util.GetSuspectedCLConfidenceScore(
confidences, build) or 0)
return suspected_cls
def UpdateSuspectedCL(repo_name, revision, commit_position, approach,
master_name, builder_name, build_number, cl_failure_type,
failures, top_score):
suspected_cl = (
WfSuspectedCL.Get(repo_name, revision) or
WfSuspectedCL.Create(repo_name, revision, commit_position))
if not suspected_cl.identified_time: # pragma: no cover.
suspected_cl.identified_time = time_util.GetUTCNow()
suspected_cl.updated_time = time_util.GetUTCNow()
if approach not in suspected_cl.approaches:
suspected_cl.approaches.append(approach)
if cl_failure_type not in suspected_cl.failure_type:
suspected_cl.failure_type.append(cl_failure_type)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
if build_key not in suspected_cl.builds:
suspected_cl.builds[build_key] = {
'approaches': [approach],
'failure_type': cl_failure_type,
'failures': failures,
'status': _GetsStatusFromSameFailure(suspected_cl.builds, failures),
'top_score': top_score
}
else:
build = suspected_cl.builds[build_key]
if approach not in build['approaches']:
build['approaches'].append(approach)
suspected_cl.put()
def _GetAllSwarmingTasks(self, failure_result_map):
"""Returns all swarming tasks related to one build.
Args:
A dict to map each step/test with the key to the build when it failed the
first time.
{
'step1': 'm/b/1',
'step2': {
'test1': 'm/b/1',
'test2': 'm/b/2'
}
}
Returns:
A dict of swarming tasks like below:
{
'step1': {
'm/b/1': WfSwarmingTask(
key=Key('WfBuild', 'm/b/1', 'WfSwarmingTask', 'step1'),...)
},
...
}
"""
if not failure_result_map:
return {}
swarming_tasks = defaultdict(dict)
for step_name, step_map in failure_result_map.iteritems():
if isinstance(step_map, basestring):
swarming_tasks[step_name][step_map] = (WfSwarmingTask.Get(
*BaseBuildModel.GetBuildInfoFromBuildKey(step_map),
step_name=step_name))
else:
for task_key in step_map.values():
if not swarming_tasks[step_name].get(task_key):
swarming_tasks[step_name][task_key] = (
WfSwarmingTask.Get(
*BaseBuildModel.GetBuildInfoFromBuildKey(
task_key),
step_name=step_name))
return swarming_tasks
def _HasBuildKeyForBuildInfoInFailureResultMap(master_name, builder_name,
build_number):
"""Checks if there is any first failed test."""
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
failure_result_map = analysis.failure_result_map
current_build_key = BaseBuildModel.CreateBuildKey(master_name,
builder_name,
build_number)
for step_keys in failure_result_map.itervalues():
for test_key in step_keys.itervalues():
if test_key == current_build_key:
return True
return False
def GetFirstTimeFailedSteps(master_name, builder_name, build_number):
"""Gets steps that have tests failed first time in the build."""
current_build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
assert analysis, "Cannot get WfAnalysis entity for %s" % current_build_key
first_failed_steps = []
for step, tests in (analysis.failure_result_map or {}).iteritems():
if isinstance(tests, basestring): # Non-swarming.
continue
if current_build_key in tests.values():
first_failed_steps.append(step)
return first_failed_steps
def _NeedANewCompileTryJob(master_name, builder_name, build_number,
failure_info):
compile_failure = failure_info.failed_steps.get('compile') or None
if compile_failure:
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
analysis.failure_result_map['compile'] = BaseBuildModel.CreateBuildKey(
master_name, builder_name, compile_failure.first_failure)
analysis.put()
if (compile_failure.supported and compile_failure.first_failure
== compile_failure.current_failure):
return True
return False
def _UpdateSuspectedCLAndAnalysis(master_name, builder_name, build_number,
cl_info, cl_status, user_name):
repo_name, revision = GetCLInfo(cl_info)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
success = (_UpdateSuspectedCL(repo_name, revision, build_key, cl_status)
and _UpdateAnalysis(master_name, builder_name, build_number,
repo_name, revision, cl_status))
if success:
_AppendTriageHistoryRecord(master_name, builder_name, build_number,
cl_info, cl_status, user_name)
return success
def RunImpl(self, pipeline_input):
culprits_should_take_actions = (
test_culprit_action.GetCulpritsShouldTakeActions(pipeline_input))
if not culprits_should_take_actions:
return
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
culprits = pipeline_input.culprits
build_failure_type = failure_type.TEST
for culprit_revision in culprits_should_take_actions:
culprit_key = culprits.get(culprit_revision)
assert culprit_key, (
'Failed to get culprit_key for culprit {} when analyzing failures'
' at build {}/{}/{}'.format(culprit_revision, master_name,
builder_name, build_number))
revert_status = constants.SKIPPED
if test_culprit_action.CanAutoCreateRevert(
culprit_key, pipeline_input): # pragma: no branch.
revert_status = yield CreateRevertCLPipeline(
CreateRevertCLParameters(cl_key=culprit_key,
build_key=build_key,
failure_type=build_failure_type))
if test_culprit_action.CanAutoCommitRevertByFindit():
submit_revert_pipeline_input = self.CreateInputObjectInstance(
SubmitRevertCLParameters,
cl_key=culprit_key,
revert_status=revert_status,
failure_type=build_failure_type)
yield SubmitRevertCLPipeline(submit_revert_pipeline_input)
# Checks if any of the culprits was also found by heuristic analysis,
# if so send notification right away.
send_notification_to_culprit_input = self.CreateInputObjectInstance(
SendNotificationForCulpritParameters,
cl_key=culprit_key,
force_notify=culprit_action.ShouldForceNotify(
culprit_key, pipeline_input),
revert_status=revert_status,
failure_type=build_failure_type)
yield SendNotificationForCulpritPipeline(
send_notification_to_culprit_input)
def RunImpl(self, pipeline_input):
if not compile_culprit_action.ShouldTakeActionsOnCulprit(
pipeline_input):
return
master_name, builder_name, build_number = (
pipeline_input.build_key.GetParts())
culprits = pipeline_input.culprits
culprit = culprits.values()[0]
force_notify = culprit_action.ShouldForceNotify(
culprit, pipeline_input)
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
build_failure_type = failure_type.COMPILE
revert_status = constants.SKIPPED
commit_status = constants.SKIPPED
if compile_culprit_action.CanAutoCreateRevert():
revert_status = yield CreateRevertCLPipeline(
CreateRevertCLParameters(cl_key=culprit,
build_key=build_key,
failure_type=build_failure_type))
if compile_culprit_action.CanAutoCommitRevertByFindit():
submit_revert_pipeline_input = self.CreateInputObjectInstance(
SubmitRevertCLParameters,
cl_key=culprit,
revert_status=revert_status,
failure_type=build_failure_type)
commit_status = yield SubmitRevertCLPipeline(
submit_revert_pipeline_input)
send_notification_to_irc_input = self.CreateInputObjectInstance(
SendNotificationToIrcParameters,
cl_key=culprit,
revert_status=revert_status,
commit_status=commit_status,
failure_type=build_failure_type)
yield SendNotificationToIrcPipeline(send_notification_to_irc_input)
send_notification_to_culprit_input = self.CreateInputObjectInstance(
SendNotificationForCulpritParameters,
cl_key=culprit,
force_notify=force_notify,
revert_status=revert_status,
failure_type=build_failure_type)
yield SendNotificationForCulpritPipeline(
send_notification_to_culprit_input)
def GetListOfTryJobBuilds(builds):
displayed_builds = {}
for build_key, build in builds.iteritems():
if analysis_approach_type.TRY_JOB not in build.get(
'approaches', []):
continue
build_info = BaseBuildModel.GetBuildInfoFromBuildKey(build_key)
master_name = build_info[0]
builder_name = build_info[1]
build_number = build_info[2]
if (not displayed_builds.get((master_name, builder_name))
or displayed_builds[(master_name, builder_name)][0] >
build_number):
displayed_builds[(master_name,
builder_name)] = (build_number, )
return [k + v for k, v in displayed_builds.iteritems()]
def testCreateWfSuspectedCL(self):
approach = analysis_approach_type.HEURISTIC
master_name = 'm'
builder_name = 'b'
build_number = 123
compile_failure_type = failure_type.COMPILE
repo_name = 'chromium'
revision = 'r1'
commit_position = 1
failures = {'compile': []}
top_score = 5
mocked_utcnow = datetime(2016, 10, 4, 0, 0, 0)
self.MockUTCNow(mocked_utcnow)
self.assertIsNone(WfSuspectedCL.Get(repo_name, revision))
suspected_cl_util.UpdateSuspectedCL(repo_name, revision,
commit_position, approach,
master_name, builder_name,
build_number, compile_failure_type,
failures, top_score)
expected_builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [approach],
'failure_type': compile_failure_type,
'failures': failures,
'status': None,
'top_score': top_score
}
}
suspected_cl = WfSuspectedCL.Get(repo_name, revision)
self.assertIsNotNone(suspected_cl)
self.assertEqual([analysis_approach_type.HEURISTIC],
suspected_cl.approaches)
self.assertEqual([compile_failure_type], suspected_cl.failure_type)
self.assertEqual(expected_builds, suspected_cl.builds)
self.assertEqual(mocked_utcnow, suspected_cl.identified_time)
self.assertEqual(mocked_utcnow, suspected_cl.updated_time)
def _PrepareTryJobDataForCompileFailure(analysis):
try_job_data = {}
if not (analysis.failure_result_map and # pragma: no branch.
constants.COMPILE_STEP_NAME in analysis.failure_result_map):
return try_job_data # pragma: no cover.
referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(
analysis.failure_result_map[constants.COMPILE_STEP_NAME])
try_job = WfTryJob.Get(*referred_build_keys)
if not try_job or not try_job.compile_results:
return try_job_data # pragma: no cover.
result = try_job.compile_results[-1]
try_job_data['status'] = analysis_status.STATUS_TO_DESCRIPTION.get(
try_job.status, 'unknown').lower()
try_job_data['url'] = result.get('url')
try_job_data['completed'] = try_job.completed
try_job_data['failed'] = try_job.failed
try_job_data['culprit'] = result.get('culprit',
{}).get(constants.COMPILE_STEP_NAME)
return try_job_data
def _GetAdditionalInformationForCL(self, repo_name, revision, confidences,
build, reference_build_key):
"""Gets additional information for a cl.
Currently additional information contains:
confidence of the result;
approaches that found this cl: HEURISTIC, TRY_JOB or both;
revert_cl_url if the cl has been reverted by Findit;
if the revert has been committed.
"""
additional_info = {}
cl = WfSuspectedCL.Get(repo_name, revision)
if not cl:
return additional_info
master_name = buildbot.GetMasterNameFromUrl(build.master_url)
builder_name = build.builder_name
current_build = build.build_number
# If the CL is found by a try job, only the first failure will be recorded.
# So we might need to go to the first failure to get CL information.
build_info = cl.GetBuildInfo(master_name, builder_name, current_build)
first_build_info = None if not reference_build_key else cl.GetBuildInfo(
*BaseBuildModel.GetBuildInfoFromBuildKey(reference_build_key))
additional_info['confidence'], additional_info['cl_approach'] = (
suspected_cl_util.GetSuspectedCLConfidenceScoreAndApproach(
confidences, build_info, first_build_info))
# Gets the revert_cl_url for the CL if there is one.
if cl.revert_cl_url:
additional_info['revert_cl_url'] = cl.revert_cl_url
additional_info['revert_committed'] = (
cl.revert_submission_status == analysis_status.COMPLETED)
return additional_info
def _GetTryJobResultForCompile(failure_result_map):
try_job_key = failure_result_map['compile']
referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(try_job_key)
culprit_info = defaultdict(lambda: defaultdict(list))
try_job = WfTryJob.Get(*referred_build_keys)
if not try_job or try_job.test_results:
return culprit_info
try_job_result = (
try_job.compile_results[-1] if try_job.compile_results else None)
compile_try_job = {'try_job_key': try_job_key, 'status': try_job.status}
if try_job_result:
if try_job_result.get('url'):
compile_try_job['try_job_url'] = try_job_result['url']
compile_try_job['try_job_build_number'] = (
_GetTryJobBuildNumber(try_job_result))
if try_job_result.get('culprit', {}).get('compile'):
compile_try_job['culprit'] = try_job_result['culprit']['compile']
culprit_info['compile']['try_jobs'].append(compile_try_job)
return culprit_info
def MockedGetAllTryJobResults(master_name, builder_name, build_number,
_):
build_key = BaseBuildModel.CreateBuildKey(master_name,
builder_name,
build_number)
return SAMPLE_TRY_JOB_INFO.get(build_key, None)
def _CreateKey(master_name, builder_name, build_number):
return ndb.Key(
'WfTryJob',
BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number))
test_results = []
compile_results = []
while more:
analyses, cursor, more = WfAnalysis.query(
ndb.AND(WfAnalysis.build_start_time >= start,
WfAnalysis.build_start_time < end)).fetch_page(
100, start_cursor=cursor)
for analysis in analyses:
if not analysis.completed or not analysis.result:
continue
build_key = analysis.key.pairs()[0][1]
master, builder_name, build_number = (
BaseBuildModel.GetBuildInfoFromBuildKey(build_key))
build_number = int(build_number)
try_job = WfTryJob.Get(master, builder_name, build_number)
for failure in analysis.result.get('failures', {}):
result = None
step_name = failure['step_name']
culprits = _GetTestTryJobCulprit(try_job, step_name)
if failure.get('tests'):
for test in failure['tests']:
if test['first_failure'] != build_number:
# Not first time failure.
continue
def _GenerateSwarmingTasksData(failure_result_map):
"""Collects info for all related swarming tasks.
Returns: A dict as below:
{
'step1': {
'swarming_tasks': {
'm/b/121': {
'task_info': {
'status': 'Completed',
'task_id': 'task1',
'task_url': ('https://chromium-swarm.appspot.com/user'
'/task/task1')
},
'all_tests': ['test2', 'test3', 'test4'],
'reliable_tests': ['test2'],
'flaky_tests': ['test3', 'test4']
}
}
},
'step2': {
'swarming_tasks': {
'm/b/121': {
'task_info': {
'status': 'Pending'
},
'all_tests': ['test1']
}
}
},
'step3': {
'swarming_tasks': {
'm/b/121': {
'task_info': {
'status': 'No swarming rerun found'
},
'all_tests': ['test1']
}
}
}
}
"""
tasks_info = defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
swarming_server = waterfall_config.GetSwarmingSettings()['server_host']
for step_name, failure in failure_result_map.iteritems():
step_tasks_info = tasks_info[step_name]['swarming_tasks']
if isinstance(failure, dict):
# Only swarming test failures have swarming re-runs.
swarming_task_keys = set(failure.values())
for key in swarming_task_keys:
task_dict = step_tasks_info[key]
referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(key)
task = WfSwarmingTask.Get(*referred_build_keys, step_name=step_name)
all_tests = _GetAllTestsForASwarmingTask(key, failure)
task_dict['all_tests'] = all_tests
if not task: # In case task got manually removed from data store.
task_info = {'status': result_status.NO_SWARMING_TASK_FOUND}
else:
task_info = {'status': task.status}
# Get the step name without platform.
# This value should have been saved in task.parameters;
# in case of no such value saved, split the step_name.
task_dict['ref_name'] = (
step_name.split()[0]
if not task.parameters or not task.parameters.get('ref_name') else
task.parameters['ref_name'])
if task.task_id: # Swarming rerun has started.
task_info['task_id'] = task.task_id
task_info['task_url'] = 'https://%s/user/task/%s' % (
swarming_server, task.task_id)
if task.classified_tests:
# Swarming rerun has completed.
# Use its result to get reliable and flaky tests.
# If task has not completed, there will be no try job yet,
# the result will be grouped in unclassified failures temporarily.
reliable_tests = task.classified_tests.get('reliable_tests', [])
task_dict['reliable_tests'] = [
test for test in reliable_tests if test in all_tests
]
flaky_tests = task.classified_tests.get('flaky_tests', [])
task_dict['flaky_tests'] = [
test for test in flaky_tests if test in all_tests
]
task_dict['task_info'] = task_info
else:
step_tasks_info[failure] = {
'task_info': {
'status': result_status.NON_SWARMING_NO_RERUN
}
}
return tasks_info
def _GetCulpritInfoForTryJobResultForTest(try_job_key, culprits_info):
referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(try_job_key)
try_job = WfTryJob.Get(*referred_build_keys)
if not try_job or try_job.compile_results:
return
try_job_result = try_job.test_results[-1] if try_job.test_results else None
for step_try_jobs in culprits_info.values():
# If try job found different culprits for each test, split tests by culprit.
additional_tests_culprit_info = []
for try_job_info in step_try_jobs['try_jobs']:
if (try_job_key != try_job_info['try_job_key'] or
try_job_info.get('status')):
# Conditions that try_job_info has status are:
# If there is no swarming task, there won't be try job;
# If the swarming task is not completed yet, there won't be try job yet;
# If there are flaky tests found, those tests will be marked as flaky,
# and no try job for them will be triggered.
continue
try_job_info['status'] = try_job.status
if try_job_result:
# Needs to use ref_name to match step_name in try job.
ref_name = try_job_info['ref_name']
# Saves try job information.
if try_job_result.get('url'): # pragma: no cover
try_job_info['try_job_url'] = try_job_result['url']
try_job_info['try_job_build_number'] = (
_GetTryJobBuildNumber(try_job_result))
if (try_job_result.get('culprit') and
try_job_result['culprit'].get(ref_name)):
# Saves try job culprits information.
# Uses culprits to group tests.
culprit_tests_map = _OrganizeTryJobResultByCulprits(
try_job_result['culprit'][ref_name])
ungrouped_tests = try_job_info.get('tests', [])
list_of_culprits = []
for culprit_info in culprit_tests_map.values():
failed_tests = culprit_info['failed_tests']
list_of_culprits.append(culprit_info)
# Gets tests that haven't been grouped.
ungrouped_tests = list(set(ungrouped_tests) ^ set(failed_tests))
if not ungrouped_tests:
# All tests have been grouped.
break
index_start = 1
if ungrouped_tests:
# There are tests don't have try job culprits.
# Group these tests together.
# Save them in current try_job_info.
try_job_info['tests'] = ungrouped_tests
try_job_info['culprit'] = {}
# Saves all the tests that have culprits later.
index_start = 0
else:
# Saves the first culprit in current try_job_info.
# Saves all the other culprits later.
try_job_info['culprit'] = {
'revision':
list_of_culprits[0]['revision'],
'commit_position':
list_of_culprits[0]['commit_position'],
'review_url':
list_of_culprits[0].get(
'url', list_of_culprits[0].get('review_url', None))
}
try_job_info['tests'] = list_of_culprits[0]['failed_tests']
for n in xrange(index_start, len(list_of_culprits)):
# Appends the rest of test groups to step_try_jobs['try_jobs'].
iterate_culprit = list_of_culprits[n]
tmp_try_job_info = copy.deepcopy(try_job_info)
tmp_try_job_info['culprit'] = {
'revision':
iterate_culprit['revision'],
'commit_position':
iterate_culprit['commit_position'],
'review_url':
iterate_culprit.get('url',
iterate_culprit.get('review_url', None))
}
tmp_try_job_info['tests'] = iterate_culprit['failed_tests']
additional_tests_culprit_info.append(tmp_try_job_info)
if additional_tests_culprit_info:
step_try_jobs['try_jobs'].extend(additional_tests_culprit_info)
def _CreateKey(master_name, builder_name,
build_number): # pragma: no cover
return ndb.Key(
'WfAnalysis',
BaseBuildModel.CreateBuildId(master_name, builder_name,
build_number))
def Create(master_name, builder_name, build_number):
key = ndb.Key(
'M',
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number),
'_DummyModel', build_number)
return _DummyModel(key=key)
def _CreateKey(master_name, builder_name,
build_number): # pragma: no cover
return ndb.Key(
'WfFailureGroup',
BaseBuildModel.CreateBuildId(master_name, builder_name,
build_number))
def testGetsFirstFailureAtTestLevelNoFailureResultMap(self):
master_name = 'm'
builder_name = 'b'
build_number = 2
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'failed': True,
'master_name': 'm',
'builder_name': 'b',
'build_number': 2,
'chromium_revision': None,
'builds': {
2: {
'blame_list': [],
'chromium_revision': None
}
},
'failed_steps': {
'abc_test': {
'current_failure': 2,
'first_failure': 1,
'last_pass': 0,
'tests': {
'Unittest2.Subtest1': {
'current_failure': 2,
'first_failure': 2,
'last_pass': 1,
'base_test_name': 'Unittest2.Subtest1'
},
'Unittest3.Subtest2': {
'current_failure': 2,
'first_failure': 1,
'last_pass': 0,
'base_test_name': 'Unittest3.Subtest2'
}
},
'supported': True
},
'a_test': {
'current_failure': 2,
'first_failure': 1,
'last_pass': 0,
'supported': True
}
},
'failure_type': failure_type.TEST
}
expected_result = {'abc_test': ['Unittest2.Subtest1']}
expected_failure_result_map = {
'abc_test': {
'Unittest2.Subtest1':
BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number),
'Unittest3.Subtest2':
BaseBuildModel.CreateBuildKey(master_name, builder_name, 1)
}
}
result = (test_failure_analysis.GetsFirstFailureAtTestLevel(
master_name, builder_name, build_number,
TestFailureInfo.FromSerializable(failure_info), False))
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
self.assertEqual(result, expected_result)
self.assertEqual(analysis.failure_result_map,
expected_failure_result_map)
def _CreateKey(
master_name, builder_name, build_number, step_name): # pragma: no cover
build_id = BaseBuildModel.CreateBuildId(
master_name, builder_name, build_number)
return ndb.Key('WfBuild', build_id, 'WfStep', step_name)
def GetBuildInfo(self, master_name, builder_name, build_number):
build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
build_number)
return self.builds.get(build_key)
def testTestLevelResultIsReturned(self, mock_fn, _):
master_name = 'm'
builder_name = 'b'
build_number = 11
master_url = 'https://build.chromium.org/p/%s' % master_name
builds = {
'builds': [{
'master_url': master_url,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': ['a', 'b on platform']
}]
}
task = WfSwarmingTask.Create(master_name, builder_name, 4,
'b on platform')
task.parameters['ref_name'] = 'b'
task.status = analysis_status.COMPLETED
task.put()
try_job = WfTryJob.Create(master_name, builder_name, 4)
try_job.status = analysis_status.COMPLETED
try_job.test_results = [{
'culprit': {
'a': {
'repo_name': 'chromium',
'revision': 'r4_2',
'commit_position': 42,
'url': None,
},
'b': {
'tests': {
'Unittest3.Subtest1': {
'repo_name': 'chromium',
'revision': 'r4_10',
'commit_position': 410,
'url': None,
},
}
}
},
}]
try_job.put()
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.status = analysis_status.COMPLETED
analysis.failure_result_map = {
'a': '/'.join([master_name, builder_name, '4']),
'b on platform': {
'Unittest1.Subtest1':
'/'.join([master_name, builder_name, '3']),
'Unittest2.Subtest1':
'/'.join([master_name, builder_name, '4']),
'Unittest3.Subtest1':
'/'.join([master_name, builder_name, '4']),
},
}
analysis.result = {
'failures': [{
'step_name':
'a',
'first_failure':
4,
'last_pass':
3,
'supported':
True,
'suspected_cls': [{
'build_number': 4,
'repo_name': 'chromium',
'revision': 'r4_2_failed',
'commit_position': None,
'url': None,
'score': 2,
'hints': {
'modified f4_2.cc (and it was in log)': 2,
},
}],
}, {
'step_name':
'b on platform',
'first_failure':
3,
'last_pass':
2,
'supported':
True,
'suspected_cls': [{
'build_number': 3,
'repo_name': 'chromium',
'revision': 'r3_1',
'commit_position': None,
'url': None,
'score': 5,
'hints': {
'added x/y/f3_1.cc (and it was in log)': 5,
},
}, {
'build_number': 4,
'repo_name': 'chromium',
'revision': 'r4_1',
'commit_position': None,
'url': None,
'score': 2,
'hints': {
'modified f4.cc (and it was in log)': 2,
},
}],
'tests': [{
'test_name':
'Unittest1.Subtest1',
'first_failure':
3,
'last_pass':
2,
'suspected_cls': [{
'build_number': 2,
'repo_name': 'chromium',
'revision': 'r2_1',
'commit_position': None,
'url': None,
'score': 5,
'hints': {
'added x/y/f99_1.cc (and it was in log)': 5,
},
}]
}, {
'test_name':
'Unittest2.Subtest1',
'first_failure':
4,
'last_pass':
2,
'suspected_cls': [{
'build_number': 2,
'repo_name': 'chromium',
'revision': 'r2_1',
'commit_position': None,
'url': None,
'score': 5,
'hints': {
'added x/y/f99_1.cc (and it was in log)': 5,
},
}]
}, {
'test_name': 'Unittest3.Subtest1',
'first_failure': 4,
'last_pass': 2,
'suspected_cls': []
}]
}, {
'step_name': 'c',
'first_failure': 4,
'last_pass': 3,
'supported': False,
'suspected_cls': [],
}]
}
analysis.put()
suspected_cl_42 = WfSuspectedCL.Create('chromium', 'r4_2', 42)
suspected_cl_42.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, 5): {
'approaches': [analysis_approach_type.TRY_JOB]
}
}
suspected_cl_42.put()
suspected_cl_21 = WfSuspectedCL.Create('chromium', 'r2_1', None)
suspected_cl_21.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, 3): {
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
},
BaseBuildModel.CreateBuildKey(master_name, builder_name, 4): {
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
},
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
}
}
suspected_cl_21.put()
suspected_cl_410 = WfSuspectedCL.Create('chromium', 'r4_10', None)
suspected_cl_410.builds = {
BaseBuildModel.CreateBuildKey(master_name, builder_name, 4): {
'approaches': [
analysis_approach_type.HEURISTIC,
analysis_approach_type.TRY_JOB
],
'top_score':
5
},
BaseBuildModel.CreateBuildKey(master_name, builder_name, build_number):
{
'approaches': [analysis_approach_type.HEURISTIC],
'top_score': 5
}
}
revert_cl = RevertCL()
revert_cl.revert_cl_url = 'revert_cl_url'
suspected_cl_410.revert_cl = revert_cl
suspected_cl_410.put()
def confidence_side_effect(_, build_info, first_build_info):
if (first_build_info and first_build_info.get('approaches') == [
analysis_approach_type.HEURISTIC,
analysis_approach_type.TRY_JOB
]):
return 100, analysis_approach_type.TRY_JOB
if build_info and build_info.get('top_score'):
return 90, analysis_approach_type.HEURISTIC
return 98, analysis_approach_type.TRY_JOB
mock_fn.side_effect = confidence_side_effect
expected_results = [{
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'a',
'is_sub_test':
False,
'first_known_failed_build_number':
4,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r4_2',
'commit_position': 42,
'confidence': 98,
'analysis_approach': 'TRY_JOB',
'revert_committed': False
}],
'analysis_approach':
'TRY_JOB',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'b on platform',
'is_sub_test':
True,
'test_name':
'Unittest1.Subtest1',
'first_known_failed_build_number':
3,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r2_1',
'confidence': 90,
'analysis_approach': 'HEURISTIC',
'revert_committed': False
}],
'analysis_approach':
'HEURISTIC',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'b on platform',
'is_sub_test':
True,
'test_name':
'Unittest2.Subtest1',
'first_known_failed_build_number':
4,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r2_1',
'confidence': 90,
'analysis_approach': 'HEURISTIC',
'revert_committed': False
}],
'analysis_approach':
'HEURISTIC',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url':
master_url,
'builder_name':
builder_name,
'build_number':
build_number,
'step_name':
'b on platform',
'is_sub_test':
True,
'test_name':
'Unittest3.Subtest1',
'first_known_failed_build_number':
4,
'suspected_cls': [{
'repo_name': 'chromium',
'revision': 'r4_10',
'commit_position': 410,
'analysis_approach': 'TRY_JOB',
'confidence': 100,
'revert_cl_url': 'revert_cl_url',
'revert_committed': False
}],
'analysis_approach':
'TRY_JOB',
'is_flaky_test':
False,
'try_job_status':
'FINISHED',
'has_findings':
True,
'is_finished':
True,
'is_supported':
True,
}, {
'master_url': master_url,
'builder_name': builder_name,
'build_number': build_number,
'step_name': 'c',
'is_sub_test': False,
'analysis_approach': 'HEURISTIC',
'is_flaky_test': False,
'has_findings': False,
'is_finished': True,
'is_supported': False,
}]
self._MockMasterIsSupported(supported=True)
response = self.call_api('AnalyzeBuildFailures', body=builds)
self.assertEqual(200, response.status_int)
self.assertItemsEqual(expected_results,
response.json_body.get('results'))
