def testGetMergedFailureWithSavedMergedFailure(self):
dummy_merged_failure = CompileFailure.Create(
ndb.Key(LuciFailedBuild, 9876543201), 'compile', ['target1.o'])
dummy_merged_failure.put()
failure = self.target_entities[0]
failure.merged_failure_key = dummy_merged_failure.key
self.assertEqual(dummy_merged_failure, failure.GetMergedFailure())
def setUp(self):
super(CompileFailureTest, self).setUp()
self.build_id = 9876543210
self.edges = [
(['target1.o'], 'CXX'),
(['target2.o'], 'ACTION'),
]
build = LuciFailedBuild.Create(
luci_project='chromium',
luci_bucket='ci',
luci_builder='Linux Builder',
build_id=self.build_id,
legacy_build_number=12345,
gitiles_host='chromium.googlesource.com',
gitiles_project='chromium/src',
gitiles_ref='refs/heads/master',
gitiles_id='git_hash',
commit_position=65450,
status=20,
create_time=datetime(2019, 3, 28),
start_time=datetime(2019, 3, 28, 0, 1),
end_time=datetime(2019, 3, 28, 1),
build_failure_type=StepTypeEnum.COMPILE)
build.put()
self.target_entities = []
for output_targets, rule in self.edges:
target = CompileFailure.Create(build.key, 'compile',
output_targets, rule)
target.put()
self.target_entities.append(target)
def testGetMergedFailure(self):
failure = CompileFailure.Create(ndb.Key(LuciFailedBuild, 9876543234),
'compile', ['target1.o'])
failure.first_failed_build_id = self.build_id
failure.put()
self.assertEqual(self.target_entities[0], failure.GetMergedFailure())
def testGroupFailuresByRegerssionRange(self):
compile_failure_3 = CompileFailure.Create(self.build.key, 'compile',
['c.o'], 'CC')
compile_failure_3.put()
failures_with_range = [{
'failure': self.compile_failure_1,
'last_passed_commit': self.commits[5],
'first_failed_commit': self.commits[6],
}, {
'failure': self.compile_failure_2,
'last_passed_commit': self.commits[7],
'first_failed_commit': self.commits[8],
}, {
'failure': compile_failure_3,
'last_passed_commit': self.commits[5],
'first_failed_commit': self.commits[6],
}]
expected_result = [
{
'failures': [self.compile_failure_1, compile_failure_3],
'last_passed_commit': self.commits[5],
'first_failed_commit': self.commits[6],
},
{
'failures': [self.compile_failure_2],
'last_passed_commit': self.commits[7],
'first_failed_commit': self.commits[8],
},
]
result = analysis_util.GroupFailuresByRegerssionRange(
failures_with_range)
self.assertItemsEqual(expected_result, result)
def testGetFailuresWithMatchingCompileFailureGroupsWithExistingGroup(self):
build_id = 8000000000122
build = Build(builder=self.builder, number=122, id=build_id)
build.input.gitiles_commit.host = 'gitiles.host.com'
build.input.gitiles_commit.project = 'project/name'
build.input.gitiles_commit.ref = 'ref/heads/master'
build.input.gitiles_commit.id = 'git_sha'
last_passed_build_info = {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha_121'
}
first_failures_in_current_build = {
'failures': {
'install packages': {
'output_targets': [
frozenset(['target1']),
],
'last_passed_build': last_passed_build_info,
},
},
'last_passed_build': last_passed_build_info
}
compile_failure = CompileFailure.Create(
self.group_build.key,
'install packages', ['target1'],
'CXX',
first_failed_build_id=self.group_build_id,
last_passed_build_id=8000000000160)
compile_failure.put()
CompileFailureGroup.Create(
luci_project=self.context.luci_project_name,
luci_bucket=build.builder.bucket,
build_id=self.group_build_id,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
last_passed_gitiles_id=last_passed_build_info['commit_id'],
last_passed_commit_position=654321,
first_failed_gitiles_id=self.first_failed_commit_id,
first_failed_commit_position=654340,
compile_failure_keys=[compile_failure.key]).put()
expected_failures_with_existing_group = {
'install packages': {
frozenset(['target1']): self.group_build_id
}
}
self.assertEqual(
expected_failures_with_existing_group,
ChromeOSProjectAPI().GetFailuresWithMatchingCompileFailureGroups(
self.context, build, first_failures_in_current_build))
def testGetCulpritsForFailures(self):
culprit = CulpritNdb.Create(self.gitiles_host, self.gitiles_project,
self.gitiles_ref, 'git_hash_123', 123)
culprit.put()
failure1 = CompileFailure.Create(self.build.key, 'compile', ['a.o'],
'CC')
failure1.culprit_commit_key = culprit.key
failure1.put()
failure2 = CompileFailure.Create(self.build.key, 'compile', ['b.o'],
'CC')
failure2.culprit_commit_key = culprit.key
failure2.put()
culprits = analysis_util.GetCulpritsForFailures([failure1, failure2])
self.assertEqual(1, len(culprits))
self.assertEqual('git_hash_123', culprits[0].commit.id)
def testGetMergedFailureForNonFirstFailure(self):
first_failed_build_id = 9876543201
dummy_merged_failure = CompileFailure.Create(
ndb.Key(LuciFailedBuild, first_failed_build_id),
'compile', ['target1.o'],
first_failed_build_id=first_failed_build_id,
failure_group_build_id=first_failed_build_id)
dummy_merged_failure.put()
failure = self.target_entities[0]
failure.first_failed_build_id = first_failed_build_id
self.assertEqual(dummy_merged_failure, failure.GetMergedFailure())
def testOnCompileFailureAnalysisResultRequestedAnalysisRunning(self):
build_id = 800000000123
request = findit_result.BuildFailureAnalysisRequest(
build_id=build_id, failed_steps=[self.compile_step_name])
build = LuciFailedBuild.Create(
luci_project='chromium',
luci_bucket='ci',
luci_builder='Linux Builder',
build_id=build_id,
legacy_build_number=12345,
gitiles_host='chromium.googlesource.com',
gitiles_project='chromium/src',
gitiles_ref='refs/heads/master',
gitiles_id='git_hash',
commit_position=65450,
status=20,
create_time=datetime(2019, 3, 28),
start_time=datetime(2019, 3, 28, 0, 1),
end_time=datetime(2019, 3, 28, 1),
build_failure_type=StepTypeEnum.COMPILE)
build.put()
compile_failure = CompileFailure.Create(build.key,
self.compile_step_name,
['target1'], 'CXX')
compile_failure.first_failed_build_id = build.build_id
compile_failure.failure_group_build_id = build.build_id
compile_failure.put()
analysis = CompileFailureAnalysis.Create(
luci_project=self.context.luci_project_name,
luci_bucket='postsubmit',
luci_builder='Linux Builder',
build_id=build_id,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
last_passed_gitiles_id='last_passed_git_hash',
last_passed_commit_position=65430,
first_failed_gitiles_id='git_hash',
first_failed_commit_position=65450,
rerun_builder_id='chromeos/postsubmit/builder-bisect',
compile_failure_keys=[])
analysis.status = analysis_status.RUNNING
analysis.Save()
responses = compile_api.OnCompileFailureAnalysisResultRequested(
request, build)
self.assertEqual(1, len(responses))
self.assertEqual(0, len(responses[0].culprits))
self.assertFalse(responses[0].is_finished)
self.assertTrue(responses[0].is_supported)
def _CreateFailure(self, failed_build_key, step_ui_name,
first_failed_build_id, last_passed_build_id,
merged_failure_key, atomic_failure, properties):
"""Creates a CompileFailure entity."""
return CompileFailure.Create(
failed_build_key=failed_build_key,
step_ui_name=step_ui_name,
output_targets=list(atomic_failure or []),
rule=(properties or {}).get('rule'),
first_failed_build_id=first_failed_build_id,
last_passed_build_id=last_passed_build_id,
# Default to first_failed_build_id, will be updated later if matching
# group exists.
failure_group_build_id=first_failed_build_id,
merged_failure_key=merged_failure_key)
def setUp(self):
super(AnalysisUtilTest, self).setUp()
self.gitiles_host = 'gitiles.host.com'
self.gitiles_project = 'project/name'
self.gitiles_ref = 'ref/heads/master'
self.build = LuciFailedBuild.Create(
luci_project='chromium',
luci_bucket='ci',
luci_builder='Linux Builder',
build_id=9876543210,
legacy_build_number=12345,
gitiles_host='chromium.googlesource.com',
gitiles_project='chromium/src',
gitiles_ref='refs/heads/master',
gitiles_id='git_hash',
commit_position=65450,
status=20,
create_time=datetime(2019, 3, 28),
start_time=datetime(2019, 3, 28, 0, 1),
end_time=datetime(2019, 3, 28, 1),
build_failure_type=StepTypeEnum.COMPILE)
self.build.put()
self.compile_failure_1 = CompileFailure.Create(self.build.key,
'compile', ['a.o'],
'CC')
self.compile_failure_1.put()
self.compile_failure_2 = CompileFailure.Create(self.build.key,
'compile', ['b.o'],
'CC')
self.compile_failure_2.put()
self.commits = []
for i in xrange(0, 11):
self.commits.append(self._CreateGitilesCommit('r%d' % i, 100 + i))
def testSaveCompileFailuresOnlyStepLevelFailures(self, _):
detailed_compile_failures = {
'compile': {
'failures': {},
'first_failed_build': {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha'
},
'last_passed_build': {
'id': 8000000000120,
'number': 120,
'commit_id': 'git_sha'
},
},
}
# Prepares data for first failed build.
first_failed_build = self._MockBuild(8000000000121, 121, 'git_sha_121')
first_failed_build_entity = luci_build.SaveFailedBuild(
self.context, first_failed_build, StepTypeEnum.COMPILE)
first_failure = CompileFailure.Create(first_failed_build_entity.key,
'compile', None, 'CXX')
first_failure.put()
pre_compile_analysis.SaveCompileFailures(self.context, self.build,
detailed_compile_failures)
build_entity = LuciFailedBuild.get_by_id(self.build_id)
self.assertIsNotNone(build_entity)
compile_failures = CompileFailure.query(
ancestor=build_entity.key).fetch()
self.assertEqual(1, len(compile_failures))
self.assertEqual(8000000000121,
compile_failures[0].first_failed_build_id)
self.assertEqual([], compile_failures[0].output_targets)
self.assertEqual(first_failure.key,
compile_failures[0].merged_failure_key)
def setUp(self):
super(CompileFailureRerunAnalysisTest, self).setUp()
self.build_id = 8000000000123
self.build_number = 123
self.builder = BuilderID(project='chromium',
bucket='ci',
builder='linux-rel')
self.build = Build(id=self.build_id,
builder=self.builder,
number=self.build_number,
status=common_pb2.FAILURE)
self.build.input.gitiles_commit.host = 'gitiles.host.com'
self.build.input.gitiles_commit.project = 'project/name'
self.build.input.gitiles_commit.ref = 'ref/heads/master'
self.build.input.gitiles_commit.id = 'git_sha_123'
self.build.create_time.FromDatetime(datetime(2019, 4, 9))
self.build.start_time.FromDatetime(datetime(2019, 4, 9, 0, 1))
self.build.end_time.FromDatetime(datetime(2019, 4, 9, 1))
self.context = Context(luci_project_name='chromium',
gitiles_host='gitiles.host.com',
gitiles_project='project/name',
gitiles_ref='ref/heads/master',
gitiles_id='git_sha')
build_entity = LuciFailedBuild.Create(
luci_project=self.context.luci_project_name,
luci_bucket=self.build.builder.bucket,
luci_builder=self.build.builder.builder,
build_id=self.build_id,
legacy_build_number=self.build_number,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
gitiles_id=self.context.gitiles_id,
commit_position=6000005,
status=20,
create_time=datetime(2019, 3, 28),
start_time=datetime(2019, 3, 28, 0, 1),
end_time=datetime(2019, 3, 28, 1),
build_failure_type=StepTypeEnum.COMPILE)
build_entity.put()
self.compile_failure = CompileFailure.Create(
failed_build_key=build_entity.key,
step_ui_name='compile',
output_targets=['a.o'],
first_failed_build_id=self.build_id,
failure_group_build_id=None)
self.compile_failure.put()
self.analysis = CompileFailureAnalysis.Create(
luci_project=self.context.luci_project_name,
luci_bucket=self.build.builder.bucket,
luci_builder=self.build.builder.builder,
build_id=self.build_id,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
last_passed_gitiles_id='left_sha',
last_passed_commit_position=6000000,
first_failed_gitiles_id=self.context.gitiles_id,
first_failed_commit_position=6000005,
rerun_builder_id='chromium/findit/findit-variables',
compile_failure_keys=[self.compile_failure.key])
self.analysis.Save()
def testGetFirstFailuresInCurrentBuildWithoutGroup(self, *_):
build_121_info = {
'id': 8000000000121,
'number': self.build_number - 2,
'commit_id': 'git_sha_121'
}
first_failures_in_current_build = {
'failures': {
'compile': {
'output_targets':
[frozenset(['target1']),
frozenset(['target2'])],
'last_passed_build':
build_121_info,
},
},
'last_passed_build': build_121_info
}
# Creates and saves entities of the existing group.
detailed_compile_failures = {
'compile': {
'failures': {
frozenset(['target1']): {
'rule': 'CXX',
'first_failed_build': self.build_info,
'last_passed_build': build_121_info,
},
frozenset(['target2']): {
'rule': 'ACTION',
'first_failed_build': self.build_info,
'last_passed_build': build_121_info,
},
},
'first_failed_build': self.build_info,
'last_passed_build': build_121_info,
},
}
pre_compile_analysis.SaveCompileFailures(self.context, self.build,
detailed_compile_failures)
# Prepares data for existing failure group.
group_build = self._MockBuild(8000000000134,
12134,
'git_sha_134',
builder_name='Mac')
group_build_entity = luci_build.SaveFailedBuild(
self.context, group_build, StepTypeEnum.COMPILE)
group_failure1 = CompileFailure.Create(group_build_entity.key,
'compile', ['target1'], 'CXX')
group_failure1.put()
group_failure2 = CompileFailure.Create(group_build_entity.key,
'compile', ['target2'],
'ACTION')
group_failure2.put()
self.assertEqual(
{
'failures': {},
'last_passed_build': None
},
pre_compile_analysis.GetFirstFailuresInCurrentBuildWithoutGroup(
self.context, self.build, first_failures_in_current_build))
build = LuciFailedBuild.get_by_id(self.build_id)
compile_failures = CompileFailure.query(ancestor=build.key).fetch()
self.assertEqual(2, len(compile_failures))
for failure in compile_failures:
if failure.output_targets == ['target1']:
self.assertEqual(group_failure1.key,
failure.merged_failure_key)
else:
self.assertEqual(group_failure2.key,
failure.merged_failure_key)
def testSaveCompileFailures(self, _):
detailed_compile_failures = {
'compile': {
'failures': {
frozenset(['target1', 'target2']): {
'rule': 'CXX',
'first_failed_build': {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha_121'
},
'last_passed_build': {
'id': 8000000000120,
'number': 120,
'commit_id': 'git_sha'
},
},
},
'first_failed_build': {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha_121'
},
'last_passed_build': {
'id': 8000000000120,
'number': 120,
'commit_id': 'git_sha'
},
},
}
# Prepares data for existing failure group.
group_build = self._MockBuild(8000003400121,
12134,
'git_sha_121',
builder_name='Mac')
group_build_entity = luci_build.SaveFailedBuild(
self.context, group_build, StepTypeEnum.COMPILE)
group_failure = CompileFailure.Create(group_build_entity.key,
'compile',
['target1', 'target2'], 'CXX')
group_failure.put()
# Prepares data for first failed build.
first_failed_build = self._MockBuild(8000000000121, 121, 'git_sha_121')
first_failed_build_entity = luci_build.SaveFailedBuild(
self.context, first_failed_build, StepTypeEnum.COMPILE)
first_failure = CompileFailure.Create(first_failed_build_entity.key,
'compile',
['target1', 'target2'], 'CXX')
first_failure.merged_failure_key = group_failure.key
first_failure.put()
pre_compile_analysis.SaveCompileFailures(self.context, self.build,
detailed_compile_failures)
build = LuciFailedBuild.get_by_id(self.build_id)
self.assertIsNotNone(build)
compile_failures = CompileFailure.query(ancestor=build.key).fetch()
self.assertEqual(1, len(compile_failures))
self.assertEqual(8000000000121,
compile_failures[0].first_failed_build_id)
self.assertEqual(group_failure.key,
compile_failures[0].merged_failure_key)
def testGetFailuresWithMatchingCompileFailureGroupsFailureNotExactlySame(
self):
build_id = 8000000000122
build = Build(builder=self.builder, number=122, id=build_id)
build.input.gitiles_commit.host = 'gitiles.host.com'
build.input.gitiles_commit.project = 'project/name'
build.input.gitiles_commit.ref = 'ref/heads/master'
build.input.gitiles_commit.id = 'git_sha'
output_target1 = json.dumps({
'category': 'chromeos-base',
'packageName': 'target1'
})
output_target2 = json.dumps({
'category': 'chromeos-base',
'packageName': 'target2'
})
last_passed_build_info = {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha_121'
}
first_failures_in_current_build = {
'failures': {
'install packages': {
'output_targets':
[frozenset([output_target1]),
frozenset([output_target2])],
'last_passed_build':
last_passed_build_info,
},
},
'last_passed_build': last_passed_build_info
}
compile_failure = CompileFailure.Create(
self.group_build.key,
'install packages', [output_target1],
'CXX',
first_failed_build_id=self.group_build_id,
last_passed_build_id=8000000000160)
compile_failure.put()
CompileFailureGroup.Create(
luci_project=self.context.luci_project_name,
luci_bucket=build.builder.bucket,
build_id=self.group_build_id,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
last_passed_gitiles_id=last_passed_build_info['commit_id'],
last_passed_commit_position=654321,
first_failed_gitiles_id=self.first_failed_commit_id,
first_failed_commit_position=654340,
compile_failure_keys=[compile_failure.key]).put()
self.assertEqual(
{},
ChromeOSProjectAPI().GetFailuresWithMatchingCompileFailureGroups(
self.context, build, first_failures_in_current_build))
def SaveCompileFailures(context, build, detailed_compile_failures):
"""Saves the failed build and compile failures in data store.
Args:
context (findit_v2.services.context.Context): Scope of the analysis.
build (buildbucket build.proto): ALL info about the build.
detailed_compile_failures (dict): A dict of detailed compile failures.
{
'step_name': {
'failures': {
frozenset(['target1', 'target2']): {
'rule': 'emerge',
'first_failed_build': {
'id': 8765432109,
'number': 123,
'commit_id': 654321
},
'last_passed_build': None
},
...
},
'first_failed_build': {
'id': 8765432109,
'number': 123,
'commit_id': 654321
},
'last_passed_build': None
},
}
"""
build_entity = luci_build.SaveFailedBuild(context, build,
StepTypeEnum.COMPILE)
assert build_entity, 'Failed to create failure entity for build {}'.format(
build.id)
failed_build_key = build_entity.key
compile_failure_entities = []
first_failures = {}
for step_ui_name, step_info in detailed_compile_failures.iteritems():
failures = step_info['failures']
if not failures:
logging.warning(
'Cannot get detailed compile failure info for build %d,'
' saving step level info only.', build.id)
first_failed_build_id = step_info.get('first_failed_build', {}).get('id')
merged_failure_key = compile_failure.GetMergedFailureKey(
first_failures, first_failed_build_id, step_ui_name, None)
new_entity = CompileFailure.Create(
failed_build_key=failed_build_key,
step_ui_name=step_ui_name,
output_targets=None,
first_failed_build_id=first_failed_build_id,
last_passed_build_id=step_info.get('last_passed_build', {}).get('id'),
# Default to first_failed_build_id, will be updated later if matching
# group exists.
failure_group_build_id=first_failed_build_id,
merged_failure_key=merged_failure_key)
compile_failure_entities.append(new_entity)
continue
for output_targets, failure in failures.iteritems():
first_failed_build_id = failure.get('first_failed_build', {}).get('id')
merged_failure_key = compile_failure.GetMergedFailureKey(
first_failures, first_failed_build_id, step_ui_name, output_targets)
new_entity = CompileFailure.Create(
failed_build_key=failed_build_key,
step_ui_name=step_ui_name,
output_targets=list(output_targets),
first_failed_build_id=first_failed_build_id,
last_passed_build_id=(failure.get('last_passed_build') or
{}).get('id'),
# Default to first_failed_build_id, will be updated later if matching
# group exists.
failure_group_build_id=first_failed_build_id,
rule=failure.get('rule'),
dependencies=failure.get('dependencies'),
merged_failure_key=merged_failure_key)
compile_failure_entities.append(new_entity)
ndb.put_multi(compile_failure_entities)