def _CreateCompileRerunBuild(self, commit_position=6000002):
rerun_commit = GitilesCommit(
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
gitiles_id=str(commit_position),
commit_position=commit_position)
rerun_builder = BuilderID(project='chromium',
bucket='findit',
builder='findit-variables')
rerun_build = CompileRerunBuild.Create(
luci_project=rerun_builder.project,
luci_bucket=rerun_builder.bucket,
luci_builder=rerun_builder.builder,
build_id=8000000000789,
legacy_build_number=60789,
gitiles_host=rerun_commit.gitiles_host,
gitiles_project=rerun_commit.gitiles_project,
gitiles_ref=rerun_commit.gitiles_ref,
gitiles_id=rerun_commit.gitiles_id,
commit_position=rerun_commit.commit_position,
status=1,
create_time=datetime(2019, 3, 28),
parent_key=self.analysis.key)
rerun_build.put()
return rerun_build
def testProcessRerunBuildResultBuildPassed(self, mock_compile_failures,
mock_analysis):
build_id = 8000000000123
build_number = 123
builder = BuilderID(project='chromeos',
bucket='postsubmit',
builder='findit-variable')
build = Build(id=build_id,
builder=builder,
number=build_number,
status=common_pb2.SUCCESS,
tags=[{
'key': 'analyzed_build_id',
'value': str(self.analyzed_build_id)
}])
build.input.gitiles_commit.host = 'gitiles.host.com'
build.input.gitiles_commit.project = 'project/name'
build.input.gitiles_commit.ref = 'ref/heads/master'
build.input.gitiles_commit.id = 'git_sha_6543221'
build.create_time.FromDatetime(datetime(2019, 4, 9))
step1 = Step(name='s1', status=common_pb2.SUCCESS)
step2 = Step(name=self.compile_step_name, status=common_pb2.SUCCESS)
build.steps.extend([step1, step2])
CompileRerunBuild.Create(luci_project=self.context.luci_project_name,
luci_bucket=build.builder.bucket,
luci_builder=build.builder.builder,
build_id=build_id,
legacy_build_number=build_number,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
gitiles_id='git_sha_6543221',
commit_position=6543221,
status=build.status,
create_time=build.create_time.ToDatetime(),
parent_key=self.analysis.key).put()
self.assertTrue(
compile_analysis.OnCompileRerunBuildCompletion(
self.context, build))
self.assertFalse(mock_compile_failures.called)
rerun_build = CompileRerunBuild.get_by_id(build_id,
parent=self.analysis.key)
self.assertEqual({}, rerun_build.GetFailuresInBuild())
self.assertTrue(mock_analysis.called)
def testLuciRerunBuildSearch(self):
build_id = 1234567890
commit_position = 65432
commit = GitilesCommit(gitiles_host='chromium.googlesource.com',
gitiles_project='chromium/src',
gitiles_ref='refs/heads/master',
gitiles_id='git_hash',
commit_position=commit_position)
analysis = self._CreateCompileFailureAnalysis()
self._CreateCompileRerunBuild(build_id, commit_position, analysis.key)
rerun_builds = CompileRerunBuild.SearchBuildOnCommit(
analysis.key, commit)
self.assertEqual(1, len(rerun_builds))
self.assertEqual(build_id, rerun_builds[0].build_id)
def _CreateRerunBuild(self, rerun_builder, new_build, rerun_commit,
analysis_key):
return CompileRerunBuild.Create(
luci_project=rerun_builder.project,
luci_bucket=rerun_builder.bucket,
luci_builder=rerun_builder.builder,
build_id=new_build.id,
legacy_build_number=new_build.number,
gitiles_host=rerun_commit.gitiles_host,
gitiles_project=rerun_commit.gitiles_project,
gitiles_ref=rerun_commit.gitiles_ref,
gitiles_id=rerun_commit.gitiles_id,
commit_position=rerun_commit.commit_position,
status=new_build.status,
create_time=new_build.create_time.ToDatetime(),
parent_key=analysis_key)
def testLuciRerunBuildGetFailedTargets(self):
build_id = 1234567890
commit_position = 65432
analysis = self._CreateCompileFailureAnalysis()
self._CreateCompileRerunBuild(build_id, commit_position, analysis.key)
rerun_build = CompileRerunBuild.get_by_id(build_id,
parent=analysis.key)
rerun_build.results = []
for target in self.target_entities:
result = CompileFailureInRerunBuild(
step_ui_name=target.step_ui_name,
output_targets=target.output_targets)
rerun_build.failures.append(result)
result = rerun_build.GetFailuresInBuild()
self.assertItemsEqual(['target1.o', 'target2.o'], result['compile'])
def _CreateCompileRerunBuild(self, build_id, commit_position,
analysis_key):
build = CompileRerunBuild.Create(
luci_project='chromium',
luci_bucket='ci',
luci_builder='Linux Builder',
build_id=build_id,
legacy_build_number=11111,
gitiles_host='chromium.googlesource.com',
gitiles_project='chromium/src',
gitiles_ref='refs/heads/master',
gitiles_id='git_hash',
commit_position=commit_position,
status=1,
create_time=datetime(2019, 3, 28),
parent_key=analysis_key)
build.put()
def testLuciRerunBuild(self):
build_id = 1234567890
commit_position = 65432
analysis = self._CreateCompileFailureAnalysis()
self._CreateCompileRerunBuild(build_id, commit_position, analysis.key)
rerun_build = CompileRerunBuild.get_by_id(build_id,
parent=analysis.key)
self.assertIsNotNone(rerun_build)
detailed_compile_failures = {
'compile': {
'failures': {
frozenset(['target1.o', 'target2.o']): {
'rule': 'CXX',
'first_failed_build': {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha'
},
'last_passed_build': {
'id': 8000000000120,
'number': 120,
'commit_id': 'git_sha'
},
},
},
'first_failed_build': {
'id': 8000000000121,
'number': 121,
'commit_id': 'git_sha'
},
'last_passed_build': {
'id': 8000000000120,
'number': 120,
'commit_id': 'git_sha'
},
},
}
rerun_build.SaveRerunBuildResults(common_pb2.FAILURE,
detailed_compile_failures)
result = rerun_build.GetFailedTargets()
self.assertItemsEqual(['target1.o', 'target2.o'], result['compile'])
def _ProcessAndSaveRerunBuildResult(context, analyzed_build_id, rerun_build):
"""Gets results of a completed rerun build and save it in datastore.
Args:
context (findit_v2.services.context.Context): Scope of the analysis.
analyzed_build_id (int): Id of the failed ci/post_submit build that's being
analyzed.
rerun_build (buildbucket build.proto): ALL info about the rerun build.
Returns:
True if the rerun build entity is updated, otherwise False.
"""
project_api = projects.GetProjectAPI(context.luci_project_name)
analysis = CompileFailureAnalysis.GetVersion(analyzed_build_id)
if not analysis:
logging.error('CompileFailureAnalysis missing for %d.',
analyzed_build_id)
return False
rerun_build_entity = CompileRerunBuild.get_by_id(rerun_build.id,
parent=analysis.key)
if not rerun_build_entity:
logging.error('CompileRerunBuild entity for build %d missing.',
rerun_build.id)
return False
detailed_compile_failures = {}
if rerun_build.status == common_pb2.FAILURE:
failed_steps = build_util.GetFailedStepsInBuild(context, rerun_build)
compile_steps = [
fs[0] for fs in failed_steps if fs[1] == StepTypeEnum.COMPILE
]
detailed_compile_failures = project_api.GetCompileFailures(
rerun_build, compile_steps) if compile_steps else {}
_SaveRerunBuildResults(rerun_build_entity, rerun_build.status,
detailed_compile_failures,
rerun_build.end_time.ToDatetime())
rerun_build_entity.put()
return True
def testRerunBasedAnalysisContinueWithNextRerunBuild(
self, mock_revisions, mock_trigger_build, _):
mock_revisions.return_value = {
n: str(n)
for n in xrange(6000000, 6000005)
}
mock_rerun_build = Build(id=8000055000123, number=78990)
mock_rerun_build.create_time.FromDatetime(datetime(2019, 4, 30))
mock_trigger_build.return_value = mock_rerun_build
compile_failure_rerun_analysis.RerunBasedAnalysis(
self.context, self.build_id, self.build)
self.assertTrue(mock_trigger_build.called)
analysis = CompileFailureAnalysis.GetVersion(self.build_id)
self.assertEqual(analysis_status.RUNNING, analysis.status)
rerun_builds = CompileRerunBuild.query(
ancestor=self.analysis.key).fetch()
self.assertEqual(1, len(rerun_builds))
self.assertEqual(6000002,
rerun_builds[0].gitiles_commit.commit_position)
def testTriggerRerunBuild(self, mock_trigger_build, _):
new_build_id = 800000024324
new_build = Build(id=new_build_id, number=300)
new_build.status = common_pb2.SCHEDULED
new_build.create_time.FromDatetime(datetime(2019, 4, 20))
rerun_builder = BuilderID(project='chromium',
bucket='findit',
builder='findit-variables')
rerun_commit = GitilesCommit(
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
gitiles_id='6000002',
commit_position=6000002)
output_targets = {'compile': ['a.o']}
mock_trigger_build.return_value = new_build
compile_failure_rerun_analysis.TriggerRerunBuild(
self.context, self.build_id, self.build, self.analysis.key,
rerun_builder, rerun_commit, output_targets)
rerun_build = CompileRerunBuild.get_by_id(new_build_id,
parent=self.analysis.key)
self.assertIsNotNone(rerun_build)
mock_trigger_build.assert_called_once_with(
rerun_builder,
common_pb2.GitilesCommit(project=rerun_commit.gitiles_project,
host=rerun_commit.gitiles_host,
ref=rerun_commit.gitiles_ref,
id=rerun_commit.gitiles_id),
{'recipe': 'compile'},
tags=[{
'value': 'compile-failure-culprit-finding',
'key': 'purpose'
}, {
'value': str(self.build.id),
'key': 'analyzed_build_id'
}])
def testTriggerRerunBuildFailedToTriggerBuild(self, mock_trigger_build, _):
"""This test is for the case where there's already an existing rerun build,
so no new rerun-build should be scheduled."""
rerun_commit = GitilesCommit(
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
gitiles_id='6000002',
commit_position=6000002)
rerun_builder = BuilderID(project='chromium',
bucket='findit',
builder='findit-variables')
output_targets = {'compile': ['a.o']}
compile_failure_rerun_analysis.TriggerRerunBuild(
self.context, self.build_id, self.build, self.analysis.key,
rerun_builder, rerun_commit, output_targets)
self.assertTrue(mock_trigger_build.called)
rerun_builds = CompileRerunBuild.query(
ancestor=self.analysis.key).fetch()
self.assertEqual([], rerun_builds)
def _GetExistingRerunBuild(self, analysis_key, rerun_commit):
return CompileRerunBuild.SearchBuildOnCommit(analysis_key,
rerun_commit)
def _FetchRerunBuildsOfAnalysis(self, analysis):
return CompileRerunBuild.query(ancestor=analysis.key).order(
CompileRerunBuild.gitiles_commit.commit_position).fetch()
def _GetRegressionRangesForCompileFailures(analysis):
"""Gets updated regression ranges and failures having that range.
Uses completed rerun builds in this analysis to narrow down regression
ranges for each failures.
For example, if initially the regression range for the analysis is (r0, r10]
and compile failures in the analysis have output_targets ['a.o', 'b.o']
and ['c.o'] respectively (below use output_targets to identify a compile
failure).
1. When there's no rerun build, all targets have the same range (r0, r10]
2. 1st rerun build on r5, all passed. Then all failures have a smaller
range (r5, r10]
3. 2nd rerun build on r7, ['a.o', 'b.o'] failed, ['c.o'] passed. So now the
regression range for ['a.o', 'b.o'] is (r5, r7], and for ['c.o'] is
(r7, r10].
4. 3rd rerun build on r6, and it only compiles ['a.o', 'b.o']. and both of
them failed. The regression range is updated to (r5, r6].
6. 4th rerun build on r8 and it only compiles 'c.o', and it failed. So the
regression range is updated to (r7, r8].
Returns:
(list of dict): Failures with the same regression range and the range.
[
{
'failures': [CompileFailure(output_targets=['a.o', 'b.o'])],
'last_passed_commit': GitilesCommit(gitiles_id=r5),
'first_failed_commit': GitilesCommit(gitiles_id=r6)
},
{
'failures': [CompileFailure(output_targets=['c.o'])],
'last_passed_commit': GitilesCommit(gitiles_id=r7),
'first_failed_commit': GitilesCommit(gitiles_id=r8)
},
]
"""
compile_failures = ndb.get_multi(analysis.compile_failure_keys)
rerun_builds = CompileRerunBuild.query(ancestor=analysis.key).order(
CompileRerunBuild.gitiles_commit.commit_position).fetch()
if not rerun_builds:
return [{
'failures': compile_failures,
'last_passed_commit': analysis.last_passed_commit,
'first_failed_commit': analysis.first_failed_commit
}]
# Gets rerun builds results.
# Specifically, if a rerun build failed, gets its failed targets.
# Otherwise just keep an empty failed targets indicating a successful build.
rerun_builds_info = [
(rerun_build.gitiles_commit, rerun_build.GetFailedTargets())
for rerun_build in rerun_builds
if rerun_build.status in [common_pb2.FAILURE, common_pb2.SUCCESS]
]
# A list for regression ranges of each failure.
# Initially all failures have the same (and the widest) range. By checking
# rerun build results, each failure's regression range could be narrower and
# different from others.
failures_with_range = []
for failure in compile_failures:
if failure.culprit_commit_key:
# Skips the failures if it already found the culprit.
continue
failures_with_range.append({
'failure': failure,
'last_passed_commit': analysis.last_passed_commit,
'first_failed_commit': analysis.first_failed_commit
})
# Updates regression range for each failed targets.
analysis_util.UpdateFailureRegressionRanges(rerun_builds_info,
failures_with_range)
# Groups failed targets with the same regression range, and returns these
# groups along with their regression range.
return analysis_util.GroupFailuresByRegerssionRange(failures_with_range)
def TriggerRerunBuild(context, analyzed_build_id, referred_build, analysis_key,
rerun_builder, rerun_commit, output_targets):
"""Triggers a rerun build if there's no existing one.
Creates and saves a CompileRerunBuild entity if a new build is triggered.
Checking for existing build and saving new build are in one transaction to
make sure no duplicated rerun builds can be triggered.
Args:
context (findit_v2.services.context.Context): Scope of the analysis.
analyzed_build_id (int): Build id of the build that's being analyzed.
referred_build (buildbucket build.proto): Info about the build being
referred to trigger new rerun builds. This build could be the analyzed
build or a previous rerun build in the same analysis.
analysis_key (Key to CompileFailureAnalysis): Key to the running analysis.
rerun_builder (BuilderId): Builder to rerun the build.
rerun_commit (GitilesCommit): Gitiles commit the build runs on.
output_targets (dict): A dict of targets the rerun build should re-compile.
{
'compile': ['target1.o', ...]
}
"""
# Check if there's a running build on that commit already.
existing_builds = CompileRerunBuild.SearchBuildOnCommit(
analysis_key, rerun_commit)
if existing_builds:
# TODO(crbug/957760): Re-trigger the build if the existing one(s) ended with
# unexpected failures.
logging.debug('Found existing rerun build for analysis %s on commit %d.',
analysis_key.urlsafe(), rerun_commit.commit_position)
return
rerun_tags = _GetRerunBuildTags(analyzed_build_id)
input_properties = _GetRerunBuildInputProperties(context, referred_build,
output_targets)
if not input_properties:
logging.error(
'Failed to get input properties to trigger rerun build'
'for build %d.', analyzed_build_id)
return
gitiles_commit_pb = common_pb2.GitilesCommit(
project=rerun_commit.gitiles_project,
host=rerun_commit.gitiles_host,
ref=rerun_commit.gitiles_ref,
id=rerun_commit.gitiles_id)
new_build = buildbucket_client.TriggerV2Build(
rerun_builder, gitiles_commit_pb, input_properties, tags=rerun_tags)
if not new_build:
logging.error(
'Failed to trigger rerun build for %s in build %d,'
'on commit %s', output_targets, analyzed_build_id,
rerun_commit.gitiles_id)
return
CompileRerunBuild.Create(
luci_project=rerun_builder.project,
luci_bucket=rerun_builder.bucket,
luci_builder=rerun_builder.builder,
build_id=new_build.id,
legacy_build_number=new_build.number,
gitiles_host=rerun_commit.gitiles_host,
gitiles_project=rerun_commit.gitiles_project,
gitiles_ref=rerun_commit.gitiles_ref,
gitiles_id=rerun_commit.gitiles_id,
commit_position=rerun_commit.commit_position,
status=new_build.status,
create_time=new_build.create_time.ToDatetime(),
parent_key=analysis_key).put()
def testProcessRerunBuildResult(self, mock_compile_failures):
build_id = 8000000000123
build_number = 123
builder = BuilderID(project='chromeos',
bucket='postsubmit',
builder='findit-variable')
build = Build(id=build_id,
builder=builder,
number=build_number,
status=common_pb2.FAILURE,
tags=[{
'key': 'analyzed_build_id',
'value': str(self.analyzed_build_id)
}])
build.input.gitiles_commit.host = 'gitiles.host.com'
build.input.gitiles_commit.project = 'project/name'
build.input.gitiles_commit.ref = 'ref/heads/master'
build.input.gitiles_commit.id = 'git_sha_6543221'
build.create_time.FromDatetime(datetime(2019, 4, 9))
step1 = Step(name='s1', status=common_pb2.SUCCESS)
step2 = Step(name=self.compile_step_name, status=common_pb2.FAILURE)
build.steps.extend([step1, step2])
output_target = json.dumps({
'category': 'chromeos-base',
'packageName': 'target2'
})
build.output.properties['build_compile_failure_output'] = {
'failures': [
{
'output_targets': [output_target],
'rule': 'emerge'
},
],
'failed_step': self.compile_step_name
}
mock_compile_failures.return_value = {
self.compile_step_name: {
'failures': {
frozenset(['target1.o', 'target2.o']): {
'rule': 'CXX',
'first_failed_build': {
'id': build_id,
'number': build_number,
'commit_id': 'git_sha_6543221'
},
'last_passed_build': None
},
},
'first_failed_build': {
'id': build_id,
'number': build_number,
'commit_id': 'git_sha_6543221'
},
'last_passed_build': None
},
}
CompileRerunBuild.Create(luci_project=self.context.luci_project_name,
luci_bucket=build.builder.bucket,
luci_builder=build.builder.builder,
build_id=build_id,
legacy_build_number=build_number,
gitiles_host=self.context.gitiles_host,
gitiles_project=self.context.gitiles_project,
gitiles_ref=self.context.gitiles_ref,
gitiles_id='git_sha_6543221',
commit_position=6543221,
status=build.status,
create_time=build.create_time.ToDatetime(),
parent_key=self.analysis.key).put()
self.assertTrue(
compile_api._ProcessAndSaveRerunBuildResult(
self.context, self.analyzed_build_id, build))
rerun_build = CompileRerunBuild.get_by_id(build_id,
parent=self.analysis.key)
self.assertItemsEqual(
['target1.o', 'target2.o'],
rerun_build.GetFailedTargets()[self.compile_step_name])