def ExtractBuildInfo(master_name, builder_name, build_number, build_data):
"""Extracts and returns build information as an instance of BuildInfo."""
build_info = BuildInfo(master_name, builder_name, build_number)
data_json = json.loads(build_data)
chromium_revision = GetBuildProperty(data_json.get('properties', []),
'got_revision')
commit_position_line = GetBuildProperty(data_json.get('properties', []),
'got_revision_cp')
parent_buildername = GetBuildProperty(data_json.get('properties', []),
'parent_buildername')
parent_mastername = GetBuildProperty(data_json.get('properties', []),
'parent_mastername')
build_info.build_start_time = GetBuildStartTime(data_json)
build_info.build_end_time = GetBuildEndTime(data_json)
build_info.chromium_revision = chromium_revision
build_info.commit_position = _GetCommitPosition(commit_position_line)
build_info.completed = data_json.get('currentStep') is None
build_info.result = GetBuildResult(data_json)
build_info.parent_buildername = parent_buildername
build_info.parent_mastername = parent_mastername
changes = data_json.get('sourceStamp', {}).get('changes', [])
for change in changes:
if change['revision'] not in build_info.blame_list:
build_info.blame_list.append(change['revision'])
# Step categories:
# 1. A step is passed if it is in SUCCESS or WARNINGS status.
# 2. A step is failed if it is in FAILED status.
# 3. A step is not passed if it is not in SUCCESS or WARNINGS status. This
# category includes steps in statuses: FAILED, SKIPPED, EXCEPTION, RETRY,
# CANCELLED, etc.
steps = data_json.get('steps', [])
for step_data in steps:
step_name = step_data['name']
if not step_data.get('isFinished', False):
# Skip steps that haven't started yet or are still running.
continue
step_result = GetStepResult(step_data)
if step_result not in (SUCCESS, WARNINGS):
build_info.not_passed_steps.append(step_name)
step_logs = step_data.get('logs')
if step_logs and 'preamble' == step_logs[0][0]:
# Skip a annotating step like "steps" or "slave_steps", which wraps other
# steps. A failed annotated step like "content_browsertests" will make
# the annotating step like "steps" fail too. Such annotating steps have a
# log with name "preamble".
continue
if step_result in (SUCCESS, WARNINGS):
build_info.passed_steps.append(step_name)
elif step_result == FAILURE:
build_info.failed_steps.append(step_name)
return build_info
def ExtractBuildInfoFromV2Build(master_name, builder_name, build_number, build):
"""Generates BuildInfo using bb v2 build info.
This conversion is needed to keep Findit v1 running, will be deprecated in
v2 (TODO: crbug.com/966982).
Args:
master_name (str): The name of the master.
builder_name (str): The name of the builder.
build_number (int): The build number.
build (build_pb2.Build): All info about the build.
Returns:
(BuildInfo)
"""
build_info = BuildInfo(master_name, builder_name, build_number)
input_properties = json_format.MessageToDict(build.input.properties)
chromium_revision = build.input.gitiles_commit.id
runtime = input_properties.get('$recipe_engine/runtime') or {}
build_info.chromium_revision = chromium_revision
repo_url = git.GetRepoUrlFromV2Build(build)
build_info.commit_position = git.GetCommitPositionFromRevision(
build.input.gitiles_commit.id, repo_url=repo_url)
build_info.build_start_time = build.create_time.ToDatetime()
build_info.build_end_time = build.end_time.ToDatetime()
build_info.completed = bool(build_info.build_end_time)
build_info.result = build.status
build_info.parent_buildername = input_properties.get('parent_buildername')
build_info.parent_mastername = input_properties.get('parent_mastername')
build_info.buildbucket_id = str(build.id)
build_info.buildbucket_bucket = build.builder.bucket
build_info.is_luci = runtime.get('is_luci')
build_info.blame_list = GetBlameListForV2Build(build)
# Step categories:
# 1. A step is passed if it is in SUCCESS status.
# 2. A step is failed if it is in FAILURE status.
# 3. A step is not passed if it is not in SUCCESS status. This category
# includes steps in statuses: FAILURE, INFRA_FAILURE, CANCELED, etc.
for step in build.steps:
step_name = step.name
step_status = step.status
if step_status in [
common_pb2.STATUS_UNSPECIFIED, common_pb2.SCHEDULED, common_pb2.STARTED
]:
continue
if step_status != common_pb2.SUCCESS:
build_info.not_passed_steps.append(step_name)
if step_name == 'Failure reason':
# 'Failure reason' is always red when the build breaks or has exception,
# but it is not a failed step.
continue
if not step.logs:
# Skip wrapping steps.
continue
if step_status == common_pb2.SUCCESS:
build_info.passed_steps.append(step_name)
elif step_status == common_pb2.FAILURE:
build_info.failed_steps.append(step_name)
return build_info
def testGetIsolateShaForCommitPositionPipelineMatchingTarget(
self, mocked_reference_build):
master_name = 'm'
builder_name = 'b'
parent_mastername = 'p_m'
parent_buildername = 'p_b'
build_number = 100
build_id = 123
test_name = 't'
requested_commit_position = 1000
requested_revision = 'r1000'
expected_sha = 'sha1'
build_url = 'url'
luci_name = 'chromium'
bucket_name = 'ci'
gitiles_host = 'chromium.googlesource.com'
gitiles_project = 'chromium/src'
gitiles_ref = 'refs/heads/master'
gerrit_patch = ''
isolate_target_name = 'browser_tests'
step_name = 's'
isolated_hash = 'isolated_hash'
expected_output = GetIsolateShaOutput(isolate_sha=expected_sha,
build_number=None,
build_url=build_url,
try_job_url=None)
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name,
test_name)
analysis.Save()
build = BuildInfo(master_name, builder_name, build_number)
build.commit_position = requested_commit_position
build.parent_mastername = parent_mastername
build.parent_buildername = parent_buildername
mocked_reference_build.return_value = build
isolated_target = IsolatedTarget.Create(
build_id, luci_name, bucket_name, parent_mastername,
parent_buildername, gitiles_host, gitiles_project, gitiles_ref,
gerrit_patch, isolate_target_name, isolated_hash,
requested_commit_position, requested_revision)
isolated_target.put()
step_metadata = StepMetadata(canonical_step_name=None,
dimensions=None,
full_step_name=None,
isolate_target_name=isolate_target_name,
patched=True,
swarm_task_ids=None,
waterfall_buildername=None,
waterfall_mastername=None)
get_sha_input = GetIsolateShaForCommitPositionParameters(
analysis_urlsafe_key=unicode(analysis.key.urlsafe()),
commit_position=requested_commit_position,
dimensions=ListOfBasestring.FromSerializable([]),
revision=requested_revision,
step_metadata=step_metadata,
upper_bound_build_number=analysis.build_number)
get_sha_for_target_input = GetIsolateShaForTargetInput(
isolated_target_urlsafe_key=isolated_target.key.urlsafe())
self.MockSynchronousPipeline(GetIsolateShaForTargetPipeline,
get_sha_for_target_input, expected_output)
pipeline_job = GetIsolateShaForCommitPositionPipeline(get_sha_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(
pipeline_job.pipeline_id)
pipeline_output = pipeline_job.outputs.default.value
self.assertEqual(expected_output.ToSerializable(), pipeline_output)
def testNextCommitPositionPipelineContinueAnalysis(
self, mock_reference_build, mock_heuristic, mock_next_commit,
mock_bound_commits):
master_name = 'm'
builder_name = 'b'
parent_mastername = 'p_m'
parent_buildername = 'p_b'
build_number = 100
build_id = 10000
step_name = 's'
test_name = 't'
start_commit_position = 1000
expected_next_commit_id = CommitID(commit_position=990, revision='r990')
reference_build = BuildInfo(master_name, builder_name, build_number)
reference_build.commit_position = start_commit_position
reference_build.parent_mastername = parent_mastername
reference_build.parent_buildername = parent_buildername
mock_reference_build.return_value = reference_build
mock_heuristic.return_value = False
calculated_next_commit_id = CommitID(commit_position=999, revision='r999')
mock_next_commit.return_value = (calculated_next_commit_id, None)
target_name = 'browser_tests'
step_metadata = StepMetadata(
canonical_step_name=None,
dimensions=None,
full_step_name=None,
isolate_target_name=target_name,
patched=True,
swarm_task_ids=None,
waterfall_buildername=None,
waterfall_mastername=None)
luci_name = 'chromium'
bucket_name = 'ci'
gitiles_host = 'chromium.googlesource.com'
gitiles_project = 'chromium/src'
gitiles_ref = 'refs/heads/master'
gerrit_patch = ''
lower_bound_target = IsolatedTarget.Create(
build_id - 1, luci_name, bucket_name, parent_mastername,
parent_buildername, gitiles_host, gitiles_project, gitiles_ref,
gerrit_patch, target_name, 'hash_1',
expected_next_commit_id.commit_position, None)
lower_bound_target.put()
upper_bound_target = IsolatedTarget.Create(
build_id, luci_name, bucket_name, parent_mastername, parent_buildername,
gitiles_host, gitiles_project, gitiles_ref, gerrit_patch, target_name,
'hash_2', start_commit_position, None)
upper_bound_target.put()
mock_bound_commits.return_value = (
expected_next_commit_id,
CommitID(commit_position=start_commit_position, revision='r1000'))
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name, test_name)
analysis.data_points = [
DataPoint.Create(commit_position=start_commit_position)
]
analysis.Save()
next_commit_position_input = NextCommitPositionInput(
analysis_urlsafe_key=analysis.key.urlsafe(),
commit_position_range=IntRange(lower=None, upper=start_commit_position),
step_metadata=step_metadata)
pipeline_job = NextCommitPositionPipeline(next_commit_position_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(pipeline_job.pipeline_id)
next_commit_position_output = pipeline_job.outputs.default.value
self.assertFalse(pipeline_job.was_aborted)
self.assertIsNone(next_commit_position_output['culprit_commit_id'])
self.assertEqual(expected_next_commit_id.ToSerializable(),
next_commit_position_output['next_commit_id'])
mock_bound_commits.assert_called_once_with(
analysis.data_points, lower_bound_target, upper_bound_target)