def testGetV2Build(self, mock_post):
build_id = '8945610992972640896'
mock_build = Build()
mock_build.id = int(build_id)
mock_build.status = 12
mock_build.output.properties['mastername'] = 'chromium.linux'
mock_build.output.properties['buildername'] = 'Linux Builder'
mock_build.output.properties.get_or_create_struct(
'swarm_hashes_ref/heads/mockmaster(at){#123}'
)['mock_target'] = 'mock_hash'
gitiles_commit = mock_build.input.gitiles_commit
gitiles_commit.host = 'gitiles.host'
gitiles_commit.project = 'gitiles/project'
gitiles_commit.ref = 'refs/heads/mockmaster'
mock_build.builder.project = 'mock_luci_project'
mock_build.builder.bucket = 'mock_bucket'
mock_build.builder.builder = 'Linux Builder'
mock_headers = {'X-Prpc-Grpc-Code': '0'}
binary_data = mock_build.SerializeToString()
mock_post.return_value = (200, binary_data, mock_headers)
build = buildbucket_client.GetV2Build(build_id)
self.assertIsNotNone(build)
self.assertEqual(mock_build.id, build.id)
mock_headers = {'X-Prpc-Grpc-Code': '4'}
binary_data = mock_build.SerializeToString()
mock_post.return_value = (404, binary_data, mock_headers)
self.assertIsNone(buildbucket_client.GetV2Build(build_id))
def GetBuildAndContextForAnalysis(project, build_id):
"""Gets all information about a build and generates context from it.
Args:
project (str): Luci project of the build.
build_id (int): Id of the build.
Returns:
(buildbucket build.proto): ALL info about the build.
(Context): Context of an analysis.
"""
build = buildbucket_client.GetV2Build(build_id,
fields=FieldMask(paths=['*']))
if not build:
logging.error('Failed to get build info for build %d.', build_id)
return None, None
if (build.input.gitiles_commit.host !=
projects.GERRIT_PROJECTS[project]['gitiles-host']
or build.input.gitiles_commit.project !=
projects.GERRIT_PROJECTS[project]['name']):
logging.warning('Unexpected gitiles project for build: %r', build_id)
return None, None
context = Context(luci_project_name=project,
gitiles_host=build.input.gitiles_commit.host,
gitiles_project=build.input.gitiles_commit.project,
gitiles_ref=build.input.gitiles_commit.ref,
gitiles_id=build.input.gitiles_commit.id)
return build, context
def GetStepLogForLuciBuild(build_id,
full_step_name,
http_client,
log_name='stdout'):
"""Returns specific log of the specified step in a LUCI build.
Args:
build_id(str): Buildbucket id.
full_step_name(str): Full name of the step.
http_client(FinditHttpClient): Http_client to make the request.
log_name(str): Name of the log.
Returns:
Requested Log after processing based on the log_name.
- return the log as it is if the log name is 'stdout' or
'json.output[ninja_info]'
- return the deserialized log otherwise.
"""
build = buildbucket_client.GetV2Build(build_id, FieldMask(paths=['steps']))
if not build:
logging.exception('Error retrieving buildbucket build id: %s' %
build_id)
return None
return GetStepLogFromBuildObject(build, full_step_name, http_client,
log_name)
def GetBuildInfo(master_name, builder_name, build_number):
"""Gets build info given a master, builder, and build number.
Args:
master_name (str): The name of the master.
builder_name (str): The name of the builder.
build_number (int): The build number.
Returns:
Build information as an instance of BuildInfo.
"""
build = DownloadBuildData(master_name, builder_name, build_number)
if not build.build_id:
return None
bb_build = buildbucket_client.GetV2Build(build.build_id,
fields=FieldMask(paths=['*']))
build_info = buildbot.ExtractBuildInfoFromV2Build(master_name,
builder_name,
build_number, bb_build)
if not build.completed:
build.start_time = build_info.build_start_time
build.completed = build_info.completed
build.result = build_info.result
build.put()
return build_info
def _GetPreviousCompileStepsAndFailuresInPreviousBuild(
project_api, prev_build, detailed_compile_failures):
"""Gets compile failures in the previous build.
Args:
project_api (ProjectAPI): API for project specific logic.
prev_build (buildbucket build.proto): SIMPLE info about the build.
detailed_compile_failures (dict): A dict of detailed compile failures.
"""
detailed_prev_build = buildbucket_client.GetV2Build(
prev_build.id, fields=FieldMask(paths=['*']))
# Looks for compile steps in previous build. Here only the failed compile
# steps in current build are relevant.
prev_compile_steps = {
s.name: s
for s in detailed_prev_build.steps
if s.name in detailed_compile_failures
}
# Looks for compile steps that failed in both current build and this build.
prev_failed_compile_steps = {
step_name: step
for step_name, step in prev_compile_steps.iteritems()
if step.status == common_pb2.FAILURE
}
prev_failures = project_api.GetCompileFailures(
detailed_prev_build,
prev_failed_compile_steps.values()) if prev_failed_compile_steps else {}
return prev_compile_steps, prev_failures
def GetBuilderInfoForLUCIBuild(build_id):
"""Gets a build's project and bucket info.
Args:
build_id(str): Buildbucket id of a LUCI build.
"""
build_proto = buildbucket_client.GetV2Build(
build_id, fields=FieldMask(paths=['builder']))
if not build_proto:
logging.exception('Error retrieving buildbucket build id: %s',
build_id)
return None, None
return build_proto.builder.project, build_proto.builder.bucket
def GetTestRerunBuildInputProperties(self, tests, analyzed_build_id):
properties = {}
build = buildbucket_client.GetV2Build(
analyzed_build_id,
fields=FieldMask(paths=['input.properties', 'builder']))
properties['target_builder'] = {
'master': build.input.properties['mastername'],
'builder': build.builder.builder
}
properties['mastername'] = build.input.properties['mastername']
properties['tests'] = {
s: [t['name'] for t in tests_in_suite['tests']
] for s, tests_in_suite in tests.iteritems()
}
return properties
def GetRerunDimensions(self, analyzed_build_id):
# Copy these dimensions from the analyzed builder to the rerun job request.
dimension_whitelist = set([
'os',
'gpu',
])
build = buildbucket_client.GetV2Build(
analyzed_build_id,
fields=FieldMask(paths=['infra.swarming.taskDimensions']))
assert build, "Could not get analyzed build %d" % analyzed_build_id
dimension_dicts = []
for dimension_proto in build.infra.swarming.task_dimensions:
if dimension_proto.key in dimension_whitelist:
# Explicitly include only the 'key' and 'value' keys to discard
# 'expiration' which is unrelated.
dimension_dicts.append({
'key': dimension_proto.key,
'value': dimension_proto.value
})
return dimension_dicts
def GetCompileRerunBuildInputProperties(self, failed_targets,
analyzed_build_id):
all_targets = set()
for step, targets in failed_targets.iteritems():
# Assume the step is a compile step.
assert step == 'compile'
all_targets |= set(targets)
properties = {}
build = buildbucket_client.GetV2Build(
analyzed_build_id,
fields=FieldMask(paths=['input.properties', 'builder']))
# The recipe depends on the mastername being set.
# TODO(crbug.com/1008119): Fix the recipe so that it populates this property
# based on target_builder.
properties['mastername'] = build.input.properties['mastername']
properties['target_builder'] = {
'master': build.input.properties['mastername'],
'builder': build.builder.builder
}
properties['compile_targets'] = list(all_targets)
return properties
def ProcessBuildForFlakes(task_param):
"""Detects a type of flakes from a build.
Args:
task_param(DetectFlakesFromFlakyCQBuildParam): Parameters of the task to
detect cq false rejection or retry with patch flakes from a flaky cq
build.
"""
build_id = task_param.build_id
flake_type_enum = DESCRIPTION_TO_FLAKE_TYPE.get(task_param.flake_type_desc)
build_pb = buildbucket_client.GetV2Build(
build_id, FieldMask(paths=['number', 'builder', 'input', 'steps']))
assert build_pb, 'Error retrieving buildbucket build id: {}'.format(
build_id)
luci_project = build_pb.builder.project
luci_bucket = build_pb.builder.bucket
luci_builder = build_pb.builder.builder
legacy_master_name = build_pb.input.properties['mastername']
legacy_build_number = build_pb.number
gerrit_changes = build_pb.input.gerrit_changes
gerrit_cl_id = None
gerrit_project = None
if gerrit_changes:
gerrit_cl_id = build_pb.input.gerrit_changes[0].change
gerrit_project = build_pb.input.gerrit_changes[0].project
# Fall-back approach to get gerrit_cl_id and gerrit_project
gerrit_cl_id = gerrit_cl_id or (build_pb.input.properties['patch_issue']
if 'patch_issue'
in build_pb.input.properties else None)
gerrit_project = gerrit_project or (
build_pb.input.properties['patch_project']
if 'patch_project' in build_pb.input.properties else None)
if not gerrit_cl_id:
return
flake_info = GetFlakesFromFlakyCQBuild(build_id, build_pb, flake_type_enum)
row = {
'luci_project': luci_project,
'luci_bucket': luci_bucket,
'luci_builder': luci_builder,
'legacy_master_name': legacy_master_name,
'legacy_build_number': legacy_build_number,
'build_id': build_id,
'gerrit_project': gerrit_project,
'gerrit_cl_id': gerrit_cl_id
}
new_flakes = []
new_occurrences = []
for step_ui_name, tests in flake_info.iteritems():
# Uses the start time of a step as the flake happen time.
step_start_time, _ = step_util.GetStepStartAndEndTime(
build_pb, step_ui_name)
row['step_ui_name'] = step_ui_name
row['test_start_msec'] = step_start_time
for test in tests:
row['test_name'] = test
new_flakes.append(_CreateFlakeFromRow(row))
new_occurrences.append(
_CreateFlakeOccurrenceFromRow(row, flake_type_enum))
_StoreMultipleLocalEntities(new_flakes)
_StoreMultipleLocalEntities(new_occurrences)
_UpdateFlakeMetadata(new_occurrences)
def _GetV2AnalysisResultFromV1(self, request):
"""Constructs v2 analysis results based on v1 analysis.
This is a temporary work around to make sure Findit's analysis results for
chromium build failures are still available on SoM during v1 to v2
migration.
Args:
request (findit_result.BuildFailureAnalysisRequest)
Returns:
[findit_result.BuildFailureAnalysisResponse] for results of a v1 analysis,
otherwise return None.
"""
if (request.build_alternative_id
and request.build_alternative_id.project != 'chromium'):
return None
build = None
if request.build_id:
build = buildbucket_client.GetV2Build(
request.build_id,
fields=FieldMask(
paths=['id', 'number', 'builder', 'output.properties']))
elif request.build_alternative_id:
build = buildbucket_client.GetV2BuildByBuilderAndBuildNumber(
request.build_alternative_id.project,
request.build_alternative_id.bucket,
request.build_alternative_id.builder,
request.build_alternative_id.number,
fields=FieldMask(
paths=['id', 'number', 'builder', 'output.properties']))
if not build:
logging.error('Failed to download build when requesting for %s',
request)
return None
if build.builder.project != 'chromium':
return None
properties = json_format.MessageToDict(build.output.properties)
build_number = build.number
master_name = properties.get('target_mastername',
properties.get('mastername'))
if not build_number or not master_name:
logging.error('Missing master_name or build_number for build %d',
build.id)
return None
heuristic_analysis = WfAnalysis.Get(master_name, build.builder.builder,
build_number)
if not heuristic_analysis:
return None
results = []
v1_build_request = _BuildFailure(builder_name=build.builder.builder,
build_number=build_number)
self._GenerateResultsForBuild(v1_build_request, heuristic_analysis,
results, None)
return self._GetV2ResultFromV1(request, results)