def _ExtractBuildInfo(self, master_name, builder_name, build_number):
"""Returns a BuildInfo instance for the specified build."""
build = build_util.DownloadBuildData(master_name, builder_name,
build_number)
if build is None: # pragma: no cover
raise pipeline.Retry('Too many download from %s' % master_name)
if not build.data: # pragma: no cover
return None
build_info = buildbot.ExtractBuildInfo(master_name, builder_name,
build_number, build.data)
if not build.completed:
build.start_time = build_info.build_start_time
build.completed = build_info.completed
build.result = build_info.result
build.put()
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
if analysis and not analysis.build_start_time:
analysis.build_start_time = build_info.build_start_time
analysis.put()
return build_info
def testGetBuildDataFromBuildMaster(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = build_util.DownloadBuildData(master_name, builder_name,
build_number)
expected_build_data = 'Test get build data from build master'
self.assertEqual(expected_build_data, build.data)
def CheckForFirstKnownFailure(master_name, builder_name, build_number,
failure_info):
"""Checks for first known failures of the given failed steps.
Args:
master_name (str): master of the failed build.
builder_name (str): builder of the failed build.
build_number (int): builder number of the current failed build.
failure_info (CompileFailureInfo, TestFailureInfo): information of the build
failure.
Returns:
failure_info (CompileFailureInfo, TestFailureInfo): updated failure_info.
"""
build = build_util.DownloadBuildData(master_name, builder_name, build_number)
failed_steps = failure_info.failed_steps
failure_info.builds = _UpdateStringTypedBuildKeyToInt(failure_info.builds)
# Look back for first known failures.
for previous_build in build_util.IteratePreviousBuildsFrom(
master_name, builder_name, build.build_id, _MAX_BUILDS_TO_CHECK):
build_info = buildbot.ExtractBuildInfoFromV2Build(
master_name, builder_name, previous_build.number, previous_build)
failure_info.builds[build_info.build_number] = (
FailureInfoBuild.FromSerializable(
_GetBlameListAndRevisionForBuild(build_info)))
if build_info.result == common_pb2.SUCCESS:
for step_name in failed_steps:
if failed_steps[step_name].last_pass is None:
failed_steps[step_name].last_pass = build_info.build_number
# All steps passed, so stop looking back.
return failure_info
else:
# If a step is not run due to some bot exception, we are not sure
# whether the step could pass or not. So we only check failed/passed
# steps here.
for step_name in build_info.failed_steps:
if (step_name in failed_steps and
failed_steps[step_name].last_pass is None):
failed_steps[step_name].first_failure = build_info.build_number
for step_name in failed_steps:
if (step_name in build_info.passed_steps and
failed_steps[step_name].last_pass is None):
failed_steps[step_name].last_pass = build_info.build_number
if all(step_info.last_pass is not None
for step_info in failed_steps.values()):
# All failed steps passed in this build cycle.
return failure_info
return failure_info
def testDownloadBuildDataSourceFromBM(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.put()
self.UpdateUnitTestConfigSettings('download_build_data_settings',
{'use_chrome_build_extract': False})
build_util.DownloadBuildData(master_name, builder_name, build_number)
self.assertEqual(build.data_source, build_util.BUILDBOT_MASTER)
def testGetBuildDataNotDownloadAgain(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = 'dummy'
build.completed = False
build.last_crawled_time = self._TimeBeforeNowBySeconds(60)
build.put()
build_util.DownloadBuildData(master_name, builder_name, build_number)
expected_build_data = 'dummy'
self.assertEqual(expected_build_data, build.data)
def testDownloadBuildDataSourceFromBMUpateBuildData(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = 'Original build data'
build.last_crawled_time = self._TimeBeforeNowBySeconds(360)
build.put()
self.UpdateUnitTestConfigSettings('download_build_data_settings',
{'use_chrome_build_extract': False})
build_util.DownloadBuildData(master_name, builder_name, build_number)
self.assertEqual(build.data_source, build_util.BUILDBOT_MASTER)
self.assertEqual(build.data, 'Test get build data from build master')
def testGetBuildDataFromArchive(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.put()
self._MockUrlfetchWithBuildDataFromArchive(
master_name,
builder_name,
build_number,
build_data='Test get build data')
build_util.DownloadBuildData(master_name, builder_name, build_number)
expected_build_data = 'Test get build data from archive'
self.assertEqual(expected_build_data, build.data)
def testDownloadBuildDataSourceFromCBE(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.put()
self.UpdateUnitTestConfigSettings('download_build_data_settings',
{'use_chrome_build_extract': True})
self._MockUrlfetchWithBuildDataFromArchive(
master_name,
builder_name,
build_number,
build_data='Test get build data')
build_util.DownloadBuildData(master_name, builder_name, build_number)
self.assertEqual(build.data_source, build_util.CHROME_BUILD_EXTRACT)
def GetWaterfallBuildStepLog(master_name,
builder_name,
build_number,
full_step_name,
http_client,
log_name='stdout'):
"""Returns specific log of the specified step."""
build = build_util.DownloadBuildData(master_name, builder_name,
build_number)
if build.build_id:
# This build should be a LUCI build.
return GetStepLogForLuciBuild(build.build_id, full_step_name,
http_client, log_name)
# This build is a buildbot build, fall back to the legacy way of getting log.
data = logdog_util.GetStepLogLegacy(build.log_location, full_step_name,
log_name, http_client)
return _ParseStepLogIfAppropriate(data, log_name)
def testDownloadBuildData(self, mock_get_build):
mock_get_build.return_value = Build(id=8000000123)
build = build_util.DownloadBuildData('m', 'b', 123)
self.assertIsNotNone(build)
self.assertEqual('8000000123', build.build_id)
def testDownloadBuildDataNoNeed(self, mock_get_build):
build = WfBuild.Create('m', 'b', 123)
build.build_id = '8000000123'
build.put()
build_util.DownloadBuildData('m', 'b', 123)
self.assertFalse(mock_get_build.called)
def _CheckReverts(master_name, builder_name, current_build_number):
"""Checks each cl in current build to see if some of them are reverted.
Returns:
{
'c9cc182781484f9010f062859cda048afef': {
'action': 'Reverted',
'fixed_cl_commit_position': '341992',
'fixed_revision': 'c9cc182781484f9010f062859cda048afef',
'fixed_cl_review_url': (
'https://codereview.chromium.org/1278653002'),
'fixing_build_number': 0,
'fixing_build_url': (
'https://luci-milo.appspot.com/buildbot/m/b/0')
'fixing_revision': '208c65020aecfcf305d524058f7ca89363',
'fixing_cl_commit_position': '342013',
'fixing_cl_review_url': (
'https://codereview.chromium.org/1278653005'),
'fixing_build_number': 2,
'fixing_build_url': (
'https://luci-milo.appspot.com/buildbot/m/b/2')
},
...
}
"""
data = {}
reverted_cls = {}
blamed_cls = {}
steps_pass = False
build_number, current_failed_steps = _GetFirstFailedBuild(
master_name, builder_name, current_build_number)
if not build_number:
return data
while not steps_pass:
# Breaks the loop after the first green build
# or all the current failed steps pass.
build = build_util.DownloadBuildData(
master_name, builder_name, build_number)
if not build or not build.data:
return data
build_info = buildbot.ExtractBuildInfo(
master_name, builder_name, build_number, build.data)
if build_number <= current_build_number:
# All the cls in builds prior to the current build(included)
# should be checked for reverts.
for blamed_revision in build_info.blame_list:
blamed_cls[blamed_revision] = build_number
if (build_info.result == 0 or
_AllFailedStepsPassed(build_info.passed_steps, current_failed_steps)):
steps_pass = True
for cl_in_blame_list in build_info.blame_list:
cls_info = GetPossibleRevertInfoFromRevision(cl_in_blame_list)
if not cls_info:
continue
fixed_revision = cls_info['fixed_revision']
if (fixed_revision in blamed_cls and
build_number > blamed_cls[fixed_revision] and
build_number > current_build_number):
# If a CL and its reverting cl are in the same build,
# it doesn't have any impact on the build failure.
# And possible fix should take effect after the current build.
cls_info['fixed_build_number'] = blamed_cls[fixed_revision]
cls_info['fixed_build_url'] = (
buildbot.CreateBuildUrl(
master_name, builder_name, blamed_cls[fixed_revision]))
cls_info['fixing_build_number'] = build_number
cls_info['fixing_build_url'] = (
buildbot.CreateBuildUrl(master_name, builder_name, build_number))
reverted_cls[fixed_revision] = cls_info
build_number += 1
if reverted_cls:
data = reverted_cls
return data
