def DownloadBuildData(master_name, builder_name, build_number):
"""Downloads build data and returns a WfBuild instance."""
build = WfBuild.Get(master_name, builder_name, build_number)
if not build:
build = WfBuild.Create(master_name, builder_name, build_number)
# Cache the data to avoid pulling from master again.
if _BuildDataNeedUpdating(build):
use_cbe = waterfall_config.GetDownloadBuildDataSettings().get(
'use_chrome_build_extract')
if use_cbe:
# Retrieve build data from build archive first.
build.data = buildbot.GetBuildDataFromArchive(
master_name, builder_name, build_number,
HTTP_CLIENT_NO_404_ERROR)
if build.data:
build.data_source = CHROME_BUILD_EXTRACT
elif not lock_util.WaitUntilDownloadAllowed(
master_name): # pragma: no cover
return None
if not build.data or not use_cbe:
# Retrieve build data from build master.
build.data = buildbot.GetBuildDataFromBuildMaster(
master_name, builder_name, build_number,
HTTP_CLIENT_LOGGING_ERRORS)
build.data_source = BUILDBOT_MASTER
build.last_crawled_time = time_util.GetUTCNow()
build.put()
return build
def _DownloadBuildData(self, master_name, builder_name, build_number):
"""Downloads build data and returns a WfBuild instance."""
build = WfBuild.Get(master_name, builder_name, build_number)
if not build:
build = WfBuild.Create(master_name, builder_name, build_number)
# Cache the data to avoid pulling from master again.
if self._BuildDataNeedUpdating(build):
# Retrieve build data from build archive first.
build.data = buildbot.GetBuildDataFromArchive(
build.master_name, build.builder_name, build.build_number,
self.HTTP_CLIENT_NO_404_ERROR)
if build.data is None:
if not lock_util.WaitUntilDownloadAllowed(
master_name): # pragma: no cover
raise pipeline.Retry('Too many download from %s' %
master_name)
# Retrieve build data from build master.
build.data = buildbot.GetBuildDataFromBuildMaster(
build.master_name, build.builder_name, build.build_number,
self.HTTP_CLIENT_LOGGING_ERRORS)
build.last_crawled_time = datetime.utcnow()
build.put()
return build
def _MockDownloadBuildData(self, master_name, builder_name, build_number):
build = WfBuild.Get(master_name, builder_name, build_number)
if not build: # pragma: no cover
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = self._GetBuildInfo(master_name, builder_name,
build_number)
build.put()
return build
def testTriggerTryJob(self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 1
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {
'properties': {
'parent_mastername': 'pm',
'parent_buildername': 'pb'
}
}
build.put()
response = {
'build': {
'id': '1',
'url': 'url',
'status': 'SCHEDULED',
}
}
results = [(None,
buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
build_id = ScheduleFlakeTryJobPipeline()._TriggerTryJob(
master_name, builder_name, {}, [],
failure_type.GetDescriptionForFailureType(failure_type.FLAKY_TEST),
None, None)
self.assertEqual(build_id, '1')
def testBailOutForTryJobWithOutdatedTimestamp(self, mock_fn):
master_name = 'master1'
builder_name = 'builder1'
build_number = 223
WfAnalysis.Create(master_name, builder_name, build_number).put()
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed_steps': {
'compile': {
'current_failure': 221,
'first_failure': 221,
'last_pass': 220
}
},
'failure_type': failure_type.COMPILE
}
yesterday = datetime.utcnow() - timedelta(days=1)
build = WfBuild.Create(master_name, builder_name, build_number)
build.start_time = yesterday
build.put()
mock_fn.return_value = True
need_try_job, try_job_key = try_job_util.NeedANewWaterfallTryJob(
master_name, builder_name, build_number, failure_info, None, None)
self.assertFalse(need_try_job)
self.assertIsNone(try_job_key)
def testLegacyGetStepMetadataFromLUCIBuild(self, mock_build, _):
build = WfBuild.Create('m', 'b', 123)
build.build_id = '8948240770002521488'
build.put()
mock_build.return_value = build
step_metadata = step_util.GetWaterfallBuildStepLog(
'm', 'b', 123, 's', None, 'step_metadata')
self.assertEqual(step_metadata, wf_testcase.SAMPLE_STEP_METADATA)
def testShouldBailOutforOutdatedBuild(self):
yesterday = datetime.utcnow() - timedelta(days=1)
build = WfBuild.Create('m', 'b', 1)
build.start_time = yesterday
self.assertTrue(try_job_util._ShouldBailOutForOutdatedBuild(build))
build.start_time = yesterday + timedelta(hours=1)
self.assertFalse(try_job_util._ShouldBailOutForOutdatedBuild(build))
def DownloadBuildData(master_name, builder_name, build_number):
"""Downloads build data and returns a WfBuild instance."""
build = WfBuild.Get(master_name, builder_name, build_number)
if not build:
build = WfBuild.Create(master_name, builder_name, build_number)
if build.build_id:
return build
luci_project, luci_bucket = buildbot.GetLuciProjectAndBucketForMaster(
master_name)
bb_build = buildbucket_client.GetV2BuildByBuilderAndBuildNumber(
luci_project, luci_bucket, builder_name, build_number)
build.last_crawled_time = time_util.GetUTCNow()
build.build_id = str(bb_build.id)
build.put()
return build
def testLookBackUntilGreenBuild(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number,
wf_analysis_status.ANALYZING)
# Setup build data for builds:
# 123: mock urlfetch to ensure it is fetched.
self._MockUrlfetchWithBuildData(master_name, builder_name, 123)
# 122: mock a build in datastore to ensure it is not fetched again.
build = WfBuild.Create(master_name, builder_name, 122)
build.data = self._GetBuildData(master_name, builder_name, 122)
build.completed = True
build.put()
self._MockUrlfetchWithBuildData(master_name,
builder_name,
122,
build_data='Blow up if used!')
# 121: mock a build in datastore to ensure it is updated.
build = WfBuild.Create(master_name, builder_name, 121)
build.data = 'Blow up if used!'
build.last_crawled_time = self._TimeBeforeNowBySeconds(7200)
build.completed = False
build.put()
self._MockUrlfetchWithBuildData(master_name, builder_name, 121)
pipeline = DetectFirstFailurePipeline()
failure_info = pipeline.run(master_name, builder_name, build_number)
expected_failed_steps = {
'net_unittests': {
'last_pass': 122,
'current_failure': 123,
'first_failure': 123
},
'unit_tests': {
'last_pass': 121,
'current_failure': 123,
'first_failure': 122
}
}
self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testGetBuildFailureInfo(self, mock_build, *_):
master_name = 'm'
builder_name = 'b'
build_number = 223
self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number,
analysis_status.PENDING)
build = WfBuild.Create(master_name, builder_name, build_number)
build.build_id = '80000000223'
build.completed = True
build.put()
build_223 = Build(
id=80000000223, number=build_number, status=common_pb2.FAILURE)
build_223.input.gitiles_commit.id = 'rev223'
step1 = Step(name='compile', status=common_pb2.SUCCESS)
log = step1.logs.add()
log.name = 'stdout'
step2 = Step(name='abc_test', status=common_pb2.FAILURE)
log = step2.logs.add()
log.name = 'stdout'
build_223.steps.extend([step1, step2])
mock_build.return_value = build_223
failure_info, should_proceed = ci_failure.GetBuildFailureInfo(
master_name, builder_name, build_number)
expected_failure_info = {
'failed': True,
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'is_luci': None,
'buildbucket_bucket': '',
'buildbucket_id': '80000000223',
'chromium_revision': 'rev223',
'builds': {
build_number: {
'blame_list': ['rev223'],
'chromium_revision': 'rev223'
}
},
'failed_steps': {
'abc_test': {
'current_failure': build_number,
'first_failure': build_number,
'supported': True
}
},
'failure_type': failure_type.TEST,
'parent_mastername': None,
'parent_buildername': None,
}
self.assertEqual(expected_failure_info, failure_info)
self.assertTrue(should_proceed)
def testGetBuildInfoBuildNotAvailable(self, mocked_fn):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {}
mocked_fn.return_value = build
self.assertIsNone(
build_util.GetBuildInfo(master_name, builder_name, build_number))
def testGetBuildInfo(self, mocked_fn):
build = WfBuild.Create('m', 'b', 123)
build.data = json.dumps({
'properties': [['got_revision', 'a_git_hash'],
['got_revision_cp', 'refs/heads/master@{#12345}']],
})
mocked_fn.return_value = build
build_info = build_util.GetBuildInfo('m', 'b', 123)
self.assertEqual(build_info.chromium_revision, 'a_git_hash')
def testGetBuildFailureInfoBuildSuccess(self, mock_monitoring, mock_build,
*_):
master_name = 'm'
builder_name = 'b'
build_number = 121
self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number,
analysis_status.PENDING)
build = WfBuild.Create(master_name, builder_name, build_number)
build.build_id = '80000000223'
build.completed = True
build.put()
build_121 = Build(
id=80000000121, number=build_number, status=common_pb2.FAILURE)
build_121.input.gitiles_commit.id = 'rev121'
step1 = Step(name='net_unittests', status=common_pb2.SUCCESS)
log = step1.logs.add()
log.name = 'stdout'
step2 = Step(name='unit_tests', status=common_pb2.SUCCESS)
log = step2.logs.add()
log.name = 'stdout'
build_121.steps.extend([step1, step2])
mock_build.return_value = build_121
failure_info, should_proceed = ci_failure.GetBuildFailureInfo(
master_name, builder_name, build_number)
expected_failure_info = {
'failed': False,
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'chromium_revision': 'rev121',
'builds': {},
'failed_steps': {},
'failure_type': failure_type.UNKNOWN,
'parent_mastername': None,
'parent_buildername': None,
'is_luci': None,
'buildbucket_bucket': '',
'buildbucket_id': '80000000121',
}
self.assertEqual(expected_failure_info, failure_info)
self.assertFalse(should_proceed)
mock_monitoring.assert_called_once_with(
master_name='m',
builder_name='b',
failure_type='unknown',
canonical_step_name='unknown',
isolate_target_name='unknown',
status='Completed',
analysis_type='Pre-Analysis')
def testSuccessfullyScheduleNewTryJobForCompileWithSuspectedRevisions(
self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 223
good_revision = 'rev1'
bad_revision = 'rev2'
build_id = '1'
url = 'url'
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {
'properties': {
'parent_mastername': 'pm',
'parent_buildername': 'pb'
}
}
build.put()
response = {
'build': {
'id': build_id,
'url': url,
'status': 'SCHEDULED',
}
}
results = [(None,
buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
WfTryJob.Create(master_name, builder_name, build_number).put()
try_job_pipeline = ScheduleCompileTryJobPipeline()
try_job_id = try_job_pipeline.run(master_name, builder_name,
build_number, good_revision,
bad_revision, failure_type.COMPILE,
None, ['r5'], None, None)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
try_job_data = WfTryJobData.Get(build_id)
expected_try_job_id = '1'
self.assertEqual(expected_try_job_id, try_job_id)
self.assertEqual(expected_try_job_id,
try_job.compile_results[-1]['try_job_id'])
self.assertTrue(expected_try_job_id in try_job.try_job_ids)
self.assertIsNotNone(try_job_data)
self.assertEqual(try_job_data.master_name, master_name)
self.assertEqual(try_job_data.builder_name, builder_name)
self.assertEqual(try_job_data.build_number, build_number)
self.assertEqual(
try_job_data.try_job_type,
failure_type.GetDescriptionForFailureType(failure_type.COMPILE))
self.assertFalse(try_job_data.has_compile_targets)
self.assertTrue(try_job_data.has_heuristic_results)
def testBuildDataNeedUpdating(self):
build = WfBuild.Create('m', 'b', 1)
# Build data is not available.
self.assertTrue(build_util._BuildDataNeedUpdating(build))
# Build was not completed and data is not recent.
build.data = 'dummy'
build.completed = False
build.last_crawled_time = self._TimeBeforeNowBySeconds(360)
self.assertTrue(build_util._BuildDataNeedUpdating(build))
def testGetBuildInfo(self, mocked_fn, mock_build_info, _):
build = WfBuild.Create('m', 'b', 123)
build.build_id = '8000000123'
mocked_fn.return_value = build
expected_build_info = BuildInfo('m', 'b', 123)
expected_build_info.chromium_revision = 'a_git_hash'
mock_build_info.return_value = expected_build_info
build_info = build_util.GetBuildInfo('m', 'b', 123)
self.assertEqual(build_info.chromium_revision, 'a_git_hash')
def testDownloadBuildDataSourceFromBM(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.put()
self.UpdateUnitTestConfigSettings('download_build_data_settings',
{'use_chrome_build_extract': False})
build_util.DownloadBuildData(master_name, builder_name, build_number)
self.assertEqual(build.data_source, build_util.BUILDBOT_MASTER)
def testHelpTriageHandlerReturnNoneForGreenBuild(self):
build_url = buildbot.CreateBuildUrl(self.master_name,
self.builder_name, 123)
build = WfBuild.Create(self.master_name, self.builder_name, 123)
build.data = self._GetBuildInfo(self.master_name, self.builder_name,
123)
build.put()
response = self.test_app.get('/help-triage', params={'url': build_url})
expected_results = {}
self.assertEqual(200, response.status_int)
self.assertEqual(expected_results, response.json_body)
def NeedANewWaterfallTryJob(master_name,
builder_name,
build_number,
failure_info,
signals,
heuristic_result,
force_try_job=False):
tryserver_mastername, tryserver_buildername = (
waterfall_config.GetWaterfallTrybot(master_name, builder_name))
try_job_type = failure_info['failure_type']
if not tryserver_mastername or not tryserver_buildername:
logging.info('%s, %s is not supported yet.', master_name, builder_name)
return False, None
if not force_try_job:
build = WfBuild.Get(master_name, builder_name, build_number)
if _ShouldBailOutForOutdatedBuild(build):
logging.error('Build time %s is more than 24 hours old. '
'Try job will not be triggered.' % build.start_time)
return False, None
if try_job_type == failure_type.COMPILE:
need_new_try_job = _NeedANewCompileTryJob(master_name, builder_name,
build_number, failure_info)
else:
need_new_try_job = _NeedANewTestTryJob(master_name, builder_name,
build_number, failure_info,
force_try_job)
# TODO(chanli): enable the feature to trigger single try job for a group
# when notification is ready.
# We still call _IsBuildFailureUniqueAcrossPlatforms just so we have data for
# failure groups.
# TODO(chanli): Add checking for culprits of the group when enabling
# single try job: add current build to suspected_cl.builds if the try job for
# this group has already completed.
if need_new_try_job:
_IsBuildFailureUniqueAcrossPlatforms(
master_name, builder_name, build_number, try_job_type,
failure_info['builds'][str(build_number)]['blame_list'],
failure_info['failed_steps'], signals, heuristic_result)
try_job_was_created, try_job_key = _ReviveOrCreateTryJobEntity(
master_name, builder_name, build_number, force_try_job)
need_new_try_job = need_new_try_job and try_job_was_created
return need_new_try_job, try_job_key
def testBuildDataNotNeedUpdating(self):
build = WfBuild.Create('m', 'b', 1)
# Build is not completed yet but data is recent.
build.data = 'dummy'
build.completed = False
build.last_crawled_time = self._TimeBeforeNowBySeconds(60)
self.assertFalse(build_util._BuildDataNeedUpdating(build))
# Build was completed and data is final.
build.data = 'dummy'
build.completed = True
build.last_crawled_time = self._TimeBeforeNowBySeconds(360)
self.assertFalse(build_util._BuildDataNeedUpdating(build))
def testGetBuildDataNotDownloadAgain(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = 'dummy'
build.completed = False
build.last_crawled_time = self._TimeBeforeNowBySeconds(60)
build.put()
build_util.DownloadBuildData(master_name, builder_name, build_number)
expected_build_data = 'dummy'
self.assertEqual(expected_build_data, build.data)
def testDownloadBuildDataSourceFromBMUpateBuildData(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = 'Original build data'
build.last_crawled_time = self._TimeBeforeNowBySeconds(360)
build.put()
self.UpdateUnitTestConfigSettings('download_build_data_settings',
{'use_chrome_build_extract': False})
build_util.DownloadBuildData(master_name, builder_name, build_number)
self.assertEqual(build.data_source, build_util.BUILDBOT_MASTER)
self.assertEqual(build.data, 'Test get build data from build master')
def testScheduleFlakeTryJob(self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 1
step_name = 's'
test_name = 't'
git_hash = 'a1b2c3d4'
build_id = '1'
url = 'url'
analysis_key = ndb.Key('key', 1)
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {
'properties': {
'parent_mastername': 'pm',
'parent_buildername': 'pb'
}
}
build.put()
response = {
'build': {
'id': build_id,
'url': url,
'status': 'SCHEDULED',
}
}
results = [(None,
buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
FlakeTryJob.Create(master_name, builder_name, step_name, test_name,
git_hash).put()
try_job_pipeline = ScheduleFlakeTryJobPipeline()
try_job_id = try_job_pipeline.run(master_name, builder_name, step_name,
test_name, git_hash,
analysis_key.urlsafe(), None, None)
try_job = FlakeTryJob.Get(master_name, builder_name, step_name,
test_name, git_hash)
try_job_data = FlakeTryJobData.Get(build_id)
self.assertEqual(build_id, try_job_id)
self.assertEqual(build_id, try_job.flake_results[-1]['try_job_id'])
self.assertTrue(build_id in try_job.try_job_ids)
self.assertEqual(try_job_data.try_job_key, try_job.key)
self.assertEqual(analysis_key, try_job_data.analysis_key)
def testGetBuildDataFromArchive(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.put()
self._MockUrlfetchWithBuildDataFromArchive(
master_name,
builder_name,
build_number,
build_data='Test get build data')
build_util.DownloadBuildData(master_name, builder_name, build_number)
expected_build_data = 'Test get build data from archive'
self.assertEqual(expected_build_data, build.data)
def testDownloadBuildDataSourceFromCBE(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.put()
self.UpdateUnitTestConfigSettings('download_build_data_settings',
{'use_chrome_build_extract': True})
self._MockUrlfetchWithBuildDataFromArchive(
master_name,
builder_name,
build_number,
build_data='Test get build data')
build_util.DownloadBuildData(master_name, builder_name, build_number)
self.assertEqual(build.data_source, build_util.CHROME_BUILD_EXTRACT)
def testSuccessfullyScheduleNewTryJobForTest(self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 223
good_revision = 'rev1'
bad_revision = 'rev2'
targeted_tests = ['a on platform', ['a', ['test1', 'test2']]]
build_id = '1'
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {'properties': {'parent_mastername': 'pm',
'parent_buildername': 'pb'}}
build.put()
response = {
'build': {
'id': build_id,
'url': 'url',
'status': 'SCHEDULED',
}
}
results = [(None, buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
WfTryJob.Create(master_name, builder_name, build_number).put()
try_job_pipeline = ScheduleTestTryJobPipeline()
try_job_id = try_job_pipeline.run(
master_name, builder_name, build_number, good_revision, bad_revision,
failure_type.TEST, None, None, None, targeted_tests)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(try_job_id, build_id)
self.assertEqual(try_job.test_results[-1]['try_job_id'], build_id)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNotNone(try_job_data)
self.assertEqual(try_job_data.master_name, master_name)
self.assertEqual(try_job_data.builder_name, builder_name)
self.assertEqual(try_job_data.build_number, build_number)
self.assertEqual(
try_job_data.try_job_type,
failure_type.GetDescriptionForFailureType(failure_type.TEST))
self.assertFalse(try_job_data.has_compile_targets)
self.assertFalse(try_job_data.has_heuristic_results)
def _FetchBuildsFromAnalyses(analyses):
"""Fetch builds that have the same keys as analyses.
Args:
analyses: A list of WfAnalyses objects for which to get builds.
Returns:
A dict of builds in {build_id: build} format. Example:
{'m/b/1': build_1, 'm/b/2': build_2}
"""
build_keys = []
for analysis in analyses:
build_keys.append(
WfBuild._CreateKey(analysis.master_name, analysis.builder_name,
analysis.build_number))
builds = ndb.get_multi(build_keys)
return {_GetBuildId(build): build for build in builds}
def testStoreDetectedCIFlakes(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.build_id = '87654321'
build.put()
flaky_tests = {'s': ['t1', 't2']}
detect_flake_occurrences.StoreDetectedCIFlakes(master_name, builder_name,
build_number, flaky_tests)
flake = Flake.Get('chromium', 'normalized_step_name', 't1')
self.assertIsNotNone(flake)
occurrences = FlakeOccurrence.query(ancestor=flake.key).fetch()
self.assertEqual(1, len(occurrences))
self.assertEqual(FlakeType.CI_FAILED_STEP, occurrences[0].flake_type)
def testGetBuildEndTime(self):
cases = {
'null': None,
'1467740016': datetime.datetime(2016, 7, 5, 17, 33, 36),
}
for end_time, expected_time in cases.iteritems():
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = '{"times": [1467738821, %s]}' % end_time
build.completed = True
build.last_crawled_time = self._TimeBeforeNowBySeconds(10)
build.put()
self.assertEqual(
expected_time,
build_util.GetBuildEndTime(master_name, builder_name,
build_number))
def NeedANewWaterfallTryJob(master_name,
builder_name,
build_number,
force_try_job,
build_completed=True):
"""Preliminary check if a new try job is needed.
Don't need try job if build not completed yet, only runs for builds started
within 24 hours, unless it's a forced rerun.
"""
if not build_completed:
return False
if not force_try_job:
build = WfBuild.Get(master_name, builder_name, build_number)
if _ShouldBailOutForOutdatedBuild(build):
logging.error('Build time %s is more than 24 hours old. '
'Try job will not be triggered.' % build.start_time)
return False
return True
