def _CheckFYIBisectJob(job, issue_tracker):
try:
if job.use_buildbucket and not job.buildbucket_job_id:
job.key.delete()
return
if not _IsBisectJobCompleted(job):
return
if not job.results_data:
raise BisectJobFailure(
'Bisect job completed, but results data is not '
'found, bot might have failed to post results.')
error_message = bisect_fyi.VerifyBisectFYIResults(job)
_PostSuccessfulResult(job, issue_tracker)
if not bisect_fyi.IsBugUpdated(job, issue_tracker):
error_message += '\nFailed to update bug with bisect results.'
except BisectJobFailure as e:
error_message = 'Bisect job failed because, %s' % e
except BugUpdateFailure as e:
error_message = 'Failed to update bug with bisect results: %s' % e
finally:
job_info = buildbucket_service.GetJobStatus(job.buildbucket_job_id)
job_info = job_info.get('build', {})
if not job.results_data:
job.results_data = {}
job.results_data['buildbot_log_url'] = str(job_info.get('url'))
# When the job fails before getting to the point where it post bisect results
# to the dashboard, the tryjob's results_data is not set.
# As a special case for Bisect FYI jobs, we query buildbucket to get the
# bisect job's status.
if ((job.results_data and job.results_data.get('status') == FAILED)
or error_message):
job.SetFailed()
_SendFYIBisectEmail(job, error_message)
def testGetJobStatus(self):
fake_id = '1234567890'
with mock.patch('apiclient.discovery.build',
mock.MagicMock(return_value=self.fake_service)):
_ = buildbucket_service.GetJobStatus(fake_id)
# Ensure the given job id was passed to the request.
request = self.fake_service.bodies[0]
self.assertIn('id', request)
self.assertEqual(fake_id, request['id'])
def CheckFailureFromBuildBucket(self):
# Buildbucket job id is not always set.
if not self.buildbucket_job_id:
return
job_info = buildbucket_service.GetJobStatus(self.buildbucket_job_id)
data = job_info.get('build', {})
# Since the job is completed successfully, results_data must
# have been set appropriately by the bisector.
# The buildbucket job's 'status' and 'result' fields are documented here:
# https://goto.google.com/bb_status
if data.get('status') == 'COMPLETED' and data.get(
'result') == 'SUCCESS':
return
# Proceed if the job failed or cancelled
logging.info('Job failed. Buildbucket id %s', self.buildbucket_job_id)
data['result_details'] = json.loads(data['result_details_json'])
# There are various failure and cancellation reasons for a buildbucket
# job to fail as listed in https://goto.google.com/bb_status.
job_updates = {
'status':
'failed',
'failure_reason': (data.get('cancelation_reason')
or data.get('failure_reason')),
'buildbot_log_url':
data.get('url')
}
details = data.get('result_details')
if details:
properties = details.get('properties')
if properties:
job_updates['bisect_bot'] = properties.get('buildername')
job_updates['extra_result_code'] = properties.get(
'extra_result_code')
bisect_config = properties.get('bisect_config')
if bisect_config:
job_updates['try_job_id'] = bisect_config.get('try_job_id')
job_updates['bug_id'] = bisect_config.get('bug_id')
job_updates['command'] = bisect_config.get('command')
job_updates['test_type'] = bisect_config.get('test_type')
job_updates['metric'] = bisect_config.get('metric')
job_updates['good_revision'] = bisect_config.get(
'good_revision')
job_updates['bad_revision'] = bisect_config.get(
'bad_revision')
if not self.results_data:
self.results_data = {}
self.results_data.update(job_updates)
self.status = 'failed'
self.last_ran_timestamp = datetime.datetime.fromtimestamp(
float(data['updated_ts']) / 1000000)
self.put()
logging.info('updated status to failed.')
def _IsBuildBucketJobCompleted(job):
"""Checks whether the bisect job is completed."""
job_info = buildbucket_service.GetJobStatus(job.buildbucket_job_id)
if not job_info:
return False
data = job_info.get('build', {})
# The status of the build can be one of [STARTED, SCHEDULED or COMPLETED]
# The buildbucket 'status' fields are documented here:
# https://goto.google.com/bb_status
if data.get('status') != 'COMPLETED':
return False
return True
def CheckFailureFromBuildBucket(self):
# Buildbucket job id is not always set.
if not self.buildbucket_job_id:
return
job_info = buildbucket_service.GetJobStatus(self.buildbucket_job_id)
data = job_info.get('build', {})
# Proceed if the job failed and the job status has
# not been updated
if data.get('result') != 'FAILURE' or self.status == 'failed':
return
data['result_details'] = json.loads(data['result_details_json'])
job_updates = {
'status': 'failed',
'failure_reason': data.get('failure_reason'),
'buildbot_log_url': data.get('url')
}
details = data.get('result_details')
if details:
properties = details.get('properties')
if properties:
job_updates['bisect_bot'] = properties.get('buildername')
job_updates['extra_result_code'] = properties.get(
'extra_result_code')
bisect_config = properties.get('bisect_config')
if bisect_config:
job_updates['try_job_id'] = bisect_config.get('try_job_id')
job_updates['bug_id'] = bisect_config.get('bug_id')
job_updates['command'] = bisect_config.get('command')
job_updates['test_type'] = bisect_config.get('test_type')
job_updates['metric'] = bisect_config.get('metric')
job_updates['good_revision'] = bisect_config.get(
'good_revision')
job_updates['bad_revision'] = bisect_config.get(
'bad_revision')
if not self.results_data:
self.results_data = {}
self.results_data.update(job_updates)
self.status = 'failed'
self.last_ran_timestamp = datetime.datetime.fromtimestamp(
float(data['updated_ts']) / 1000000)
self.put()
def get(self, job_id):
original_status = buildbucket_service.GetJobStatus(job_id)
error = 'error' in original_status
if error:
error_reason = original_status['error'].get('reason')
status_text = json.dumps(original_status, sort_keys=True, indent=4)
else:
clean_status = _ConvertTimes(
_ParseJsonKeys(original_status.get('build')))
status_text = json.dumps(clean_status, sort_keys=True, indent=4)
self.RenderHtml(
'buildbucket_job_status.html', {
'job_id': job_id,
'status_text': 'DATA:' + status_text,
'build': None if error else clean_status,
'error': error_reason if error else None,
'original_response': original_status,
})
def _GetBisectResults(job):
"""Gets bisect results for a bisect job.
Args:
job: TryJob entity.
Returns:
A dictionary containing status, results, buildbot_log_url, and
issue_url for this bisect job. The issue_url may be a link to a Rietveld
issue or to Buildbucket job info.
"""
results = {}
# Fetch bisect bot results from Rietveld server.
if job.use_buildbucket:
try_job_info = _ValidateAndConvertBuildbucketResponse(
buildbucket_service.GetJobStatus(job.buildbucket_job_id), job)
hostname = app_identity.get_default_version_hostname()
job_id = job.buildbucket_job_id
issue_url = 'https://%s/buildbucket_job_status/%s' % (hostname, job_id)
else:
response = _FetchRietveldIssueJSON(job)
issue_url = _RietveldIssueURL(job)
try_job_info = _ValidateRietveldResponse(response)
results['buildbot_log_url'] = str(try_job_info['url'])
results['issue_url'] = str(issue_url)
# Check whether the bisect job is finished or not and fetch the output.
result = int(try_job_info['result'])
if result not in OK + FAIL:
return None
results_url = '%s/steps/Results/logs/stdio/text' % try_job_info['url']
response = _FetchURL(results_url, skip_status_code=True)
results['bisect_bot'] = try_job_info['builder']
# We don't see content for "Result" step. Bot probably did not get there.
if not response or response.status_code != 200:
results['status'] = 'Failure'
results['results'] = ''
build_data = _FetchBuildData(try_job_info['url'])
if build_data:
_CheckBisectBotForInfraFailure(job.bug_id, build_data,
try_job_info['url'])
results['results'] = _GetBotFailureInfo(build_data)
partial_result = _GetPartialBisectResult(build_data,
try_job_info['url'])
if partial_result:
results['status'] = 'Failure with partial results'
results['results'] += partial_result
return results
# Clean result.
# If the bisect_results string contains any non-ASCII characters,
# converting to string should prevent an error from being raised.
bisect_result = _BeautifyContent(str(response.content))
# Bisect is considered success if result is provided.
# "BISECTION ABORTED" is added when a job is ealy aborted because the
# associated issue was closed.
# TODO(robertocn): Make sure we are outputting this string
if ('BISECT JOB RESULTS' in bisect_result
or 'BISECTION ABORTED' in bisect_result):
results['status'] = 'Completed'
else:
results['status'] = 'Failure'
results['results'] = bisect_result
return results
def _IsBisectJobCompleted(job):
return _ValidateBuildbucketResponse(
buildbucket_service.GetJobStatus(job.buildbucket_job_id))
