def _CheckFYIBisectJob(job, issue_tracker): try: if job.use_buildbucket and not job.buildbucket_job_id: job.key.delete() return if not _IsBisectJobCompleted(job): return if not job.results_data: raise BisectJobFailure( 'Bisect job completed, but results data is not ' 'found, bot might have failed to post results.') error_message = bisect_fyi.VerifyBisectFYIResults(job) _PostSuccessfulResult(job, issue_tracker) if not bisect_fyi.IsBugUpdated(job, issue_tracker): error_message += '\nFailed to update bug with bisect results.' except BisectJobFailure as e: error_message = 'Bisect job failed because, %s' % e except BugUpdateFailure as e: error_message = 'Failed to update bug with bisect results: %s' % e finally: job_info = buildbucket_service.GetJobStatus(job.buildbucket_job_id) job_info = job_info.get('build', {}) if not job.results_data: job.results_data = {} job.results_data['buildbot_log_url'] = str(job_info.get('url')) # When the job fails before getting to the point where it post bisect results # to the dashboard, the tryjob's results_data is not set. # As a special case for Bisect FYI jobs, we query buildbucket to get the # bisect job's status. if ((job.results_data and job.results_data.get('status') == FAILED) or error_message): job.SetFailed() _SendFYIBisectEmail(job, error_message)
def testGetJobStatus(self): fake_id = '1234567890' with mock.patch('apiclient.discovery.build', mock.MagicMock(return_value=self.fake_service)): _ = buildbucket_service.GetJobStatus(fake_id) # Ensure the given job id was passed to the request. request = self.fake_service.bodies[0] self.assertIn('id', request) self.assertEqual(fake_id, request['id'])
def CheckFailureFromBuildBucket(self): # Buildbucket job id is not always set. if not self.buildbucket_job_id: return job_info = buildbucket_service.GetJobStatus(self.buildbucket_job_id) data = job_info.get('build', {}) # Since the job is completed successfully, results_data must # have been set appropriately by the bisector. # The buildbucket job's 'status' and 'result' fields are documented here: # https://goto.google.com/bb_status if data.get('status') == 'COMPLETED' and data.get( 'result') == 'SUCCESS': return # Proceed if the job failed or cancelled logging.info('Job failed. Buildbucket id %s', self.buildbucket_job_id) data['result_details'] = json.loads(data['result_details_json']) # There are various failure and cancellation reasons for a buildbucket # job to fail as listed in https://goto.google.com/bb_status. job_updates = { 'status': 'failed', 'failure_reason': (data.get('cancelation_reason') or data.get('failure_reason')), 'buildbot_log_url': data.get('url') } details = data.get('result_details') if details: properties = details.get('properties') if properties: job_updates['bisect_bot'] = properties.get('buildername') job_updates['extra_result_code'] = properties.get( 'extra_result_code') bisect_config = properties.get('bisect_config') if bisect_config: job_updates['try_job_id'] = bisect_config.get('try_job_id') job_updates['bug_id'] = bisect_config.get('bug_id') job_updates['command'] = bisect_config.get('command') job_updates['test_type'] = bisect_config.get('test_type') job_updates['metric'] = bisect_config.get('metric') job_updates['good_revision'] = bisect_config.get( 'good_revision') job_updates['bad_revision'] = bisect_config.get( 'bad_revision') if not self.results_data: self.results_data = {} self.results_data.update(job_updates) self.status = 'failed' self.last_ran_timestamp = datetime.datetime.fromtimestamp( float(data['updated_ts']) / 1000000) self.put() logging.info('updated status to failed.')
def _IsBuildBucketJobCompleted(job): """Checks whether the bisect job is completed.""" job_info = buildbucket_service.GetJobStatus(job.buildbucket_job_id) if not job_info: return False data = job_info.get('build', {}) # The status of the build can be one of [STARTED, SCHEDULED or COMPLETED] # The buildbucket 'status' fields are documented here: # https://goto.google.com/bb_status if data.get('status') != 'COMPLETED': return False return True
def CheckFailureFromBuildBucket(self): # Buildbucket job id is not always set. if not self.buildbucket_job_id: return job_info = buildbucket_service.GetJobStatus(self.buildbucket_job_id) data = job_info.get('build', {}) # Proceed if the job failed and the job status has # not been updated if data.get('result') != 'FAILURE' or self.status == 'failed': return data['result_details'] = json.loads(data['result_details_json']) job_updates = { 'status': 'failed', 'failure_reason': data.get('failure_reason'), 'buildbot_log_url': data.get('url') } details = data.get('result_details') if details: properties = details.get('properties') if properties: job_updates['bisect_bot'] = properties.get('buildername') job_updates['extra_result_code'] = properties.get( 'extra_result_code') bisect_config = properties.get('bisect_config') if bisect_config: job_updates['try_job_id'] = bisect_config.get('try_job_id') job_updates['bug_id'] = bisect_config.get('bug_id') job_updates['command'] = bisect_config.get('command') job_updates['test_type'] = bisect_config.get('test_type') job_updates['metric'] = bisect_config.get('metric') job_updates['good_revision'] = bisect_config.get( 'good_revision') job_updates['bad_revision'] = bisect_config.get( 'bad_revision') if not self.results_data: self.results_data = {} self.results_data.update(job_updates) self.status = 'failed' self.last_ran_timestamp = datetime.datetime.fromtimestamp( float(data['updated_ts']) / 1000000) self.put()
def get(self, job_id): original_status = buildbucket_service.GetJobStatus(job_id) error = 'error' in original_status if error: error_reason = original_status['error'].get('reason') status_text = json.dumps(original_status, sort_keys=True, indent=4) else: clean_status = _ConvertTimes( _ParseJsonKeys(original_status.get('build'))) status_text = json.dumps(clean_status, sort_keys=True, indent=4) self.RenderHtml( 'buildbucket_job_status.html', { 'job_id': job_id, 'status_text': 'DATA:' + status_text, 'build': None if error else clean_status, 'error': error_reason if error else None, 'original_response': original_status, })
def _GetBisectResults(job): """Gets bisect results for a bisect job. Args: job: TryJob entity. Returns: A dictionary containing status, results, buildbot_log_url, and issue_url for this bisect job. The issue_url may be a link to a Rietveld issue or to Buildbucket job info. """ results = {} # Fetch bisect bot results from Rietveld server. if job.use_buildbucket: try_job_info = _ValidateAndConvertBuildbucketResponse( buildbucket_service.GetJobStatus(job.buildbucket_job_id), job) hostname = app_identity.get_default_version_hostname() job_id = job.buildbucket_job_id issue_url = 'https://%s/buildbucket_job_status/%s' % (hostname, job_id) else: response = _FetchRietveldIssueJSON(job) issue_url = _RietveldIssueURL(job) try_job_info = _ValidateRietveldResponse(response) results['buildbot_log_url'] = str(try_job_info['url']) results['issue_url'] = str(issue_url) # Check whether the bisect job is finished or not and fetch the output. result = int(try_job_info['result']) if result not in OK + FAIL: return None results_url = '%s/steps/Results/logs/stdio/text' % try_job_info['url'] response = _FetchURL(results_url, skip_status_code=True) results['bisect_bot'] = try_job_info['builder'] # We don't see content for "Result" step. Bot probably did not get there. if not response or response.status_code != 200: results['status'] = 'Failure' results['results'] = '' build_data = _FetchBuildData(try_job_info['url']) if build_data: _CheckBisectBotForInfraFailure(job.bug_id, build_data, try_job_info['url']) results['results'] = _GetBotFailureInfo(build_data) partial_result = _GetPartialBisectResult(build_data, try_job_info['url']) if partial_result: results['status'] = 'Failure with partial results' results['results'] += partial_result return results # Clean result. # If the bisect_results string contains any non-ASCII characters, # converting to string should prevent an error from being raised. bisect_result = _BeautifyContent(str(response.content)) # Bisect is considered success if result is provided. # "BISECTION ABORTED" is added when a job is ealy aborted because the # associated issue was closed. # TODO(robertocn): Make sure we are outputting this string if ('BISECT JOB RESULTS' in bisect_result or 'BISECTION ABORTED' in bisect_result): results['status'] = 'Completed' else: results['status'] = 'Failure' results['results'] = bisect_result return results
def _IsBisectJobCompleted(job): return _ValidateBuildbucketResponse( buildbucket_service.GetJobStatus(job.buildbucket_job_id))