def testGetStepLogBuildbucketError(self, *_):
self.assertIsNone(
swarming_util.GetStepLog(self.buildbucket_id, self.step_name,
self.http_client))
def testGetStepLogStdioNoAnnotations(self, *_):
self.assertIsNone(
swarming_util.GetStepLog(self.buildbucket_id, self.step_name,
self.http_client))
def testGetStepLogStdio(self, *_):
self.assertEqual(
'log1/nlog2',
swarming_util.GetStepLog(self.buildbucket_id, self.step_name,
self.http_client))
def testGetStepMetadataStreamNone(self, *_):
step_metadata = swarming_util.GetStepLog(self.buildbucket_id,
self.step_name,
self.http_client,
'step_metadata')
self.assertIsNone(step_metadata)
def testGetStepMetadata(self, *_):
step_metadata = swarming_util.GetStepLog(self.buildbucket_id,
self.step_name,
self.http_client,
'step_metadata')
self.assertEqual(step_metadata, wf_testcase.SAMPLE_STEP_METADATA)
def _callback(self, callback_params, pipeline_id=None):
"""Updates the TryJobData entities with status from buildbucket."""
# callback_params may have been serialized if the callback was converted to
# a URL.
if isinstance(callback_params, basestring):
callback_params = json.loads(callback_params)
self.last_params = callback_params
_ = pipeline_id # We do nothing with this id.
try_job_id = callback_params['try_job_id']
assert try_job_id
urlsafe_try_job_key = callback_params['urlsafe_try_job_key']
try_job_type = callback_params['try_job_type']
deadline = callback_params['deadline']
already_set_started = callback_params['already_set_started']
error_count = callback_params['error_count']
max_error_times = callback_params['max_error_times']
default_pipeline_wait_seconds = callback_params[
'default_pipeline_wait_seconds']
timeout_hours = callback_params['timeout_hours']
backoff_time = callback_params['backoff_time']
if try_job_type == failure_type.FLAKY_TEST:
try_job_data = FlakeTryJobData.Get(try_job_id)
else:
try_job_data = WfTryJobData.Get(try_job_id)
error, build = buildbucket_client.GetTryJobs([try_job_id])[0]
if error:
if error_count < max_error_times:
error_count += 1
self.delay_callback(
backoff_time,
callback_params={
'try_job_id': try_job_id,
'try_job_type': try_job_type,
'urlsafe_try_job_key': urlsafe_try_job_key,
'deadline': deadline,
'already_set_started': already_set_started,
'error_count': error_count,
'max_error_times': max_error_times,
'default_pipeline_wait_seconds': default_pipeline_wait_seconds,
'timeout_hours': timeout_hours,
'backoff_time': backoff_time * 2,
}
)
return
else: # pragma: no cover
# Buildbucket has responded error more than 5 times, retry pipeline.
_UpdateTryJobMetadata(
try_job_data, try_job_type, build, error, False)
raise pipeline.Retry(
'Error "%s" occurred. Reason: "%s"' % (error.message,
error.reason))
elif build.status == BuildbucketBuild.COMPLETED:
swarming_task_id = buildbot.GetSwarmingTaskIdFromUrl(
build.url)
if swarming_task_id:
try:
report = json.loads(swarming_util.GetStepLog(
try_job_id, 'report', HttpClientAppengine(), 'report'))
except (ValueError, TypeError) as e: # pragma: no cover
report = {}
logging.exception(
'Failed to load result report for swarming/%s '
'due to exception %s.' % (swarming_task_id, e.message))
else:
try_job_master_name, try_job_builder_name, try_job_build_number = (
buildbot.ParseBuildUrl(build.url))
try:
report = json.loads(buildbot.GetStepLog(
try_job_master_name, try_job_builder_name, try_job_build_number,
'report', HttpClientAppengine(), 'report'))
except (ValueError, TypeError) as e: # pragma: no cover
report = {}
logging.exception(
'Failed to load result report for %s/%s/%s due to exception %s.'
% (try_job_master_name, try_job_builder_name,
try_job_build_number, e.message))
_UpdateTryJobMetadata(
try_job_data, try_job_type, build, error, False,
report if report else {})
result_to_update = self._UpdateTryJobResult(
urlsafe_try_job_key, try_job_type, try_job_id,
build.url, BuildbucketBuild.COMPLETED, report)
self.complete(result_to_update[-1])
return
else:
error_count = 0
backoff_time = default_pipeline_wait_seconds
if build.status == BuildbucketBuild.STARTED and not (
already_set_started):
# It is possible this branch is skipped if a fast build goes from
# 'SCHEDULED' to 'COMPLETED' between queries, so start_time may be
# unavailable.
start_time = time_util.MicrosecondsToDatetime(build.updated_time)
self._UpdateTryJobResult(
urlsafe_try_job_key, try_job_type, try_job_id,
build.url, BuildbucketBuild.STARTED)
already_set_started = True
# Update as much try job metadata as soon as possible to avoid data
# loss in case of errors.
try_job_data.start_time = start_time
try_job_data.request_time = (
time_util.MicrosecondsToDatetime(build.request_time))
try_job_data.try_job_url = build.url
try_job_data.callback_url = self.get_callback_url(
callback_params=json.dumps({
'try_job_id': try_job_id,
'try_job_type': try_job_type,
'urlsafe_try_job_key': urlsafe_try_job_key,
'deadline': deadline,
'already_set_started': already_set_started,
'error_count': error_count,
'max_error_times': max_error_times,
'default_pipeline_wait_seconds': default_pipeline_wait_seconds,
'timeout_hours': timeout_hours,
'backoff_time': backoff_time,
})
)
try_job_data.put()
if time.time() > deadline: # pragma: no cover
_UpdateTryJobMetadata(
try_job_data, try_job_type, build, error, True)
# Explicitly abort the whole pipeline.
raise pipeline.Abort(
'Try job %s timed out after %d hours.' % (
try_job_id, timeout_hours))
# Ensure last_buildbucket_response is always the most recent
# whenever available during intermediate queries.
_UpdateLastBuildbucketResponse(try_job_data, build)