def _WaitAndStreamLogs(self, build_op, logs_bucket, build_id, logs_uri):
"""Wait for a Cloud Build to finish, optionally streaming logs."""
log.status.Print(
'Started cloud build [{build_id}].'.format(build_id=build_id))
if logs_bucket:
log_object = self.CLOUDBUILD_LOGFILE_FMT_STRING.format(
build_id=build_id)
log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket,
obj=log_object)
log_loc = None
if logs_uri:
log.status.Print('To see logs in the Cloud Console: ' +
logs_uri)
log_loc = 'at ' + logs_uri
else:
log.status.Print('Logs can be found in the Cloud Console.')
log_loc = 'in the Cloud Console.'
op = self.WaitForOperation(operation=build_op,
retry_callback=log_tailer.Poll)
# Poll the logs one final time to ensure we have everything. We know this
# final poll will get the full log contents because GCS is strongly
# consistent and Container Builder waits for logs to finish pushing before
# marking the build complete.
log_tailer.Poll(is_last=True)
else:
op = self.WaitForOperation(operation=build_op)
final_status = _GetStatusFromOp(op)
if final_status != self.CLOUDBUILD_SUCCESS:
raise BuildFailedError('Cloud build failed with status ' +
final_status + '. Check logs ' + log_loc)
def testPoll503(self):
# If the service is unavailable (err 503), we do best effort polling and
# then move on without finishing the log.
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=0-'},
body='Some log text\n',
headers={'status': 206})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=14-'},
body='I am a server and Im really overloaded',
headers={'status': 503})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=14-'},
body='So much load. So service unavailable.',
headers={'status': 503})
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
tailer.Poll() # 206 (content)
tailer.Poll() # 503
tailer.Poll(True) # 503
expected = ('--------- REMOTE BUILD OUTPUT ----------\n'
'Some log text\n'
'-------- (possibly incomplete) ---------\n\n')
self.assertEqual(self.mock_log_content, expected)
def testPoll429(self):
# If the service is pushing back (err 429), we do best effort polling and
# then move on without finishing the log.
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=0-'},
body='Some log text\n',
headers={'status': 206})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=14-'},
body='stop calling me so often!',
headers={'status': 429})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=14-'},
body='too much! server is pushing back with 429',
headers={'status': 429})
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
tailer.Poll() # 206 (content)
tailer.Poll() # 429
tailer.Poll(True) # 429
expected = ('--------- REMOTE BUILD OUTPUT ----------\n'
'Some log text\n'
'-------- (possibly incomplete) ---------\n\n')
self.assertEqual(self.mock_log_content, expected)
def testPollError(self):
# Any other errors will result in hard failure.
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=0-'},
body='you got no permission',
headers={'status': 403})
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
try:
tailer.Poll() # 403
except api_exceptions.HttpError:
# Success.
return
self.fail('Expected a HttpError')
def WaitAndStreamLogs(self, build_op):
"""Wait for a Cloud Build to finish, streaming logs if possible."""
build_id = GetBuildProp(build_op, 'id', required=True)
logs_uri = GetBuildProp(build_op, 'logUrl')
logs_bucket = GetBuildProp(build_op, 'logsBucket')
log.status.Print(
'Started cloud build [{build_id}].'.format(build_id=build_id))
log_loc = 'in the Cloud Console.'
log_tailer = None
if logs_bucket:
log_object = self.CLOUDBUILD_LOGFILE_FMT_STRING.format(
build_id=build_id)
log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket,
obj=log_object)
if logs_uri:
log.status.Print('To see logs in the Cloud Console: ' +
logs_uri)
log_loc = 'at ' + logs_uri
else:
log.status.Print('Logs can be found in the Cloud Console.')
callback = None
if log_tailer:
callback = log_tailer.Poll
try:
op = self.WaitForOperation(operation=build_op,
retry_callback=callback)
except OperationTimeoutError:
log.debug('', exc_info=True)
raise BuildFailedError('Cloud build timed out. Check logs ' +
log_loc)
# Poll the logs one final time to ensure we have everything. We know this
# final poll will get the full log contents because GCS is strongly
# consistent and Container Builder waits for logs to finish pushing before
# marking the build complete.
if log_tailer:
log_tailer.Poll(is_last=True)
final_status = _GetStatusFromOp(op)
if final_status != self.CLOUDBUILD_SUCCESS:
message = requests.ExtractErrorMessage(
encoding.MessageToPyValue(op.error))
raise BuildFailedError('Cloud build failed. Check logs ' +
log_loc + ' Failure status: ' +
final_status + ': ' + message)
def testPoll(self):
# This test covers the happy path flow ...
# 1) logfile hasn't started yet and returns 404's
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=0-'},
body='not found here',
headers={'status': 404})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=0-'},
body='not found yet',
headers={'status': 404})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=0-'},
body='Some log text\n',
headers={'status': 206})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=14-'},
body='couldnt do the range you asked me for.',
headers={'status': 416})
self.AddHTTPResponse(self._LOG_URL,
expected_params={'alt': 'media'},
request_headers={'Range': 'bytes=14-'},
body='Some more text\nAnd another line\n',
headers={'status': 206})
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
tailer.Poll() # 404
tailer.Poll() # 404
# The log separator line doesn't print until the log starts.
self.assertEqual(self.mock_log_content, '')
tailer.Poll() # 206 (content)
tailer.Poll() # 416
tailer.Poll(True) # 206 (content)
expected = ('--------- REMOTE BUILD OUTPUT ----------\n'
'Some log text\n'
'Some more text\nAnd another line\n'
'----------------------------------------\n\n')
self.assertEqual(self.mock_log_content, expected)
def testPollNoneHTTPError(self):
self.SetNextException(ValueError('bad stuff'))
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
with self.assertRaisesRegex(ValueError, 'bad stuff'):
tailer.Poll() # non-HttpLib2Errors are raised
def testPollHTTPError_IsLast(self):
self.SetNextException(httplib2.HttpLib2Error('bad stuff'))
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
with self.assertRaisesRegex(api_exceptions.CommunicationError,
'bad stuff'):
tailer.Poll(is_last=True) # the last poll will raise the error
def testPollHTTPError(self):
self.SetNextException(httplib2.HttpLib2Error('bad stuff'))
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
tailer.Poll() # no error raised
def ExecuteCloudBuild(project, bucket_ref, object_name, output_image):
"""Execute a call to CloudBuild service and wait for it to finish.
Args:
project: the cloud project ID.
bucket_ref: Reference to GCS bucket containing source to build.
object_name: GCS object name containing source to build.
output_image: GCR location for the output docker image;
eg, gcr.io/test-gae/hardcoded-output-tag.
Raises:
BuildFailedError: when the build fails.
"""
builder = properties.VALUES.app.container_builder_image.Get()
log.debug('Using builder image: [{0}]'.format(builder))
logs_bucket = bucket_ref.bucket
cloud_build_timeout = properties.VALUES.app.cloud_build_timeout.Get()
if cloud_build_timeout is not None:
timeout_str = cloud_build_timeout + 's'
else:
timeout_str = None
cloudbuild_client = core_apis.GetClientInstance('cloudbuild', 'v1')
cloudbuild_messages = core_apis.GetMessagesModule('cloudbuild', 'v1')
build_op = cloudbuild_client.projects_builds.Create(
cloudbuild_messages.CloudbuildProjectsBuildsCreateRequest(
projectId=project,
build=cloudbuild_messages.Build(
timeout=timeout_str,
source=cloudbuild_messages.Source(
storageSource=cloudbuild_messages.StorageSource(
bucket=bucket_ref.bucket,
object=object_name,
), ),
steps=[
cloudbuild_messages.BuildStep(
name=builder, args=['build', '-t', output_image, '.'])
],
images=[output_image],
logsBucket=logs_bucket,
),
))
# Find build ID from operation metadata and print the logs URL.
build_id = None
logs_uri = None
if build_op.metadata is not None:
for prop in build_op.metadata.additionalProperties:
if prop.key == 'build':
for build_prop in prop.value.object_value.properties:
if build_prop.key == 'id':
build_id = build_prop.value.string_value
if logs_uri is not None:
break
if build_prop.key == 'logUrl':
logs_uri = build_prop.value.string_value
if build_id is not None:
break
break
if build_id is None:
raise BuildFailedError('Could not determine build ID')
log.status.Print(
'Started cloud build [{build_id}].'.format(build_id=build_id))
log_object = CLOUDBUILD_LOGFILE_FMT_STRING.format(build_id=build_id)
log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket, obj=log_object)
log_loc = None
if logs_uri:
log.status.Print('To see logs in the Cloud Console: ' + logs_uri)
log_loc = 'at ' + logs_uri
else:
log.status.Print('Logs can be found in the Cloud Console.')
log_loc = 'in the Cloud Console.'
op = operations.WaitForOperation(
operation_service=cloudbuild_client.operations,
operation=build_op,
retry_interval=1,
max_retries=60 * 60,
retry_callback=log_tailer.Poll)
# Poll the logs one final time to ensure we have everything. We know this
# final poll will get the full log contents because GCS is strongly consistent
# and Container Builder waits for logs to finish pushing before marking the
# build complete.
log_tailer.Poll(is_last=True)
final_status = _GetStatusFromOp(op)
if final_status != CLOUDBUILD_SUCCESS:
raise BuildFailedError('Cloud build failed with status ' +
final_status + '. Check logs ' + log_loc)