def _WaitAndStreamLogs(self, build_op, logs_bucket, build_id, logs_uri): """Wait for a Cloud Build to finish, optionally streaming logs.""" log.status.Print( 'Started cloud build [{build_id}].'.format(build_id=build_id)) if logs_bucket: log_object = self.CLOUDBUILD_LOGFILE_FMT_STRING.format( build_id=build_id) log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket, obj=log_object) log_loc = None if logs_uri: log.status.Print('To see logs in the Cloud Console: ' + logs_uri) log_loc = 'at ' + logs_uri else: log.status.Print('Logs can be found in the Cloud Console.') log_loc = 'in the Cloud Console.' op = self.WaitForOperation(operation=build_op, retry_callback=log_tailer.Poll) # Poll the logs one final time to ensure we have everything. We know this # final poll will get the full log contents because GCS is strongly # consistent and Container Builder waits for logs to finish pushing before # marking the build complete. log_tailer.Poll(is_last=True) else: op = self.WaitForOperation(operation=build_op) final_status = _GetStatusFromOp(op) if final_status != self.CLOUDBUILD_SUCCESS: raise BuildFailedError('Cloud build failed with status ' + final_status + '. Check logs ' + log_loc)
def testPoll503(self): # If the service is unavailable (err 503), we do best effort polling and # then move on without finishing the log. self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=0-'}, body='Some log text\n', headers={'status': 206}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=14-'}, body='I am a server and Im really overloaded', headers={'status': 503}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=14-'}, body='So much load. So service unavailable.', headers={'status': 503}) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') tailer.Poll() # 206 (content) tailer.Poll() # 503 tailer.Poll(True) # 503 expected = ('--------- REMOTE BUILD OUTPUT ----------\n' 'Some log text\n' '-------- (possibly incomplete) ---------\n\n') self.assertEqual(self.mock_log_content, expected)
def testPoll429(self): # If the service is pushing back (err 429), we do best effort polling and # then move on without finishing the log. self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=0-'}, body='Some log text\n', headers={'status': 206}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=14-'}, body='stop calling me so often!', headers={'status': 429}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=14-'}, body='too much! server is pushing back with 429', headers={'status': 429}) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') tailer.Poll() # 206 (content) tailer.Poll() # 429 tailer.Poll(True) # 429 expected = ('--------- REMOTE BUILD OUTPUT ----------\n' 'Some log text\n' '-------- (possibly incomplete) ---------\n\n') self.assertEqual(self.mock_log_content, expected)
def testPollError(self): # Any other errors will result in hard failure. self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=0-'}, body='you got no permission', headers={'status': 403}) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') try: tailer.Poll() # 403 except api_exceptions.HttpError: # Success. return self.fail('Expected a HttpError')
def WaitAndStreamLogs(self, build_op): """Wait for a Cloud Build to finish, streaming logs if possible.""" build_id = GetBuildProp(build_op, 'id', required=True) logs_uri = GetBuildProp(build_op, 'logUrl') logs_bucket = GetBuildProp(build_op, 'logsBucket') log.status.Print( 'Started cloud build [{build_id}].'.format(build_id=build_id)) log_loc = 'in the Cloud Console.' log_tailer = None if logs_bucket: log_object = self.CLOUDBUILD_LOGFILE_FMT_STRING.format( build_id=build_id) log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket, obj=log_object) if logs_uri: log.status.Print('To see logs in the Cloud Console: ' + logs_uri) log_loc = 'at ' + logs_uri else: log.status.Print('Logs can be found in the Cloud Console.') callback = None if log_tailer: callback = log_tailer.Poll try: op = self.WaitForOperation(operation=build_op, retry_callback=callback) except OperationTimeoutError: log.debug('', exc_info=True) raise BuildFailedError('Cloud build timed out. Check logs ' + log_loc) # Poll the logs one final time to ensure we have everything. We know this # final poll will get the full log contents because GCS is strongly # consistent and Container Builder waits for logs to finish pushing before # marking the build complete. if log_tailer: log_tailer.Poll(is_last=True) final_status = _GetStatusFromOp(op) if final_status != self.CLOUDBUILD_SUCCESS: message = requests.ExtractErrorMessage( encoding.MessageToPyValue(op.error)) raise BuildFailedError('Cloud build failed. Check logs ' + log_loc + ' Failure status: ' + final_status + ': ' + message)
def testPoll(self): # This test covers the happy path flow ... # 1) logfile hasn't started yet and returns 404's self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=0-'}, body='not found here', headers={'status': 404}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=0-'}, body='not found yet', headers={'status': 404}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=0-'}, body='Some log text\n', headers={'status': 206}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=14-'}, body='couldnt do the range you asked me for.', headers={'status': 416}) self.AddHTTPResponse(self._LOG_URL, expected_params={'alt': 'media'}, request_headers={'Range': 'bytes=14-'}, body='Some more text\nAnd another line\n', headers={'status': 206}) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') tailer.Poll() # 404 tailer.Poll() # 404 # The log separator line doesn't print until the log starts. self.assertEqual(self.mock_log_content, '') tailer.Poll() # 206 (content) tailer.Poll() # 416 tailer.Poll(True) # 206 (content) expected = ('--------- REMOTE BUILD OUTPUT ----------\n' 'Some log text\n' 'Some more text\nAnd another line\n' '----------------------------------------\n\n') self.assertEqual(self.mock_log_content, expected)
def testPollNoneHTTPError(self): self.SetNextException(ValueError('bad stuff')) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') with self.assertRaisesRegex(ValueError, 'bad stuff'): tailer.Poll() # non-HttpLib2Errors are raised
def testPollHTTPError_IsLast(self): self.SetNextException(httplib2.HttpLib2Error('bad stuff')) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') with self.assertRaisesRegex(api_exceptions.CommunicationError, 'bad stuff'): tailer.Poll(is_last=True) # the last poll will raise the error
def testPollHTTPError(self): self.SetNextException(httplib2.HttpLib2Error('bad stuff')) tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt') tailer.Poll() # no error raised
def ExecuteCloudBuild(project, bucket_ref, object_name, output_image): """Execute a call to CloudBuild service and wait for it to finish. Args: project: the cloud project ID. bucket_ref: Reference to GCS bucket containing source to build. object_name: GCS object name containing source to build. output_image: GCR location for the output docker image; eg, gcr.io/test-gae/hardcoded-output-tag. Raises: BuildFailedError: when the build fails. """ builder = properties.VALUES.app.container_builder_image.Get() log.debug('Using builder image: [{0}]'.format(builder)) logs_bucket = bucket_ref.bucket cloud_build_timeout = properties.VALUES.app.cloud_build_timeout.Get() if cloud_build_timeout is not None: timeout_str = cloud_build_timeout + 's' else: timeout_str = None cloudbuild_client = core_apis.GetClientInstance('cloudbuild', 'v1') cloudbuild_messages = core_apis.GetMessagesModule('cloudbuild', 'v1') build_op = cloudbuild_client.projects_builds.Create( cloudbuild_messages.CloudbuildProjectsBuildsCreateRequest( projectId=project, build=cloudbuild_messages.Build( timeout=timeout_str, source=cloudbuild_messages.Source( storageSource=cloudbuild_messages.StorageSource( bucket=bucket_ref.bucket, object=object_name, ), ), steps=[ cloudbuild_messages.BuildStep( name=builder, args=['build', '-t', output_image, '.']) ], images=[output_image], logsBucket=logs_bucket, ), )) # Find build ID from operation metadata and print the logs URL. build_id = None logs_uri = None if build_op.metadata is not None: for prop in build_op.metadata.additionalProperties: if prop.key == 'build': for build_prop in prop.value.object_value.properties: if build_prop.key == 'id': build_id = build_prop.value.string_value if logs_uri is not None: break if build_prop.key == 'logUrl': logs_uri = build_prop.value.string_value if build_id is not None: break break if build_id is None: raise BuildFailedError('Could not determine build ID') log.status.Print( 'Started cloud build [{build_id}].'.format(build_id=build_id)) log_object = CLOUDBUILD_LOGFILE_FMT_STRING.format(build_id=build_id) log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket, obj=log_object) log_loc = None if logs_uri: log.status.Print('To see logs in the Cloud Console: ' + logs_uri) log_loc = 'at ' + logs_uri else: log.status.Print('Logs can be found in the Cloud Console.') log_loc = 'in the Cloud Console.' op = operations.WaitForOperation( operation_service=cloudbuild_client.operations, operation=build_op, retry_interval=1, max_retries=60 * 60, retry_callback=log_tailer.Poll) # Poll the logs one final time to ensure we have everything. We know this # final poll will get the full log contents because GCS is strongly consistent # and Container Builder waits for logs to finish pushing before marking the # build complete. log_tailer.Poll(is_last=True) final_status = _GetStatusFromOp(op) if final_status != CLOUDBUILD_SUCCESS: raise BuildFailedError('Cloud build failed with status ' + final_status + '. Check logs ' + log_loc)