def _upload_to_gcs(self, local_file_path, gcs_location):
gcs_bucket, gcs_object = self._get_gcs_bucket_and_name(gcs_location)
request = storage.StorageObjectsInsertRequest(bucket=gcs_bucket,
name=gcs_object)
_LOGGER.info('Starting GCS upload to %s...', gcs_location)
total_size = os.path.getsize(local_file_path)
from apitools.base.py import exceptions
try:
with open(local_file_path, 'rb') as stream:
upload = storage.Upload(stream, 'application/octet-stream',
total_size)
self._storage_client.objects.Insert(request, upload=upload)
except exceptions.HttpError as e:
reportable_errors = {
403: 'access denied',
404: 'bucket not found',
}
if e.status_code in reportable_errors:
raise IOError(
('Could not upload to GCS path %s: %s. Please verify '
'that credentials are valid and that you have write '
'access to the specified path.') %
(gcs_location, reportable_errors[e.status_code]))
raise
_LOGGER.info('Completed GCS upload to %s.', gcs_location)
def stage_file(self,
gcs_or_local_path,
file_name,
stream,
mime_type='application/octet-stream'):
"""Stages a file at a GCS or local path with stream-supplied contents."""
if not gcs_or_local_path.startswith('gs://'):
local_path = FileSystems.join(gcs_or_local_path, file_name)
logging.info('Staging file locally to %s', local_path)
with open(local_path, 'wb') as f:
f.write(stream.read())
return
gcs_location = FileSystems.join(gcs_or_local_path, file_name)
bucket, name = gcs_location[5:].split('/', 1)
request = storage.StorageObjectsInsertRequest(bucket=bucket, name=name)
logging.info('Starting GCS upload to %s...', gcs_location)
upload = storage.Upload(stream, mime_type)
try:
response = self._storage_client.objects.Insert(request,
upload=upload)
except exceptions.HttpError as e:
reportable_errors = {
403: 'access denied',
404: 'bucket not found',
}
if e.status_code in reportable_errors:
raise IOError(
('Could not upload to GCS path %s: %s. Please verify '
'that credentials are valid and that you have write '
'access to the specified path.') %
(gcs_or_local_path, reportable_errors[e.status_code]))
raise
logging.info('Completed GCS upload to %s', gcs_location)
return response
