def read(self, amt=None): # pylint: disable=invalid-name
"""Exposes a stream from the in-memory buffer to the upload."""
if self.position == self.src_obj_size or amt == 0:
# If there is no data left or 0 bytes were requested, return an empty
# string so callers can call still call len() and read(0).
return ''
if amt is None or amt > TRANSFER_BUFFER_SIZE:
raise BadRequestException(
'Invalid HTTP read size %s during daisy chain operation, '
'expected <= %s.' % (amt, TRANSFER_BUFFER_SIZE))
while True:
with self.lock:
if self.buffer:
break
with self.download_exception_lock:
if self.download_exception:
# Download thread died, so we will never recover. Raise the
# exception that killed it.
raise self.download_exception # pylint: disable=raising-bad-type
# Buffer was empty, yield thread priority so the download thread can fill.
time.sleep(0)
with self.lock:
data = self.buffer.popleft()
self.last_position = self.position
self.last_data = data
data_len = len(data)
self.position += data_len
self.bytes_buffered -= data_len
if data_len > amt:
raise BadRequestException(
'Invalid read during daisy chain operation, got data of size '
'%s, expected size %s.' % (data_len, amt))
return data
def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name
restart_download = False
if whence == os.SEEK_END:
if offset:
raise BadRequestException(
'Invalid seek during daisy chain operation. Non-zero offset %s '
'from os.SEEK_END is not supported' % offset)
with self.lock:
self.last_position = self.position
self.last_data = None
# Safe because we check position against src_obj_size in read.
self.position = self.src_obj_size
elif whence == os.SEEK_SET:
with self.lock:
if offset == self.position:
pass
elif offset == self.last_position:
self.position = self.last_position
if self.last_data:
# If we seek to end and then back, we won't have last_data; we'll
# get it on the next call to read.
self.buffer.appendleft(self.last_data)
self.bytes_buffered += len(self.last_data)
else:
# Once a download is complete, boto seeks to 0 and re-reads to
# compute the hash if an md5 isn't already present (for example a GCS
# composite object), so we have to re-download the whole object.
# Also, when daisy-chaining to a resumable upload, on error the
# service may have received any number of the bytes; the download
# needs to be restarted from that point.
restart_download = True
if restart_download:
self.stop_download.set()
# Consume any remaining bytes in the download thread so that
# the thread can exit, then restart the thread at the desired position.
while self.download_thread.is_alive():
with self.lock:
while self.bytes_buffered:
self.bytes_buffered -= len(self.buffer.popleft())
time.sleep(0)
with self.lock:
self.position = offset
self.buffer = deque()
self.bytes_buffered = 0
self.last_position = 0
self.last_data = None
self.StartDownloadThread(
start_byte=offset,
progress_callback=self.progress_callback)
else:
raise IOError('Daisy-chain download wrapper does not support '
'seek mode %s' % whence)
def read(amt=None): # pylint: disable=invalid-name
"""Overrides HTTPConnection.getresponse.read.
This function only supports reads of TRANSFER_BUFFER_SIZE or smaller.
Args:
amt: Integer n where 0 < n <= TRANSFER_BUFFER_SIZE. This is a
keyword argument to match the read function it overrides,
but it is required.
Returns:
Data read from HTTPConnection.
"""
if not amt or amt > TRANSFER_BUFFER_SIZE:
raise BadRequestException(
'Invalid HTTP read size %s during download, expected %s.'
% (amt, TRANSFER_BUFFER_SIZE))
else:
amt = amt or TRANSFER_BUFFER_SIZE
if not self.processed_initial_bytes:
self.processed_initial_bytes = True
if self.outer_progress_callback:
self.callback_processor = ProgressCallbackWithBackoff(
self.outer_total_size,
self.outer_progress_callback)
self.callback_processor.Progress(
self.outer_bytes_downloaded_container.
bytes_transferred)
data = orig_read_func(amt)
read_length = len(data)
if self.callback_processor:
self.callback_processor.Progress(read_length)
if self.outer_digesters:
for alg in self.outer_digesters:
self.outer_digesters[alg].update(data)
return data
def _TranslateApitoolsException(self, e, service_account_id=None):
"""Translates apitools exceptions into their gsutil equivalents.
Args:
e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS.
service_account_id: Optional service account ID that caused the exception.
Returns:
CloudStorageApiServiceException for translatable exceptions, None
otherwise.
"""
if isinstance(e, apitools_exceptions.HttpError):
message = self._GetMessageFromHttpError(e)
if e.status_code == 400:
# It is possible that the Project ID is incorrect. Unfortunately the
# JSON API does not give us much information about what part of the
# request was bad.
return BadRequestException(message or 'Bad Request',
status=e.status_code)
elif e.status_code == 401:
if 'Login Required' in str(e):
return AccessDeniedException(
message or 'Access denied: login required.',
status=e.status_code)
elif 'insufficient_scope' in str(e):
# If the service includes insufficient scope error detail in the
# response body, this check can be removed.
return AccessDeniedException(
_INSUFFICIENT_OAUTH2_SCOPE_MESSAGE,
status=e.status_code,
body=self._GetAcceptableScopesFromHttpError(e))
elif e.status_code == 403:
# Messaging for when the the originating credentials don't have access
# to impersonate a service account.
if 'The caller does not have permission' in str(e):
return AccessDeniedException(
'Service account impersonation failed. Please go to the Google '
'Cloud Platform Console (https://cloud.google.com/console), '
'select IAM & admin, then Service Accounts, and grant your '
'originating account the Service Account Token Creator role on '
'the target service account.')
# The server's errors message when IAM Credentials API aren't enabled
# are pretty great so we just display them.
if 'IAM Service Account Credentials API has not been used' in str(
e):
return AccessDeniedException(message)
if 'The account for the specified project has been disabled' in str(
e):
return AccessDeniedException(message
or 'Account disabled.',
status=e.status_code)
elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e):
return AccessDeniedException(
message or 'Access denied: quota exceeded. '
'Is your project ID valid?',
status=e.status_code)
elif 'User Rate Limit Exceeded' in str(e):
return AccessDeniedException(
'Rate limit exceeded. Please retry this '
'request later.',
status=e.status_code)
elif 'Access Not Configured' in str(e):
return AccessDeniedException(
'Access Not Configured. Please go to the Google Cloud Platform '
'Console (https://cloud.google.com/console#/project) for your '
'project, select APIs & services, and enable the Google Cloud '
'IAM Credentials API.',
status=e.status_code)
elif 'insufficient_scope' in str(e):
# If the service includes insufficient scope error detail in the
# response body, this check can be removed.
return AccessDeniedException(
_INSUFFICIENT_OAUTH2_SCOPE_MESSAGE,
status=e.status_code,
body=self._GetAcceptableScopesFromHttpError(e))
else:
return AccessDeniedException(message or e.message
or service_account_id,
status=e.status_code)
elif e.status_code == 404:
return NotFoundException(message or e.message,
status=e.status_code)
elif e.status_code == 409 and service_account_id:
return ServiceException('The key %s already exists.' %
service_account_id,
status=e.status_code)
elif e.status_code == 412:
return PreconditionException(message, status=e.status_code)
return ServiceException(message, status=e.status_code)
def _TranslateApitoolsException(self, e, key_name=None):
"""Translates apitools exceptions into their gsutil equivalents.
Args:
e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS.
key_name: Optional key name in request that caused the exception.
Returns:
CloudStorageApiServiceException for translatable exceptions, None
otherwise.
"""
if isinstance(e, apitools_exceptions.HttpError):
message = self._GetMessageFromHttpError(e)
if e.status_code == 400:
# It is possible that the Project ID is incorrect. Unfortunately the
# JSON API does not give us much information about what part of the
# request was bad.
return BadRequestException(message or 'Bad Request',
status=e.status_code)
elif e.status_code == 401:
if 'Login Required' in str(e):
return AccessDeniedException(
message or 'Access denied: login required.',
status=e.status_code)
elif 'insufficient_scope' in str(e):
# If the service includes insufficient scope error detail in the
# response body, this check can be removed.
return AccessDeniedException(
_INSUFFICIENT_OAUTH2_SCOPE_MESSAGE,
status=e.status_code,
body=self._GetAcceptableScopesFromHttpError(e))
elif e.status_code == 403:
if 'The account for the specified project has been disabled' in str(
e):
return AccessDeniedException(message
or 'Account disabled.',
status=e.status_code)
elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e):
return AccessDeniedException(
message or 'Access denied: quota exceeded. '
'Is your project ID valid?',
status=e.status_code)
elif 'User Rate Limit Exceeded' in str(e):
return AccessDeniedException(
'Rate limit exceeded. Please retry this '
'request later.',
status=e.status_code)
elif 'Access Not Configured' in str(e):
return AccessDeniedException(
'Access Not Configured. Please go to the Google Cloud Platform '
'Console (https://cloud.google.com/console#/project) for your '
'project, select APIs & services, and enable the Google Cloud '
'KMS API.',
status=e.status_code)
elif 'insufficient_scope' in str(e):
# If the service includes insufficient scope error detail in the
# response body, this check can be removed.
return AccessDeniedException(
_INSUFFICIENT_OAUTH2_SCOPE_MESSAGE,
status=e.status_code,
body=self._GetAcceptableScopesFromHttpError(e))
else:
return AccessDeniedException(message or e.message
or key_name,
status=e.status_code)
elif e.status_code == 404:
return NotFoundException(message or e.message,
status=e.status_code)
elif e.status_code == 409 and key_name:
return ServiceException('The key %s already exists.' %
key_name,
status=e.status_code)
elif e.status_code == 412:
return PreconditionException(message, status=e.status_code)
return ServiceException(message, status=e.status_code)