def download_file(self,
bucket,
key,
filename,
extra_args=None,
callback=None):
"""Download an S3 object to a file.
Variants have also been injected into S3 client, Bucket and Object.
You don't have to use S3Transfer.download_file() directly.
.. seealso::
:py:meth:`S3.Client.download_file`
:py:meth:`S3.Client.download_fileobj`
"""
if not isinstance(filename, six.string_types):
raise ValueError('Filename must be a string')
subscribers = self._get_subscribers(callback)
future = self._manager.download(bucket, key, filename, extra_args,
subscribers)
try:
future.result()
# This is for backwards compatibility where when retries are
# exceeded we need to throw the same error from boto3 instead of
# s3transfer's built in RetriesExceededError as current users are
# catching the boto3 one instead of the s3transfer exception to do
# their own retries.
except S3TransferRetriesExceededError as e:
raise RetriesExceededError(e.last_exception)
def _download_range(self, bucket, key, filename,
part_size, num_parts, callback, part_index):
try:
range_param = self._calculate_range_param(
part_size, part_index, num_parts)
max_attempts = self._config.num_download_attempts
last_exception = None
for i in range(max_attempts):
try:
logger.debug("Making get_object call.")
response = self._client.get_object(
Bucket=bucket, Key=key, Range=range_param)
streaming_body = StreamReaderProgress(
response['Body'], callback)
buffer_size = 1024 * 16
current_index = part_size * part_index
for chunk in iter(lambda: streaming_body.read(buffer_size),
b''):
self._ioqueue.put((current_index, chunk))
current_index += len(chunk)
return
except (socket.timeout, socket.error,
ReadTimeoutError, IncompleteReadError) as e:
logger.debug("Retrying exception caught (%s), "
"retrying request, (attempt %s / %s)", e, i,
max_attempts, exc_info=True)
last_exception = e
continue
raise RetriesExceededError(last_exception)
finally:
logger.debug("EXITING _download_range for part: %s", part_index)
def test_download_retry_exceeded_error(caplog):
"""Tests whether a retry exceeded error is handled as expected when downloading."""
caplog.set_level(logging.DEBUG, "snowflake.connector")
mock_resource = MagicMock()
mock_resource.download_file.side_effect = RetriesExceededError(
Boto3Error())
client_meta = {
"cloud_client": mock_resource,
"stage_info": {
"location": "loc"
},
}
meta = {
"name": "f",
"src_file_name": "f",
"stage_location_type": "S3",
"client_meta": SFResourceMeta(**client_meta),
"sha256_digest": "asd",
"src_file_size": 99,
"get_callback_output_stream": None,
"show_progress_bar": False,
"get_callback": None,
}
meta = SnowflakeFileMeta(**meta)
with mock.patch(
"snowflake.connector.s3_util.SnowflakeS3Util._get_s3_object",
return_value=mock_resource,
):
SnowflakeS3Util._native_download_file(meta, "f", 4)
assert meta.last_error is mock_resource.download_file.side_effect
assert meta.result_status == ResultStatus.NEED_RETRY
def _prepare_retry(self, unprocessed_items):
self._retry_attempt = self._retry_attempt + 1
if 0 <= self._retries.max_attempts < self._retry_attempt:
raise RetriesExceededError(
None,
msg=
f"Max Retries Exceeded: retry attempt - {self._retry_attempt},"
f" max retries - {self._retries.max_attempts}")
self._items_buffer.extend(unprocessed_items)
def _get_object(self, bucket, key, filename, extra_args, callback):
# precondition: num_download_attempts > 0
max_attempts = self._config.num_download_attempts
last_exception = None
for i in range(max_attempts):
try:
return self._do_get_object(bucket, key, filename,
extra_args, callback)
except S3_RETRYABLE_ERRORS as e:
logger.debug("Retrying exception caught (%s), "
"retrying request, (attempt %s / %s)", e, i,
max_attempts, exc_info=True)
last_exception = e
continue
raise RetriesExceededError(last_exception)
def test_download_retry_exceeded_error(caplog):
"""Tests whether a retry exceeded error is handled as expected when downloading."""
mock_resource = MagicMock()
mock_resource.download_file.side_effect = RetriesExceededError(Boto3Error())
meta = {'client': mock_resource,
'sha256_digest': 'asd',
'stage_info': {'location': 'loc'},
'src_file_name': 'f',
'src_file_size': 99,
'get_callback_output_stream': None,
'show_progress_bar': False,
'get_callback': None}
with mock.patch('snowflake.connector.s3_util.SnowflakeS3Util._get_s3_object', return_value=mock_resource):
SnowflakeS3Util._native_download_file(meta, 'f', 4)
assert meta['last_error'] is mock_resource.download_file.side_effect
assert meta['result_status'] == ResultStatus.NEED_RETRY
def _get_object(self, bucket, key, filename, extra_args, callback):
# precondition: num_download_attempts > 0
max_attempts = self._config.num_download_attempts
last_exception = None
for i in range(max_attempts):
try:
return self._do_get_object(bucket, key, filename,
extra_args, callback)
except (socket.timeout, socket.error,
ReadTimeoutError, IncompleteReadError) as e:
# TODO: we need a way to reset the callback if the
# download failed.
logger.debug("Retrying exception caught (%s), "
"retrying request, (attempt %s / %s)", e, i,
max_attempts, exc_info=True)
last_exception = e
continue
raise RetriesExceededError(last_exception)
def download_file(self,
bucket,
key,
filename,
extra_args=None,
callback=None):
subscribers = self._get_subscribers(callback)
future = self._manager.download(bucket, key, filename, extra_args,
subscribers)
try:
future.result()
# This is for backwards compatibility where when retries are
# exceeded we need to throw the same error from boto3 instead of
# s3transfer's built in RetriesExceededError as current users are
# catching the boto3 one instead of the s3transfer exception to do
# their own retries.
except S3TransferRetriesExceededError as e:
raise RetriesExceededError(e.last_exception)
def test_download_retry_exceeded_error(caplog):
"""Tests whether a retry exceeded error is handled as expected when downloading."""
caplog.set_level(logging.DEBUG, 'snowflake.connector')
mock_resource = MagicMock()
mock_resource.download_file.side_effect = RetriesExceededError(Boto3Error())
client_meta = {
'cloud_client': mock_resource,
'stage_info': {'location': 'loc'},
}
meta = {'name': 'f',
'src_file_name': 'f',
'stage_location_type': 'S3',
'client_meta': SFResourceMeta(**client_meta),
'sha256_digest': 'asd',
'src_file_size': 99,
'get_callback_output_stream': None,
'show_progress_bar': False,
'get_callback': None}
meta = SnowflakeFileMeta(**meta)
with mock.patch('snowflake.connector.s3_util.SnowflakeS3Util._get_s3_object', return_value=mock_resource):
SnowflakeS3Util._native_download_file(meta, 'f', 4)
assert meta.last_error is mock_resource.download_file.side_effect
assert meta.result_status == ResultStatus.NEED_RETRY