def download_file(self, bucket, key, filename, extra_args=None, callback=None): """Download an S3 object to a file. Variants have also been injected into S3 client, Bucket and Object. You don't have to use S3Transfer.download_file() directly. .. seealso:: :py:meth:`S3.Client.download_file` :py:meth:`S3.Client.download_fileobj` """ if not isinstance(filename, six.string_types): raise ValueError('Filename must be a string') subscribers = self._get_subscribers(callback) future = self._manager.download(bucket, key, filename, extra_args, subscribers) try: future.result() # This is for backwards compatibility where when retries are # exceeded we need to throw the same error from boto3 instead of # s3transfer's built in RetriesExceededError as current users are # catching the boto3 one instead of the s3transfer exception to do # their own retries. except S3TransferRetriesExceededError as e: raise RetriesExceededError(e.last_exception)
def _download_range(self, bucket, key, filename, part_size, num_parts, callback, part_index): try: range_param = self._calculate_range_param( part_size, part_index, num_parts) max_attempts = self._config.num_download_attempts last_exception = None for i in range(max_attempts): try: logger.debug("Making get_object call.") response = self._client.get_object( Bucket=bucket, Key=key, Range=range_param) streaming_body = StreamReaderProgress( response['Body'], callback) buffer_size = 1024 * 16 current_index = part_size * part_index for chunk in iter(lambda: streaming_body.read(buffer_size), b''): self._ioqueue.put((current_index, chunk)) current_index += len(chunk) return except (socket.timeout, socket.error, ReadTimeoutError, IncompleteReadError) as e: logger.debug("Retrying exception caught (%s), " "retrying request, (attempt %s / %s)", e, i, max_attempts, exc_info=True) last_exception = e continue raise RetriesExceededError(last_exception) finally: logger.debug("EXITING _download_range for part: %s", part_index)
def test_download_retry_exceeded_error(caplog): """Tests whether a retry exceeded error is handled as expected when downloading.""" caplog.set_level(logging.DEBUG, "snowflake.connector") mock_resource = MagicMock() mock_resource.download_file.side_effect = RetriesExceededError( Boto3Error()) client_meta = { "cloud_client": mock_resource, "stage_info": { "location": "loc" }, } meta = { "name": "f", "src_file_name": "f", "stage_location_type": "S3", "client_meta": SFResourceMeta(**client_meta), "sha256_digest": "asd", "src_file_size": 99, "get_callback_output_stream": None, "show_progress_bar": False, "get_callback": None, } meta = SnowflakeFileMeta(**meta) with mock.patch( "snowflake.connector.s3_util.SnowflakeS3Util._get_s3_object", return_value=mock_resource, ): SnowflakeS3Util._native_download_file(meta, "f", 4) assert meta.last_error is mock_resource.download_file.side_effect assert meta.result_status == ResultStatus.NEED_RETRY
def _prepare_retry(self, unprocessed_items): self._retry_attempt = self._retry_attempt + 1 if 0 <= self._retries.max_attempts < self._retry_attempt: raise RetriesExceededError( None, msg= f"Max Retries Exceeded: retry attempt - {self._retry_attempt}," f" max retries - {self._retries.max_attempts}") self._items_buffer.extend(unprocessed_items)
def _get_object(self, bucket, key, filename, extra_args, callback): # precondition: num_download_attempts > 0 max_attempts = self._config.num_download_attempts last_exception = None for i in range(max_attempts): try: return self._do_get_object(bucket, key, filename, extra_args, callback) except S3_RETRYABLE_ERRORS as e: logger.debug("Retrying exception caught (%s), " "retrying request, (attempt %s / %s)", e, i, max_attempts, exc_info=True) last_exception = e continue raise RetriesExceededError(last_exception)
def test_download_retry_exceeded_error(caplog): """Tests whether a retry exceeded error is handled as expected when downloading.""" mock_resource = MagicMock() mock_resource.download_file.side_effect = RetriesExceededError(Boto3Error()) meta = {'client': mock_resource, 'sha256_digest': 'asd', 'stage_info': {'location': 'loc'}, 'src_file_name': 'f', 'src_file_size': 99, 'get_callback_output_stream': None, 'show_progress_bar': False, 'get_callback': None} with mock.patch('snowflake.connector.s3_util.SnowflakeS3Util._get_s3_object', return_value=mock_resource): SnowflakeS3Util._native_download_file(meta, 'f', 4) assert meta['last_error'] is mock_resource.download_file.side_effect assert meta['result_status'] == ResultStatus.NEED_RETRY
def _get_object(self, bucket, key, filename, extra_args, callback): # precondition: num_download_attempts > 0 max_attempts = self._config.num_download_attempts last_exception = None for i in range(max_attempts): try: return self._do_get_object(bucket, key, filename, extra_args, callback) except (socket.timeout, socket.error, ReadTimeoutError, IncompleteReadError) as e: # TODO: we need a way to reset the callback if the # download failed. logger.debug("Retrying exception caught (%s), " "retrying request, (attempt %s / %s)", e, i, max_attempts, exc_info=True) last_exception = e continue raise RetriesExceededError(last_exception)
def download_file(self, bucket, key, filename, extra_args=None, callback=None): subscribers = self._get_subscribers(callback) future = self._manager.download(bucket, key, filename, extra_args, subscribers) try: future.result() # This is for backwards compatibility where when retries are # exceeded we need to throw the same error from boto3 instead of # s3transfer's built in RetriesExceededError as current users are # catching the boto3 one instead of the s3transfer exception to do # their own retries. except S3TransferRetriesExceededError as e: raise RetriesExceededError(e.last_exception)
def test_download_retry_exceeded_error(caplog): """Tests whether a retry exceeded error is handled as expected when downloading.""" caplog.set_level(logging.DEBUG, 'snowflake.connector') mock_resource = MagicMock() mock_resource.download_file.side_effect = RetriesExceededError(Boto3Error()) client_meta = { 'cloud_client': mock_resource, 'stage_info': {'location': 'loc'}, } meta = {'name': 'f', 'src_file_name': 'f', 'stage_location_type': 'S3', 'client_meta': SFResourceMeta(**client_meta), 'sha256_digest': 'asd', 'src_file_size': 99, 'get_callback_output_stream': None, 'show_progress_bar': False, 'get_callback': None} meta = SnowflakeFileMeta(**meta) with mock.patch('snowflake.connector.s3_util.SnowflakeS3Util._get_s3_object', return_value=mock_resource): SnowflakeS3Util._native_download_file(meta, 'f', 4) assert meta.last_error is mock_resource.download_file.side_effect assert meta.result_status == ResultStatus.NEED_RETRY