def _make_s3_transfer(self, method_name, config=None, *args, **kwargs):
"""
Creates a boto3 ``S3Transfer`` object for doing multipart uploads
and downloads and executes the given method.
"""
transfer = _get_s3_transfer(config=config)
method = getattr(transfer, method_name)
try:
return method(*args, **kwargs)
except boto3_exceptions.S3UploadFailedError as e:
raise exceptions.FailedUploadError(str(e), e) from e
except boto3_exceptions.RetriesExceededError as e:
raise exceptions.FailedDownloadError(str(e), e) from e
def download(self, dest, condition=None, use_manifest=False, **kwargs):
"""Downloads a directory from S3 to a destination directory.
Args:
dest (str): The destination path to download file to. If downloading to a directory,
there must be a trailing slash. The directory will be created if it doesn't exist.
condition (function(results) -> bool): The method will only return
when the results of download matches the condition.
Returns:
List[S3Path]: A list of the downloaded objects.
Notes:
- The destination directory will be created automatically if it doesn't exist.
- This method downloads to paths relative to the current directory.
"""
utils.validate_condition(condition)
if use_manifest:
object_names = utils.get_data_manifest_contents(self)
manifest_cond = partial(utils.validate_manifest_list, object_names)
condition = (utils.join_conditions(condition, manifest_cond)
if condition else manifest_cond)
source = utils.with_trailing_slash(self)
files_to_download = [{
'source': file,
'dest': dest
} for file in source.list()]
options = settings.get()['s3:download']
segment_size = utils.str_to_bytes(options.get('segment_size'))
transfer_config = {
'multipart_threshold': segment_size,
'max_concurrency': options.get('segment_threads'),
'multipart_chunksize': segment_size
}
download_w_config = partial(self._download_object_worker,
config=transfer_config)
downloaded = {'completed': [], 'failed': []}
with S3DownloadLogger(len(files_to_download)) as dl:
pool = ThreadPool(options['object_threads'])
try:
result_iter = pool.imap_unordered(download_w_config,
files_to_download)
while True:
try:
result = result_iter.next(0xFFFF)
if result['success']:
dl.add_result(result)
downloaded['completed'].append(result)
else:
downloaded['failed'].append(result)
except StopIteration:
break
pool.close()
except BaseException:
pool.terminate()
raise
finally:
pool.join()
if downloaded['failed']:
raise exceptions.FailedDownloadError(
'an error occurred while downloading', downloaded)
utils.check_condition(condition,
[r['source'] for r in downloaded['completed']])
return downloaded