def test_create_transfer_manager_with_no_threads(self):
client = object()
config = TransferConfig()
config.use_threads = False
with mock.patch('ibm_boto3.s3.transfer.TransferManager') as manager:
create_transfer_manager(client, config)
assert manager.call_args == mock.call(client, config, None,
NonThreadedExecutor)
def test_alias_max_io_queue(self):
ref_value = 10
config = TransferConfig(max_io_queue=ref_value)
self.assert_value_of_actual_and_alias(config, 'max_io_queue_size',
'max_io_queue', ref_value)
# Set a new value using the alias
new_value = 15
config.max_io_queue = new_value
# Make sure it sets the value for both the alias and the actual
# value that will be used in the TransferManager
self.assert_value_of_actual_and_alias(config, 'max_io_queue_size',
'max_io_queue', new_value)
def test_download_progress(self):
self.contents = b'A' * 55
self.stub_multipart_download(contents=self.contents,
part_size=5,
num_parts=11)
transfer_config = TransferConfig(multipart_chunksize=5,
multipart_threshold=1,
max_concurrency=1)
def progress_callback(amount):
self.progress += amount
self.progress_times_called += 1
with self.stubber:
self.s3.meta.client.download_fileobj(Bucket=self.bucket,
Key=self.key,
Fileobj=self.fileobj,
Config=transfer_config,
Callback=progress_callback)
# Assert that the progress callback was called the correct number of
# times with the correct amounts.
self.assertEqual(self.progress_times_called, 11)
self.assertEqual(self.progress, 55)
self.stubber.assert_no_pending_responses()
def test_create_transfer_manager(self):
client = object()
config = TransferConfig()
osutil = OSUtils()
with mock.patch('ibm_boto3.s3.transfer.TransferManager') as manager:
create_transfer_manager(client, config, osutil)
assert manager.call_args == mock.call(client, config, osutil, None)
def test_copy_progress(self):
chunksize = 8 * (1024**2)
self.stub_multipart_copy(chunksize, 3)
transfer_config = TransferConfig(
multipart_chunksize=chunksize,
multipart_threshold=1,
max_concurrency=1,
)
def progress_callback(amount):
self.progress += amount
self.progress_times_called += 1
with self.stubber:
self.s3.meta.client.copy(
Bucket=self.bucket,
Key=self.key,
CopySource=self.copy_source,
Config=transfer_config,
Callback=progress_callback,
)
# Assert that the progress callback was called the correct number of
# times with the correct amounts.
assert self.progress_times_called == 3
assert self.progress == chunksize * 3
def download_fileobj(self, Bucket, Key, Fileobj, ExtraArgs=None,
Callback=None, Config=None):
"""Download an object from S3 to a file-like object.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart download in
multiple threads if necessary.
Usage::
import ibm_boto3
s3 = ibm_boto3.client('s3')
with open('filename', 'wb') as data:
s3.download_fileobj('mybucket', 'mykey', data)
:type Fileobj: a file-like object
:param Fileobj: A file-like object to download into. At a minimum, it must
implement the `write` method and must accept bytes.
:type Bucket: str
:param Bucket: The name of the bucket to download from.
:type Key: str
:param Key: The name of the key to download from.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: ibm_boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
download.
"""
if not hasattr(Fileobj, 'write'):
raise ValueError('Fileobj must implement write')
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with create_transfer_manager(self, config) as manager:
future = manager.download(
bucket=Bucket, key=Key, fileobj=Fileobj,
extra_args=ExtraArgs, subscribers=subscribers)
return future.result()
def __init__(self):
try:
with open(os.path.join(sys.path[0], "config.json")) as config_file:
config_json = json.loads(config_file.read())
except IOError:
raise ConversionOperationException("INIT", "Config file not found")
except json.JSONDecodeError:
raise ConversionOperationException("INIT", "Malformed config file")
for required in [
"IMAGE_FILE_NAME", "DESTINATION_IMAGE_FORMAT",
"IBM_API_KEY_ID", "IAM_SERVICE_ID", "S3_ENDPOINT",
"BUCKET_NAME"
]:
if required not in config_json:
raise ConversionOperationException(
"INIT", "Required key {} not found in config file".format(
required))
self.bucket_name = config_json["BUCKET_NAME"]
self.image_file_name = config_json["IMAGE_FILE_NAME"]
self.source_image_format = None
if config_json[
"DESTINATION_IMAGE_FORMAT"] not in self.SUPPORTED_DEST_FORMATS:
raise ConversionOperationException(
"INIT", "Conversion to {} format not supported".format(
config_json["DESTINATION_IMAGE_FORMAT"]))
self.destination_image_format = config_json["DESTINATION_IMAGE_FORMAT"]
self.converted_image_name = self.clear_extension(
self.image_file_name) + "." + self.destination_image_format
self.image_path_download = config_json.get("DOWNLOAD_PATH") or './'
if self.image_path_download[-1] != "/":
self.image_path_download += "/"
self.image_path_convert = config_json.get("CONVERT_PATH") or './'
if self.image_path_convert[-1] != "/":
self.image_path_convert += "/"
client = ibm_boto3.client(
service_name='s3',
ibm_api_key_id=config_json["IBM_API_KEY_ID"],
ibm_service_instance_id=config_json["IAM_SERVICE_ID"],
ibm_auth_endpoint="https://iam.cloud.ibm.com/identity/token",
config=Config(signature_version="oauth"),
endpoint_url=config_json["S3_ENDPOINT"])
self.transfer_manager = TransferManager(client, TransferConfig())
def test_multipart_download(self):
self.contents = b'A' * 55
self.stub_multipart_download(
contents=self.contents, part_size=5, num_parts=11)
transfer_config = TransferConfig(
multipart_chunksize=5, multipart_threshold=1,
max_concurrency=1)
with self.stubber:
self.s3.meta.client.download_fileobj(
Bucket=self.bucket, Key=self.key, Fileobj=self.fileobj,
Config=transfer_config)
self.assertEqual(self.fileobj.getvalue(), self.contents)
self.stubber.assert_no_pending_responses()
def test_multipart_upload(self):
chunksize = 8 * (1024 ** 2)
contents = six.BytesIO(b'0' * (chunksize * 3))
self.stub_multipart_upload(num_parts=3)
transfer_config = TransferConfig(
multipart_chunksize=chunksize, multipart_threshold=1,
max_concurrency=1)
with self.stubber:
# The stubber will assert that all the right parameters are called.
self.s3.meta.client.upload_fileobj(
Fileobj=contents, Bucket=self.bucket, Key=self.key,
Config=transfer_config)
self.stubber.assert_no_pending_responses()
def _upload_if_needed(src_path,
storage,
sm_storage,
storage_type,
s3_client=None,
use_db_mutex=True):
"""
Uploads the object from `src_path` if it doesn't already exist in its translated COS path.
Returns a CloudObject for the COS object
"""
bucket, key = _choose_cos_location(src_path, sm_storage, storage_type)
with ExitStack() as stack:
if use_db_mutex:
# Lock during upload to prevent parallel jobs upload the same file simultaneously
stack.enter_context(DBMutex().lock(bucket + key, timeout=1200))
try:
storage.head_object(bucket, key)
logger.debug(f'{src_path} already uploaded')
return CloudObject(storage.backend, bucket, key)
except StorageNoSuchKeyError:
logger.info(f'Uploading {src_path}...')
if src_path.startswith('s3a://'):
assert s3_client, 'S3 client must be supplied to support s3a:// paths'
src_bucket, src_key = split_s3_path(src_path)
obj = s3_client.get_object(Bucket=src_bucket, Key=src_key)
if hasattr(storage.get_client(), 'upload_fileobj'):
# Try streaming upload to IBM COS
transfer_config = TransferConfig(multipart_chunksize=20 *
MB,
max_concurrency=20,
io_chunksize=1 * MB)
storage.get_client().upload_fileobj(Fileobj=obj['Body'],
Bucket=bucket,
Key=key,
Config=transfer_config)
cobject = CloudObject(storage.backend, bucket, key)
else:
# Fall back to buffering the entire object in memory for other backends
cobject = storage.put_cloudobject(obj['Body'].read(),
bucket, key)
else:
cobject = storage.put_cloudobject(open(src_path, 'rb'), bucket,
key)
logger.info(f'Uploading {src_path}...Done')
return cobject
def test_transferconfig_parameters(self):
config = TransferConfig(
multipart_threshold=8 * MB,
max_concurrency=10,
multipart_chunksize=8 * MB,
num_download_attempts=5,
max_io_queue=100,
io_chunksize=256 * KB,
use_threads=True,
max_bandwidth=1024 * KB,
)
assert config.multipart_threshold == 8 * MB
assert config.multipart_chunksize == 8 * MB
assert config.max_request_concurrency == 10
assert config.num_download_attempts == 5
assert config.max_io_queue_size == 100
assert config.io_chunksize == 256 * KB
assert config.use_threads is True
assert config.max_bandwidth == 1024 * KB
def test_can_create_with_extra_configurations(self):
transfer = S3Transfer(client=mock.Mock(),
config=TransferConfig(),
osutil=OSUtils())
assert isinstance(transfer, S3Transfer)
def copy(self,
CopySource,
Bucket,
Key,
ExtraArgs=None,
Callback=None,
SourceClient=None,
Config=None):
"""Copy an object from one S3 location to another.
This is a managed transfer which will perform a multipart copy in
multiple threads if necessary.
Usage::
import ibm_boto3
s3 = ibm_boto3.resource('s3')
copy_source = {
'Bucket': 'mybucket',
'Key': 'mykey'
}
s3.meta.client.copy(copy_source, 'otherbucket', 'otherkey')
:type CopySource: dict
:param CopySource: The name of the source bucket, key name of the
source object. The dictionary format is:
``{'Bucket': 'bucket', 'Key': 'key'}``.
:type Bucket: str
:param Bucket: The name of the bucket to copy to
:type Key: str
:param Key: The name of the key to copy to
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation
:type Callback: method
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the copy.
:type SourceClient: ibm_botocore or ibm_boto3 Client
:param SourceClient: The client to be used for operation that
may happen at the source object. For example, this client is
used for the head_object that determines the size of the copy.
If no client is provided, the current client is used as the client
for the source object.
:type Config: ibm_boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
copy.
"""
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with create_transfer_manager(self, config) as manager:
future = manager.copy(copy_source=CopySource,
bucket=Bucket,
key=Key,
extra_args=ExtraArgs,
subscribers=subscribers,
source_client=SourceClient)
return future.result()
def download_file(self, bucket, key, filename):
tc = TransferConfig()
self.cos_resource.Bucket(bucket).download_file(Key=key,
Filename=filename,
Config=tc)
def upload_file(self, filename, bucket, key):
tc = TransferConfig()
self.cos_client.upload_file(filename, bucket, key, Config=tc)
def upload_fileobj(self,
Fileobj,
Bucket,
Key,
ExtraArgs=None,
Callback=None,
Config=None):
"""Upload a file-like object to S3.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart upload in
multiple threads if necessary.
Usage::
import ibm_boto3
s3 = ibm_boto3.client('s3')
with open('filename', 'rb') as data:
s3.upload_fileobj(data, 'mybucket', 'mykey')
:type Fileobj: a file-like object
:param Fileobj: A file-like object to upload. At a minimum, it must
implement the `read` method, and must return bytes.
:type Bucket: str
:param Bucket: The name of the bucket to upload to.
:type Key: str
:param Key: The name of the key to upload to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
ibm_boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: ibm_boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
upload.
"""
if not hasattr(Fileobj, 'read'):
raise ValueError('Fileobj must implement read')
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with create_transfer_manager(self, config) as manager:
future = manager.upload(
fileobj=Fileobj,
bucket=Bucket,
key=Key,
extra_args=ExtraArgs,
subscribers=subscribers,
)
return future.result()