def test_upload__path_doesnt_exist(self, mock_os):
mock_os.path.isfile.return_value = False
upload_path = '/tmp/upload_file'
with pytest.raises(ValueError) as ex_cm:
S3ClientWrapper.upload_file('foo_bucket',
'http://foo.s3.amazon.com',
'/foo/bar/baz', upload_path)
assert 'does not exist' in str(ex_cm.value)
mock_os.path.isfile.assert_called_once_with(upload_path)
def upload_client_auth_s3(syn, file_path, bucket, endpoint_url, key_prefix, storage_location_id, mimetype=None):
profile = syn._get_client_authenticated_s3_profile(endpoint_url, bucket)
file_key = key_prefix + '/' + os.path.basename(file_path)
S3ClientWrapper.upload_file(bucket, endpoint_url, file_key, file_path, profile_name=profile)
file_handle = syn._createExternalObjectStoreFileHandle(file_key, file_path, storage_location_id, mimetype=mimetype)
syn.cache.add(file_handle['id'], file_path)
return file_handle
def test_download__import_error(self, mock_attempt_import):
"""Verify an error importing boto3 is raised as expected"""
mock_attempt_import.side_effect = ImportError("can't import boto3")
bucket_name = 'foo_bucket'
remote_file_key = 'foo/bar/baz'
download_file_path = '/tmp/download'
endpoint_url = 'http://foo.s3.amazon.com'
with pytest.raises(ImportError):
S3ClientWrapper.download_file(bucket_name, endpoint_url,
remote_file_key, download_file_path)
def _upload_test(**kwargs):
bucket_name = 'foo_bucket'
remote_file_key = 'foo/bar/baz'
upload_file_path = '/tmp/upload_file'
endpoint_url = 'http://foo.s3.amazon.com'
with mock.patch('boto3.session.Session') as mock_boto_session,\
mock.patch.object(S3ClientWrapper, '_create_progress_callback_func') as mock_create_progress_callback,\
mock.patch('boto3.s3.transfer.TransferConfig') as mock_TransferConfig,\
mock.patch.object(remote_file_storage_wrappers, 'os') as mock_os:
returned_upload_path = S3ClientWrapper.upload_file(
bucket_name, endpoint_url, remote_file_key, upload_file_path,
**kwargs)
if 'profile_name' in kwargs:
mock_boto_session.assert_called_once_with(
profile_name=kwargs['profile_name'])
else:
mock_boto_session.assert_called_once_with(
**kwargs['credentials'])
resource = mock_boto_session.return_value.resource
resource.assert_called_once_with('s3', endpoint_url=endpoint_url)
s3 = resource.return_value
s3.Bucket.assert_called_once_with(bucket_name)
s3_bucket = s3.Bucket.return_value
mock_TransferConfig.assert_called_once_with(
**kwargs.get('transfer_config_kwargs', {}))
transfer_config = mock_TransferConfig.return_value
progress_callback = None
if kwargs.get('show_progress', True):
mock_os.stat.assert_called_once_with(upload_file_path)
file_size = mock_os.stat.return_value
mock_os.path.basename.assert_called_once_with(upload_file_path)
filename = mock_os.path.basename(upload_file_path)
progress_callback = S3ClientWrapper._create_progress_callback_func(
file_size, filename, prefix='Uploading')
else:
assert not mock_create_progress_callback.called
s3_bucket.upload_file.assert_called_once_with(
upload_file_path,
remote_file_key,
Callback=progress_callback,
Config=transfer_config)
# why do we return something we passed...?
assert upload_file_path == returned_upload_path
def upload_fn(credentials):
return S3ClientWrapper.upload_file(
bucket_name,
None,
remote_file_key,
local_path,
credentials=credentials,
transfer_config_kwargs={'max_concurrency': syn.max_threads}
)
def _download_error_test(exception, raised_type):
bucket_name = 'foo_bucket'
remote_file_key = '/foo/bar/baz'
download_file_path = '/tmp/download'
endpoint_url = 'http://foo.s3.amazon.com'
with mock.patch('boto3.session.Session') as mock_boto_session:
resource = mock_boto_session.return_value.resource
s3 = resource.return_value
s3.Object.side_effect = exception
with pytest.raises(raised_type):
S3ClientWrapper.download_file(
bucket_name,
endpoint_url,
remote_file_key,
download_file_path,
)
def _download_test(**kwargs):
bucket_name = 'foo_bucket'
remote_file_key = 'foo/bar/baz'
download_file_path = '/tmp/download'
endpoint_url = 'http://foo.s3.amazon.com'
with mock.patch('boto3.session.Session') as mock_boto_session,\
mock.patch.object(S3ClientWrapper, '_create_progress_callback_func') as mock_create_progress_callback,\
mock.patch('boto3.s3.transfer.TransferConfig') as mock_TransferConfig:
returned_download_file_path = S3ClientWrapper.download_file(
bucket_name, endpoint_url, remote_file_key, download_file_path,
**kwargs)
if 'profile_name' in kwargs:
mock_boto_session.assert_called_once_with(
profile_name=kwargs['profile_name'])
else:
mock_boto_session.assert_called_once_with(
**kwargs['credentials'])
resource = mock_boto_session.return_value.resource
resource.assert_called_once_with('s3', endpoint_url=endpoint_url)
s3 = resource.return_value
s3.Object.assert_called_once_with(bucket_name, remote_file_key)
s3_object = s3.Object.return_value
mock_TransferConfig.assert_called_once_with(
**kwargs.get('transfer_config_kwargs', {}))
transfer_config = mock_TransferConfig.return_value
progress_callback = None
if kwargs.get('show_progress', True):
s3_object.load.assert_called_once_with()
mock_create_progress_callback.assert_called_once_with(
s3_object.content_length,
os.path.basename(download_file_path),
prefix='Downloading')
progress_callback = mock_create_progress_callback.return_value
else:
assert not mock_create_progress_callback.called
s3_object.download_file.assert_called_once_with(
download_file_path,
Callback=progress_callback,
Config=transfer_config)
# why do we return something we passed...?
assert download_file_path == returned_download_file_path