def test_dry_run(self):
self.cli_params['dryrun'] = True
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
fileinfo = self.create_file_info(self.key)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
self.assertIsInstance(result, DryRunResult)
self.assertEqual(result.transfer_type, 'download')
self.assertTrue(result.dest.endswith(self.filename))
self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key)
def test_dry_run(self):
self.cli_params['dryrun'] = True
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
fileinfo = self.create_file_info(self.key)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
self.assertIsInstance(result, DryRunResult)
self.assertEqual(result.transfer_type, 'download')
self.assertTrue(result.dest.endswith(self.filename))
self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key)
def setUp(self):
super(TestDownloadRequestSubmitter, self).setUp()
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
class TestDownloadRequestSubmitter(BaseTransferRequestSubmitterTest):
def setUp(self):
super(TestDownloadRequestSubmitter, self).setUp()
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
def assert_no_downloads_happened(self):
self.assertEqual(len(self.transfer_manager.download.call_args_list), 0)
def create_file_info(self, key, associated_response_data=None):
kwargs = {
'src': self.bucket + '/' + key,
'src_type': 's3',
'dest': self.filename,
'dest_type': 'local',
'operation_name': 'download',
'compare_key': key,
}
if associated_response_data is not None:
kwargs['associated_response_data'] = associated_response_data
return FileInfo(**kwargs)
def test_can_submit(self):
fileinfo = FileInfo(src=self.bucket + '/' + self.key,
dest=self.filename,
operation_name='download')
self.assertTrue(self.transfer_request_submitter.can_submit(fileinfo))
fileinfo.operation_name = 'foo'
self.assertFalse(self.transfer_request_submitter.can_submit(fileinfo))
def test_submit(self):
fileinfo = self.create_file_info(self.key)
future = self.transfer_request_submitter.submit(fileinfo)
self.assertIs(self.transfer_manager.download.return_value, future)
download_call_kwargs = self.transfer_manager.download.call_args[1]
self.assertEqual(download_call_kwargs['fileobj'], self.filename)
self.assertEqual(download_call_kwargs['bucket'], self.bucket)
self.assertEqual(download_call_kwargs['key'], self.key)
self.assertEqual(download_call_kwargs['extra_args'], {})
# Make sure the subscriber applied are of the correct type and order
ref_subscribers = [
ProvideSizeSubscriber, DirectoryCreatorSubscriber,
ProvideLastModifiedTimeSubscriber, DownloadResultSubscriber
]
actual_subscribers = download_call_kwargs['subscribers']
self.assertEqual(len(ref_subscribers), len(actual_subscribers))
for i, actual_subscriber in enumerate(actual_subscribers):
self.assertIsInstance(actual_subscriber, ref_subscribers[i])
def test_submit_with_extra_args(self):
fileinfo = self.create_file_info(self.key)
self.cli_params['sse_c'] = 'AES256'
self.cli_params['sse_c_key'] = 'mykey'
self.transfer_request_submitter.submit(fileinfo)
# Set some extra argument like sse_c to make sure cli
# params get mapped to request parameters.
download_call_kwargs = self.transfer_manager.download.call_args[1]
self.assertEqual(download_call_kwargs['extra_args'], {
'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'mykey'
})
def test_warn_glacier_for_incompatible(self):
fileinfo = FileInfo(src=self.bucket + '/' + self.key,
dest=self.filename,
operation_name='download',
associated_response_data={
'StorageClass': 'GLACIER',
})
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have been submitted because it is a non-restored
# glacier object.
warning_result = self.result_queue.get()
self.assertIsInstance(warning_result, WarningResult)
self.assertIn(
'Unable to perform download operations on GLACIER objects',
warning_result.message)
# The transfer should have been skipped.
self.assertIsNone(future)
self.assert_no_downloads_happened()
def test_not_warn_glacier_for_compatible(self):
fileinfo = self.create_file_info(self.key,
associated_response_data={
'StorageClass': 'GLACIER',
'Restore':
'ongoing-request="false"'
})
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have not been submitted because it is a restored
# glacier object.
self.assertTrue(self.result_queue.empty())
# And the transfer should not have been skipped.
self.assertIs(self.transfer_manager.download.return_value, future)
self.assertEqual(len(self.transfer_manager.download.call_args_list), 1)
def test_warn_glacier_force_glacier(self):
self.cli_params['force_glacier_transfer'] = True
fileinfo = self.create_file_info(self.key,
associated_response_data={
'StorageClass': 'GLACIER',
})
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have not been submitted because it is glacier
# transfers were forced.
self.assertTrue(self.result_queue.empty())
self.assertIs(self.transfer_manager.download.return_value, future)
self.assertEqual(len(self.transfer_manager.download.call_args_list), 1)
def test_warn_glacier_ignore_glacier_warnings(self):
self.cli_params['ignore_glacier_warnings'] = True
fileinfo = FileInfo(src=self.bucket + '/' + self.key,
dest=self.filename,
operation_name='download',
associated_response_data={
'StorageClass': 'GLACIER',
})
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have not been submitted because it was specified
# to ignore glacier warnings.
self.assertTrue(self.result_queue.empty())
# But the transfer still should have been skipped.
self.assertIsNone(future)
self.assert_no_downloads_happened()
def test_warn_and_ignore_on_parent_dir_reference(self):
fileinfo = self.create_file_info('../foo.txt')
future = self.transfer_request_submitter.submit(fileinfo)
warning_result = self.result_queue.get()
self.assertIsInstance(warning_result, WarningResult)
self.assert_no_downloads_happened()
def test_warn_and_ignore_with_leading_chars(self):
fileinfo = self.create_file_info('a/../../foo.txt')
future = self.transfer_request_submitter.submit(fileinfo)
warning_result = self.result_queue.get()
self.assertIsInstance(warning_result, WarningResult)
self.assert_no_downloads_happened()
def test_allow_double_dots_that_dont_escape_cwd(self):
self.cli_params['dryrun'] = True
# This is fine because it's 'foo.txt'.
fileinfo = self.create_file_info('a/../foo.txt')
future = self.transfer_request_submitter.submit(fileinfo)
self.assertIsInstance(self.result_queue.get(), DryRunResult)
def test_dry_run(self):
self.cli_params['dryrun'] = True
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
fileinfo = self.create_file_info(self.key)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
self.assertIsInstance(result, DryRunResult)
self.assertEqual(result.transfer_type, 'download')
self.assertTrue(result.dest.endswith(self.filename))
self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key)
def test_submit_move_adds_delete_source_subscriber(self):
fileinfo = self.create_file_info(self.key)
self.cli_params['guess_mime_type'] = True # Default settings
self.cli_params['is_move'] = True
self.transfer_request_submitter.submit(fileinfo)
ref_subscribers = [
ProvideSizeSubscriber,
DirectoryCreatorSubscriber,
ProvideLastModifiedTimeSubscriber,
DeleteSourceObjectSubscriber,
DownloadResultSubscriber,
]
download_call_kwargs = self.transfer_manager.download.call_args[1]
actual_subscribers = download_call_kwargs['subscribers']
self.assertEqual(len(ref_subscribers), len(actual_subscribers))
for i, actual_subscriber in enumerate(actual_subscribers):
self.assertIsInstance(actual_subscriber, ref_subscribers[i])
def setUp(self):
super(TestDownloadRequestSubmitter, self).setUp()
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
class TestDownloadRequestSubmitter(BaseTransferRequestSubmitterTest):
def setUp(self):
super(TestDownloadRequestSubmitter, self).setUp()
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
def test_can_submit(self):
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename,
operation_name='download')
self.assertTrue(
self.transfer_request_submitter.can_submit(fileinfo))
fileinfo.operation_name = 'foo'
self.assertFalse(
self.transfer_request_submitter.can_submit(fileinfo))
def test_submit(self):
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename)
future = self.transfer_request_submitter.submit(fileinfo)
self.assertIs(self.transfer_manager.download.return_value, future)
download_call_kwargs = self.transfer_manager.download.call_args[1]
self.assertEqual(download_call_kwargs['fileobj'], self.filename)
self.assertEqual(download_call_kwargs['bucket'], self.bucket)
self.assertEqual(download_call_kwargs['key'], self.key)
self.assertEqual(download_call_kwargs['extra_args'], {})
# Make sure the subscriber applied are of the correct type and order
ref_subscribers = [
ProvideSizeSubscriber,
DirectoryCreatorSubscriber,
ProvideLastModifiedTimeSubscriber,
DownloadResultSubscriber
]
actual_subscribers = download_call_kwargs['subscribers']
self.assertEqual(len(ref_subscribers), len(actual_subscribers))
for i, actual_subscriber in enumerate(actual_subscribers):
self.assertIsInstance(actual_subscriber, ref_subscribers[i])
def test_submit_with_extra_args(self):
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename)
self.cli_params['sse_c'] = 'AES256'
self.cli_params['sse_c_key'] = 'mykey'
self.transfer_request_submitter.submit(fileinfo)
# Set some extra argument like sse_c to make sure cli
# params get mapped to request parameters.
download_call_kwargs = self.transfer_manager.download.call_args[1]
self.assertEqual(
download_call_kwargs['extra_args'],
{'SSECustomerAlgorithm': 'AES256', 'SSECustomerKey': 'mykey'}
)
def test_warn_glacier_for_incompatible(self):
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename,
operation_name='download',
associated_response_data={
'StorageClass': 'GLACIER',
}
)
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have been submitted because it is a non-restored
# glacier object.
warning_result = self.result_queue.get()
self.assertIsInstance(warning_result, WarningResult)
self.assertIn(
'Unable to perform download operations on GLACIER objects',
warning_result.message)
# The transfer should have been skipped.
self.assertIsNone(future)
self.assertEqual(len(self.transfer_manager.download.call_args_list), 0)
def test_not_warn_glacier_for_compatible(self):
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename,
operation_name='download',
associated_response_data={
'StorageClass': 'GLACIER',
'Restore': 'ongoing-request="false"'
}
)
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have not been submitted because it is a restored
# glacier object.
self.assertTrue(self.result_queue.empty())
# And the transfer should not have been skipped.
self.assertIs(self.transfer_manager.download.return_value, future)
self.assertEqual(len(self.transfer_manager.download.call_args_list), 1)
def test_warn_glacier_force_glacier(self):
self.cli_params['force_glacier_transfer'] = True
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename,
operation_name='download',
associated_response_data={
'StorageClass': 'GLACIER',
}
)
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have not been submitted because it is glacier
# transfers were forced.
self.assertTrue(self.result_queue.empty())
self.assertIs(self.transfer_manager.download.return_value, future)
self.assertEqual(len(self.transfer_manager.download.call_args_list), 1)
def test_warn_glacier_ignore_glacier_warnings(self):
self.cli_params['ignore_glacier_warnings'] = True
fileinfo = FileInfo(
src=self.bucket+'/'+self.key, dest=self.filename,
operation_name='download',
associated_response_data={
'StorageClass': 'GLACIER',
}
)
future = self.transfer_request_submitter.submit(fileinfo)
# A warning should have not been submitted because it was specified
# to ignore glacier warnings.
self.assertTrue(self.result_queue.empty())
# But the transfer still should have been skipped.
self.assertIsNone(future)
self.assertEqual(len(self.transfer_manager.download.call_args_list), 0)
def test_dry_run(self):
self.cli_params['dryrun'] = True
self.transfer_request_submitter = DownloadRequestSubmitter(
self.transfer_manager, self.result_queue, self.cli_params)
fileinfo = FileInfo(
dest=self.filename, dest_type='local', operation_name='download',
src=self.bucket + '/' + self.key, src_type='s3')
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
self.assertIsInstance(result, DryRunResult)
self.assertEqual(result.transfer_type, 'download')
self.assertTrue(result.dest.endswith(self.filename))
self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key)
def test_submit_move_adds_delete_source_subscriber(self):
fileinfo = FileInfo(
dest=self.filename, src=self.bucket+'/'+self.key)
self.cli_params['guess_mime_type'] = True # Default settings
self.cli_params['is_move'] = True
self.transfer_request_submitter.submit(fileinfo)
ref_subscribers = [
ProvideSizeSubscriber,
DirectoryCreatorSubscriber,
ProvideLastModifiedTimeSubscriber,
DeleteSourceObjectSubscriber,
DownloadResultSubscriber,
]
download_call_kwargs = self.transfer_manager.download.call_args[1]
actual_subscribers = download_call_kwargs['subscribers']
self.assertEqual(len(ref_subscribers), len(actual_subscribers))
for i, actual_subscriber in enumerate(actual_subscribers):
self.assertIsInstance(actual_subscriber, ref_subscribers[i])