def test_dry_run(self): self.cli_params['dryrun'] = True self.transfer_request_submitter = DownloadRequestSubmitter( self.transfer_manager, self.result_queue, self.cli_params) fileinfo = self.create_file_info(self.key) self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() self.assertIsInstance(result, DryRunResult) self.assertEqual(result.transfer_type, 'download') self.assertTrue(result.dest.endswith(self.filename)) self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key)
def setUp(self): super(TestDownloadRequestSubmitter, self).setUp() self.transfer_request_submitter = DownloadRequestSubmitter( self.transfer_manager, self.result_queue, self.cli_params)
class TestDownloadRequestSubmitter(BaseTransferRequestSubmitterTest): def setUp(self): super(TestDownloadRequestSubmitter, self).setUp() self.transfer_request_submitter = DownloadRequestSubmitter( self.transfer_manager, self.result_queue, self.cli_params) def assert_no_downloads_happened(self): self.assertEqual(len(self.transfer_manager.download.call_args_list), 0) def create_file_info(self, key, associated_response_data=None): kwargs = { 'src': self.bucket + '/' + key, 'src_type': 's3', 'dest': self.filename, 'dest_type': 'local', 'operation_name': 'download', 'compare_key': key, } if associated_response_data is not None: kwargs['associated_response_data'] = associated_response_data return FileInfo(**kwargs) def test_can_submit(self): fileinfo = FileInfo(src=self.bucket + '/' + self.key, dest=self.filename, operation_name='download') self.assertTrue(self.transfer_request_submitter.can_submit(fileinfo)) fileinfo.operation_name = 'foo' self.assertFalse(self.transfer_request_submitter.can_submit(fileinfo)) def test_submit(self): fileinfo = self.create_file_info(self.key) future = self.transfer_request_submitter.submit(fileinfo) self.assertIs(self.transfer_manager.download.return_value, future) download_call_kwargs = self.transfer_manager.download.call_args[1] self.assertEqual(download_call_kwargs['fileobj'], self.filename) self.assertEqual(download_call_kwargs['bucket'], self.bucket) self.assertEqual(download_call_kwargs['key'], self.key) self.assertEqual(download_call_kwargs['extra_args'], {}) # Make sure the subscriber applied are of the correct type and order ref_subscribers = [ ProvideSizeSubscriber, DirectoryCreatorSubscriber, ProvideLastModifiedTimeSubscriber, DownloadResultSubscriber ] actual_subscribers = download_call_kwargs['subscribers'] self.assertEqual(len(ref_subscribers), len(actual_subscribers)) for i, actual_subscriber in enumerate(actual_subscribers): self.assertIsInstance(actual_subscriber, ref_subscribers[i]) def test_submit_with_extra_args(self): fileinfo = self.create_file_info(self.key) self.cli_params['sse_c'] = 'AES256' self.cli_params['sse_c_key'] = 'mykey' self.transfer_request_submitter.submit(fileinfo) # Set some extra argument like sse_c to make sure cli # params get mapped to request parameters. download_call_kwargs = self.transfer_manager.download.call_args[1] self.assertEqual(download_call_kwargs['extra_args'], { 'SSECustomerAlgorithm': 'AES256', 'SSECustomerKey': 'mykey' }) def test_warn_glacier_for_incompatible(self): fileinfo = FileInfo(src=self.bucket + '/' + self.key, dest=self.filename, operation_name='download', associated_response_data={ 'StorageClass': 'GLACIER', }) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have been submitted because it is a non-restored # glacier object. warning_result = self.result_queue.get() self.assertIsInstance(warning_result, WarningResult) self.assertIn( 'Unable to perform download operations on GLACIER objects', warning_result.message) # The transfer should have been skipped. self.assertIsNone(future) self.assert_no_downloads_happened() def test_not_warn_glacier_for_compatible(self): fileinfo = self.create_file_info(self.key, associated_response_data={ 'StorageClass': 'GLACIER', 'Restore': 'ongoing-request="false"' }) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have not been submitted because it is a restored # glacier object. self.assertTrue(self.result_queue.empty()) # And the transfer should not have been skipped. self.assertIs(self.transfer_manager.download.return_value, future) self.assertEqual(len(self.transfer_manager.download.call_args_list), 1) def test_warn_glacier_force_glacier(self): self.cli_params['force_glacier_transfer'] = True fileinfo = self.create_file_info(self.key, associated_response_data={ 'StorageClass': 'GLACIER', }) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have not been submitted because it is glacier # transfers were forced. self.assertTrue(self.result_queue.empty()) self.assertIs(self.transfer_manager.download.return_value, future) self.assertEqual(len(self.transfer_manager.download.call_args_list), 1) def test_warn_glacier_ignore_glacier_warnings(self): self.cli_params['ignore_glacier_warnings'] = True fileinfo = FileInfo(src=self.bucket + '/' + self.key, dest=self.filename, operation_name='download', associated_response_data={ 'StorageClass': 'GLACIER', }) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have not been submitted because it was specified # to ignore glacier warnings. self.assertTrue(self.result_queue.empty()) # But the transfer still should have been skipped. self.assertIsNone(future) self.assert_no_downloads_happened() def test_warn_and_ignore_on_parent_dir_reference(self): fileinfo = self.create_file_info('../foo.txt') future = self.transfer_request_submitter.submit(fileinfo) warning_result = self.result_queue.get() self.assertIsInstance(warning_result, WarningResult) self.assert_no_downloads_happened() def test_warn_and_ignore_with_leading_chars(self): fileinfo = self.create_file_info('a/../../foo.txt') future = self.transfer_request_submitter.submit(fileinfo) warning_result = self.result_queue.get() self.assertIsInstance(warning_result, WarningResult) self.assert_no_downloads_happened() def test_allow_double_dots_that_dont_escape_cwd(self): self.cli_params['dryrun'] = True # This is fine because it's 'foo.txt'. fileinfo = self.create_file_info('a/../foo.txt') future = self.transfer_request_submitter.submit(fileinfo) self.assertIsInstance(self.result_queue.get(), DryRunResult) def test_dry_run(self): self.cli_params['dryrun'] = True self.transfer_request_submitter = DownloadRequestSubmitter( self.transfer_manager, self.result_queue, self.cli_params) fileinfo = self.create_file_info(self.key) self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() self.assertIsInstance(result, DryRunResult) self.assertEqual(result.transfer_type, 'download') self.assertTrue(result.dest.endswith(self.filename)) self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key) def test_submit_move_adds_delete_source_subscriber(self): fileinfo = self.create_file_info(self.key) self.cli_params['guess_mime_type'] = True # Default settings self.cli_params['is_move'] = True self.transfer_request_submitter.submit(fileinfo) ref_subscribers = [ ProvideSizeSubscriber, DirectoryCreatorSubscriber, ProvideLastModifiedTimeSubscriber, DeleteSourceObjectSubscriber, DownloadResultSubscriber, ] download_call_kwargs = self.transfer_manager.download.call_args[1] actual_subscribers = download_call_kwargs['subscribers'] self.assertEqual(len(ref_subscribers), len(actual_subscribers)) for i, actual_subscriber in enumerate(actual_subscribers): self.assertIsInstance(actual_subscriber, ref_subscribers[i])
class TestDownloadRequestSubmitter(BaseTransferRequestSubmitterTest): def setUp(self): super(TestDownloadRequestSubmitter, self).setUp() self.transfer_request_submitter = DownloadRequestSubmitter( self.transfer_manager, self.result_queue, self.cli_params) def test_can_submit(self): fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename, operation_name='download') self.assertTrue( self.transfer_request_submitter.can_submit(fileinfo)) fileinfo.operation_name = 'foo' self.assertFalse( self.transfer_request_submitter.can_submit(fileinfo)) def test_submit(self): fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename) future = self.transfer_request_submitter.submit(fileinfo) self.assertIs(self.transfer_manager.download.return_value, future) download_call_kwargs = self.transfer_manager.download.call_args[1] self.assertEqual(download_call_kwargs['fileobj'], self.filename) self.assertEqual(download_call_kwargs['bucket'], self.bucket) self.assertEqual(download_call_kwargs['key'], self.key) self.assertEqual(download_call_kwargs['extra_args'], {}) # Make sure the subscriber applied are of the correct type and order ref_subscribers = [ ProvideSizeSubscriber, DirectoryCreatorSubscriber, ProvideLastModifiedTimeSubscriber, DownloadResultSubscriber ] actual_subscribers = download_call_kwargs['subscribers'] self.assertEqual(len(ref_subscribers), len(actual_subscribers)) for i, actual_subscriber in enumerate(actual_subscribers): self.assertIsInstance(actual_subscriber, ref_subscribers[i]) def test_submit_with_extra_args(self): fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename) self.cli_params['sse_c'] = 'AES256' self.cli_params['sse_c_key'] = 'mykey' self.transfer_request_submitter.submit(fileinfo) # Set some extra argument like sse_c to make sure cli # params get mapped to request parameters. download_call_kwargs = self.transfer_manager.download.call_args[1] self.assertEqual( download_call_kwargs['extra_args'], {'SSECustomerAlgorithm': 'AES256', 'SSECustomerKey': 'mykey'} ) def test_warn_glacier_for_incompatible(self): fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename, operation_name='download', associated_response_data={ 'StorageClass': 'GLACIER', } ) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have been submitted because it is a non-restored # glacier object. warning_result = self.result_queue.get() self.assertIsInstance(warning_result, WarningResult) self.assertIn( 'Unable to perform download operations on GLACIER objects', warning_result.message) # The transfer should have been skipped. self.assertIsNone(future) self.assertEqual(len(self.transfer_manager.download.call_args_list), 0) def test_not_warn_glacier_for_compatible(self): fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename, operation_name='download', associated_response_data={ 'StorageClass': 'GLACIER', 'Restore': 'ongoing-request="false"' } ) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have not been submitted because it is a restored # glacier object. self.assertTrue(self.result_queue.empty()) # And the transfer should not have been skipped. self.assertIs(self.transfer_manager.download.return_value, future) self.assertEqual(len(self.transfer_manager.download.call_args_list), 1) def test_warn_glacier_force_glacier(self): self.cli_params['force_glacier_transfer'] = True fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename, operation_name='download', associated_response_data={ 'StorageClass': 'GLACIER', } ) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have not been submitted because it is glacier # transfers were forced. self.assertTrue(self.result_queue.empty()) self.assertIs(self.transfer_manager.download.return_value, future) self.assertEqual(len(self.transfer_manager.download.call_args_list), 1) def test_warn_glacier_ignore_glacier_warnings(self): self.cli_params['ignore_glacier_warnings'] = True fileinfo = FileInfo( src=self.bucket+'/'+self.key, dest=self.filename, operation_name='download', associated_response_data={ 'StorageClass': 'GLACIER', } ) future = self.transfer_request_submitter.submit(fileinfo) # A warning should have not been submitted because it was specified # to ignore glacier warnings. self.assertTrue(self.result_queue.empty()) # But the transfer still should have been skipped. self.assertIsNone(future) self.assertEqual(len(self.transfer_manager.download.call_args_list), 0) def test_dry_run(self): self.cli_params['dryrun'] = True self.transfer_request_submitter = DownloadRequestSubmitter( self.transfer_manager, self.result_queue, self.cli_params) fileinfo = FileInfo( dest=self.filename, dest_type='local', operation_name='download', src=self.bucket + '/' + self.key, src_type='s3') self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() self.assertIsInstance(result, DryRunResult) self.assertEqual(result.transfer_type, 'download') self.assertTrue(result.dest.endswith(self.filename)) self.assertEqual(result.src, 's3://' + self.bucket + '/' + self.key) def test_submit_move_adds_delete_source_subscriber(self): fileinfo = FileInfo( dest=self.filename, src=self.bucket+'/'+self.key) self.cli_params['guess_mime_type'] = True # Default settings self.cli_params['is_move'] = True self.transfer_request_submitter.submit(fileinfo) ref_subscribers = [ ProvideSizeSubscriber, DirectoryCreatorSubscriber, ProvideLastModifiedTimeSubscriber, DeleteSourceObjectSubscriber, DownloadResultSubscriber, ] download_call_kwargs = self.transfer_manager.download.call_args[1] actual_subscribers = download_call_kwargs['subscribers'] self.assertEqual(len(ref_subscribers), len(actual_subscribers)) for i, actual_subscriber in enumerate(actual_subscribers): self.assertIsInstance(actual_subscriber, ref_subscribers[i])