def test_can_submit(self) -> None: """Test can_submit.""" fileinfo = FileInfo(src=self.filename, dest=None, operation_name="delete", src_type="local") assert self.transfer_request_submitter.can_submit(fileinfo) fileinfo.operation_name = "foo" assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_can_submit(self) -> None: """Test can_submit.""" fileinfo = FileInfo( src=self.filename, dest=self.bucket + "/" + self.key, operation_name="upload", ) assert self.transfer_request_submitter.can_submit(fileinfo) fileinfo.operation_name = "foo" assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_can_submit(self) -> None: """Test can_submit.""" fileinfo = FileInfo( src=self.bucket + "/" + self.key, dest=None, operation_name="delete", src_type="s3", ) assert self.transfer_request_submitter.can_submit(fileinfo) fileinfo.operation_name = "foo" assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_submit(self) -> None: """Test submit.""" fileinfo = FileInfo( src=self.source_bucket + "/" + self.source_key, dest=self.bucket + "/" + self.key, ) self.config_params["guess_mime_type"] = True future = self.transfer_request_submitter.submit(fileinfo) assert self.transfer_manager.copy.return_value is future call_kwargs = cast(Dict[str, Any], self.transfer_manager.copy.call_args[1]) assert call_kwargs["copy_source"] == { "Bucket": self.source_bucket, "Key": self.source_key, } assert call_kwargs["bucket"] == self.bucket assert call_kwargs["key"] == self.key assert call_kwargs["extra_args"] == {} ref_subscribers = [ ProvideSizeSubscriber, ProvideCopyContentTypeSubscriber, CopyResultSubscriber, ] actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_extra_args(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.filename, dest=self.bucket + "/" + self.key) self.config_params["storage_class"] = "STANDARD_IA" self.transfer_request_submitter.submit(fileinfo) call_kwargs = cast(Dict[str, Any], self.transfer_manager.upload.call_args[1]) assert call_kwargs["extra_args"] == {"StorageClass": "STANDARD_IA"}
def test_is_glacier_compatible_copy_download( self, expected: bool, storage_class: Literal["DEEP_ARCHIVE", "GLACIER", "INTELLIGENT_TIERING", "ONEZONE_IA", "OUTPOSTS", "REDUCED_REDUNDANCY", "STANDARD", "STANDARD_IA", ], ) -> None: """Test is_glacier_compatible.""" assert (FileInfo( "", response_data=build_object_data(storage_class=storage_class), operation_name="copy", ).is_glacier_compatible is expected) assert (FileInfo( "", response_data=build_object_data(storage_class=storage_class), operation_name="download", ).is_glacier_compatible is expected)
def test_can_submit(self) -> None: """Test can_submit.""" fileinfo = FileInfo( src=self.bucket + "/" + self.key, dest=self.filename, operation_name="download", ) assert self.transfer_request_submitter.can_submit(fileinfo) self.config_params["is_stream"] = False assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_warn_glacier_ignore_warning(self) -> None: """Test submit.""" self.config_params["ignore_glacier_warnings"] = True fileinfo = FileInfo( src=self.bucket + "/" + self.key, dest=self.filename, operation_name="download", response_data={"StorageClass": "GLACIER"}, ) future = self.transfer_request_submitter.submit(fileinfo) assert self.result_queue.empty() assert not future self.assert_no_downloads_happened()
def test_submit_warn_glacier_ignore_warning(self) -> None: """Test submit.""" self.config_params["ignore_glacier_warnings"] = True fileinfo = FileInfo( src=self.source_bucket + "/" + self.source_key, dest=self.bucket + "/" + self.key, operation_name="copy", response_data={"StorageClass": "GLACIER"}, ) future = self.transfer_request_submitter.submit(fileinfo) assert future is None assert self.result_queue.empty() assert len( self.transfer_manager.copy.call_args_list) == 0 # type: ignore
def test_submit_no_guess_content_mime_type(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.filename, dest=self.bucket + "/" + self.key) self.config_params["guess_mime_type"] = False self.transfer_request_submitter.submit(fileinfo) call_kwargs = cast(Dict[str, Any], self.transfer_manager.upload.call_args[1]) ref_subscribers = [ProvideSizeSubscriber, UploadResultSubscriber] actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_raise_stdin_missing(self, mocker: MockerFixture) -> None: """Test submit.""" mocker.patch("sys.stdin", None) fileinfo = FileInfo( src=self.filename, src_type="local", operation_name="upload", dest=self.bucket + "/" + self.key, dest_type="s3", ) with pytest.raises(StdinMissingError) as excinfo: self.transfer_request_submitter.submit(fileinfo) assert (excinfo.value.message == "stdin is required for this operation, but is not available")
def test_submit_content_type_specified(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.filename, dest=self.bucket + "/" + self.key) self.config_params["content_type"] = "text/plain" self.transfer_request_submitter.submit(fileinfo) call_kwargs = cast(Dict[str, Any], self.transfer_manager.upload.call_args[1]) assert call_kwargs["extra_args"] == {"ContentType": "text/plain"} ref_subscribers = [ProvideSizeSubscriber, UploadResultSubscriber] actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_warn_glacier_incompatible(self) -> None: """Test submit.""" fileinfo = FileInfo( src=self.bucket + "/" + self.key, dest=self.filename, operation_name="download", response_data={"StorageClass": "GLACIER"}, ) future = self.transfer_request_submitter.submit(fileinfo) warning_result = self.result_queue.get() assert isinstance(warning_result, PrintTask) assert ("Unable to perform download operations on GLACIER objects" in warning_result.message) assert not future self.assert_no_downloads_happened()
def test_submit_warn_too_large_transfer(self) -> None: """Test submit.""" fileinfo = FileInfo( src=self.filename, dest=self.bucket + "/" + self.key, size=MAX_UPLOAD_SIZE + 1, ) future = self.transfer_request_submitter.submit(fileinfo) warning_result = self.result_queue.get() assert isinstance(warning_result, PrintTask) assert "exceeds s3 upload limit" in warning_result.message assert self.transfer_manager.upload.return_value is future assert len( self.transfer_manager.upload.call_args_list) == 1 # type: ignore
def test_submit_expected_size_provided(self) -> None: """Test submit.""" provided_size = 100 self.config_params["expected_size"] = provided_size fileinfo = FileInfo(src=self.filename, dest=self.bucket + "/" + self.key) self.transfer_request_submitter.submit(fileinfo) call_kwargs = cast(Dict[str, Any], self.transfer_manager.upload.call_args[1]) ref_subscribers = [ProvideSizeSubscriber, UploadStreamResultSubscriber] actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i]) assert actual_subscribers[0].size == provided_size
def create_file_info( self, key: str, response_data: Optional[Dict[str, Any]] = None) -> FileInfo: """Create FileInfo.""" kwargs: Dict[str, Any] = { "src": self.bucket + "/" + key, "src_type": "s3", "dest": self.filename, "dest_type": "local", "operation_name": "download", "compare_key": key, } if response_data is not None: kwargs["response_data"] = response_data return FileInfo(**kwargs)
def test_init_default(self, tmp_path: Path) -> None: """Test __init__.""" obj = FileInfo(tmp_path) assert obj.src == tmp_path assert not obj.src_type assert not obj.operation_name assert not obj.client assert not obj.dest assert not obj.dest_type assert not obj.compare_key assert not obj.size assert obj.last_update == EPOCH_TIME assert obj.parameters == {} assert not obj.source_client assert not obj.is_stream assert not obj.associated_response_data
def test_dry_run(self) -> None: """Test submit.""" self.config_params["dryrun"] = True fileinfo = FileInfo( src=self.filename, src_type="local", dest=self.filename, dest_type="local", operation_name="delete", ) self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() assert isinstance(result, DryRunResult) assert result.transfer_type == "delete" assert result.src.endswith(self.filename) # type: ignore assert result.dest is None
def test_submit_warn_glacier_incompatible(self) -> None: """Test submit.""" fileinfo = FileInfo( src=self.source_bucket + "/" + self.source_key, dest=self.bucket + "/" + self.key, operation_name="copy", response_data={"StorageClass": "GLACIER"}, ) future = self.transfer_request_submitter.submit(fileinfo) warning_result = self.result_queue.get() assert isinstance(warning_result, PrintTask) assert ("Unable to perform copy operations on GLACIER objects" in warning_result.message) assert future is None assert len( self.transfer_manager.copy.call_args_list) == 0 # type: ignore
def test_submit_dry_run(self) -> None: """Test submit.""" self.config_params["dryrun"] = True fileinfo = FileInfo( src=self.filename, src_type="local", operation_name="upload", dest=self.bucket + "/" + self.key, dest_type="s3", ) self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() assert isinstance(result, DryRunResult) assert result.transfer_type == "upload" assert result.dest == "s3://" + self.bucket + "/" + self.key assert result.src == "-"
def test_dry_run_move(self) -> None: """Test submit.""" self.config_params["dryrun"] = True self.config_params["is_move"] = True self.transfer_request_submitter = UploadRequestSubmitter( self.transfer_manager, self.result_queue, self.config_params) fileinfo = FileInfo( src=self.filename, src_type="local", operation_name="upload", dest=self.bucket + "/" + self.key, dest_type="s3", ) self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() assert isinstance(result, DryRunResult) assert result.transfer_type == "move"
def test_call(self, mock_submitters: MockSubmitters, tmp_path: Path) -> None: """Test call.""" mock_submitters.instances["copy"].can_submit.return_value = True mock_submitters.instances["delete"].can_submit.return_value = True self.result_command_recorder.get_command_result.return_value = "success" # type: ignore handler = S3TransferHandler(self.transfer_manager, self.config_params, self.result_command_recorder) fileinfos = [FileInfo(src=tmp_path)] assert handler.call(fileinfos) == "success" # type: ignore mock_submitters.instances["copy"].can_submit.assert_called_once_with( fileinfos[0]) mock_submitters.instances["copy"].submit.assert_called_once_with( fileinfos[0]) self.result_command_recorder.notify_total_submissions.assert_called_once_with( 1) # type: ignore # noqa self.result_command_recorder.get_command_result.assert_called_once_with( ) # type: ignore
def test_submit_move_adds_delete_source_subscriber(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.filename, dest=self.bucket + "/" + self.key) self.config_params["guess_mime_type"] = True self.config_params["is_move"] = True self.transfer_request_submitter.submit(fileinfo) ref_subscribers = [ ProvideSizeSubscriber, ProvideUploadContentTypeSubscriber, DeleteSourceFileSubscriber, UploadResultSubscriber, ] call_kwargs = cast(Dict[str, Any], self.transfer_manager.upload.call_args[1]) actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.filename, dest=self.bucket + "/" + self.key) future = self.transfer_request_submitter.submit(fileinfo) assert self.transfer_manager.upload.return_value is future call_kwargs = cast(Dict[str, Any], self.transfer_manager.upload.call_args[1]) assert isinstance(call_kwargs["fileobj"], NonSeekableStream) assert call_kwargs["bucket"] == self.bucket assert call_kwargs["key"] == self.key assert call_kwargs["extra_args"] == {} ref_subscribers = [UploadStreamResultSubscriber] actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i])
def test_is_glacier_compatible_restored( self, storage_class: Literal["DEEP_ARCHIVE", "GLACIER", "INTELLIGENT_TIERING", "ONEZONE_IA", "OUTPOSTS", "REDUCED_REDUNDANCY", "STANDARD", "STANDARD_IA", ], ) -> None: """Test is_glacier_compatible.""" assert ( FileInfo( "", operation_name="copy", response_data={ # type: ignore "Restore": 'ongoing-request="false"', **build_object_data(storage_class=storage_class), }, ).is_glacier_compatible is True )
def test_submit(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.bucket + "/" + self.key, dest=None, operation_name="delete") future = self.transfer_request_submitter.submit(fileinfo) assert self.transfer_manager.delete.return_value is future call_kwargs = cast(Dict[str, Any], self.transfer_manager.delete.call_args[1]) assert call_kwargs["bucket"] == self.bucket assert call_kwargs["key"] == self.key assert call_kwargs["extra_args"] == {} ref_subscribers = [DeleteResultSubscriber] actual_subscribers = call_kwargs["subscribers"] assert len(ref_subscribers) == len(actual_subscribers) for i, actual_subscriber in enumerate(actual_subscribers): assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_dry_run(self) -> None: """Test submit.""" self.config_params["dryrun"] = True self.transfer_request_submitter = DeleteRequestSubmitter( self.transfer_manager, self.result_queue, self.config_params) fileinfo = FileInfo( src=self.bucket + "/" + self.key, src_type="s3", dest=self.bucket + "/" + self.key, dest_type="s3", operation_name="delete", ) self.transfer_request_submitter.submit(fileinfo) result = self.result_queue.get() assert isinstance(result, DryRunResult) assert result.transfer_type == "delete" assert result.src == "s3://" + self.bucket + "/" + self.key assert not result.dest
def test_submit_with_exception(self) -> None: """Test submit.""" fileinfo = FileInfo(src=self.filename, dest=None, operation_name="delete", src_type="local") result = self.transfer_request_submitter.submit(fileinfo) assert result queued_result = self.result_queue.get() assert isinstance(queued_result, QueuedResult) assert queued_result.transfer_type == "delete" assert queued_result.src.endswith(self.filename) # type: ignore assert queued_result.dest is None assert queued_result.total_transfer_size == 0 failure_result = self.result_queue.get() assert isinstance(failure_result, FailureResult) assert failure_result.transfer_type == "delete" assert failure_result.src.endswith(self.filename) # type: ignore assert failure_result.dest is None
def test_submit(self, tmp_path: Path) -> None: """Test submit.""" full_filename = tmp_path / self.filename full_filename.write_text("content") fileinfo = FileInfo(src=full_filename, dest=None, operation_name="delete", src_type="local") result = self.transfer_request_submitter.submit(fileinfo) assert result queued_result = self.result_queue.get() assert isinstance(queued_result, QueuedResult) assert queued_result.transfer_type == "delete" assert queued_result.src.endswith(self.filename) # type: ignore assert queued_result.dest is None assert queued_result.total_transfer_size == 0 failure_result = self.result_queue.get() assert isinstance(failure_result, SuccessResult) assert failure_result.transfer_type == "delete" assert failure_result.src.endswith(self.filename) # type: ignore assert failure_result.dest is None assert not full_filename.exists()
def test_is_glacier_compatible_no_response_data(self) -> None: """Test is_glacier_compatible.""" assert FileInfo("").is_glacier_compatible