def test_can_submit_remote_deletes(self) -> None:
"""Test can_submit."""
fileinfo = FileInfo(src=self.filename,
dest=None,
operation_name="delete",
src_type="s3")
assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_submit(self) -> None:
"""Test submit."""
fileinfo = FileInfo(
src=self.source_bucket + "/" + self.source_key,
dest=self.bucket + "/" + self.key,
)
self.config_params["guess_mime_type"] = True
future = self.transfer_request_submitter.submit(fileinfo)
assert self.transfer_manager.copy.return_value is future
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.copy.call_args[1])
assert call_kwargs["copy_source"] == {
"Bucket": self.source_bucket,
"Key": self.source_key,
}
assert call_kwargs["bucket"] == self.bucket
assert call_kwargs["key"] == self.key
assert call_kwargs["extra_args"] == {}
ref_subscribers = [
ProvideSizeSubscriber,
ProvideCopyContentTypeSubscriber,
CopyResultSubscriber,
]
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
def test_can_submit_local_delete(self) -> None:
"""Test can_submit."""
fileinfo = FileInfo(
src=self.bucket + "/" + self.key,
dest=None,
operation_name="delete",
src_type="local",
)
assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_submit_extra_args(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.filename,
dest=self.bucket + "/" + self.key)
self.config_params["storage_class"] = "STANDARD_IA"
self.transfer_request_submitter.submit(fileinfo)
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.upload.call_args[1])
assert call_kwargs["extra_args"] == {"StorageClass": "STANDARD_IA"}
def test_is_glacier_compatible_copy_download(
self,
expected: bool,
storage_class: Literal["DEEP_ARCHIVE", "GLACIER",
"INTELLIGENT_TIERING", "ONEZONE_IA", "OUTPOSTS",
"REDUCED_REDUNDANCY", "STANDARD",
"STANDARD_IA", ],
) -> None:
"""Test is_glacier_compatible."""
assert (FileInfo(
"",
response_data=build_object_data(storage_class=storage_class),
operation_name="copy",
).is_glacier_compatible is expected)
assert (FileInfo(
"",
response_data=build_object_data(storage_class=storage_class),
operation_name="download",
).is_glacier_compatible is expected)
def test_can_submit(self) -> None:
"""Test can_submit."""
fileinfo = FileInfo(
src=self.bucket + "/" + self.key,
dest=self.filename,
operation_name="download",
)
assert self.transfer_request_submitter.can_submit(fileinfo)
self.config_params["is_stream"] = False
assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_can_submit(self) -> None:
"""Test can_submit."""
fileinfo = FileInfo(
src=self.filename,
dest=self.bucket + "/" + self.key,
operation_name="upload",
)
assert self.transfer_request_submitter.can_submit(fileinfo)
fileinfo.operation_name = "foo"
assert not self.transfer_request_submitter.can_submit(fileinfo)
def test_warn_glacier_ignore_warning(self) -> None:
"""Test submit."""
self.config_params["ignore_glacier_warnings"] = True
fileinfo = FileInfo(
src=self.bucket + "/" + self.key,
dest=self.filename,
operation_name="download",
response_data={"StorageClass": "GLACIER"},
)
future = self.transfer_request_submitter.submit(fileinfo)
assert self.result_queue.empty()
assert not future
self.assert_no_downloads_happened()
def test_submit_warn_glacier_ignore_warning(self) -> None:
"""Test submit."""
self.config_params["ignore_glacier_warnings"] = True
fileinfo = FileInfo(
src=self.source_bucket + "/" + self.source_key,
dest=self.bucket + "/" + self.key,
operation_name="copy",
response_data={"StorageClass": "GLACIER"},
)
future = self.transfer_request_submitter.submit(fileinfo)
assert future is None
assert self.result_queue.empty()
assert len(
self.transfer_manager.copy.call_args_list) == 0 # type: ignore
def test_submit_no_guess_content_mime_type(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.filename,
dest=self.bucket + "/" + self.key)
self.config_params["guess_mime_type"] = False
self.transfer_request_submitter.submit(fileinfo)
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.upload.call_args[1])
ref_subscribers = [ProvideSizeSubscriber, UploadResultSubscriber]
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_raise_stdin_missing(self, mocker: MockerFixture) -> None:
"""Test submit."""
mocker.patch("sys.stdin", None)
fileinfo = FileInfo(
src=self.filename,
src_type="local",
operation_name="upload",
dest=self.bucket + "/" + self.key,
dest_type="s3",
)
with pytest.raises(StdinMissingError) as excinfo:
self.transfer_request_submitter.submit(fileinfo)
assert (excinfo.value.message ==
"stdin is required for this operation, but is not available")
def test_submit_content_type_specified(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.filename,
dest=self.bucket + "/" + self.key)
self.config_params["content_type"] = "text/plain"
self.transfer_request_submitter.submit(fileinfo)
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.upload.call_args[1])
assert call_kwargs["extra_args"] == {"ContentType": "text/plain"}
ref_subscribers = [ProvideSizeSubscriber, UploadResultSubscriber]
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_warn_glacier_incompatible(self) -> None:
"""Test submit."""
fileinfo = FileInfo(
src=self.bucket + "/" + self.key,
dest=self.filename,
operation_name="download",
response_data={"StorageClass": "GLACIER"},
)
future = self.transfer_request_submitter.submit(fileinfo)
warning_result = self.result_queue.get()
assert isinstance(warning_result, PrintTask)
assert ("Unable to perform download operations on GLACIER objects"
in warning_result.message)
assert not future
self.assert_no_downloads_happened()
def test_submit_warn_too_large_transfer(self) -> None:
"""Test submit."""
fileinfo = FileInfo(
src=self.filename,
dest=self.bucket + "/" + self.key,
size=MAX_UPLOAD_SIZE + 1,
)
future = self.transfer_request_submitter.submit(fileinfo)
warning_result = self.result_queue.get()
assert isinstance(warning_result, PrintTask)
assert "exceeds s3 upload limit" in warning_result.message
assert self.transfer_manager.upload.return_value is future
assert len(
self.transfer_manager.upload.call_args_list) == 1 # type: ignore
def test_submit_expected_size_provided(self) -> None:
"""Test submit."""
provided_size = 100
self.config_params["expected_size"] = provided_size
fileinfo = FileInfo(src=self.filename,
dest=self.bucket + "/" + self.key)
self.transfer_request_submitter.submit(fileinfo)
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.upload.call_args[1])
ref_subscribers = [ProvideSizeSubscriber, UploadStreamResultSubscriber]
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
assert actual_subscribers[0].size == provided_size
def create_file_info(
self,
key: str,
response_data: Optional[Dict[str, Any]] = None) -> FileInfo:
"""Create FileInfo."""
kwargs: Dict[str, Any] = {
"src": self.bucket + "/" + key,
"src_type": "s3",
"dest": self.filename,
"dest_type": "local",
"operation_name": "download",
"compare_key": key,
}
if response_data is not None:
kwargs["response_data"] = response_data
return FileInfo(**kwargs)
def test_init_default(self, tmp_path: Path) -> None:
"""Test __init__."""
obj = FileInfo(tmp_path)
assert obj.src == tmp_path
assert not obj.src_type
assert not obj.operation_name
assert not obj.client
assert not obj.dest
assert not obj.dest_type
assert not obj.compare_key
assert not obj.size
assert obj.last_update == EPOCH_TIME
assert obj.parameters == {}
assert not obj.source_client
assert not obj.is_stream
assert not obj.associated_response_data
def test_dry_run(self) -> None:
"""Test submit."""
self.config_params["dryrun"] = True
fileinfo = FileInfo(
src=self.filename,
src_type="local",
dest=self.filename,
dest_type="local",
operation_name="delete",
)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
assert isinstance(result, DryRunResult)
assert result.transfer_type == "delete"
assert result.src.endswith(self.filename) # type: ignore
assert result.dest is None
def test_submit_warn_glacier_incompatible(self) -> None:
"""Test submit."""
fileinfo = FileInfo(
src=self.source_bucket + "/" + self.source_key,
dest=self.bucket + "/" + self.key,
operation_name="copy",
response_data={"StorageClass": "GLACIER"},
)
future = self.transfer_request_submitter.submit(fileinfo)
warning_result = self.result_queue.get()
assert isinstance(warning_result, PrintTask)
assert ("Unable to perform copy operations on GLACIER objects"
in warning_result.message)
assert future is None
assert len(
self.transfer_manager.copy.call_args_list) == 0 # type: ignore
def test_submit_dry_run(self) -> None:
"""Test submit."""
self.config_params["dryrun"] = True
fileinfo = FileInfo(
src=self.filename,
src_type="local",
operation_name="upload",
dest=self.bucket + "/" + self.key,
dest_type="s3",
)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
assert isinstance(result, DryRunResult)
assert result.transfer_type == "upload"
assert result.dest == "s3://" + self.bucket + "/" + self.key
assert result.src == "-"
def test_dry_run_move(self) -> None:
"""Test submit."""
self.config_params["dryrun"] = True
self.config_params["is_move"] = True
self.transfer_request_submitter = UploadRequestSubmitter(
self.transfer_manager, self.result_queue, self.config_params)
fileinfo = FileInfo(
src=self.filename,
src_type="local",
operation_name="upload",
dest=self.bucket + "/" + self.key,
dest_type="s3",
)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
assert isinstance(result, DryRunResult)
assert result.transfer_type == "move"
def test_call(self, mock_submitters: MockSubmitters,
tmp_path: Path) -> None:
"""Test call."""
mock_submitters.instances["copy"].can_submit.return_value = True
mock_submitters.instances["delete"].can_submit.return_value = True
self.result_command_recorder.get_command_result.return_value = "success" # type: ignore
handler = S3TransferHandler(self.transfer_manager, self.config_params,
self.result_command_recorder)
fileinfos = [FileInfo(src=tmp_path)]
assert handler.call(fileinfos) == "success" # type: ignore
mock_submitters.instances["copy"].can_submit.assert_called_once_with(
fileinfos[0])
mock_submitters.instances["copy"].submit.assert_called_once_with(
fileinfos[0])
self.result_command_recorder.notify_total_submissions.assert_called_once_with(
1) # type: ignore # noqa
self.result_command_recorder.get_command_result.assert_called_once_with(
) # type: ignore
def test_submit_move_adds_delete_source_subscriber(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.filename,
dest=self.bucket + "/" + self.key)
self.config_params["guess_mime_type"] = True
self.config_params["is_move"] = True
self.transfer_request_submitter.submit(fileinfo)
ref_subscribers = [
ProvideSizeSubscriber,
ProvideUploadContentTypeSubscriber,
DeleteSourceFileSubscriber,
UploadResultSubscriber,
]
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.upload.call_args[1])
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.filename,
dest=self.bucket + "/" + self.key)
future = self.transfer_request_submitter.submit(fileinfo)
assert self.transfer_manager.upload.return_value is future
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.upload.call_args[1])
assert isinstance(call_kwargs["fileobj"], NonSeekableStream)
assert call_kwargs["bucket"] == self.bucket
assert call_kwargs["key"] == self.key
assert call_kwargs["extra_args"] == {}
ref_subscribers = [UploadStreamResultSubscriber]
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
def test_is_glacier_compatible_restored(
self,
storage_class: Literal["DEEP_ARCHIVE", "GLACIER",
"INTELLIGENT_TIERING", "ONEZONE_IA", "OUTPOSTS",
"REDUCED_REDUNDANCY", "STANDARD",
"STANDARD_IA", ],
) -> None:
"""Test is_glacier_compatible."""
assert (
FileInfo(
"",
operation_name="copy",
response_data={ # type: ignore
"Restore": 'ongoing-request="false"',
**build_object_data(storage_class=storage_class),
},
).is_glacier_compatible
is True
)
def test_submit(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.bucket + "/" + self.key,
dest=None,
operation_name="delete")
future = self.transfer_request_submitter.submit(fileinfo)
assert self.transfer_manager.delete.return_value is future
call_kwargs = cast(Dict[str, Any],
self.transfer_manager.delete.call_args[1])
assert call_kwargs["bucket"] == self.bucket
assert call_kwargs["key"] == self.key
assert call_kwargs["extra_args"] == {}
ref_subscribers = [DeleteResultSubscriber]
actual_subscribers = call_kwargs["subscribers"]
assert len(ref_subscribers) == len(actual_subscribers)
for i, actual_subscriber in enumerate(actual_subscribers):
assert isinstance(actual_subscriber, ref_subscribers[i])
def test_submit_dry_run(self) -> None:
"""Test submit."""
self.config_params["dryrun"] = True
self.transfer_request_submitter = DeleteRequestSubmitter(
self.transfer_manager, self.result_queue, self.config_params)
fileinfo = FileInfo(
src=self.bucket + "/" + self.key,
src_type="s3",
dest=self.bucket + "/" + self.key,
dest_type="s3",
operation_name="delete",
)
self.transfer_request_submitter.submit(fileinfo)
result = self.result_queue.get()
assert isinstance(result, DryRunResult)
assert result.transfer_type == "delete"
assert result.src == "s3://" + self.bucket + "/" + self.key
assert not result.dest
def test_submit_with_exception(self) -> None:
"""Test submit."""
fileinfo = FileInfo(src=self.filename,
dest=None,
operation_name="delete",
src_type="local")
result = self.transfer_request_submitter.submit(fileinfo)
assert result
queued_result = self.result_queue.get()
assert isinstance(queued_result, QueuedResult)
assert queued_result.transfer_type == "delete"
assert queued_result.src.endswith(self.filename) # type: ignore
assert queued_result.dest is None
assert queued_result.total_transfer_size == 0
failure_result = self.result_queue.get()
assert isinstance(failure_result, FailureResult)
assert failure_result.transfer_type == "delete"
assert failure_result.src.endswith(self.filename) # type: ignore
assert failure_result.dest is None
def test_submit(self, tmp_path: Path) -> None:
"""Test submit."""
full_filename = tmp_path / self.filename
full_filename.write_text("content")
fileinfo = FileInfo(src=full_filename,
dest=None,
operation_name="delete",
src_type="local")
result = self.transfer_request_submitter.submit(fileinfo)
assert result
queued_result = self.result_queue.get()
assert isinstance(queued_result, QueuedResult)
assert queued_result.transfer_type == "delete"
assert queued_result.src.endswith(self.filename) # type: ignore
assert queued_result.dest is None
assert queued_result.total_transfer_size == 0
failure_result = self.result_queue.get()
assert isinstance(failure_result, SuccessResult)
assert failure_result.transfer_type == "delete"
assert failure_result.src.endswith(self.filename) # type: ignore
assert failure_result.dest is None
assert not full_filename.exists()
def test_is_glacier_compatible_no_response_data(self) -> None:
"""Test is_glacier_compatible."""
assert FileInfo("").is_glacier_compatible