def test__given_copy_instructions__when_error_during_copy__should_rollback_copied_files(
error_type):
copy_instructions = [
CopyInstruction("myfile.txt", "mycopy.txt"),
CopyInstruction("myfile.txt", "mycopy.txt"),
]
local_fs = filesystem_raising_on_copy(error_type, required_copy_calls=2)
ssh_fs = create_autospec(spec=Filesystem)
factory = MemoryFilesystemFactoryStub(local_fs, ssh_fs)
run_prepare_stage(factory, copy_instructions)
ssh_fs.delete.assert_called_with("mycopy.txt")
def test__given_collect_instructions__when_file_not_found__should_still_collect_remaining_files(
):
ssh_fs = MemoryFilesystemFake(files=["myfile.txt"])
factory = MemoryFilesystemFactoryStub(ssh_fs=ssh_fs)
files_to_collect = [
CopyInstruction("invalid", "_"),
CopyInstruction("myfile.txt", "collected.txt")
]
run_finalize_stage(factory, files_to_collect, [])
local_fs = factory.local_filesystem
assert local_fs.exists("collected.txt") is True
def test__given_collect_instructions__when_file_exists__should_still_collect_remaining_files(
):
local_fs = MemoryFilesystemFake(files=["existing.txt"])
ssh_fs = MemoryFilesystemFake(files=["myfile.txt"])
factory = MemoryFilesystemFactoryStub(local_fs, ssh_fs)
files_to_collect = [
CopyInstruction("myfile.txt", "existing.txt"),
CopyInstruction("myfile.txt", "collected.txt")
]
run_finalize_stage(factory, files_to_collect, [])
assert local_fs.exists("collected.txt") is True
def test__given_files_to_collect__when_collect__should_copy_to_source_fs():
source_fs_spy = new_mock_filesystem(["copy_file.txt"])
target_fs = new_mock_filesystem(["file.txt", "funny.gif"])
sut = EnvironmentPreparation(source_fs_spy, target_fs)
sut.files_to_collect([
CopyInstruction("file.txt", "copy_file.txt", True),
CopyInstruction("funny.gif", "copy_funny.gif", False),
])
sut.collect()
assert source_fs_spy.exists("copy_file.txt")
assert source_fs_spy.exists("copy_funny.gif")
def test__given_files_to_copy__when_preparing__should_copy_files():
source_fs = new_mock_filesystem(["file.txt", "funny.gif"])
target_fs = new_mock_filesystem(["evenfunnier.gif"])
sut = EnvironmentPreparation(source_fs, target_fs)
sut.files_to_copy([
CopyInstruction("file.txt", "filecopy.txt"),
CopyInstruction("funny.gif", "evenfunnier.gif", overwrite=True)
])
sut.prepare()
assert target_fs.exists("filecopy.txt")
assert target_fs.exists("evenfunnier.gif")
def test__given_files_to_collect_with_file_already_existing_on_source_fs__when_collecting__should_still_collect_remaining_files(
):
source_fs_spy = new_mock_filesystem(["copy_file.txt"])
target_fs = new_mock_filesystem(["file.txt", "funny.gif"])
sut = EnvironmentPreparation(source_fs_spy, target_fs)
sut.files_to_collect([
CopyInstruction("file.txt", "copy_file.txt", False),
CopyInstruction("funny.gif", "copy_funny.gif", False),
])
sut.collect()
assert source_fs_spy.exists("copy_funny.gif")
def _collect_copy_instructions(
copy_list: List[Dict[str, str]]) -> List[CopyInstruction]:
return [
CopyInstruction(os.path.expandvars(cp["from"]),
os.path.expandvars(cp["to"]),
bool(cp.get("overwrite", False))) for cp in copy_list
]
def test__given_config_with_files_to_copy__when_running__should_copy_files_to_remote_filesystem(
osfs_type_mock, sshfs_type_mock):
opts = launch_options(copy=[
CopyInstruction("myfile.txt", "mycopy.txt"),
CopyInstruction("otherfile.gif", "copy.gif")
])
osfs_type_mock.create("myfile.txt")
osfs_type_mock.create("otherfile.gif")
sut = make_sut(opts)
sut.run(opts)
assert sshfs_type_mock.exists(f"{HOME_DIR}/mycopy.txt")
assert sshfs_type_mock.exists(f"{HOME_DIR}/copy.gif")
def test__given_files_to_copy__but_not_preparing__should_not_do_anything():
source_fs = new_mock_filesystem(["file1.txt"])
target_fs = new_mock_filesystem()
sut = EnvironmentPreparation(source_fs, target_fs)
sut.files_to_copy([CopyInstruction("file1.txt", "file2.txt")])
assert target_fs.exists("file2.txt") is False
def test__given_files_to_collect_with_file_already_existing_on_source_fs__when_collecting__should_log_error_to_ui(
):
source_fs_spy = new_mock_filesystem(["copy_file.txt"])
target_fs = new_mock_filesystem(["file.txt", "funny.gif"])
ui_spy = MagicMock()
sut = EnvironmentPreparation(source_fs_spy, target_fs, ui_spy)
sut.files_to_collect([
CopyInstruction("file.txt", "copy_file.txt", False),
CopyInstruction("funny.gif", "copy_funny.gif", False),
])
sut.collect()
ui_spy.error.assert_called_with(
"FileExistsError: Cannot copy file 'file.txt'")
def test__given_config_with_non_existing_file_to_copy__when_running__should_print_to_ui(
osfs_type_mock):
opts = launch_options(copy=[
CopyInstruction("myfile.txt", "mycopy.txt"),
CopyInstruction("otherfile.gif", "copy.gif")
])
osfs_type_mock.return_value.create("myfile.txt")
ui_spy = Mock()
sut = make_sut(opts, ui_spy)
sut.run(opts)
assert call.error(
"FileNotFoundError: otherfile.gif") in ui_spy.method_calls
def test__given_config_with_non_existing_file_to_copy__when_running__should_perform_rollback_and_exit(
osfs_type_mock, sshfs_type_mock):
opts = launch_options(watch=True,
copy=[
CopyInstruction("myfile.txt", "mycopy.txt"),
CopyInstruction("otherfile.gif", "copy.gif")
])
osfs_type_mock.return_value.create("myfile.txt")
sut = make_sut(opts)
exit_code = sut.run(opts)
assert not sshfs_type_mock.return_value.exists(f"{HOME_DIR}/mycopy.txt")
assert not sshfs_type_mock.return_value.exists(f"{HOME_DIR}/copy.gif")
assert exit_code == 1
def test__given_copied_file_not_on_target_fs__when_rolling_back__should_remove_remaining_copied_files_from_target_fs(
):
source_fs_spy = new_mock_filesystem(["file.txt", "funny.gif"])
target_fs = new_mock_filesystem()
sut = EnvironmentPreparation(source_fs_spy, target_fs)
sut.files_to_copy([
CopyInstruction("file.txt", "filecopy.txt"),
CopyInstruction("funny.gif", "evenfunnier.gif")
])
sut.prepare()
target_fs.delete("filecopy.txt")
sut.rollback()
assert target_fs.exists("evenfunnier.gif") is False
def test__given_rollback_done__when_rolling_back_again__should_not_do_anything(
):
source_fs_spy = new_mock_filesystem(["file.txt", "funny.gif"])
target_fs = MagicMock(wraps=new_mock_filesystem())
sut = EnvironmentPreparation(source_fs_spy, target_fs)
sut.files_to_copy([
CopyInstruction("file.txt", "filecopy.txt"),
CopyInstruction("funny.gif", "evenfunnier.gif")
])
sut.prepare()
sut.rollback()
target_fs.reset_mock()
sut.rollback()
target_fs.delete.assert_not_called()
def test__given_config_with_files_to_collect__when_running__should_collect_files_from_remote_filesystem_after_completing_job_and_before_cleaning(
osfs_type_mock, sshfs_type_mock):
opts = launch_options(
watch=True,
copy=[CopyInstruction("myfile.txt", "mycopy.txt")],
clean=["mycopy.txt"],
collect=[CopyInstruction("mycopy.txt", "mycopy.txt")])
local_fs = osfs_type_mock.return_value
local_fs.create("myfile.txt")
sut = make_sut(opts)
sut.run(opts)
sshfs = sshfs_type_mock.return_value
assert local_fs.exists("mycopy.txt")
assert not sshfs.exists("mycopy.txt")
def test__given_valid_launch_args__should_return_matching_config():
config = parse_cli_args([
"launch",
"--watch",
"test/testconfig/config.yml"
])
assert config == LaunchOptions(
sbatch=REMOTE_SLURM_SCRIPT_PATH,
connection=CONNECTION_DATA,
proxyjumps=PROXYJUMPS,
copy_files=[
CopyInstruction("myfile.txt", "mycopy.txt"),
CopyInstruction(LOCAL_SLURM_SCRIPT_PATH, REMOTE_SLURM_SCRIPT_PATH, True)],
clean_files=["mycopy.txt", REMOTE_SLURM_SCRIPT_PATH],
collect_files=[CopyInstruction(REMOTE_RESULT_FILEPATH, "result.txt", True)],
watch=True
)
def test__given_files_to_copy_with_non_existing_file__when_preparing_then_rollback__should_remove_copied_files_from_target_fs(
):
source_fs_spy = new_mock_filesystem(["funny.gif"])
target_fs = new_mock_filesystem()
sut = EnvironmentPreparation(source_fs_spy, target_fs)
sut.files_to_copy([
CopyInstruction("file.txt", "filecopy.txt"),
CopyInstruction("funny.gif", "evenfunnier.gif")
])
with pytest.raises(FileNotFoundError):
sut.prepare()
sut.rollback()
assert target_fs.exists("filecopy.txt") is False
def test__given_copy_instructions__when_error_during_copy__should_return_false(
):
copy_instructions = [CopyInstruction("myfile.txt", "mycopy.txt")]
local_fs = filesystem_raising_on_copy(FileNotFoundError,
required_copy_calls=1)
factory = MemoryFilesystemFactoryStub(local_fs)
actual = run_prepare_stage(factory, copy_instructions)
assert actual == False
def test__given_collect_instruction__when_running__should_copy_collect_item_to_local_filesystem(
):
ssh_fs = MemoryFilesystemFake(files=["myfile.txt"])
factory = MemoryFilesystemFactoryStub(ssh_fs=ssh_fs)
run_finalize_stage(factory,
[CopyInstruction("myfile.txt", "collected.txt")], [])
local_fs = factory.local_filesystem
assert local_fs.exists("collected.txt") is True
def test__given_rollback_done_with_file_not_found__when_rolling_back_again__should_try_to_delete_remaining_files(
):
source_fs_spy = new_mock_filesystem(["file.txt", "funny.gif"])
target_fs_spy = MagicMock(wraps=new_mock_filesystem())
sut = EnvironmentPreparation(source_fs_spy, target_fs_spy)
sut.files_to_copy([
CopyInstruction("file.txt", "filecopy.txt"),
CopyInstruction("funny.gif", "evenfunnier.gif")
])
sut.prepare()
target_fs_spy.delete("filecopy.txt")
sut.rollback()
target_fs_spy.reset_mock()
sut.rollback()
target_fs_spy.delete.assert_has_calls([call("filecopy.txt")])
def test__given_copy_instructions__when_running__should_copy_files_to_remote_with_given_overwrite_settings(
):
copy_instructions = [CopyInstruction("myfile.txt", "mycopy.txt", True)]
local_fs_mock = create_autospec(spec=Filesystem)
factory = MemoryFilesystemFactoryStub(local_fs=local_fs_mock)
run_prepare_stage(factory, copy_instructions)
local_fs_mock.copy.assert_called_with(source="myfile.txt",
target="mycopy.txt",
overwrite=True,
filesystem=factory.ssh_filesystem)
def test__given_config_with_files_to_clean__when_running__should_remove_files_from_remote_filesystem(
osfs_type_mock, sshfs_type_mock):
opts = launch_options(watch=True,
copy=[CopyInstruction("myfile.txt", "mycopy.txt")],
clean=["mycopy.txt"])
osfs_type_mock.return_value.create("myfile.txt")
sut = make_sut(opts)
sut.run(opts)
assert not sshfs_type_mock.return_value.exists(f"{HOME_DIR}/mycopy.txt")