def test__given_copy_instructions__when_error_during_copy__should_rollback_copied_files(
        error_type):
    copy_instructions = [
        CopyInstruction("myfile.txt", "mycopy.txt"),
        CopyInstruction("myfile.txt", "mycopy.txt"),
    ]

    local_fs = filesystem_raising_on_copy(error_type, required_copy_calls=2)
    ssh_fs = create_autospec(spec=Filesystem)
    factory = MemoryFilesystemFactoryStub(local_fs, ssh_fs)

    run_prepare_stage(factory, copy_instructions)

    ssh_fs.delete.assert_called_with("mycopy.txt")
Esempio n. 2
0
def test__given_collect_instructions__when_file_not_found__should_still_collect_remaining_files(
):
    ssh_fs = MemoryFilesystemFake(files=["myfile.txt"])
    factory = MemoryFilesystemFactoryStub(ssh_fs=ssh_fs)

    files_to_collect = [
        CopyInstruction("invalid", "_"),
        CopyInstruction("myfile.txt", "collected.txt")
    ]

    run_finalize_stage(factory, files_to_collect, [])

    local_fs = factory.local_filesystem
    assert local_fs.exists("collected.txt") is True
Esempio n. 3
0
def test__given_collect_instructions__when_file_exists__should_still_collect_remaining_files(
):
    local_fs = MemoryFilesystemFake(files=["existing.txt"])
    ssh_fs = MemoryFilesystemFake(files=["myfile.txt"])
    factory = MemoryFilesystemFactoryStub(local_fs, ssh_fs)

    files_to_collect = [
        CopyInstruction("myfile.txt", "existing.txt"),
        CopyInstruction("myfile.txt", "collected.txt")
    ]

    run_finalize_stage(factory, files_to_collect, [])

    assert local_fs.exists("collected.txt") is True
Esempio n. 4
0
def test__given_files_to_collect__when_collect__should_copy_to_source_fs():
    source_fs_spy = new_mock_filesystem(["copy_file.txt"])
    target_fs = new_mock_filesystem(["file.txt", "funny.gif"])

    sut = EnvironmentPreparation(source_fs_spy, target_fs)

    sut.files_to_collect([
        CopyInstruction("file.txt", "copy_file.txt", True),
        CopyInstruction("funny.gif", "copy_funny.gif", False),
    ])

    sut.collect()

    assert source_fs_spy.exists("copy_file.txt")
    assert source_fs_spy.exists("copy_funny.gif")
Esempio n. 5
0
def test__given_files_to_copy__when_preparing__should_copy_files():
    source_fs = new_mock_filesystem(["file.txt", "funny.gif"])
    target_fs = new_mock_filesystem(["evenfunnier.gif"])

    sut = EnvironmentPreparation(source_fs, target_fs)

    sut.files_to_copy([
        CopyInstruction("file.txt", "filecopy.txt"),
        CopyInstruction("funny.gif", "evenfunnier.gif", overwrite=True)
    ])

    sut.prepare()

    assert target_fs.exists("filecopy.txt")
    assert target_fs.exists("evenfunnier.gif")
Esempio n. 6
0
def test__given_files_to_collect_with_file_already_existing_on_source_fs__when_collecting__should_still_collect_remaining_files(
):
    source_fs_spy = new_mock_filesystem(["copy_file.txt"])
    target_fs = new_mock_filesystem(["file.txt", "funny.gif"])

    sut = EnvironmentPreparation(source_fs_spy, target_fs)

    sut.files_to_collect([
        CopyInstruction("file.txt", "copy_file.txt", False),
        CopyInstruction("funny.gif", "copy_funny.gif", False),
    ])

    sut.collect()

    assert source_fs_spy.exists("copy_funny.gif")
Esempio n. 7
0
def _collect_copy_instructions(
        copy_list: List[Dict[str, str]]) -> List[CopyInstruction]:
    return [
        CopyInstruction(os.path.expandvars(cp["from"]),
                        os.path.expandvars(cp["to"]),
                        bool(cp.get("overwrite", False))) for cp in copy_list
    ]
Esempio n. 8
0
def test__given_config_with_files_to_copy__when_running__should_copy_files_to_remote_filesystem(
        osfs_type_mock, sshfs_type_mock):
    opts = launch_options(copy=[
        CopyInstruction("myfile.txt", "mycopy.txt"),
        CopyInstruction("otherfile.gif", "copy.gif")
    ])

    osfs_type_mock.create("myfile.txt")
    osfs_type_mock.create("otherfile.gif")

    sut = make_sut(opts)

    sut.run(opts)

    assert sshfs_type_mock.exists(f"{HOME_DIR}/mycopy.txt")
    assert sshfs_type_mock.exists(f"{HOME_DIR}/copy.gif")
Esempio n. 9
0
def test__given_files_to_copy__but_not_preparing__should_not_do_anything():
    source_fs = new_mock_filesystem(["file1.txt"])
    target_fs = new_mock_filesystem()

    sut = EnvironmentPreparation(source_fs, target_fs)
    sut.files_to_copy([CopyInstruction("file1.txt", "file2.txt")])

    assert target_fs.exists("file2.txt") is False
Esempio n. 10
0
def test__given_files_to_collect_with_file_already_existing_on_source_fs__when_collecting__should_log_error_to_ui(
):
    source_fs_spy = new_mock_filesystem(["copy_file.txt"])
    target_fs = new_mock_filesystem(["file.txt", "funny.gif"])

    ui_spy = MagicMock()
    sut = EnvironmentPreparation(source_fs_spy, target_fs, ui_spy)

    sut.files_to_collect([
        CopyInstruction("file.txt", "copy_file.txt", False),
        CopyInstruction("funny.gif", "copy_funny.gif", False),
    ])

    sut.collect()

    ui_spy.error.assert_called_with(
        "FileExistsError: Cannot copy file 'file.txt'")
Esempio n. 11
0
def test__given_config_with_non_existing_file_to_copy__when_running__should_print_to_ui(
        osfs_type_mock):

    opts = launch_options(copy=[
        CopyInstruction("myfile.txt", "mycopy.txt"),
        CopyInstruction("otherfile.gif", "copy.gif")
    ])

    osfs_type_mock.return_value.create("myfile.txt")

    ui_spy = Mock()
    sut = make_sut(opts, ui_spy)

    sut.run(opts)

    assert call.error(
        "FileNotFoundError: otherfile.gif") in ui_spy.method_calls
Esempio n. 12
0
def test__given_config_with_non_existing_file_to_copy__when_running__should_perform_rollback_and_exit(
        osfs_type_mock, sshfs_type_mock):
    opts = launch_options(watch=True,
                          copy=[
                              CopyInstruction("myfile.txt", "mycopy.txt"),
                              CopyInstruction("otherfile.gif", "copy.gif")
                          ])

    osfs_type_mock.return_value.create("myfile.txt")

    sut = make_sut(opts)

    exit_code = sut.run(opts)

    assert not sshfs_type_mock.return_value.exists(f"{HOME_DIR}/mycopy.txt")
    assert not sshfs_type_mock.return_value.exists(f"{HOME_DIR}/copy.gif")
    assert exit_code == 1
Esempio n. 13
0
def test__given_copied_file_not_on_target_fs__when_rolling_back__should_remove_remaining_copied_files_from_target_fs(
):
    source_fs_spy = new_mock_filesystem(["file.txt", "funny.gif"])
    target_fs = new_mock_filesystem()

    sut = EnvironmentPreparation(source_fs_spy, target_fs)
    sut.files_to_copy([
        CopyInstruction("file.txt", "filecopy.txt"),
        CopyInstruction("funny.gif", "evenfunnier.gif")
    ])

    sut.prepare()
    target_fs.delete("filecopy.txt")

    sut.rollback()

    assert target_fs.exists("evenfunnier.gif") is False
Esempio n. 14
0
def test__given_rollback_done__when_rolling_back_again__should_not_do_anything(
):
    source_fs_spy = new_mock_filesystem(["file.txt", "funny.gif"])
    target_fs = MagicMock(wraps=new_mock_filesystem())

    sut = EnvironmentPreparation(source_fs_spy, target_fs)
    sut.files_to_copy([
        CopyInstruction("file.txt", "filecopy.txt"),
        CopyInstruction("funny.gif", "evenfunnier.gif")
    ])
    sut.prepare()
    sut.rollback()
    target_fs.reset_mock()

    sut.rollback()

    target_fs.delete.assert_not_called()
Esempio n. 15
0
def test__given_config_with_files_to_collect__when_running__should_collect_files_from_remote_filesystem_after_completing_job_and_before_cleaning(
        osfs_type_mock, sshfs_type_mock):
    opts = launch_options(
        watch=True,
        copy=[CopyInstruction("myfile.txt", "mycopy.txt")],
        clean=["mycopy.txt"],
        collect=[CopyInstruction("mycopy.txt", "mycopy.txt")])

    local_fs = osfs_type_mock.return_value
    local_fs.create("myfile.txt")

    sut = make_sut(opts)

    sut.run(opts)

    sshfs = sshfs_type_mock.return_value
    assert local_fs.exists("mycopy.txt")
    assert not sshfs.exists("mycopy.txt")
Esempio n. 16
0
def test__given_valid_launch_args__should_return_matching_config():
    config = parse_cli_args([
        "launch",
        "--watch",
        "test/testconfig/config.yml"
    ])

    assert config == LaunchOptions(
        sbatch=REMOTE_SLURM_SCRIPT_PATH,
        connection=CONNECTION_DATA,
        proxyjumps=PROXYJUMPS,
        copy_files=[
            CopyInstruction("myfile.txt", "mycopy.txt"),
            CopyInstruction(LOCAL_SLURM_SCRIPT_PATH, REMOTE_SLURM_SCRIPT_PATH, True)],
        clean_files=["mycopy.txt", REMOTE_SLURM_SCRIPT_PATH],
        collect_files=[CopyInstruction(REMOTE_RESULT_FILEPATH, "result.txt", True)],
        watch=True
    )
Esempio n. 17
0
def test__given_files_to_copy_with_non_existing_file__when_preparing_then_rollback__should_remove_copied_files_from_target_fs(
):
    source_fs_spy = new_mock_filesystem(["funny.gif"])
    target_fs = new_mock_filesystem()

    sut = EnvironmentPreparation(source_fs_spy, target_fs)

    sut.files_to_copy([
        CopyInstruction("file.txt", "filecopy.txt"),
        CopyInstruction("funny.gif", "evenfunnier.gif")
    ])

    with pytest.raises(FileNotFoundError):
        sut.prepare()

    sut.rollback()

    assert target_fs.exists("filecopy.txt") is False
def test__given_copy_instructions__when_error_during_copy__should_return_false(
):
    copy_instructions = [CopyInstruction("myfile.txt", "mycopy.txt")]
    local_fs = filesystem_raising_on_copy(FileNotFoundError,
                                          required_copy_calls=1)
    factory = MemoryFilesystemFactoryStub(local_fs)

    actual = run_prepare_stage(factory, copy_instructions)

    assert actual == False
Esempio n. 19
0
def test__given_collect_instruction__when_running__should_copy_collect_item_to_local_filesystem(
):
    ssh_fs = MemoryFilesystemFake(files=["myfile.txt"])
    factory = MemoryFilesystemFactoryStub(ssh_fs=ssh_fs)

    run_finalize_stage(factory,
                       [CopyInstruction("myfile.txt", "collected.txt")], [])

    local_fs = factory.local_filesystem
    assert local_fs.exists("collected.txt") is True
Esempio n. 20
0
def test__given_rollback_done_with_file_not_found__when_rolling_back_again__should_try_to_delete_remaining_files(
):
    source_fs_spy = new_mock_filesystem(["file.txt", "funny.gif"])
    target_fs_spy = MagicMock(wraps=new_mock_filesystem())

    sut = EnvironmentPreparation(source_fs_spy, target_fs_spy)
    sut.files_to_copy([
        CopyInstruction("file.txt", "filecopy.txt"),
        CopyInstruction("funny.gif", "evenfunnier.gif")
    ])

    sut.prepare()
    target_fs_spy.delete("filecopy.txt")
    sut.rollback()

    target_fs_spy.reset_mock()
    sut.rollback()

    target_fs_spy.delete.assert_has_calls([call("filecopy.txt")])
def test__given_copy_instructions__when_running__should_copy_files_to_remote_with_given_overwrite_settings(
):
    copy_instructions = [CopyInstruction("myfile.txt", "mycopy.txt", True)]

    local_fs_mock = create_autospec(spec=Filesystem)
    factory = MemoryFilesystemFactoryStub(local_fs=local_fs_mock)

    run_prepare_stage(factory, copy_instructions)

    local_fs_mock.copy.assert_called_with(source="myfile.txt",
                                          target="mycopy.txt",
                                          overwrite=True,
                                          filesystem=factory.ssh_filesystem)
Esempio n. 22
0
def test__given_config_with_files_to_clean__when_running__should_remove_files_from_remote_filesystem(
        osfs_type_mock, sshfs_type_mock):
    opts = launch_options(watch=True,
                          copy=[CopyInstruction("myfile.txt", "mycopy.txt")],
                          clean=["mycopy.txt"])

    osfs_type_mock.return_value.create("myfile.txt")

    sut = make_sut(opts)

    sut.run(opts)

    assert not sshfs_type_mock.return_value.exists(f"{HOME_DIR}/mycopy.txt")