def test_e2e_mkdir(helper: Helper) -> None: helper.run_cli( ["storage", "mkdir", "--parents", helper.tmpstorage + "folder"]) helper.check_dir_exists_on_storage("folder", "") # Create existing directory with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli(["storage", "mkdir", helper.tmpstorage + "folder"]) assert cm.value.returncode == EX_OSFILE helper.mkdir("folder", exist_ok=True) # Create a subdirectory in existing directory helper.run_cli( ["storage", "mkdir", helper.tmpstorage + "folder/subfolder"]) helper.check_dir_exists_on_storage("subfolder", "folder") # Create a subdirectory in non-existing directory with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli( ["storage", "mkdir", helper.tmpstorage + "parent/child"]) assert cm.value.returncode == EX_OSFILE helper.check_dir_absent_on_storage("parent", "") helper.run_cli( ["storage", "mkdir", "--parents", helper.tmpstorage + "parent/child"]) helper.check_dir_exists_on_storage("parent", "") helper.check_dir_exists_on_storage("child", "parent")
def test_e2e_copy_recursive_to_platform(helper: Helper, nested_data: Tuple[str, str, str], tmp_path: Path) -> None: helper.mkdir("") srcfile, checksum, dir_path = nested_data target_file_name = Path(srcfile).name # Upload local file captured = helper.run_cli( ["storage", "cp", "-r", dir_path, helper.tmpstorage]) # stderr has logs like "Using path ..." # assert not captured.err assert not captured.out helper.check_file_exists_on_storage(target_file_name, f"nested/directory/for/test", FILE_SIZE_B // 3) # Download into local directory and confirm checksum targetdir = tmp_path / "bar" targetdir.mkdir() helper.run_cli( ["storage", "cp", "-r", f"{helper.tmpstorage}", str(targetdir)]) targetfile = targetdir / "nested" / "directory" / "for" / "test" / target_file_name print("source file", srcfile) print("target file", targetfile) assert helper.hash_hex(targetfile) == checksum
def test_e2e_storage(data: Tuple[Path, str], tmp_path: Path, helper: Helper) -> None: srcfile, checksum = data # Create directory for the test helper.mkdir("folder", parents=True) # Upload local file helper.check_upload_file_to_storage("foo", "folder", str(srcfile)) # Confirm file has been uploaded helper.check_file_exists_on_storage("foo", "folder", FILE_SIZE_B) # Download into local file and confirm checksum helper.check_file_on_storage_checksum("foo", "folder", checksum, str(tmp_path), "bar") # Rename file on the storage helper.check_rename_file_on_storage("foo", "folder", "bar", "folder") helper.check_file_exists_on_storage("bar", "folder", FILE_SIZE_B) # Rename directory on the storage helper.check_rename_directory_on_storage("folder", "folder2") helper.check_file_exists_on_storage("bar", "folder2", FILE_SIZE_B) # Non-recursive removing should not have any effect with pytest.raises(IsADirectoryError, match=".+Target is a directory") as cm: helper.rm("folder2", recursive=False) assert cm.value.errno == errno.EISDIR helper.check_file_exists_on_storage("bar", "folder2", FILE_SIZE_B)
def test_e2e_rename(helper: Helper) -> None: helper.mkdir("folder", parents=True) helper.run_cli([ "storage", "mv", helper.tmpstorage + "folder", helper.tmpstorage + "otherfolder", ]) helper.check_dir_absent_on_storage("folder", "") helper.check_dir_exists_on_storage("otherfolder", "")
def test_e2e_glob(tmp_path: Path, helper: Helper) -> None: # Create files and directories and copy them to storage helper.mkdir("") folder = tmp_path / "folder" folder.mkdir() (folder / "subfolder").mkdir() (folder / "foo").write_bytes(b"foo") (folder / "bar").write_bytes(b"bar") (folder / "baz").write_bytes(b"baz") helper.run_cli([ "storage", "cp", "-r", tmp_path.as_uri() + "/f*", helper.tmpstorage + "/folder", ]) captured = helper.run_cli(["storage", "ls", helper.tmpstorage + "/folder"]) assert sorted( captured.out.splitlines()) == ["bar", "baz", "foo", "subfolder"] # Move files with pattern helper.run_cli([ "storage", "mv", helper.tmpstorage + "/folder/[bf]*", helper.tmpstorage + "/folder/subfolder", ]) captured = helper.run_cli( ["storage", "ls", helper.tmpstorage + "/folder/subfolder"]) assert sorted(captured.out.splitlines()) == ["bar", "baz", "foo"] # Download files with pattern download = tmp_path / "download" download.mkdir() helper.run_cli( ["storage", "cp", helper.tmpstorage + "/**/b*", str(download)]) assert sorted(download.iterdir()) == [download / "bar", download / "baz"] # Remove files with pattern helper.run_cli(["storage", "rm", helper.tmpstorage + "/**/b*"]) captured = helper.run_cli( ["storage", "ls", helper.tmpstorage + "/folder/subfolder"]) assert sorted(captured.out.splitlines()) == ["foo"] # Test subcommand "glob" captured = helper.run_cli(["storage", "glob", helper.tmpstorage + "/**"]) prefix = (f"storage://{helper.cluster_name}/{helper.username}/" f"{URL(helper.tmpstorage).path}") assert sorted(captured.out.splitlines()) == [ prefix, prefix + "folder", prefix + "folder/subfolder", prefix + "folder/subfolder/foo", ]
def test_copy_local_file_to_platform_directory(helper: Helper, data2: _Data) -> None: srcfile, checksum = data2 file_name = str(PurePath(srcfile).name) helper.mkdir("folder", parents=True) # Upload local file to existing directory helper.run_cli(["storage", "cp", srcfile, helper.tmpstorage + "/folder"]) # Ensure file is there helper.check_file_exists_on_storage(file_name, "folder", FILE_SIZE_B // 3)
def test_e2e_move_content_to_directory(helper: Helper) -> None: helper.mkdir("folder/subfolder", parents=True) helper.mkdir("otherfolder", parents=True) helper.run_cli([ "storage", "mv", "-T", helper.tmpstorage + "folder", helper.tmpstorage + "otherfolder", ]) helper.check_dir_absent_on_storage("folder", "") helper.check_dir_exists_on_storage("subfolder", "otherfolder")
def test_e2e_cp_filter(tmp_path: Path, helper: Helper) -> None: # Create files and directories and copy them to storage helper.mkdir("") folder = tmp_path / "folder" folder.mkdir() (folder / "subfolder").mkdir() (folder / "foo").write_bytes(b"foo") (folder / "bar").write_bytes(b"bar") (folder / "baz").write_bytes(b"baz") helper.run_cli([ "storage", "cp", "-r", "--exclude", "*", "--include", "b??", "--exclude", "*z", tmp_path.as_uri() + "/folder", helper.tmpstorage + "/filtered", ]) captured = helper.run_cli( ["storage", "ls", helper.tmpstorage + "/filtered"]) assert captured.out.splitlines() == ["bar"] # Copy all files to storage helper.run_cli([ "storage", "cp", "-r", tmp_path.as_uri() + "/folder", helper.tmpstorage + "/folder", ]) # Copy filtered files from storage helper.run_cli([ "storage", "cp", "-r", "--exclude", "*", "--include", "b??", "--exclude", "*z", helper.tmpstorage + "/folder", tmp_path.as_uri() + "/filtered", ]) assert os.listdir(tmp_path / "filtered") == ["bar"]
def test_e2e_ls_skip_hidden(tmp_path: Path, helper: Helper) -> None: # Create files and directories and copy them to storage helper.mkdir("") folder = tmp_path / "folder" folder.mkdir() (folder / "foo").write_bytes(b"foo") (folder / ".bar").write_bytes(b"bar") helper.run_cli([ "storage", "cp", "-r", tmp_path.as_uri() + "/folder", helper.tmpstorage ]) captured = helper.run_cli(["storage", "ls", helper.tmpstorage + "/folder"]) assert captured.out.splitlines() == ["foo"]
def test_e2e_copy_recursive_file(helper: Helper, tmp_path: Path) -> None: helper.mkdir("") srcfile = tmp_path / "testfile" dstfile = tmp_path / "copyfile" srcfile.write_bytes(b"abc") captured = helper.run_cli( ["storage", "cp", "-r", str(srcfile), helper.tmpstorage]) assert not captured.out captured = helper.run_cli( ["storage", "cp", "-r", helper.tmpstorage + "testfile", str(dstfile)]) assert not captured.out assert dstfile.read_bytes() == b"abc"
def test_copy_local_to_platform_single_file_3(helper: Helper, data: _Data) -> None: # case when copy happens with rename to 'different_name.txt' srcfile, checksum = data helper.mkdir("") # Upload local file to non existing directory with pytest.raises(subprocess.CalledProcessError, match=str(EX_OSFILE)): captured = helper.run_cli([ "storage", "cp", srcfile, helper.tmpstorage + "/non_existing_dir/" ]) assert not captured.err assert captured.out == "" # Ensure dir is not created helper.check_dir_absent_on_storage("non_existing_dir", "")
def test_copy_local_single_file_to_platform_file(helper: Helper, data: _Data) -> None: # case when copy happens with rename to 'different_name.txt' srcfile, checksum = data file_name = str(PurePath(srcfile).name) helper.mkdir("folder", parents=True) # Upload local file to platform helper.run_cli([ "storage", "cp", srcfile, helper.tmpstorage + "/folder/different_name.txt" ]) # Ensure file is there helper.check_file_exists_on_storage("different_name.txt", "folder", FILE_SIZE_B) helper.check_file_absent_on_storage(file_name, "folder")
def test_copy_and_remove_multiple_files(helper: Helper, data2: _Data, data3: _Data, tmp_path: Path) -> None: helper.mkdir("") # case when copy happens with rename to 'different_name.txt' srcfile, checksum = data2 srcfile2, checksum2 = data3 srcname = os.path.basename(srcfile) srcname2 = os.path.basename(srcfile2) # Upload local files captured = helper.run_cli( ["storage", "cp", srcfile, srcfile2, helper.tmpstorage]) assert captured.out == "" # Confirm files has been uploaded helper.check_file_exists_on_storage(srcname, "", FILE_SIZE_B // 3) helper.check_file_exists_on_storage(srcname2, "", FILE_SIZE_B // 5) # Download into local directory and confirm checksum targetdir = tmp_path / "bar" targetdir.mkdir() helper.run_cli([ "storage", "cp", f"{helper.tmpstorage}/{srcname}", f"{helper.tmpstorage}/{srcname2}", str(targetdir), ]) assert helper.hash_hex(targetdir / srcname) == checksum assert helper.hash_hex(targetdir / srcname2) == checksum2 # Remove the files from platform captured = helper.run_cli([ "storage", "rm", f"{helper.tmpstorage}/{srcname}", f"{helper.tmpstorage}/{srcname2}", ]) assert captured.out == "" # Ensure files are not there helper.check_file_absent_on_storage(srcname, "") helper.check_file_absent_on_storage(srcname2, "")
def test_e2e_no_glob(tmp_path: Path, helper: Helper) -> None: # Create files and directories and copy them to storage helper.mkdir("") dir = tmp_path / "[d]" dir.mkdir() (dir / "f").write_bytes(b"f") (dir / "[df]").write_bytes(b"[df]") helper.run_cli([ "storage", "cp", "-r", "--no-glob", tmp_path.as_uri() + "/[d]", helper.tmpstorage + "/d", ]) # Move files with literal path helper.run_cli([ "storage", "mv", "--no-glob", helper.tmpstorage + "/d/[df]", helper.tmpstorage ]) captured = helper.run_cli(["storage", "ls", helper.tmpstorage]) assert sorted(captured.out.splitlines()) == ["[df]", "d"] # Download files with literal path download = tmp_path / "download" download.mkdir() helper.run_cli([ "storage", "cp", "--no-glob", helper.tmpstorage + "/[df]", str(download) ]) assert sorted(download.iterdir()) == [download / "[df]"] # Remove files with literal path helper.run_cli(["storage", "rm", "--no-glob", helper.tmpstorage + "/[df]"]) captured = helper.run_cli(["storage", "ls", helper.tmpstorage]) assert sorted(captured.out.splitlines()) == ["d"]
def test_ls_directory_itself(helper: Helper) -> None: helper.mkdir("") captured = helper.run_cli( ["storage", "ls", "--directory", helper.tmpstorage]) _, _, name = helper.tmpstoragename.rpartition("/") assert captured.out.splitlines() == [name]
def test_empty_directory_ls_output(helper: Helper) -> None: helper.mkdir("") # Ensure output of ls - empty directory shall print nothing. captured = helper.run_cli(["storage", "ls", helper.tmpstorage]) assert not captured.out