def test_e2e_mkdir(helper: Helper) -> None: helper.run_cli( ["storage", "mkdir", "--parents", helper.tmpstorage + "folder"]) helper.check_dir_exists_on_storage("folder", "") # Create existing directory with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli(["storage", "mkdir", helper.tmpstorage + "folder"]) assert cm.value.returncode == EX_OSFILE helper.mkdir("folder", exist_ok=True) # Create a subdirectory in existing directory helper.run_cli( ["storage", "mkdir", helper.tmpstorage + "folder/subfolder"]) helper.check_dir_exists_on_storage("subfolder", "folder") # Create a subdirectory in non-existing directory with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli( ["storage", "mkdir", helper.tmpstorage + "parent/child"]) assert cm.value.returncode == EX_OSFILE helper.check_dir_absent_on_storage("parent", "") helper.run_cli( ["storage", "mkdir", "--parents", helper.tmpstorage + "parent/child"]) helper.check_dir_exists_on_storage("parent", "") helper.check_dir_exists_on_storage("child", "parent")
def test_images_complete_lifecycle( helper: Helper, image: str, tag: str, loop: asyncio.AbstractEventLoop, docker: aiodocker.Docker, ) -> None: # Let`s push image captured = helper.run_cli(["image", "push", image]) # stderr has "Used image ..." lines # assert not captured.err image_full_str = f"image://{helper.cluster_name}/{helper.username}/{image}" assert captured.out.endswith(image_full_str) image_url = URL(image_full_str) # Check if image available on registry captured = helper.run_cli(["image", "ls", "--full-uri"]) image_urls = [URL(line) for line in captured.out.splitlines() if line] for url in image_urls: assert url.scheme == "image" image_url_without_tag = image_url.with_path( image_url.path.replace(f":{tag}", "")) assert image_url_without_tag in image_urls # delete local loop.run_until_complete(docker.images.delete(image, force=True)) docker_ls_output = loop.run_until_complete(docker.images.list()) local_images = parse_docker_ls_output(docker_ls_output) assert image not in local_images # Pull image as with another tag captured = helper.run_cli(["image", "pull", f"image:{image}"]) # stderr has "Used image ..." lines # assert not captured.err assert captured.out.endswith(image) # check pulled locally, delete for cleanup docker_ls_output = loop.run_until_complete(docker.images.list()) local_images = parse_docker_ls_output(docker_ls_output) assert image in local_images # Execute image and check result captured = helper.run_cli([ "-q", "submit", *JOB_TINY_CONTAINER_PARAMS, "--non-preemptible", "--no-wait-start", str(image_url), ]) assert not captured.err job_id = captured.out assert job_id.startswith("job-") helper.wait_job_change_state_to(job_id, JobStatus.SUCCEEDED, JobStatus.FAILED) helper.check_job_output(job_id, re.escape(tag))
def test_image_tags(helper: Helper, image: str, tag: str) -> None: # push image captured = helper.run_cli(["image", "push", image]) image_full_str = f"image://{helper.username}/{image}" assert captured.out.endswith(image_full_str) # check the tag is present now image_full_str_no_tag = image_full_str.replace(f":{tag}", "") captured = helper.run_cli(["image", "tags", image_full_str_no_tag]) assert tag in captured.out cmd = f"neuro image tags {image_full_str}" result = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) assertion_msg = f"Command {cmd} should fail: {result.stdout} {result.stderr}" assert result.returncode, assertion_msg image_full_str_latest_tag = image_full_str.replace(f":{tag}", ":latest") cmd = f"neuro image tags {image_full_str_latest_tag}" result = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) assertion_msg = f"Command {cmd} should fail: {result.stdout} {result.stderr}" assert result.returncode, assertion_msg
def test_remove_cluster_user_does_not_exist(helper: Helper) -> None: username = "******" with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli( ["admin", "remove-cluster-user", helper.cluster_name, username]) assert cm.value.returncode == 72 assert f"User 'some-clearly-invalid-username' not found" in cm.value.stderr
def test_e2e_copy_recursive_to_platform(helper: Helper, nested_data: Tuple[str, str, str], tmp_path: Path) -> None: helper.mkdir("") srcfile, checksum, dir_path = nested_data target_file_name = Path(srcfile).name # Upload local file captured = helper.run_cli( ["storage", "cp", "-r", dir_path, helper.tmpstorage]) # stderr has logs like "Using path ..." # assert not captured.err assert not captured.out helper.check_file_exists_on_storage(target_file_name, f"nested/directory/for/test", FILE_SIZE_B // 3) # Download into local directory and confirm checksum targetdir = tmp_path / "bar" targetdir.mkdir() helper.run_cli( ["storage", "cp", "-r", f"{helper.tmpstorage}", str(targetdir)]) targetfile = targetdir / "nested" / "directory" / "for" / "test" / target_file_name print("source file", srcfile) print("target file", targetfile) assert helper.hash_hex(targetfile) == checksum
def test_tree(helper: Helper, data: _Data, tmp_path: Path) -> None: folder = tmp_path / "folder" folder.mkdir() (folder / "foo").write_bytes(b"foo") (folder / "bar").write_bytes(b"bar") subfolder = folder / "folder" subfolder.mkdir() (subfolder / "baz").write_bytes(b"baz") helper.run_cli(["storage", "cp", "-r", folder.as_uri(), helper.tmpstorage]) capture = helper.run_cli(["storage", "tree", helper.tmpstorage]) assert capture.err == "" expected = textwrap.dedent(f"""\ '{helper.tmpstorage}' ├── 'bar' ├── 'folder' │ └── 'baz' └── 'foo' 1 directories, 3 files""") if sys.platform == "win32": trans = str.maketrans("".join(TreeFormatter.ANSI_DELIMS), "".join(TreeFormatter.SIMPLE_DELIMS)) expected = expected.translate(trans) assert capture.out == expected
def test_revoke_no_effect(helper: Helper) -> None: uri = f"storage://{helper.username}/{uuid4()}" with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli(["-v", "acl", "revoke", uri, "public"]) assert cm.value.returncode == 127 assert "Operation has no effect" in cm.value.stderr assert f"Using resource '{uri}'" in cm.value.stderr
def test_docker_helper(helper: Helper, image: str, tag: str, nmrc_path: Path, monkeypatch: Any) -> None: monkeypatch.setenv(CONFIG_ENV_NAME, str(nmrc_path or DEFAULT_CONFIG_PATH)) helper.run_cli(["config", "docker"]) registry = helper.registry_url.host username = helper.username full_tag = f"{registry}/{username}/{image}" tag_cmd = f"docker tag {image} {full_tag}" result = subprocess.run(tag_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) assert ( not result.returncode ), f"Command {tag_cmd} failed: {result.stdout!r} {result.stderr!r} " push_cmd = f"docker push {full_tag}" result = subprocess.run(push_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) assert ( not result.returncode ), f"Command {push_cmd} failed: {result.stdout!r} {result.stderr!r} " # Run image and check output image_url = f"image://{helper.cluster_name}/{username}/{image}" job_id = helper.run_job_and_wait_state(image_url, "", wait_state=JobStatus.SUCCEEDED, stop_state=JobStatus.FAILED) helper.check_job_output(job_id, re.escape(tag))
def test_e2e_copy_target_directory_no_target_directory(helper: Helper, tmp_path: Path) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli( ["storage", "cp", "-t", helper.tmpstorage, "-T", str(tmp_path)]) assert "Cannot combine" in cm.value.stderr
def test_e2e_copy_non_existing_platform_to_____existing_local( helper: Helper, tmp_path: Path) -> None: # Try downloading non existing file with pytest.raises(subprocess.CalledProcessError, match=str(EX_OSFILE)): helper.run_cli( ["storage", "cp", helper.tmpstorage + "/foo", str(tmp_path)])
def test_grant_image_with_tag_fails(request: Any, helper: Helper) -> None: uri = f"image://{helper.username}/{uuid4()}:latest" another_test_user = "******" with pytest.raises(subprocess.CalledProcessError) as cm: request.addfinalizer(lambda: revoke(helper, uri, another_test_user)) helper.run_cli(["acl", "grant", uri, another_test_user, "read"]) assert cm.value.returncode == 127 assert "tag is not allowed" in cm.value.stderr
def test_e2e_copy_no_target_directory_extra_operand(helper: Helper, tmp_path: Path) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "storage", "cp", "-T", str(tmp_path), helper.tmpstorage, str(tmp_path) ]) assert "Extra operand after " in cm.value.stderr
def test_remove_cluster_user_remove_oneself(helper: Helper) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "admin", "remove-cluster-user", helper.cluster_name, helper.username ]) assert cm.value.returncode == 65 assert ("Illegal argument(s) (Cluster users cannot remove themselves)" in cm.value.stderr)
def test_add_cluster_user_invalid_role(helper: Helper) -> None: username = "******" with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "admin", "add-cluster-user", helper.cluster_name, username, "my_role" ]) assert cm.value.returncode == 2 assert "invalid choice:" in cm.value.stderr assert "(choose from admin, manager, user)" in cm.value.stderr
def test_e2e_rename(helper: Helper) -> None: helper.mkdir("folder", parents=True) helper.run_cli([ "storage", "mv", helper.tmpstorage + "folder", helper.tmpstorage + "otherfolder", ]) helper.check_dir_absent_on_storage("folder", "") helper.check_dir_exists_on_storage("otherfolder", "")
def test_add_cluster_user_already_exists(helper: Helper) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "admin", "add-cluster-user", helper.cluster_name, helper.username, "user" ]) assert cm.value.returncode == 65 assert ( f"Illegal argument(s) (User '{helper.username}' already exists in cluster " f"'{helper.cluster_name}')" in cm.value.stderr)
def test_e2e_move_no_target_directory_extra_operand(helper: Helper) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "storage", "mv", "-T", helper.tmpstorage + "/foo", helper.tmpstorage + "/bar", helper.tmpstorage + "/baz", ]) assert "Extra operand after " in cm.value.stderr
def test_e2e_move_target_directory_no_target_directory(helper: Helper) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "storage", "mv", "-t", helper.tmpstorage + "/foo", "-T", helper.tmpstorage + "/bar", ]) assert "Cannot combine" in cm.value.stderr
def test_copy_local_file_to_platform_directory(helper: Helper, data2: _Data) -> None: srcfile, checksum = data2 file_name = str(PurePath(srcfile).name) helper.mkdir("folder", parents=True) # Upload local file to existing directory helper.run_cli(["storage", "cp", srcfile, helper.tmpstorage + "/folder"]) # Ensure file is there helper.check_file_exists_on_storage(file_name, "folder", FILE_SIZE_B // 3)
def test_e2e_move_content_to_directory(helper: Helper) -> None: helper.mkdir("folder/subfolder", parents=True) helper.mkdir("otherfolder", parents=True) helper.run_cli([ "storage", "mv", "-T", helper.tmpstorage + "folder", helper.tmpstorage + "otherfolder", ]) helper.check_dir_absent_on_storage("folder", "") helper.check_dir_exists_on_storage("subfolder", "otherfolder")
def test_e2e_blob_storage_copy_file_explicit_directory( helper: Helper, data: _Data, tmp_bucket: str ) -> None: srcfile, checksum = data file_name = str(PurePath(srcfile).name) key = f"folder/{file_name}" # Upload local file to existing directory with explocit -t param helper.run_cli(["blob", "cp", "-t", f"blob:{tmp_bucket}/folder", srcfile]) # Ensure file is there helper.check_blob_size(tmp_bucket, key, FILE_SIZE_B)
def test_e2e_blob_storage_copy_file_implicit_directory( helper: Helper, data: _Data, tmp_bucket: str ) -> None: srcfile, checksum = data file_name = str(PurePath(srcfile).name) key = f"folder/{file_name}" # Upload local file to a directory defined by trailing slash helper.run_cli(["blob", "cp", srcfile, f"blob:{tmp_bucket}/folder/"]) # Ensure file is there helper.check_blob_size(tmp_bucket, key, FILE_SIZE_B)
def test_e2e_ls_skip_hidden(tmp_path: Path, helper: Helper) -> None: # Create files and directories and copy them to storage helper.mkdir("") folder = tmp_path / "folder" folder.mkdir() (folder / "foo").write_bytes(b"foo") (folder / ".bar").write_bytes(b"bar") helper.run_cli([ "storage", "cp", "-r", tmp_path.as_uri() + "/folder", helper.tmpstorage ]) captured = helper.run_cli(["storage", "ls", helper.tmpstorage + "/folder"]) assert captured.out.splitlines() == ["foo"]
def test_e2e_blob_storage_copy_recursive_file( helper: Helper, nested_data: Tuple[str, str, str], tmp_path: Path, tmp_bucket: str ) -> None: srcfile = tmp_path / "testfile" dstfile = tmp_path / "copyfile" srcfile.write_bytes(b"abc") captured = helper.run_cli(["blob", "cp", "-r", str(srcfile), f"blob:{tmp_bucket}"]) assert not captured.out captured = helper.run_cli( ["blob", "cp", "-r", f"blob:{tmp_bucket}/testfile", str(dstfile)] ) assert not captured.out assert dstfile.read_bytes() == b"abc"
def test_images_push_with_specified_name( helper: Helper, image: str, tag: str, loop: asyncio.AbstractEventLoop, docker: aiodocker.Docker, ) -> None: # Let`s push image image_no_tag = image.replace(f":{tag}", "") pushed_no_tag = f"{image_no_tag}-pushed" pulled_no_tag = f"{image_no_tag}-pulled" pulled = f"{pulled_no_tag}:{tag}" captured = helper.run_cli( ["image", "push", image, f"image:{pushed_no_tag}:{tag}"]) # stderr has "Used image ..." lines # assert not captured.err image_pushed_full_str = ( f"image://{helper.cluster_name}/{helper.username}/{pushed_no_tag}:{tag}" ) assert captured.out.endswith(image_pushed_full_str) image_url_without_tag = image_pushed_full_str.replace(f":{tag}", "") # Check if image available on registry captured = helper.run_cli(["image", "ls", "--full-uri"]) image_urls = captured.out.splitlines() assert image_url_without_tag in image_urls # check locally docker_ls_output = loop.run_until_complete(docker.images.list()) local_images = parse_docker_ls_output(docker_ls_output) assert pulled not in local_images # Pull image as with another name captured = helper.run_cli( ["image", "pull", f"image:{pushed_no_tag}:{tag}", pulled]) # stderr has "Used image ..." lines # assert not captured.err assert captured.out.endswith(pulled) # check locally docker_ls_output = loop.run_until_complete(docker.images.list()) local_images = parse_docker_ls_output(docker_ls_output) assert pulled in local_images # TODO (A.Yushkovskiy): delete the pushed image in GCR # delete locally loop.run_until_complete(docker.images.delete(pulled, force=True))
def test_e2e_copy_recursive_file(helper: Helper, tmp_path: Path) -> None: helper.mkdir("") srcfile = tmp_path / "testfile" dstfile = tmp_path / "copyfile" srcfile.write_bytes(b"abc") captured = helper.run_cli( ["storage", "cp", "-r", str(srcfile), helper.tmpstorage]) assert not captured.out captured = helper.run_cli( ["storage", "cp", "-r", helper.tmpstorage + "testfile", str(dstfile)]) assert not captured.out assert dstfile.read_bytes() == b"abc"
def test_grant_image_no_tag(request: Any, helper: Helper) -> None: rel_path = str(uuid4()) rel_uri = f"image:{rel_path}" uri = f"image://{helper.username}/{rel_path}" another_test_user = "******" request.addfinalizer(lambda: revoke(helper, rel_uri, another_test_user)) captured = helper.run_cli( ["-v", "acl", "grant", rel_uri, another_test_user, "read"]) assert captured.out == "" expected_err = f"Using resource '{uri}'" assert expected_err in captured.err captured = helper.run_cli( ["-v", "acl", "revoke", rel_uri, another_test_user]) assert captured.out == "" assert expected_err in captured.err
def test_copy_local_single_file_to_platform_file(helper: Helper, data: _Data) -> None: # case when copy happens with rename to 'different_name.txt' srcfile, checksum = data file_name = str(PurePath(srcfile).name) helper.mkdir("folder", parents=True) # Upload local file to platform helper.run_cli([ "storage", "cp", srcfile, helper.tmpstorage + "/folder/different_name.txt" ]) # Ensure file is there helper.check_file_exists_on_storage("different_name.txt", "folder", FILE_SIZE_B) helper.check_file_absent_on_storage(file_name, "folder")
def test_list_role(request: Any, helper: Helper) -> None: captured = helper.run_cli(["acl", "list", "-s", "role"]) assert captured.err == "" result = captured.out.splitlines() self_role_uri = f"role://{helper.username}" role = helper.username for line in result: uri, *_ = line.split() assert uri.startswith("role://") if uri != self_role_uri: role = uri[len("role://"):] print(f"Test using role {role!r}") captured = helper.run_cli(["acl", "list", "-u", role]) assert captured.err == "" captured = helper.run_cli(["acl", "list", "-u", role, "--shared"]) assert captured.err == ""
def test_e2e_blob_storage_cp_filter( helper: Helper, nested_data: Tuple[str, str, str], tmp_path: Path, tmp_bucket: str ) -> None: # Create files and directories and copy them to storage folder = tmp_path / "folder" folder.mkdir() (folder / "subfolder").mkdir() (folder / "foo").write_bytes(b"foo") (folder / "bar").write_bytes(b"bar") (folder / "baz").write_bytes(b"baz") helper.run_cli( [ "blob", "cp", "-r", "--exclude", "*", "--include", "b??", "--exclude", "*z", tmp_path.as_uri() + "/folder", f"blob:{tmp_bucket}/filtered", ] ) captured = helper.run_cli(["blob", "ls", f"blob:{tmp_bucket}/filtered/"]) prefix = f"blob:{tmp_bucket}/filtered/" assert sorted(captured.out.splitlines()) == [prefix, prefix + "bar"] # Copy all files to storage helper.run_cli( [ "blob", "cp", "-r", tmp_path.as_uri() + "/folder", f"blob:{tmp_bucket}/folder", ] ) # Copy filtered files from storage helper.run_cli( [ "blob", "cp", "-r", "--exclude", "*", "--include", "b??", "--exclude", "*z", f"blob:{tmp_bucket}" + "/folder", tmp_path.as_uri() + "/filtered", ] ) assert os.listdir(tmp_path / "filtered") == ["bar"]