def test_image_tags(helper: Helper, image: str, tag: str) -> None:
# push image
captured = helper.run_cli(["image", "push", image])
image_full_str = f"image://{helper.username}/{image}"
assert captured.out.endswith(image_full_str)
# check the tag is present now
image_full_str_no_tag = image_full_str.replace(f":{tag}", "")
captured = helper.run_cli(["image", "tags", image_full_str_no_tag])
assert tag in captured.out
cmd = f"neuro image tags {image_full_str}"
result = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
)
assertion_msg = f"Command {cmd} should fail: {result.stdout} {result.stderr}"
assert result.returncode, assertion_msg
image_full_str_latest_tag = image_full_str.replace(f":{tag}", ":latest")
cmd = f"neuro image tags {image_full_str_latest_tag}"
result = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
)
assertion_msg = f"Command {cmd} should fail: {result.stdout} {result.stderr}"
assert result.returncode, assertion_msg
def test_e2e_copy_target_directory_no_target_directory(helper: Helper,
tmp_path: Path) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli(
["storage", "cp", "-t", helper.tmpstorage, "-T",
str(tmp_path)])
assert "Cannot combine" in cm.value.stderr
def test_remove_cluster_user_does_not_exist(helper: Helper) -> None:
username = "******"
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli(
["admin", "remove-cluster-user", helper.cluster_name, username])
assert cm.value.returncode == 72
assert f"User 'some-clearly-invalid-username' not found" in cm.value.stderr
def test_revoke_no_effect(helper: Helper) -> None:
uri = f"storage://{helper.username}/{uuid4()}"
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli(["-v", "acl", "revoke", uri, "public"])
assert cm.value.returncode == 127
assert "Operation has no effect" in cm.value.stderr
assert f"Using resource '{uri}'" in cm.value.stderr
def test_e2e_copy_non_existing_platform_to_____existing_local(
helper: Helper, tmp_path: Path) -> None:
# Try downloading non existing file
with pytest.raises(subprocess.CalledProcessError, match=str(EX_OSFILE)):
helper.run_cli(
["storage", "cp", helper.tmpstorage + "/foo",
str(tmp_path)])
def test_docker_helper(helper: Helper, image: str, tag: str, nmrc_path: Path,
monkeypatch: Any) -> None:
monkeypatch.setenv(CONFIG_ENV_NAME, str(nmrc_path or DEFAULT_CONFIG_PATH))
helper.run_cli(["config", "docker"])
registry = helper.registry_url.host
username = helper.username
full_tag = f"{registry}/{username}/{image}"
tag_cmd = f"docker tag {image} {full_tag}"
result = subprocess.run(tag_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
assert (
not result.returncode
), f"Command {tag_cmd} failed: {result.stdout!r} {result.stderr!r} "
push_cmd = f"docker push {full_tag}"
result = subprocess.run(push_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
assert (
not result.returncode
), f"Command {push_cmd} failed: {result.stdout!r} {result.stderr!r} "
# Run image and check output
image_url = f"image://{helper.cluster_name}/{username}/{image}"
job_id = helper.run_job_and_wait_state(image_url,
"",
wait_state=JobStatus.SUCCEEDED,
stop_state=JobStatus.FAILED)
helper.check_job_output(job_id, re.escape(tag))
def test_tree(helper: Helper, data: _Data, tmp_path: Path) -> None:
folder = tmp_path / "folder"
folder.mkdir()
(folder / "foo").write_bytes(b"foo")
(folder / "bar").write_bytes(b"bar")
subfolder = folder / "folder"
subfolder.mkdir()
(subfolder / "baz").write_bytes(b"baz")
helper.run_cli(["storage", "cp", "-r", folder.as_uri(), helper.tmpstorage])
capture = helper.run_cli(["storage", "tree", helper.tmpstorage])
assert capture.err == ""
expected = textwrap.dedent(f"""\
'{helper.tmpstorage}'
├── 'bar'
├── 'folder'
│ └── 'baz'
└── 'foo'
1 directories, 3 files""")
if sys.platform == "win32":
trans = str.maketrans("".join(TreeFormatter.ANSI_DELIMS),
"".join(TreeFormatter.SIMPLE_DELIMS))
expected = expected.translate(trans)
assert capture.out == expected
def test_grant_image_with_tag_fails(request: Any, helper: Helper) -> None:
uri = f"image://{helper.username}/{uuid4()}:latest"
another_test_user = "******"
with pytest.raises(subprocess.CalledProcessError) as cm:
request.addfinalizer(lambda: revoke(helper, uri, another_test_user))
helper.run_cli(["acl", "grant", uri, another_test_user, "read"])
assert cm.value.returncode == 127
assert "tag is not allowed" in cm.value.stderr
def test_remove_cluster_user_remove_oneself(helper: Helper) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"admin", "remove-cluster-user", helper.cluster_name,
helper.username
])
assert cm.value.returncode == 65
assert ("Illegal argument(s) (Cluster users cannot remove themselves)"
in cm.value.stderr)
def test_e2e_copy_no_target_directory_extra_operand(helper: Helper,
tmp_path: Path) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"storage", "cp", "-T",
str(tmp_path), helper.tmpstorage,
str(tmp_path)
])
assert "Extra operand after " in cm.value.stderr
def test_add_cluster_user_invalid_role(helper: Helper) -> None:
username = "******"
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"admin", "add-cluster-user", helper.cluster_name, username,
"my_role"
])
assert cm.value.returncode == 2
assert "invalid choice:" in cm.value.stderr
assert "(choose from admin, manager, user)" in cm.value.stderr
def test_add_cluster_user_already_exists(helper: Helper) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"admin", "add-cluster-user", helper.cluster_name, helper.username,
"user"
])
assert cm.value.returncode == 65
assert (
f"Illegal argument(s) (User '{helper.username}' already exists in cluster "
f"'{helper.cluster_name}')" in cm.value.stderr)
def test_e2e_move_no_target_directory_extra_operand(helper: Helper) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"storage",
"mv",
"-T",
helper.tmpstorage + "/foo",
helper.tmpstorage + "/bar",
helper.tmpstorage + "/baz",
])
assert "Extra operand after " in cm.value.stderr
def test_e2e_move_target_directory_no_target_directory(helper: Helper) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"storage",
"mv",
"-t",
helper.tmpstorage + "/foo",
"-T",
helper.tmpstorage + "/bar",
])
assert "Cannot combine" in cm.value.stderr
def test_e2e_job_top(helper: Helper) -> None:
def split_non_empty_parts(line: str, sep: str) -> List[str]:
return [part.strip() for part in line.split(sep) if part.strip()]
command = f"sleep 300"
job_id = helper.run_job_and_wait_state(image=UBUNTU_IMAGE_NAME, command=command)
try:
# TODO: implement progressive timeout
# even 15 secs usually enough for low-load testing
# but under high load the value should be increased
capture = helper.run_cli(["job", "top", job_id, "--timeout", "60"])
except subprocess.CalledProcessError as ex:
stdout = ex.output
stderr = ex.stderr
else:
stdout = capture.out
stderr = capture.err
helper.kill_job(job_id)
try:
header, *lines = split_non_empty_parts(stdout, sep="\n")
except ValueError:
assert False, f"cannot unpack\n{stdout}\n{stderr}"
header_parts = split_non_empty_parts(header, sep="\t")
assert header_parts == [
"TIMESTAMP",
"CPU",
"MEMORY (MB)",
"GPU (%)",
"GPU_MEMORY (MB)",
]
for line in lines:
line_parts = split_non_empty_parts(line, sep="\t")
timestamp_pattern_parts = [
("weekday", "[A-Z][a-z][a-z]"),
("month", "[A-Z][a-z][a-z]"),
("day", r"\d+"),
("day", r"\d\d:\d\d:\d\d"),
("year", "2019"),
]
timestamp_pattern = r"\s+".join([part[1] for part in timestamp_pattern_parts])
expected_parts = [
("timestamp", timestamp_pattern),
("cpu", r"\d.\d\d\d"),
("memory", r"\d.\d\d\d"),
("gpu", "0"),
("gpu memory", "0"),
]
for actual, (descr, pattern) in zip(line_parts, expected_parts):
assert re.match(pattern, actual) is not None, f"error in matching {descr}"
def test_e2e_blob_storage_copy_file_explicit_directory(
helper: Helper, data: _Data, tmp_bucket: str
) -> None:
srcfile, checksum = data
file_name = str(PurePath(srcfile).name)
key = f"folder/{file_name}"
# Upload local file to existing directory with explocit -t param
helper.run_cli(["blob", "cp", "-t", f"blob:{tmp_bucket}/folder", srcfile])
# Ensure file is there
helper.check_blob_size(tmp_bucket, key, FILE_SIZE_B)
def test_e2e_move_content_to_directory(helper: Helper) -> None:
helper.mkdir("folder/subfolder", parents=True)
helper.mkdir("otherfolder", parents=True)
helper.run_cli([
"storage",
"mv",
"-T",
helper.tmpstorage + "folder",
helper.tmpstorage + "otherfolder",
])
helper.check_dir_absent_on_storage("folder", "")
helper.check_dir_exists_on_storage("subfolder", "otherfolder")
def test_e2e_blob_storage_copy_file_implicit_directory(
helper: Helper, data: _Data, tmp_bucket: str
) -> None:
srcfile, checksum = data
file_name = str(PurePath(srcfile).name)
key = f"folder/{file_name}"
# Upload local file to a directory defined by trailing slash
helper.run_cli(["blob", "cp", srcfile, f"blob:{tmp_bucket}/folder/"])
# Ensure file is there
helper.check_blob_size(tmp_bucket, key, FILE_SIZE_B)
def test_images_complete_lifecycle(
helper: Helper,
image: str,
tag: str,
loop: asyncio.AbstractEventLoop,
docker: aiodocker.Docker,
) -> None:
# Let`s push image
captured = helper.run_cli(["image", "push", image])
# stderr has "Used image ..." lines
# assert not captured.err
image_full_str = f"image://{helper.cluster_name}/{helper.username}/{image}"
assert captured.out.endswith(image_full_str)
image_url = URL(image_full_str)
# Check if image available on registry
captured = helper.run_cli(["image", "ls", "--full-uri"])
image_urls = [URL(line) for line in captured.out.splitlines() if line]
for url in image_urls:
assert url.scheme == "image"
image_url_without_tag = image_url.with_path(
image_url.path.replace(f":{tag}", ""))
assert image_url_without_tag in image_urls
# delete local
loop.run_until_complete(docker.images.delete(image, force=True))
docker_ls_output = loop.run_until_complete(docker.images.list())
local_images = parse_docker_ls_output(docker_ls_output)
assert image not in local_images
# Pull image as with another tag
captured = helper.run_cli(["image", "pull", f"image:{image}"])
# stderr has "Used image ..." lines
# assert not captured.err
assert captured.out.endswith(image)
# check pulled locally, delete for cleanup
docker_ls_output = loop.run_until_complete(docker.images.list())
local_images = parse_docker_ls_output(docker_ls_output)
assert image in local_images
# Execute image and check result
captured = helper.run_cli([
"-q",
"submit",
*JOB_TINY_CONTAINER_PARAMS,
"--non-preemptible",
"--no-wait-start",
str(image_url),
])
assert not captured.err
job_id = captured.out
assert job_id.startswith("job-")
helper.wait_job_change_state_to(job_id, JobStatus.SUCCEEDED,
JobStatus.FAILED)
helper.check_job_output(job_id, re.escape(tag))
def test_e2e_copy_recursive_to_platform(helper: Helper,
nested_data: Tuple[str, str, str],
tmp_path: Path) -> None:
helper.mkdir("")
srcfile, checksum, dir_path = nested_data
target_file_name = Path(srcfile).name
# Upload local file
captured = helper.run_cli(
["storage", "cp", "-r", dir_path, helper.tmpstorage])
# stderr has logs like "Using path ..."
# assert not captured.err
assert not captured.out
helper.check_file_exists_on_storage(target_file_name,
f"nested/directory/for/test",
FILE_SIZE_B // 3)
# Download into local directory and confirm checksum
targetdir = tmp_path / "bar"
targetdir.mkdir()
helper.run_cli(
["storage", "cp", "-r", f"{helper.tmpstorage}",
str(targetdir)])
targetfile = targetdir / "nested" / "directory" / "for" / "test" / target_file_name
print("source file", srcfile)
print("target file", targetfile)
assert helper.hash_hex(targetfile) == checksum
def test_e2e_ls_skip_hidden(tmp_path: Path, helper: Helper) -> None:
# Create files and directories and copy them to storage
helper.mkdir("")
folder = tmp_path / "folder"
folder.mkdir()
(folder / "foo").write_bytes(b"foo")
(folder / ".bar").write_bytes(b"bar")
helper.run_cli([
"storage", "cp", "-r",
tmp_path.as_uri() + "/folder", helper.tmpstorage
])
captured = helper.run_cli(["storage", "ls", helper.tmpstorage + "/folder"])
assert captured.out.splitlines() == ["foo"]
def test_e2e_blob_storage_copy_recursive_file(
helper: Helper, nested_data: Tuple[str, str, str], tmp_path: Path, tmp_bucket: str
) -> None:
srcfile = tmp_path / "testfile"
dstfile = tmp_path / "copyfile"
srcfile.write_bytes(b"abc")
captured = helper.run_cli(["blob", "cp", "-r", str(srcfile), f"blob:{tmp_bucket}"])
assert not captured.out
captured = helper.run_cli(
["blob", "cp", "-r", f"blob:{tmp_bucket}/testfile", str(dstfile)]
)
assert not captured.out
assert dstfile.read_bytes() == b"abc"
def test_images_push_with_specified_name(
helper: Helper,
image: str,
tag: str,
loop: asyncio.AbstractEventLoop,
docker: aiodocker.Docker,
) -> None:
# Let`s push image
image_no_tag = image.replace(f":{tag}", "")
pushed_no_tag = f"{image_no_tag}-pushed"
pulled_no_tag = f"{image_no_tag}-pulled"
pulled = f"{pulled_no_tag}:{tag}"
captured = helper.run_cli(
["image", "push", image, f"image:{pushed_no_tag}:{tag}"])
# stderr has "Used image ..." lines
# assert not captured.err
image_pushed_full_str = (
f"image://{helper.cluster_name}/{helper.username}/{pushed_no_tag}:{tag}"
)
assert captured.out.endswith(image_pushed_full_str)
image_url_without_tag = image_pushed_full_str.replace(f":{tag}", "")
# Check if image available on registry
captured = helper.run_cli(["image", "ls", "--full-uri"])
image_urls = captured.out.splitlines()
assert image_url_without_tag in image_urls
# check locally
docker_ls_output = loop.run_until_complete(docker.images.list())
local_images = parse_docker_ls_output(docker_ls_output)
assert pulled not in local_images
# Pull image as with another name
captured = helper.run_cli(
["image", "pull", f"image:{pushed_no_tag}:{tag}", pulled])
# stderr has "Used image ..." lines
# assert not captured.err
assert captured.out.endswith(pulled)
# check locally
docker_ls_output = loop.run_until_complete(docker.images.list())
local_images = parse_docker_ls_output(docker_ls_output)
assert pulled in local_images
# TODO (A.Yushkovskiy): delete the pushed image in GCR
# delete locally
loop.run_until_complete(docker.images.delete(pulled, force=True))
def test_copy_local_to_platform_single_file_3(helper: Helper,
data: _Data) -> None:
# case when copy happens with rename to 'different_name.txt'
srcfile, checksum = data
helper.mkdir("")
# Upload local file to non existing directory
with pytest.raises(subprocess.CalledProcessError, match=str(EX_OSFILE)):
captured = helper.run_cli([
"storage", "cp", srcfile, helper.tmpstorage + "/non_existing_dir/"
])
assert not captured.err
assert captured.out == ""
# Ensure dir is not created
helper.check_dir_absent_on_storage("non_existing_dir", "")
def test_e2e_copy_recursive_file(helper: Helper, tmp_path: Path) -> None:
helper.mkdir("")
srcfile = tmp_path / "testfile"
dstfile = tmp_path / "copyfile"
srcfile.write_bytes(b"abc")
captured = helper.run_cli(
["storage", "cp", "-r",
str(srcfile), helper.tmpstorage])
assert not captured.out
captured = helper.run_cli(
["storage", "cp", "-r", helper.tmpstorage + "testfile",
str(dstfile)])
assert not captured.out
assert dstfile.read_bytes() == b"abc"
def test_grant_image_no_tag(request: Any, helper: Helper) -> None:
rel_path = str(uuid4())
rel_uri = f"image:{rel_path}"
uri = f"image://{helper.username}/{rel_path}"
another_test_user = "******"
request.addfinalizer(lambda: revoke(helper, rel_uri, another_test_user))
captured = helper.run_cli(
["-v", "acl", "grant", rel_uri, another_test_user, "read"])
assert captured.out == ""
expected_err = f"Using resource '{uri}'"
assert expected_err in captured.err
captured = helper.run_cli(
["-v", "acl", "revoke", rel_uri, another_test_user])
assert captured.out == ""
assert expected_err in captured.err
def test_e2e_cp_filter(tmp_path: Path, helper: Helper) -> None:
# Create files and directories and copy them to storage
helper.mkdir("")
folder = tmp_path / "folder"
folder.mkdir()
(folder / "subfolder").mkdir()
(folder / "foo").write_bytes(b"foo")
(folder / "bar").write_bytes(b"bar")
(folder / "baz").write_bytes(b"baz")
helper.run_cli([
"storage",
"cp",
"-r",
"--exclude",
"*",
"--include",
"b??",
"--exclude",
"*z",
tmp_path.as_uri() + "/folder",
helper.tmpstorage + "/filtered",
])
captured = helper.run_cli(
["storage", "ls", helper.tmpstorage + "/filtered"])
assert captured.out.splitlines() == ["bar"]
# Copy all files to storage
helper.run_cli([
"storage",
"cp",
"-r",
tmp_path.as_uri() + "/folder",
helper.tmpstorage + "/folder",
])
# Copy filtered files from storage
helper.run_cli([
"storage",
"cp",
"-r",
"--exclude",
"*",
"--include",
"b??",
"--exclude",
"*z",
helper.tmpstorage + "/folder",
tmp_path.as_uri() + "/filtered",
])
assert os.listdir(tmp_path / "filtered") == ["bar"]
def test_e2e_rename(helper: Helper) -> None:
helper.mkdir("folder", parents=True)
helper.run_cli([
"storage",
"mv",
helper.tmpstorage + "folder",
helper.tmpstorage + "otherfolder",
])
helper.check_dir_absent_on_storage("folder", "")
helper.check_dir_exists_on_storage("otherfolder", "")
def test_list_role(request: Any, helper: Helper) -> None:
captured = helper.run_cli(["acl", "list", "-s", "role"])
assert captured.err == ""
result = captured.out.splitlines()
self_role_uri = f"role://{helper.username}"
role = helper.username
for line in result:
uri, *_ = line.split()
assert uri.startswith("role://")
if uri != self_role_uri:
role = uri[len("role://"):]
print(f"Test using role {role!r}")
captured = helper.run_cli(["acl", "list", "-u", role])
assert captured.err == ""
captured = helper.run_cli(["acl", "list", "-u", role, "--shared"])
assert captured.err == ""
def test_e2e_blob_storage_cp_filter(
helper: Helper, nested_data: Tuple[str, str, str], tmp_path: Path, tmp_bucket: str
) -> None:
# Create files and directories and copy them to storage
folder = tmp_path / "folder"
folder.mkdir()
(folder / "subfolder").mkdir()
(folder / "foo").write_bytes(b"foo")
(folder / "bar").write_bytes(b"bar")
(folder / "baz").write_bytes(b"baz")
helper.run_cli(
[
"blob",
"cp",
"-r",
"--exclude",
"*",
"--include",
"b??",
"--exclude",
"*z",
tmp_path.as_uri() + "/folder",
f"blob:{tmp_bucket}/filtered",
]
)
captured = helper.run_cli(["blob", "ls", f"blob:{tmp_bucket}/filtered/"])
prefix = f"blob:{tmp_bucket}/filtered/"
assert sorted(captured.out.splitlines()) == [prefix, prefix + "bar"]
# Copy all files to storage
helper.run_cli(
[
"blob",
"cp",
"-r",
tmp_path.as_uri() + "/folder",
f"blob:{tmp_bucket}/folder",
]
)
# Copy filtered files from storage
helper.run_cli(
[
"blob",
"cp",
"-r",
"--exclude",
"*",
"--include",
"b??",
"--exclude",
"*z",
f"blob:{tmp_bucket}" + "/folder",
tmp_path.as_uri() + "/filtered",
]
)
assert os.listdir(tmp_path / "filtered") == ["bar"]
