def test_e2e_ssh_exec_tty(helper: Helper) -> None:
command = 'bash -c "sleep 15m; false"'
job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, command)
captured = helper.run_cli(
["job", "exec", "--no-key-check", "--timeout=60", job_id, "[ -t 1 ]"])
assert captured.out == ""
def test_job_secret_env(helper: Helper, secret: Tuple[str, str]) -> None:
secret_name, secret_value = secret
bash_script = f'echo "begin"$SECRET_VAR"end" | grep begin{secret_value}end'
command = f"bash -c '{bash_script}'"
captured = helper.run_cli(
[
"job",
"run",
"-e",
f"SECRET_VAR=secret:{secret_name}",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
out = captured.out
match = re.match("Job ID: (.+)", out)
assert match is not None, captured
job_id = match.group(1)
helper.wait_job_change_state_from(job_id, JobStatus.PENDING)
helper.wait_job_change_state_from(job_id, JobStatus.RUNNING)
helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_autocomplete(helper: Helper) -> None:
job_name = f"test-job-{os.urandom(5).hex()}"
helper.kill_job(job_name)
job_id = helper.run_job_and_wait_state(
ALPINE_IMAGE_NAME, "sleep 600", name=job_name
)
out = helper.autocomplete(["kill", "test-job"])
assert job_name in out
assert job_id not in out
out = helper.autocomplete(["kill", "job-"])
assert job_name in out
assert job_id in out
out = helper.autocomplete(["kill", "job:job-"])
assert job_name in out
assert job_id in out
out = helper.autocomplete(["kill", f"job:/{helper.username}/job-"])
assert job_name in out
assert job_id in out
out = helper.autocomplete(
["kill", f"job://{helper.cluster_name}/{helper.username}/job-"]
)
assert job_name in out
assert job_id in out
helper.kill_job(job_id)
def test_job_disk_volume(
helper: Helper, disk_factory: Callable[[str], ContextManager[str]]
) -> None:
with disk_factory("1G") as disk:
bash_script = 'echo "test data" > /mnt/disk/file && cat /mnt/disk/file'
command = f"bash -c '{bash_script}'"
captured = helper.run_cli(
[
"job",
"run",
"--life-span",
"1m", # Avoid completed job to block disk from cleanup
"-v",
f"disk:{disk}:/mnt/disk:rw",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
out = captured.out
match = re.match("Job ID: (.+)", out)
assert match is not None, captured
job_id = match.group(1)
helper.wait_job_change_state_from(job_id, JobStatus.PENDING)
helper.wait_job_change_state_from(job_id, JobStatus.RUNNING)
helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_e2e_multiple_env(helper: Helper) -> None:
bash_script = 'echo begin"$VAR""$VAR2"end | grep beginVALVAL2end'
command = f"bash -c '{bash_script}'"
captured = helper.run_cli(
[
"job",
"run",
"-e",
"VAR=VAL",
"-e",
"VAR2=VAL2",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
out = captured.out
match = re.match("Job ID: (.+)", out)
assert match is not None
job_id = match.group(1)
helper.wait_job_change_state_from(job_id, JobStatus.PENDING)
helper.wait_job_change_state_from(job_id, JobStatus.RUNNING)
helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_e2e_multiple_env_from_file(helper: Helper, tmp_path: Path) -> None:
env_file = tmp_path / "env_file"
env_file.write_text("VAR2=LAV2\nVAR3=VAL3\n")
bash_script = 'echo begin"$VAR""$VAR2""$VAR3"end | grep beginVALVAL2VAL3end'
command = f"bash -c '{bash_script}'"
captured = helper.run_cli(
[
"-q",
"job",
"submit",
*JOB_TINY_CONTAINER_PARAMS,
"-e",
"VAR=VAL",
"-e",
"VAR2=VAL2",
"--env-file",
str(env_file),
"--non-preemptible",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
job_id = captured.out
helper.wait_job_change_state_from(job_id, JobStatus.PENDING)
helper.wait_job_change_state_from(job_id, JobStatus.RUNNING)
helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_run_volume_all(helper: Helper) -> None:
root_mountpoint = "/var/neuro"
cmd = " && ".join(
[
f"[ -d {root_mountpoint}/{helper.username} ]",
f"[ -d {root_mountpoint}/neuromation ]", # must be public
f"[ $NEUROMATION_ROOT == {root_mountpoint} ]",
f"[ $NEUROMATION_HOME == {root_mountpoint}/{helper.username} ]",
]
)
command = f"bash -c '{cmd}'"
img = UBUNTU_IMAGE_NAME
with pytest.raises(subprocess.CalledProcessError) as cm:
# first, run without --volume=ALL
captured = helper.run_cli(["--quiet", "run", "-T", img, command])
assert cm.value.returncode == 1
# then, run with --volume=ALL
captured = helper.run_cli(["run", "-T", "--volume=ALL", img, command])
msg = (
"Storage mountpoints will be available as the environment variables:\n"
f" NEUROMATION_ROOT={root_mountpoint}\n"
f" NEUROMATION_HOME={root_mountpoint}/{helper.username}"
)
assert msg in captured.out
found_job_ids = re.findall("Job ID: (job-.+)", captured.out)
assert len(found_job_ids) == 1
job_id = found_job_ids[0]
helper.wait_job_change_state_to(
job_id, JobStatus.SUCCEEDED, stop_state=JobStatus.FAILED
)
def test_job_run_home_volumes_automount(helper: Helper,
fakebrowser: Any) -> None:
command = "[ -d /var/storage/home -a -d /var/storage/neuromation ]"
with pytest.raises(subprocess.CalledProcessError) as cm:
# first, run without --volume=HOME
helper.run_cli([
"-q",
"job",
"run",
"--detach",
"--preset=cpu-micro",
UBUNTU_IMAGE_NAME,
command,
])
assert cm.value.returncode == 125
# then, run with --volume=HOME
capture = helper.run_cli([
"-q",
"job",
"run",
"--detach",
"--preset=cpu-micro",
"--volume",
"HOME",
UBUNTU_IMAGE_NAME,
command,
])
job_id_2 = capture.out
helper.wait_job_change_state_to(job_id_2, JobStatus.SUCCEEDED,
JobStatus.FAILED)
def test_e2e_env_from_local(helper: Helper) -> None:
os.environ["VAR"] = "VAL"
bash_script = 'echo "begin"$VAR"end" | grep beginVALend'
command = f"bash -c '{bash_script}'"
captured = helper.run_cli(
[
"job",
"submit",
*JOB_TINY_CONTAINER_PARAMS,
"-e",
"VAR",
"--non-preemptible",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
out = captured.out
match = re.match("Job ID: (.+)", out)
assert match is not None
job_id = match.group(1)
helper.wait_job_change_state_from(job_id, JobStatus.PENDING)
helper.wait_job_change_state_from(job_id, JobStatus.RUNNING)
helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_submit_no_detach_failure(helper: Helper) -> None:
# Run a new job
with pytest.raises(subprocess.CalledProcessError) as exc_info:
helper.run_cli(
["-v", "job", "run", "--http", "80", UBUNTU_IMAGE_NAME, f"exit 127"]
)
assert exc_info.value.returncode == 127
def test_pass_config(image: str, helper: Helper) -> None:
# Let`s push image
captured = helper.run_cli(["image", "push", image])
image_full_str = f"image://{helper.username}/{image}"
assert captured.out.endswith(image_full_str)
command = 'bash -c "neuro config show"'
# Run a new job
captured = helper.run_cli([
"job",
"run",
"-q",
"-s",
JOB_TINY_CONTAINER_PRESET,
"--no-wait-start",
"--pass-config",
image_full_str,
command,
])
job_id = captured.out
# sleep(1)
# Wait until the job is running
helper.wait_job_change_state_to(job_id, JobStatus.SUCCEEDED)
# Verify exit code is returned
captured = helper.run_cli(["job", "status", job_id])
store_out = captured.out
assert "Exit code: 0" in store_out
def test_job_filter_by_date_range(helper: Helper) -> None:
captured = helper.run_cli(
["job", "run", "--no-wait-start", UBUNTU_IMAGE_NAME, "sleep 300"]
)
match = re.match("Job ID: (.+)", captured.out)
assert match is not None
job_id = match.group(1)
now = datetime.now()
delta = timedelta(minutes=10)
captured = helper.run_cli(["ps", "--since", (now - delta).isoformat()])
store_out_list = captured.out.split("\n")[1:]
jobs = [x.split(" ")[0] for x in store_out_list]
assert job_id in jobs
captured = helper.run_cli(["ps", "--since", (now + delta).isoformat()])
store_out_list = captured.out.split("\n")[1:]
jobs = [x.split(" ")[0] for x in store_out_list]
assert job_id not in jobs
captured = helper.run_cli(["ps", "--until", (now - delta).isoformat()])
store_out_list = captured.out.split("\n")[1:]
jobs = [x.split(" ")[0] for x in store_out_list]
assert job_id not in jobs
captured = helper.run_cli(["ps", "--until", (now + delta).isoformat()])
store_out_list = captured.out.split("\n")[1:]
jobs = [x.split(" ")[0] for x in store_out_list]
assert job_id in jobs
def test_job_secret_file(helper: Helper, secret: Tuple[str, str]) -> None:
secret_name, secret_value = secret
bash_script = (
f'test -f /secrets/secretfile && grep "^{secret_value}$" /secrets/secretfile'
)
command = f"bash -c '{bash_script}'"
captured = helper.run_cli(
[
"job",
"run",
"-v",
f"secret:{secret_name}:/secrets/secretfile",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
out = captured.out
match = re.match("Job ID: (.+)", out)
assert match is not None, captured
job_id = match.group(1)
helper.wait_job_change_state_from(job_id, JobStatus.PENDING)
helper.wait_job_change_state_from(job_id, JobStatus.RUNNING)
helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_run_with_tty(helper: Helper) -> None:
command = "test -t 0"
job_id = helper.run_job_and_wait_state(
UBUNTU_IMAGE_NAME, command, wait_state=JobStatus.SUCCEEDED, tty=True
)
captured = helper.run_cli(["job", "status", job_id])
assert "TTY: True" in captured.out
def test_e2e_ssh_exec_no_cmd(helper: Helper) -> None:
command = 'bash -c "sleep 15m; false"'
job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, command)
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"job", "exec", "--no-tty", "--no-key-check", "--timeout=60", job_id
])
assert cm.value.returncode == 2
def test_job_browse(helper: Helper, fakebrowser: Any) -> None:
# Run a new job
captured = helper.run_cli(
["-q", "job", "run", "--detach", UBUNTU_IMAGE_NAME, "true"]
)
job_id = captured.out
captured = helper.run_cli(["-v", "job", "browse", job_id])
assert "Browsing job, please open: https://job-" in captured.out
def test_job_kill_non_existing(helper: Helper) -> None:
# try to kill non existing job
phantom_id = "not-a-job-id"
expected_out = f"Cannot kill job {phantom_id}"
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli(["job", "kill", phantom_id])
assert cm.value.returncode == 1
assert cm.value.stdout == ""
killed_jobs = cm.value.stderr.splitlines()
assert len(killed_jobs) == 1, killed_jobs
assert killed_jobs[0].startswith(expected_out)
def test_job_description(helper: Helper) -> None:
# Remember original running jobs
captured = helper.run_cli(
["job", "ls", "--status", "running", "--status", "pending"]
)
store_out_list = captured.out.split("\n")[1:]
jobs_orig = [x.split(" ")[0] for x in store_out_list]
description = "Test description for a job"
# Run a new job
command = "bash -c 'sleep 10m; false'"
captured = helper.run_cli(
[
"job",
"submit",
*JOB_TINY_CONTAINER_PARAMS,
"--http",
"80",
"--description",
description,
"--non-preemptible",
"--no-wait-start",
UBUNTU_IMAGE_NAME,
command,
]
)
match = re.match("Job ID: (.+)", captured.out)
assert match is not None
job_id = match.group(1)
# Check it was not running before
assert job_id.startswith("job-")
assert job_id not in jobs_orig
# Check it is in a running,pending job list now
captured = helper.run_cli(
["job", "ls", "--status", "running", "--status", "pending"]
)
store_out_list = captured.out.split("\n")[1:]
jobs_updated = [x.split(" ")[0] for x in store_out_list]
assert job_id in jobs_updated
# Wait until the job is running
helper.wait_job_change_state_to(job_id, JobStatus.RUNNING, JobStatus.FAILED)
# Check that it is in a running job list
captured = helper.run_cli(["job", "ls", "--status", "running"])
store_out = captured.out
assert job_id in store_out
# Check that description is in the list
assert description in store_out
assert command in store_out
# Check that no description is in the list if quite
captured = helper.run_cli(["-q", "job", "ls", "--status", "running"])
store_out = captured.out
assert job_id in store_out
assert description not in store_out
assert command not in store_out
helper.kill_job(job_id, wait=False)
def secret(helper: Helper) -> Iterator[Tuple[str, str]]:
secret_name = "secret" + str(uuid.uuid4()).replace("-", "")[:10]
secret_value = str(uuid.uuid4())
# Add secret
cap = helper.run_cli(["secret", "add", secret_name, secret_value])
assert cap.err == ""
yield (secret_name, secret_value)
# Remove secret
cap = helper.run_cli(["secret", "rm", secret_name])
assert cap.err == ""
def test_e2e_ssh_exec_true(helper: Helper) -> None:
job_name = f"test-job-{str(uuid4())[:8]}"
command = 'bash -c "sleep 15m; false"'
job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME,
command,
name=job_name)
captured = helper.run_cli([
"job", "exec", "--no-tty", "--no-key-check", "--timeout=60", job_id,
"true"
])
assert captured.out == ""
def test_e2e_ssh_exec_no_job(helper: Helper) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"job",
"exec",
"--no-tty",
"--no-key-check",
"--timeout=60",
"job_id",
"true",
])
assert cm.value.returncode == 127
def test_job_submit_bad_http_auth(helper: Helper, http_auth: str) -> None:
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"job",
"submit",
*JOB_TINY_CONTAINER_PARAMS,
http_auth,
"--no-wait-start",
UBUNTU_IMAGE_NAME,
"true",
])
assert cm.value.returncode == 2
assert f"{http_auth} requires --http" in cm.value.stderr
def test_job_save(helper: Helper, docker: aiodocker.Docker) -> None:
job_name = f"test-job-save-{uuid4().hex[:6]}"
image = f"test-image:{job_name}"
image_neuro_name = f"image://{helper.cluster_name}/{helper.username}/{image}"
command = "sh -c 'echo -n 123 > /test; sleep 10m'"
job_id_1 = helper.run_job_and_wait_state(
ALPINE_IMAGE_NAME, command=command, wait_state=JobStatus.RUNNING
)
img_uri = f"image://{helper.cluster_name}/{helper.username}/{image}"
captured = helper.run_cli(["job", "save", job_id_1, image_neuro_name])
out = captured.out
assert f"Saving job '{job_id_1}' to image '{img_uri}'..." in out
assert f"Using remote image '{img_uri}'" in out
assert "Creating image from the job container" in out
assert "Image created" in out
assert f"Using local image '{helper.username}/{image}'" in out
assert "Pushing image..." in out
assert out.endswith(img_uri)
# wait to free the job name:
helper.run_cli(["job", "kill", job_id_1])
helper.wait_job_change_state_to(job_id_1, JobStatus.CANCELLED)
command = 'sh -c \'[ "$(cat /test)" = "123" ]\''
helper.run_job_and_wait_state(
image_neuro_name, command=command, wait_state=JobStatus.SUCCEEDED
)
def test_job_attach_tty(helper: Helper) -> None:
job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, "sh", tty=True)
status = helper.job_info(job_id)
assert status.container.tty
expect = helper.pexpect(["job", "attach", job_id])
expect.expect("========== Job is running in terminal mode =========")
expect.sendline("\n") # prompt may be missing after the connection.
repl = REPLWrapper(expect, "# ", None)
ret = repl.run_command("echo abc\n")
assert ret.strip() == "echo abc\r\r\nabc"
helper.kill_job(job_id)
def test_job_run_exit_code(helper: Helper) -> None:
# Run a new job
command = 'bash -c "exit 101"'
captured = helper.run_cli(
["-q", "job", "run", "--no-wait-start", UBUNTU_IMAGE_NAME, command]
)
job_id = captured.out
# Wait until the job is running
helper.wait_job_change_state_to(job_id, JobStatus.FAILED)
# Verify exit code is returned
captured = helper.run_cli(["job", "status", job_id])
store_out = captured.out
assert "Exit code: 101" in store_out
def test_job_run_volume_all(helper: Helper) -> None:
root_mountpoint = "/var/neuro"
cmd = " && ".join(
[
f"[ -d {root_mountpoint}/{helper.username} ]",
f"[ -d {root_mountpoint}/neuromation ]", # must be public
f"[ $NEUROMATION_ROOT == {root_mountpoint} ]",
f"[ $NEUROMATION_HOME == {root_mountpoint}/{helper.username} ]",
]
)
command = f"bash -c '{cmd}'"
img = UBUNTU_IMAGE_NAME
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli(["run", "-T", "--volume=ALL", img, command])
assert cm.value.returncode == 127
def test_job_run_volume_all_and_another(helper: Helper) -> None:
with pytest.raises(subprocess.CalledProcessError):
args = ["--volume", "ALL", "--volume", "storage::/home:ro"]
captured = helper.run_cli(
["job", "run", *args, UBUNTU_IMAGE_NAME, "sleep 30"])
msg = "Cannot use `--volume=ALL` together with other `--volume` options"
assert msg in captured.err
async def test_run_with_port_forward(helper: Helper) -> None:
port = unused_port()
job_id = None
secret = uuid4()
command = (
f"bash -c \"echo -n '{secret}' > /usr/share/nginx/html/secret.txt; "
f"timeout 15m /usr/sbin/nginx -g 'daemon off;'\""
)
proc = await helper.acli(
["run", "--port-forward", f"{port}:80", "nginx:latest", command]
)
try:
await asyncio.sleep(1)
url = f"http://127.0.0.1:{port}/secret.txt"
probe = await fetch_http(url, str(secret))
assert probe == 200
assert proc.returncode is None
assert proc.stdout is not None
out = await proc.stdout.read(64 * 1024)
job_id = helper.find_job_id(out.decode("utf-8", "replace"))
assert job_id is not None
finally:
proc.terminate()
await proc.wait()
if job_id is not None:
await helper.akill_job(job_id)
def test_job_browse(helper: Helper, fakebrowser: Any) -> None:
# Run a new job
captured = helper.run_cli([
"-q",
"job",
"run",
"-s",
JOB_TINY_CONTAINER_PRESET,
"--detach",
UBUNTU_IMAGE_NAME,
"true",
])
job_id = captured.out
captured = helper.run_cli(["-v", "job", "browse", job_id])
assert "Browsing https://job-" in captured.out
assert "Open job URL: https://job-" in captured.err
def test_e2e_ssh_exec_dead_job(helper: Helper) -> None:
command = "true"
job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME,
command,
wait_state=JobStatus.SUCCEEDED)
with pytest.raises(subprocess.CalledProcessError) as cm:
helper.run_cli([
"job",
"exec",
"--no-tty",
"--no-key-check",
"--timeout=60",
job_id,
"true",
])
assert cm.value.returncode == 127