def test_pass_config(image: str, helper: Helper) -> None: # Let`s push image captured = helper.run_cli(["image", "push", image]) image_full_str = f"image://{helper.username}/{image}" assert captured.out.endswith(image_full_str) command = 'bash -c "neuro config show"' # Run a new job captured = helper.run_cli([ "job", "run", "-q", "-s", JOB_TINY_CONTAINER_PRESET, "--no-wait-start", "--pass-config", image_full_str, command, ]) job_id = captured.out # sleep(1) # Wait until the job is running helper.wait_job_change_state_to(job_id, JobStatus.SUCCEEDED) # Verify exit code is returned captured = helper.run_cli(["job", "status", job_id]) store_out = captured.out assert "Exit code: 0" in store_out
def test_job_run_home_volumes_automount(helper: Helper, fakebrowser: Any) -> None: command = "[ -d /var/storage/home -a -d /var/storage/neuromation ]" with pytest.raises(subprocess.CalledProcessError) as cm: # first, run without --volume=HOME helper.run_cli([ "-q", "job", "run", "--detach", "--preset=cpu-micro", UBUNTU_IMAGE_NAME, command, ]) assert cm.value.returncode == 125 # then, run with --volume=HOME capture = helper.run_cli([ "-q", "job", "run", "--detach", "--preset=cpu-micro", "--volume", "HOME", UBUNTU_IMAGE_NAME, command, ]) job_id_2 = capture.out helper.wait_job_change_state_to(job_id_2, JobStatus.SUCCEEDED, JobStatus.FAILED)
def test_job_submit_no_detach_failure(helper: Helper) -> None: # Run a new job with pytest.raises(subprocess.CalledProcessError) as exc_info: helper.run_cli( ["-v", "job", "run", "--http", "80", UBUNTU_IMAGE_NAME, f"exit 127"] ) assert exc_info.value.returncode == 127
def test_job_save(helper: Helper, docker: aiodocker.Docker) -> None: job_name = f"test-job-save-{uuid4().hex[:6]}" image = f"test-image:{job_name}" image_neuro_name = f"image://{helper.cluster_name}/{helper.username}/{image}" command = "sh -c 'echo -n 123 > /test; sleep 10m'" job_id_1 = helper.run_job_and_wait_state( ALPINE_IMAGE_NAME, command=command, wait_state=JobStatus.RUNNING ) img_uri = f"image://{helper.cluster_name}/{helper.username}/{image}" captured = helper.run_cli(["job", "save", job_id_1, image_neuro_name]) out = captured.out assert f"Saving job '{job_id_1}' to image '{img_uri}'..." in out assert f"Using remote image '{img_uri}'" in out assert "Creating image from the job container" in out assert "Image created" in out assert f"Using local image '{helper.username}/{image}'" in out assert "Pushing image..." in out assert out.endswith(img_uri) # wait to free the job name: helper.run_cli(["job", "kill", job_id_1]) helper.wait_job_change_state_to(job_id_1, JobStatus.CANCELLED) command = 'sh -c \'[ "$(cat /test)" = "123" ]\'' helper.run_job_and_wait_state( image_neuro_name, command=command, wait_state=JobStatus.SUCCEEDED )
def test_job_filter_by_date_range(helper: Helper) -> None: captured = helper.run_cli( ["job", "run", "--no-wait-start", UBUNTU_IMAGE_NAME, "sleep 300"] ) match = re.match("Job ID: (.+)", captured.out) assert match is not None job_id = match.group(1) now = datetime.now() delta = timedelta(minutes=10) captured = helper.run_cli(["ps", "--since", (now - delta).isoformat()]) store_out_list = captured.out.split("\n")[1:] jobs = [x.split(" ")[0] for x in store_out_list] assert job_id in jobs captured = helper.run_cli(["ps", "--since", (now + delta).isoformat()]) store_out_list = captured.out.split("\n")[1:] jobs = [x.split(" ")[0] for x in store_out_list] assert job_id not in jobs captured = helper.run_cli(["ps", "--until", (now - delta).isoformat()]) store_out_list = captured.out.split("\n")[1:] jobs = [x.split(" ")[0] for x in store_out_list] assert job_id not in jobs captured = helper.run_cli(["ps", "--until", (now + delta).isoformat()]) store_out_list = captured.out.split("\n")[1:] jobs = [x.split(" ")[0] for x in store_out_list] assert job_id in jobs
def test_job_run_volume_all(helper: Helper) -> None: root_mountpoint = "/var/neuro" cmd = " && ".join( [ f"[ -d {root_mountpoint}/{helper.username} ]", f"[ -d {root_mountpoint}/neuromation ]", # must be public f"[ $NEUROMATION_ROOT == {root_mountpoint} ]", f"[ $NEUROMATION_HOME == {root_mountpoint}/{helper.username} ]", ] ) command = f"bash -c '{cmd}'" img = UBUNTU_IMAGE_NAME with pytest.raises(subprocess.CalledProcessError) as cm: # first, run without --volume=ALL captured = helper.run_cli(["--quiet", "run", "-T", img, command]) assert cm.value.returncode == 1 # then, run with --volume=ALL captured = helper.run_cli(["run", "-T", "--volume=ALL", img, command]) msg = ( "Storage mountpoints will be available as the environment variables:\n" f" NEUROMATION_ROOT={root_mountpoint}\n" f" NEUROMATION_HOME={root_mountpoint}/{helper.username}" ) assert msg in captured.out found_job_ids = re.findall("Job ID: (job-.+)", captured.out) assert len(found_job_ids) == 1 job_id = found_job_ids[0] helper.wait_job_change_state_to( job_id, JobStatus.SUCCEEDED, stop_state=JobStatus.FAILED )
def test_job_description(helper: Helper) -> None: # Remember original running jobs captured = helper.run_cli( ["job", "ls", "--status", "running", "--status", "pending"] ) store_out_list = captured.out.split("\n")[1:] jobs_orig = [x.split(" ")[0] for x in store_out_list] description = "Test description for a job" # Run a new job command = "bash -c 'sleep 10m; false'" captured = helper.run_cli( [ "job", "submit", *JOB_TINY_CONTAINER_PARAMS, "--http", "80", "--description", description, "--non-preemptible", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) match = re.match("Job ID: (.+)", captured.out) assert match is not None job_id = match.group(1) # Check it was not running before assert job_id.startswith("job-") assert job_id not in jobs_orig # Check it is in a running,pending job list now captured = helper.run_cli( ["job", "ls", "--status", "running", "--status", "pending"] ) store_out_list = captured.out.split("\n")[1:] jobs_updated = [x.split(" ")[0] for x in store_out_list] assert job_id in jobs_updated # Wait until the job is running helper.wait_job_change_state_to(job_id, JobStatus.RUNNING, JobStatus.FAILED) # Check that it is in a running job list captured = helper.run_cli(["job", "ls", "--status", "running"]) store_out = captured.out assert job_id in store_out # Check that description is in the list assert description in store_out assert command in store_out # Check that no description is in the list if quite captured = helper.run_cli(["-q", "job", "ls", "--status", "running"]) store_out = captured.out assert job_id in store_out assert description not in store_out assert command not in store_out helper.kill_job(job_id, wait=False)
def test_job_browse(helper: Helper, fakebrowser: Any) -> None: # Run a new job captured = helper.run_cli( ["-q", "job", "run", "--detach", UBUNTU_IMAGE_NAME, "true"] ) job_id = captured.out captured = helper.run_cli(["-v", "job", "browse", job_id]) assert "Browsing job, please open: https://job-" in captured.out
def test_e2e_ssh_exec_no_cmd(helper: Helper) -> None: command = 'bash -c "sleep 15m; false"' job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, command) with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "job", "exec", "--no-tty", "--no-key-check", "--timeout=60", job_id ]) assert cm.value.returncode == 2
def test_job_kill_non_existing(helper: Helper) -> None: # try to kill non existing job phantom_id = "not-a-job-id" expected_out = f"Cannot kill job {phantom_id}" with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli(["job", "kill", phantom_id]) assert cm.value.returncode == 1 assert cm.value.stdout == "" killed_jobs = cm.value.stderr.splitlines() assert len(killed_jobs) == 1, killed_jobs assert killed_jobs[0].startswith(expected_out)
def test_e2e_ssh_exec_no_job(helper: Helper) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "job", "exec", "--no-tty", "--no-key-check", "--timeout=60", "job_id", "true", ]) assert cm.value.returncode == 127
def secret(helper: Helper) -> Iterator[Tuple[str, str]]: secret_name = "secret" + str(uuid.uuid4()).replace("-", "")[:10] secret_value = str(uuid.uuid4()) # Add secret cap = helper.run_cli(["secret", "add", secret_name, secret_value]) assert cap.err == "" yield (secret_name, secret_value) # Remove secret cap = helper.run_cli(["secret", "rm", secret_name]) assert cap.err == ""
def test_job_submit_bad_http_auth(helper: Helper, http_auth: str) -> None: with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "job", "submit", *JOB_TINY_CONTAINER_PARAMS, http_auth, "--no-wait-start", UBUNTU_IMAGE_NAME, "true", ]) assert cm.value.returncode == 2 assert f"{http_auth} requires --http" in cm.value.stderr
def test_job_run_exit_code(helper: Helper) -> None: # Run a new job command = 'bash -c "exit 101"' captured = helper.run_cli( ["-q", "job", "run", "--no-wait-start", UBUNTU_IMAGE_NAME, command] ) job_id = captured.out # Wait until the job is running helper.wait_job_change_state_to(job_id, JobStatus.FAILED) # Verify exit code is returned captured = helper.run_cli(["job", "status", job_id]) store_out = captured.out assert "Exit code: 101" in store_out
def test_job_secret_env(helper: Helper, secret: Tuple[str, str]) -> None: secret_name, secret_value = secret bash_script = f'echo "begin"$SECRET_VAR"end" | grep begin{secret_value}end' command = f"bash -c '{bash_script}'" captured = helper.run_cli( [ "job", "run", "-e", f"SECRET_VAR=secret:{secret_name}", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) out = captured.out match = re.match("Job ID: (.+)", out) assert match is not None, captured job_id = match.group(1) helper.wait_job_change_state_from(job_id, JobStatus.PENDING) helper.wait_job_change_state_from(job_id, JobStatus.RUNNING) helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_e2e_ssh_exec_tty(helper: Helper) -> None: command = 'bash -c "sleep 15m; false"' job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, command) captured = helper.run_cli( ["job", "exec", "--no-key-check", "--timeout=60", job_id, "[ -t 1 ]"]) assert captured.out == ""
def test_job_disk_volume( helper: Helper, disk_factory: Callable[[str], ContextManager[str]] ) -> None: with disk_factory("1G") as disk: bash_script = 'echo "test data" > /mnt/disk/file && cat /mnt/disk/file' command = f"bash -c '{bash_script}'" captured = helper.run_cli( [ "job", "run", "--life-span", "1m", # Avoid completed job to block disk from cleanup "-v", f"disk:{disk}:/mnt/disk:rw", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) out = captured.out match = re.match("Job ID: (.+)", out) assert match is not None, captured job_id = match.group(1) helper.wait_job_change_state_from(job_id, JobStatus.PENDING) helper.wait_job_change_state_from(job_id, JobStatus.RUNNING) helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_secret_file(helper: Helper, secret: Tuple[str, str]) -> None: secret_name, secret_value = secret bash_script = ( f'test -f /secrets/secretfile && grep "^{secret_value}$" /secrets/secretfile' ) command = f"bash -c '{bash_script}'" captured = helper.run_cli( [ "job", "run", "-v", f"secret:{secret_name}:/secrets/secretfile", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) out = captured.out match = re.match("Job ID: (.+)", out) assert match is not None, captured job_id = match.group(1) helper.wait_job_change_state_from(job_id, JobStatus.PENDING) helper.wait_job_change_state_from(job_id, JobStatus.RUNNING) helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_run_volume_all_and_another(helper: Helper) -> None: with pytest.raises(subprocess.CalledProcessError): args = ["--volume", "ALL", "--volume", "storage::/home:ro"] captured = helper.run_cli( ["job", "run", *args, UBUNTU_IMAGE_NAME, "sleep 30"]) msg = "Cannot use `--volume=ALL` together with other `--volume` options" assert msg in captured.err
def test_e2e_multiple_env_from_file(helper: Helper, tmp_path: Path) -> None: env_file = tmp_path / "env_file" env_file.write_text("VAR2=LAV2\nVAR3=VAL3\n") bash_script = 'echo begin"$VAR""$VAR2""$VAR3"end | grep beginVALVAL2VAL3end' command = f"bash -c '{bash_script}'" captured = helper.run_cli( [ "-q", "job", "submit", *JOB_TINY_CONTAINER_PARAMS, "-e", "VAR=VAL", "-e", "VAR2=VAL2", "--env-file", str(env_file), "--non-preemptible", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) job_id = captured.out helper.wait_job_change_state_from(job_id, JobStatus.PENDING) helper.wait_job_change_state_from(job_id, JobStatus.RUNNING) helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_e2e_env_from_local(helper: Helper) -> None: os.environ["VAR"] = "VAL" bash_script = 'echo "begin"$VAR"end" | grep beginVALend' command = f"bash -c '{bash_script}'" captured = helper.run_cli( [ "job", "submit", *JOB_TINY_CONTAINER_PARAMS, "-e", "VAR", "--non-preemptible", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) out = captured.out match = re.match("Job ID: (.+)", out) assert match is not None job_id = match.group(1) helper.wait_job_change_state_from(job_id, JobStatus.PENDING) helper.wait_job_change_state_from(job_id, JobStatus.RUNNING) helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_e2e_multiple_env(helper: Helper) -> None: bash_script = 'echo begin"$VAR""$VAR2"end | grep beginVALVAL2end' command = f"bash -c '{bash_script}'" captured = helper.run_cli( [ "job", "run", "-e", "VAR=VAL", "-e", "VAR2=VAL2", "--no-wait-start", UBUNTU_IMAGE_NAME, command, ] ) out = captured.out match = re.match("Job ID: (.+)", out) assert match is not None job_id = match.group(1) helper.wait_job_change_state_from(job_id, JobStatus.PENDING) helper.wait_job_change_state_from(job_id, JobStatus.RUNNING) helper.assert_job_state(job_id, JobStatus.SUCCEEDED)
def test_job_run_volume_all(helper: Helper) -> None: root_mountpoint = "/var/neuro" cmd = " && ".join( [ f"[ -d {root_mountpoint}/{helper.username} ]", f"[ -d {root_mountpoint}/neuromation ]", # must be public f"[ $NEUROMATION_ROOT == {root_mountpoint} ]", f"[ $NEUROMATION_HOME == {root_mountpoint}/{helper.username} ]", ] ) command = f"bash -c '{cmd}'" img = UBUNTU_IMAGE_NAME with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli(["run", "-T", "--volume=ALL", img, command]) assert cm.value.returncode == 127
def test_job_run_with_tty(helper: Helper) -> None: command = "test -t 0" job_id = helper.run_job_and_wait_state( UBUNTU_IMAGE_NAME, command, wait_state=JobStatus.SUCCEEDED, tty=True ) captured = helper.run_cli(["job", "status", job_id]) assert "TTY: True" in captured.out
def test_job_kill_non_existing(helper: Helper) -> None: # try to kill non existing job phantom_id = "NOT_A_JOB_ID" expected_out = f"Cannot kill job {phantom_id}" captured = helper.run_cli(["job", "kill", phantom_id]) killed_jobs = [x.strip() for x in captured.out.split("\n")] assert len(killed_jobs) == 1 assert killed_jobs[0].startswith(expected_out)
def test_e2e_ssh_exec_dead_job(helper: Helper) -> None: command = "true" job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, command, wait_state=JobStatus.SUCCEEDED) with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli([ "job", "exec", "--no-tty", "--no-key-check", "--timeout=60", job_id, "true", ]) assert cm.value.returncode == 127
def test_job_browse(helper: Helper, fakebrowser: Any) -> None: # Run a new job captured = helper.run_cli([ "-q", "job", "run", "-s", JOB_TINY_CONTAINER_PRESET, "--detach", UBUNTU_IMAGE_NAME, "true", ]) job_id = captured.out captured = helper.run_cli(["-v", "job", "browse", job_id]) assert "Browsing https://job-" in captured.out assert "Open job URL: https://job-" in captured.err
def test_create_list_delete(helper: Helper, secret_name: str) -> None: cap = helper.run_cli(["secret", "ls"]) assert cap.err == "" assert secret_name not in cap.out cap = helper.run_cli(["secret", "add", secret_name, "value"]) assert cap.err == "" cap = helper.run_cli(["secret", "ls"]) assert cap.err == "" assert secret_name in cap.out cap = helper.run_cli(["secret", "rm", secret_name]) assert cap.err == "" cap = helper.run_cli(["secret", "ls"]) assert cap.err == "" assert secret_name not in cap.out
def test_e2e_ssh_exec_false(helper: Helper) -> None: command = 'bash -c "sleep 15m; false"' job_id = helper.run_job_and_wait_state(UBUNTU_IMAGE_NAME, command) with pytest.raises(subprocess.CalledProcessError) as cm: helper.run_cli( [ "job", "exec", "--no-tty", "--no-key-check", "--timeout", str(EXEC_TIMEOUT), job_id, "false", ] ) assert cm.value.returncode == 1 helper.kill_job(job_id, wait=False)
def test_job_tags(helper: Helper) -> None: tags = [f"test-tag:{uuid4()}", "test-tag:common"] tag_options = [key for pair in [("--tag", t) for t in tags] for key in pair] command = "sleep 10m" captured = helper.run_cli( ["job", "run", *tag_options, "--no-wait-start", UBUNTU_IMAGE_NAME, command] ) match = re.match("Job ID: (.+)", captured.out) assert match is not None job_id = match.group(1) captured = helper.run_cli(["ps", *tag_options]) store_out_list = captured.out.split("\n")[1:] jobs = [x.split(" ")[0] for x in store_out_list] assert job_id in jobs captured = helper.run_cli(["job", "tags"]) tags_listed = captured.out.split("\n") assert set(tags) <= set(tags_listed)