def test_basic_notebook_start_and_kill() -> None:
with cmd.interactive_command("notebook", "start") as notebook:
for line in notebook.stdout:
if "Jupyter Notebook is running" in line:
return
raise AssertionError()
def test_start_shell_with_template() -> None:
template_name = "test_start_shell_with_template"
tpl.set_template(template_name,
conf.fixtures_path("templates/template.yaml"))
with cmd.interactive_command("shell", "start", "--template", template_name,
"--detach"):
pass
def test_start_command_with_template() -> None:
template_name = "test_start_command_with_template"
tpl.set_template(template_name,
conf.fixtures_path("templates/template.yaml"))
with cmd.interactive_command("command", "run", "--template", template_name,
"--detach", "sleep infinity"):
pass
def test_cmd_kill() -> None:
"""Start a command, extract its task ID, and then kill it."""
with cmd.interactive_command(
"command", "run",
"echo hello world; echo hello world; sleep infinity") as command:
assert command.task_id is not None
for line in command.stdout:
if "hello world" in line:
assert cmd.get_num_running_commands() == 1
break
def test_link_without_agent_user(auth: Authentication) -> None:
user = create_test_user(ADMIN_CREDENTIALS, False)
expected_output = "root:0:root:0"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if expected_output in line:
break
else:
raise AssertionError(f"Did not find {expected_output} in output")
def test_link_with_existing_agent_user(auth: Authentication) -> None:
user = create_linked_user(65534, "nobody", 65534, "nogroup")
expected_output = "nobody:65534:nogroup:65534"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if expected_output in line:
break
else:
raise AssertionError(f"Did not find {expected_output} in output")
def test_link_with_large_uid(auth: Authentication) -> None:
user = create_linked_user(2000000000, "user", 2000000000, "group")
expected_regex = r".*:2000000000:.*:2000000000"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if re.match(expected_regex, line):
break
else:
raise AssertionError(f"Did not find {expected_regex} in output")
def test_start_and_write_to_shell(tmp_path: Path) -> None:
pytest.skip("This is an extremely flaky test that needs to be rewritten")
"""Start a shell, extract its task ID, and then kill it."""
with cmd.interactive_command("shell", "start") as shell:
shell.stdin.write(b"echo hello world")
shell.stdin.close()
for line in shell.stdout:
if "hello world" in line:
break
else:
pytest.fail("Did not find expected input 'hello world' in shell stdout.")
def test_start_tensorboard_for_multi_experiment(
tmp_path: Path, secrets: Dict[str, str]) -> None:
"""
Start 3 random experiments configured with the s3 and shared_fs backends,
start a TensorBoard instance pointed to the experiments and some select
trials, and kill the TensorBoard instance.
"""
with FileTree(
tmp_path,
{
"shared_fs_config.yaml": shared_fs_config(1),
"s3_config.yaml": s3_config(1, secrets),
"multi_trial_config.yaml": shared_fs_config(3),
},
) as tree:
shared_conf_path = tree.joinpath("shared_fs_config.yaml")
shared_fs_exp_id = exp.run_basic_test(str(shared_conf_path),
conf.fixtures_path("no_op"),
num_trials)
s3_conf_path = tree.joinpath("s3_config.yaml")
s3_exp_id = exp.run_basic_test(str(s3_conf_path),
conf.fixtures_path("no_op"), num_trials)
multi_trial_config = tree.joinpath("multi_trial_config.yaml")
multi_trial_exp_id = exp.run_basic_test(str(multi_trial_config),
conf.fixtures_path("no_op"), 3)
trial_ids = [
str(t["id"]) for t in exp.experiment_trials(multi_trial_exp_id)
]
command = [
"tensorboard",
"start",
str(shared_fs_exp_id),
str(s3_exp_id),
"-t",
*trial_ids,
"--no-browser",
]
with cmd.interactive_command(*command) as tensorboard:
for line in tensorboard.stdout:
if SERVICE_READY in line:
break
else:
raise AssertionError(f"Did not find {SERVICE_READY} in output")
def test_killed_pending_command_terminates() -> None:
# Specify an outrageous number of slots to be sure that it can't be scheduled.
with cmd.interactive_command("cmd", "run", "--config",
"resources.slots=1048576",
"sleep infinity") as command:
for _ in range(10):
assert cmd.get_command(command.task_id)["state"] == "PENDING"
time.sleep(1)
# The command is killed when the context is exited; now it should reach TERMINATED soon.
for _ in range(5):
if cmd.get_command(command.task_id)["state"] == "TERMINATED":
break
time.sleep(1)
else:
state = cmd.get_command(command.task_id)["state"]
raise AssertionError(
f"Task was in state {state} rather than TERMINATED")
def test_start_tensorboard_for_shared_fs_experiment(tmp_path: Path) -> None:
"""
Start a random experiment configured with the shared_fs backend, start a
TensorBoard instance pointed to the experiment, and kill the TensorBoard
instance.
"""
with FileTree(tmp_path, {"config.yaml": shared_fs_config(1)}) as tree:
config_path = tree.joinpath("config.yaml")
experiment_id = exp.run_basic_test(str(config_path),
conf.fixtures_path("no_op"),
num_trials)
command = ["tensorboard", "start", str(experiment_id), "--no-browser"]
with cmd.interactive_command(*command) as tensorboard:
for line in tensorboard.stdout:
if SERVICE_READY in line:
break
else:
raise AssertionError(f"Did not find {SERVICE_READY} in output")