def test_basic_notebook_start_and_kill() -> None:
with cmd.interactive_command("notebook", "start") as notebook:
for line in notebook.stdout:
if re.search("Jupyter Notebook .*is running at", line) is not None:
return
raise AssertionError()
def test_start_shell_with_template() -> None:
template_name = "test_start_shell_with_template"
tpl.set_template(template_name,
conf.fixtures_path("templates/template.yaml"))
with cmd.interactive_command("shell", "start", "--template", template_name,
"--detach"):
pass
def test_start_command_with_template() -> None:
template_name = "test_start_command_with_template"
tpl.set_template(template_name,
conf.fixtures_path("templates/template.yaml"))
with cmd.interactive_command("command", "run", "--template", template_name,
"--detach", "sleep infinity"):
pass
def test_basic_notebook_start_and_kill() -> None:
lines = [] # type: List[str]
with cmd.interactive_command("notebook", "start") as notebook:
for line in notebook.stdout:
if re.search("Jupyter Notebook .*is running at", line) is not None:
return
lines.append(line)
raise ValueError(lines)
def test_start_and_write_to_shell(tmp_path: Path) -> None:
with cmd.interactive_command("shell", "start") as shell:
# Call our cli to ensure that PATH and PYTHONUSERBASE are properly set.
shell.stdin.write(b"det --version\n")
shell.stdin.close()
for line in shell.stdout:
if str(det.__version__) in line:
break
else:
pytest.fail(f"Did not find expected input {det.__version__} in shell stdout.")
def test_cmd_kill() -> None:
"""Start a command, extract its task ID, and then kill it."""
with cmd.interactive_command(
"command", "run",
"echo hello world; echo hello world; sleep infinity") as command:
assert command.task_id is not None
for line in command.stdout:
if "hello world" in line:
assert cmd.get_num_running_commands() == 1
break
def test_link_without_agent_user(auth: Authentication) -> None:
user = create_test_user(ADMIN_CREDENTIALS, False)
expected_output = "root:0:root:0"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if expected_output in line:
break
else:
raise AssertionError(f"Did not find {expected_output} in output")
def test_link_with_existing_agent_user(auth: Authentication) -> None:
user = create_linked_user(65534, "nobody", 65534, "nogroup")
expected_output = "nobody:65534:nogroup:65534"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if expected_output in line:
break
else:
raise AssertionError(f"Did not find {expected_output} in output")
def test_link_with_large_uid(auth: Authentication) -> None:
user = create_linked_user(2000000000, "someuser", 2000000000, "somegroup")
expected_output = "someuser:2000000000:somegroup:2000000000"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if expected_output in line:
break
else:
raise AssertionError(f"Did not find {expected_output} in output")
def test_link_with_large_uid(auth: Authentication) -> None:
user = create_linked_user(2000000000, "user", 2000000000, "group")
expected_regex = r".*:2000000000:.*:2000000000"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
for line in cmd.stdout:
if re.match(expected_regex, line):
break
else:
raise AssertionError(f"Did not find {expected_regex} in output")
def test_start_tensorboard_for_multi_experiment(
tmp_path: Path, secrets: Dict[str, str]) -> None:
"""
Start 3 random experiments configured with the s3 and shared_fs backends,
start a TensorBoard instance pointed to the experiments and some select
trials, and kill the TensorBoard instance.
"""
with FileTree(
tmp_path,
{
"shared_fs_config.yaml": shared_fs_config(1),
"s3_config.yaml": s3_config(1, secrets),
"multi_trial_config.yaml": shared_fs_config(3),
},
) as tree:
shared_conf_path = tree.joinpath("shared_fs_config.yaml")
shared_fs_exp_id = exp.run_basic_test(str(shared_conf_path),
conf.fixtures_path("no_op"),
num_trials)
s3_conf_path = tree.joinpath("s3_config.yaml")
s3_exp_id = exp.run_basic_test(str(s3_conf_path),
conf.fixtures_path("no_op"), num_trials)
multi_trial_config = tree.joinpath("multi_trial_config.yaml")
multi_trial_exp_id = exp.run_basic_test(str(multi_trial_config),
conf.fixtures_path("no_op"), 3)
trial_ids = [
str(t["id"]) for t in exp.experiment_trials(multi_trial_exp_id)
]
command = [
"tensorboard",
"start",
str(shared_fs_exp_id),
str(s3_exp_id),
"-t",
*trial_ids,
"--no-browser",
]
with cmd.interactive_command(*command) as tensorboard:
for line in tensorboard.stdout:
if SERVICE_READY in line:
break
if AWAITING_METRICS in line:
raise AssertionError("Tensorboard did not find metrics")
else:
raise AssertionError(f"Did not find {SERVICE_READY} in output")
def test_non_root_shell(auth: Authentication, tmp_path: pathlib.Path) -> None:
user = create_linked_user(1234, "someuser", 1234, "somegroup")
expected_output = "someuser:1234:somegroup:1234"
with logged_in_user(user), command.interactive_command("shell", "start") as shell:
shell.stdin.write(b"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)\n")
shell.stdin.close()
for line in shell.stdout:
if expected_output in line:
break
else:
raise AssertionError(f"Did not find {expected_output} in output")
def test_start_and_write_to_shell(tmp_path: Path) -> None:
pytest.skip("This is an extremely flaky test that needs to be rewritten")
"""Start a shell, extract its task ID, and then kill it."""
with cmd.interactive_command("shell", "start") as shell:
shell.stdin.write(b"echo hello world")
shell.stdin.close()
for line in shell.stdout:
if "hello world" in line:
break
else:
pytest.fail(
"Did not find expected input 'hello world' in shell stdout.")
def test_link_without_agent_user(clean_auth: None) -> None:
user = create_test_user(ADMIN_CREDENTIALS, False)
expected_output = "root:0:root:0"
with logged_in_user(user), command.interactive_command(
"cmd", "run", "bash", "-c",
"echo $(id -u -n):$(id -u):$(id -g -n):$(id -g)") as cmd:
recvd = []
for line in cmd.stdout:
if expected_output in line:
break
recvd.append(line)
else:
output = "".join(recvd)
raise AssertionError(
f"Did not find {expected_output} in output:\n{output}")
def test_killed_pending_command_terminates() -> None:
# Specify an outrageous number of slots to be sure that it can't be scheduled.
with cmd.interactive_command(
"cmd", "run", "--config", "resources.slots=1048576", "sleep infinity"
) as command:
for _ in range(10):
assert cmd.get_command(command.task_id)["state"] == "STATE_PENDING"
time.sleep(1)
# The command is killed when the context is exited; now it should reach TERMINATED soon.
for _ in range(5):
if cmd.get_command(command.task_id)["state"] == "STATE_TERMINATED":
break
time.sleep(1)
else:
state = cmd.get_command(command.task_id)["state"]
raise AssertionError(f"Task was in state {state} rather than STATE_TERMINATED")
def test_start_tensorboard_for_shared_fs_experiment(tmp_path: Path) -> None:
"""
Start a random experiment configured with the shared_fs backend, start a
TensorBoard instance pointed to the experiment, and kill the TensorBoard
instance.
"""
with FileTree(tmp_path, {"config.yaml": shared_fs_config(1)}) as tree:
config_path = tree.joinpath("config.yaml")
experiment_id = exp.run_basic_test(str(config_path),
conf.fixtures_path("no_op"),
num_trials)
command = ["tensorboard", "start", str(experiment_id), "--no-browser"]
with cmd.interactive_command(*command) as tensorboard:
for line in tensorboard.stdout:
if SERVICE_READY in line:
break
else:
raise AssertionError(f"Did not find {SERVICE_READY} in output")
