def test_read_context_with_detignore(tmp_path: Path) -> None:
with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": ""}) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py", "C.py"}
with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": "", ".detignore": "\nA.py\n"}) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"B.py", "C.py"}
with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": "", ".detignore": "\n*.py\n"}) as tree:
model_def, size = context.read_context(tree)
assert model_def == []
def test_basic_workflows(tmp_path: Path) -> None:
with FileTree(tmp_path, {"hello.py": "print('hello world')"}) as tree:
_run_and_verify_exit_code_zero(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--context",
str(tree),
"python",
"hello.py",
]
)
with FileTree(tmp_path, {"hello.py": "print('hello world')"}) as tree:
link = tree.joinpath("hello-link.py")
link.symlink_to(tree.joinpath("hello.py"))
_run_and_verify_exit_code_zero(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--context",
str(tree),
"python",
"hello-link.py",
]
)
_run_and_verify_exit_code_zero(
["det", "-m", conf.make_master_url(), "cmd", "run", "python", "-c", "print('hello world')"]
)
with pytest.raises(subprocess.CalledProcessError):
_run_and_return_real_exit_status(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--context",
"non-existent-path-here",
"python",
"hello.py",
]
)
def test_checkpoint_matches_directory_contents(tmp_path: Path) -> None:
with FileTree(
tmp_path,
{
"checkpoint": """
model_checkpoint_path: "model.ckpt-9"
all_model_checkpoint_paths: "model.ckpt-1"
all_model_checkpoint_paths: "model.ckpt-nonexistent"
all_model_checkpoint_paths: "model.ckpt-9"
""",
"model.ckpt-1.data-0-of-1": "",
"model.ckpt-orphan.data-0-of-1": "",
"model.ckpt-9.data-0-of-1": "",
},
) as tree:
checkpoints = _scan_checkpoint_directory(str(tree))
assert len(checkpoints) == 1
assert [
os.path.basename(v)
for v in checkpoints[0].state.all_model_checkpoint_paths
] == [
"model.ckpt-orphan",
"model.ckpt-1",
"model.ckpt-9",
]
def test_non_root_experiment(auth: Authentication,
tmp_path: pathlib.Path) -> None:
user = create_linked_user(65534, "nobody", 65534, "nogroup")
with logged_in_user(user):
with open(conf.fixtures_path("no_op/model_def.py")) as f:
model_def_content = f.read()
with open(conf.fixtures_path("no_op/single-one-short-step.yaml")) as f:
config = yaml.safe_load(f)
# Use a user-owned path to ensure shared_fs uses the container_path and not host_path.
with non_tmp_shared_fs_path() as host_path:
config["checkpoint_storage"] = {
"type": "shared_fs",
"host_path": host_path,
}
# Call `det --version` in a startup hook to ensure that det is on the PATH.
with FileTree(
tmp_path,
{
"startup-hook.sh": "det --version || exit 77",
"const.yaml": yaml.dump(config), # type: ignore
"model_def.py": model_def_content,
},
) as tree:
exp.run_basic_test(str(tree.joinpath("const.yaml")), str(tree),
None)
def test_configs(tmp_path: Path) -> None:
with FileTree(
tmp_path,
{
"config.yaml":
"""
resources:
slots: 1
environment:
environment_variables:
- TEST=TEST
"""
},
) as tree:
config_path = tree.joinpath("config.yaml")
_run_and_verify_exit_code_zero([
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--config-file",
str(config_path),
"python",
"-c",
"""
import os
test = os.environ["TEST"]
if test != "TEST":
print("{} != {}".format(test, "TEST"))
sys.exit(1)
""",
])
def test_large_uploads(tmp_path: Path) -> None:
with pytest.raises(subprocess.CalledProcessError):
with FileTree(tmp_path, {"hello.py": "print('hello world')"}) as tree:
large = tree.joinpath("large-file.bin")
large.touch()
f = large.open(mode="w")
f.seek(1024 * 1024 * 120)
f.write("\0")
f.close()
_run_and_return_real_exit_status(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--context",
str(tree),
"python",
"hello.py",
]
)
with FileTree(tmp_path, {"hello.py": "print('hello world')", ".detignore": "*.bin"}) as tree:
large = tree.joinpath("large-file.bin")
large.touch()
f = large.open(mode="w")
f.seek(1024 * 1024 * 120)
f.write("\0")
f.close()
_run_and_verify_exit_code_zero(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--context",
str(tree),
"python",
"hello.py",
]
)
def test_read_context_ignore_pycaches(tmp_path: Path) -> None:
with FileTree(
tmp_path,
{
"__pycache__/A.cpython-37.pyc": "",
"A.py": "",
"subdir/A.py": "",
"subdir/__pycache__/A.cpython-37.pyc": "",
},
) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "subdir", "subdir/A.py"}
def test_start_tensorboard_for_multi_experiment(
tmp_path: Path, secrets: Dict[str, str]) -> None:
"""
Start 3 random experiments configured with the s3 and shared_fs backends,
start a TensorBoard instance pointed to the experiments and some select
trials, and kill the TensorBoard instance.
"""
with FileTree(
tmp_path,
{
"shared_fs_config.yaml": shared_fs_config(1),
"s3_config.yaml": s3_config(1, secrets),
"multi_trial_config.yaml": shared_fs_config(3),
},
) as tree:
shared_conf_path = tree.joinpath("shared_fs_config.yaml")
shared_fs_exp_id = exp.run_basic_test(str(shared_conf_path),
conf.fixtures_path("no_op"),
num_trials)
s3_conf_path = tree.joinpath("s3_config.yaml")
s3_exp_id = exp.run_basic_test(str(s3_conf_path),
conf.fixtures_path("no_op"), num_trials)
multi_trial_config = tree.joinpath("multi_trial_config.yaml")
multi_trial_exp_id = exp.run_basic_test(str(multi_trial_config),
conf.fixtures_path("no_op"), 3)
trial_ids = [
str(t["id"]) for t in exp.experiment_trials(multi_trial_exp_id)
]
command = [
"tensorboard",
"start",
str(shared_fs_exp_id),
str(s3_exp_id),
"-t",
*trial_ids,
"--no-browser",
]
with cmd.interactive_command(*command) as tensorboard:
for line in tensorboard.stdout:
if SERVICE_READY in line:
break
if AWAITING_METRICS in line:
raise AssertionError("Tensorboard did not find metrics")
else:
raise AssertionError(f"Did not find {SERVICE_READY} in output")
def test_create_reject_large_model_def(requests_mock: requests_mock.Mocker, tmp_path: Path) -> None:
requests_mock.get("/info", status_code=200, json={"version": "1.0"})
requests_mock.get(
"/users/me", status_code=200, json={"username": constants.DEFAULT_DETERMINED_USER}
)
requests_mock.post(
"/experiments", status_code=requests.codes.created, headers={"Location": "/experiments/1"}
)
with tempfile.NamedTemporaryFile() as model_def_file:
model_def_file.write(os.urandom(constants.MAX_CONTEXT_SIZE + 1))
with FileTree(tmp_path, {"config.yaml": MINIMAL_CONFIG}) as tree, pytest.raises(SystemExit):
cli.main(
["experiment", "create", str(tree.joinpath("config.yaml")), model_def_file.name]
)
def test_start_tensorboard_for_shared_fs_experiment(tmp_path: Path) -> None:
"""
Start a random experiment configured with the shared_fs backend, start a
TensorBoard instance pointed to the experiment, and kill the TensorBoard
instance.
"""
with FileTree(tmp_path, {"config.yaml": shared_fs_config(1)}) as tree:
config_path = tree.joinpath("config.yaml")
experiment_id = exp.run_basic_test(str(config_path),
conf.fixtures_path("no_op"),
num_trials)
command = ["tensorboard", "start", str(experiment_id), "--no-browser"]
with cmd.interactive_command(*command) as tensorboard:
for line in tensorboard.stdout:
if SERVICE_READY in line:
break
else:
raise AssertionError(f"Did not find {SERVICE_READY} in output")
def test_absolute_bind_mount(tmp_path: Path) -> None:
_run_and_verify_exit_code_zero(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--volume",
"/bin:/foo-bar",
"ls",
"/foo-bar",
]
)
with FileTree(
tmp_path,
{
"config.yaml": """
bind_mounts:
- host_path: /bin
container_path: /foo-bar
"""
},
) as tree:
config_path = tree.joinpath("config.yaml")
_run_and_verify_exit_code_zero(
[
"det",
"-m",
conf.make_master_url(),
"cmd",
"run",
"--volume",
"/bin:/foo-bar2",
"--config-file",
str(config_path),
"ls",
"/foo-bar",
"/foo-bar2",
]
)
def test_create_with_model_def(requests_mock: requests_mock.Mocker, tmp_path: Path) -> None:
requests_mock.get("/info", status_code=200, json={"version": "1.0"})
requests_mock.get(
"/users/me", status_code=200, json={"username": constants.DEFAULT_DETERMINED_USER}
)
requests_mock.post("/login", status_code=200, json={"token": "fake-token"})
requests_mock.post(
"/experiments", status_code=requests.codes.created, headers={"Location": "/experiments/1"}
)
tempfile.mkstemp(dir=str(tmp_path))
tempfile.mkstemp(dir=str(tmp_path))
tempfile.mkstemp(dir=str(tmp_path))
with FileTree(tmp_path, {"config.yaml": MINIMAL_CONFIG}) as tree:
cli.main(
["experiment", "create", "--paused", str(tree.joinpath("config.yaml")), str(tmp_path)]
)
def test_shm_size_command(tmp_path: Path, actual: str, expected: str,
use_config_file: bool) -> None:
with FileTree(
tmp_path,
{"config.yaml": f"""
resources:
shm_size: {actual}
"""},
) as tree:
config_path = tree.joinpath("config.yaml")
cmd = ["det", "-m", conf.make_master_url(), "cmd", "run"]
if use_config_file:
cmd += ["--config-file", str(config_path)]
else:
cmd += ["--config", f"resources.shm_size={actual}"]
cmd += [
f"""df /dev/shm && \
df /dev/shm | \
tail -1 | \
[ "$(awk '{{print $2}}')" = '{expected}' ]"""
]
_run_and_verify_exit_code_zero(cmd)
def test_non_root_experiment(auth: Authentication,
tmp_path: pathlib.Path) -> None:
user = create_linked_user(65534, "nobody", 65534, "nogroup")
with logged_in_user(user):
with open(conf.fixtures_path("no_op/single-one-short-step.yaml")) as f:
config_content = f.read()
with open(conf.fixtures_path("no_op/model_def.py")) as f:
model_def_content = f.read()
# Call `det --version` in a startup hook to ensure that det is on the PATH.
with FileTree(
tmp_path,
{
"startup-hook.sh": "det --version || exit 77",
"const.yaml": config_content,
"model_def.py": model_def_content,
},
) as tree:
exp.run_basic_test(str(tree.joinpath("const.yaml")), str(tree),
None)
def test_read_context(tmp_path: Path) -> None:
with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": ""}) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py", "C.py"}
def test_read_context_with_detignore_subdirs(tmp_path: Path) -> None:
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
},
) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"] for f in model_def} == {
"A.py",
"B.py",
"subdir",
"subdir/A.py",
"subdir/B.py",
}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
".detignore": "\nA.py\n",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"B.py", "subdir", "subdir/B.py"}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
".detignore": "\nsubdir/A.py\n",
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py", "subdir", "subdir/B.py"}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
".detignore": "\n*.py\n",
},
) as tree:
model_def, size = context.read_context(tree)
assert len(model_def) == 1
with FileTree(
tmp_path,
{"A.py": "", "B.py": "", "subdir/A.py": "", "subdir/B.py": "", ".detignore": "\nsubdir\n"},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py"}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
"subdir/A.py": "",
"subdir/B.py": "",
".detignore": "\nsubdir/\n",
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py"}