def test_noop_pause_of_experiment_without_trials() -> None:
"""
Walk through starting, pausing, and resuming a single no-op experiment
which will never schedule a trial.
"""
config_obj = conf.load_config(
conf.fixtures_path("no_op/single-one-short-step.yaml"))
impossibly_large = 100
config_obj["max_restarts"] = 0
config_obj["resources"] = {"slots_per_trial": impossibly_large}
with tempfile.NamedTemporaryFile() as tf:
with open(tf.name, "w") as f:
yaml.dump(config_obj, f)
experiment_id = exp.create_experiment(tf.name,
conf.fixtures_path("no_op"),
None)
exp.pause_experiment(experiment_id)
exp.wait_for_experiment_state(experiment_id, "PAUSED")
exp.activate_experiment(experiment_id)
exp.wait_for_experiment_state(experiment_id, "ACTIVE")
for _ in range(5):
assert exp.experiment_state(experiment_id) == "ACTIVE"
time.sleep(1)
exp.cancel_single(experiment_id)
def run_failure_test_with_temp_config(
config: Dict[Any, Any],
model_def_path: str,
error_str: Optional[str] = None,
) -> None:
with tempfile.NamedTemporaryFile() as tf:
with open(tf.name, "w") as f:
yaml.dump(config, f)
run_failure_test(tf.name, model_def_path, error_str=error_str)
def test_non_root_experiment(auth: Authentication,
tmp_path: pathlib.Path) -> None:
user = create_linked_user(65534, "nobody", 65534, "nogroup")
with logged_in_user(user):
with open(conf.fixtures_path("no_op/model_def.py")) as f:
model_def_content = f.read()
with open(conf.fixtures_path("no_op/single-one-short-step.yaml")) as f:
config = yaml.safe_load(f)
# Use a user-owned path to ensure shared_fs uses the container_path and not host_path.
with non_tmp_shared_fs_path() as host_path:
config["checkpoint_storage"] = {
"type": "shared_fs",
"host_path": host_path,
}
# Call `det --version` in a startup hook to ensure that det is on the PATH.
with FileTree(
tmp_path,
{
"startup-hook.sh": "det --version || exit 77",
"const.yaml": yaml.dump(config), # type: ignore
"model_def.py": model_def_content,
},
) as tree:
exp.run_basic_test(str(tree.joinpath("const.yaml")), str(tree),
None)
def run_basic_test_with_temp_config(
config: Dict[Any, Any],
model_def_path: str,
expected_trials: Optional[int],
create_args: Optional[List[str]] = None,
max_wait_secs: int = conf.DEFAULT_MAX_WAIT_SECS,
has_zeroth_step: bool = False,
) -> int:
with tempfile.NamedTemporaryFile() as tf:
with open(tf.name, "w") as f:
yaml.dump(config, f)
experiment_id = run_basic_test(
tf.name,
model_def_path,
expected_trials,
create_args,
max_wait_secs=max_wait_secs,
has_zeroth_step=has_zeroth_step,
)
return experiment_id
def test_noop_single_warm_start() -> None:
experiment_id1 = exp.run_basic_test(
conf.fixtures_path("no_op/single.yaml"), conf.fixtures_path("no_op"),
1)
trials = exp.experiment_trials(experiment_id1)
assert len(trials) == 1
first_trial = trials[0]
first_trial_id = first_trial["id"]
assert len(first_trial["steps"]) == 30
first_step = first_trial["steps"][0]
first_checkpoint_id = first_step["checkpoint"]["id"]
last_step = first_trial["steps"][29]
last_checkpoint_id = last_step["checkpoint"]["id"]
assert last_step["validation"]["metrics"]["validation_metrics"][
"validation_error"] == pytest.approx(0.9**30)
config_base = conf.load_config(conf.fixtures_path("no_op/single.yaml"))
# Test source_trial_id.
config_obj = copy.deepcopy(config_base)
# Add a source trial ID to warm start from.
config_obj["searcher"]["source_trial_id"] = first_trial_id
experiment_id2 = exp.run_basic_test_with_temp_config(
config_obj, conf.fixtures_path("no_op"), 1)
trials = exp.experiment_trials(experiment_id2)
assert len(trials) == 1
second_trial = trials[0]
assert len(second_trial["steps"]) == 30
# Second trial should have a warm start checkpoint id.
assert second_trial["warm_start_checkpoint_id"] == last_checkpoint_id
assert second_trial["steps"][29]["validation"]["metrics"][
"validation_metrics"]["validation_error"] == pytest.approx(0.9**60)
# Now test source_checkpoint_uuid.
config_obj = copy.deepcopy(config_base)
# Add a source trial ID to warm start from.
config_obj["searcher"]["source_checkpoint_uuid"] = first_step[
"checkpoint"]["uuid"]
with tempfile.NamedTemporaryFile() as tf:
with open(tf.name, "w") as f:
yaml.dump(config_obj, f)
experiment_id3 = exp.run_basic_test(tf.name,
conf.fixtures_path("no_op"), 1)
trials = exp.experiment_trials(experiment_id3)
assert len(trials) == 1
third_trial = trials[0]
assert len(third_trial["steps"]) == 30
assert third_trial["warm_start_checkpoint_id"] == first_checkpoint_id
assert third_trial["steps"][1]["validation"]["metrics"][
"validation_metrics"]["validation_error"] == pytest.approx(0.9**3)