def test_multirun_with_free_override(
sweep_runner: TSweepRunner, # noqa: F811
calling_file: str,
calling_module: str,
overrides: List[str],
) -> None:
sweep = sweep_runner(
calling_file=calling_file,
calling_module=calling_module,
config_path="conf/config.yaml",
overrides=overrides,
strict=True,
)
with sweep:
assert sweep.returns is not None and len(sweep.returns[0]) == 2
assert sweep.returns[0][0].overrides == ["free_group=opt1"]
assert sweep.returns[0][0].cfg == {
"group_opt1": True,
"free_group_opt1": True
}
assert sweep.returns[0][1].overrides == ["free_group=opt2"]
assert sweep.returns[0][1].cfg == {
"group_opt1": True,
"free_group_opt2": True
}
def test_multirun_with_free_override(
sweep_runner,
calling_file,
calling_module,
overrides # noqa: F811
):
sweep = sweep_runner(
calling_file=calling_file,
calling_module=calling_module,
config_path="conf/config.yaml",
overrides=overrides,
strict=True,
)
with sweep:
assert len(sweep.returns[0]) == 2
assert sweep.returns[0][0].overrides == ["free_group=opt1"]
assert sweep.returns[0][0].cfg == {
"group_opt1": True,
"free_group_opt1": True
}
assert sweep.returns[0][1].overrides == ["free_group=opt2"]
assert sweep.returns[0][1].cfg == {
"group_opt1": True,
"free_group_opt2": True
}
def test_jobs_configured_via_cmd(
sweep_runner: TSweepRunner, ) -> None: # noqa: F811
sweep = sweep_runner(
calling_file=os.path.dirname(os.path.abspath(__file__)),
calling_module=None,
task_function=quadratic,
config_path="tests/config",
config_name="quadratic.yaml",
overrides=[
"hydra/sweeper=ax",
"hydra/launcher=basic",
"hydra.sweeper.params.ax_config.client.random_seed=1",
"quadratic.x=-5:-2",
"quadratic.y=-2:2",
"hydra.sweeper.params.ax_config.max_trials=2",
],
strict=True,
)
with sweep:
assert sweep.returns is None
returns = OmegaConf.load(f"{sweep.temp_dir}/optimization_results.yaml")
assert isinstance(returns, DictConfig)
assert returns["optimizer"] == "ax"
assert len(returns) == 2
best_parameters = returns["ax"]
assert len(best_parameters) == 2
assert math.isclose(best_parameters["quadratic.x"], -2.0, abs_tol=1e-4)
assert math.isclose(best_parameters["quadratic.y"], 2.0, abs_tol=1e-4)
def test_configuration_set_via_cmd_and_default_config(
sweep_runner: TSweepRunner, # noqa: F811
) -> None:
sweep = sweep_runner(
calling_file=os.path.dirname(os.path.abspath(__file__)),
calling_module=None,
task_function=quadratic,
config_path="tests/config",
config_name="default_quadratic.yaml",
overrides=[
"hydra/sweeper=ax",
"hydra/launcher=basic",
"hydra.sweeper.params.ax_config.client.random_seed=1",
"hydra.sweeper.params.ax_config.max_trials=2",
"hydra.sweeper.params.ax_config.early_stop.max_epochs_without_improvement=2",
"quadratic.x=-5:-2",
"quadratic.y=-1:1",
],
)
with sweep:
ax_config = HydraConfig.instance().hydra.sweeper.params.ax_config
assert ax_config.max_trials == 2
assert ax_config.early_stop.max_epochs_without_improvement == 2
assert ax_config.experiment.minimize is True
assert sweep.returns is None
returns = OmegaConf.load(f"{sweep.temp_dir}/optimization_results.yaml")
assert isinstance(returns, DictConfig)
best_parameters = returns["ax"]
assert "quadratic.x" in best_parameters
assert "quadratic.y" in best_parameters
def test_sweep_complex_defaults(
sweep_runner, calling_file, calling_module # noqa: F811
):
with sweep_runner(
calling_file=calling_file,
calling_module=calling_module,
config_path="conf/config.yaml",
overrides=["optimizer=adam,nesterov"],
) as sweep:
assert len(sweep.returns[0]) == 2
assert sweep.returns[0][0].overrides == ["optimizer=adam"]
assert sweep.returns[0][1].overrides == ["optimizer=nesterov"]
def test_glob_jobs(sweep_runner): # noqa: F811
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
config_path="configs/compose.yaml",
overrides=[
"hydra/sweeper=range", "hydra/launcher=basic", "foo=glob(*)"
],
strict=True,
)
with sweep:
job_ret = sweep.returns[0]
print(job_ret)
def test_example_app(sweep_runner: TSweepRunner) -> None: # noqa: F811
with sweep_runner(
calling_file="example/my_app.py",
calling_module=None,
task_function=None,
config_path=None,
config_name="config",
overrides=["task=1,2,3,4"],
) as sweep:
overrides = {("task=1",), ("task=2",), ("task=3",), ("task=4",)}
assert sweep.returns is not None and len(sweep.returns[0]) == 4
for ret in sweep.returns[0]:
assert tuple(ret.overrides) in overrides
def test_launched_jobs(sweep_runner): # noqa: F811
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
config_path="configs/compose.yaml",
overrides=["hydra/sweeper=example", "hydra/launcher=basic", "foo=1,2"],
strict=True,
)
with sweep:
job_ret = sweep.returns[0]
assert len(job_ret) == 2
assert job_ret[0].overrides == ["foo=1"]
assert job_ret[0].cfg == {"foo": 1, "bar": 100}
assert job_ret[1].overrides == ["foo=2"]
assert job_ret[1].cfg == {"foo": 2, "bar": 100}
def test_sweeping_example(sweep_runner): # noqa: F811
with sweep_runner(
calling_file="examples/tutorial/5_composition/my_app.py",
calling_module=None,
config_path="conf/config.yaml",
overrides=["schema=warehouse,support", "db=mysql,postgresql"],
) as sweep:
overrides = {
("schema=warehouse", "db=mysql"),
("schema=warehouse", "db=postgresql"),
("schema=support", "db=mysql"),
("schema=support", "db=postgresql"),
}
assert len(sweep.returns[0]) == 4
for ret in sweep.returns[0]:
assert tuple(ret.overrides) in overrides
def test_launched_jobs(
sweep_runner: TSweepRunner) -> None: # noqa: F811 # type: ignore
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
config_path="configs",
config_name="compose.yaml",
task_function=None,
overrides=["hydra/sweeper=example", "hydra/launcher=basic", "foo=1,2"],
strict=True,
)
with sweep:
assert sweep.returns is not None
job_ret = sweep.returns[0]
assert len(job_ret) == 2
assert job_ret[0].overrides == ["foo=1"]
assert job_ret[0].cfg == {"foo": 1, "bar": 100}
assert job_ret[1].overrides == ["foo=2"]
assert job_ret[1].cfg == {"foo": 2, "bar": 100}
def test_launched_jobs(sweep_runner: TSweepRunner) -> None: # noqa: F811 # type: ignore
budget = 8
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
config_path="configs",
config_name="compose.yaml",
task_function=None,
overrides=[
"hydra/sweeper=nevergrad",
"hydra/launcher=basic",
f"hydra.sweeper.params.optim.budget={budget}", # small budget to test fast
"hydra.sweeper.params.optim.num_workers=3",
"foo=1,2",
"bar=4:8",
],
strict=True,
)
with sweep:
assert sweep.returns is None
