def test_multirun_with_free_override( sweep_runner: TSweepRunner, # noqa: F811 calling_file: str, calling_module: str, overrides: List[str], ) -> None: sweep = sweep_runner( calling_file=calling_file, calling_module=calling_module, config_path="conf/config.yaml", overrides=overrides, strict=True, ) with sweep: assert sweep.returns is not None and len(sweep.returns[0]) == 2 assert sweep.returns[0][0].overrides == ["free_group=opt1"] assert sweep.returns[0][0].cfg == { "group_opt1": True, "free_group_opt1": True } assert sweep.returns[0][1].overrides == ["free_group=opt2"] assert sweep.returns[0][1].cfg == { "group_opt1": True, "free_group_opt2": True }
def test_multirun_with_free_override( sweep_runner, calling_file, calling_module, overrides # noqa: F811 ): sweep = sweep_runner( calling_file=calling_file, calling_module=calling_module, config_path="conf/config.yaml", overrides=overrides, strict=True, ) with sweep: assert len(sweep.returns[0]) == 2 assert sweep.returns[0][0].overrides == ["free_group=opt1"] assert sweep.returns[0][0].cfg == { "group_opt1": True, "free_group_opt1": True } assert sweep.returns[0][1].overrides == ["free_group=opt2"] assert sweep.returns[0][1].cfg == { "group_opt1": True, "free_group_opt2": True }
def test_jobs_configured_via_cmd( sweep_runner: TSweepRunner, ) -> None: # noqa: F811 sweep = sweep_runner( calling_file=os.path.dirname(os.path.abspath(__file__)), calling_module=None, task_function=quadratic, config_path="tests/config", config_name="quadratic.yaml", overrides=[ "hydra/sweeper=ax", "hydra/launcher=basic", "hydra.sweeper.params.ax_config.client.random_seed=1", "quadratic.x=-5:-2", "quadratic.y=-2:2", "hydra.sweeper.params.ax_config.max_trials=2", ], strict=True, ) with sweep: assert sweep.returns is None returns = OmegaConf.load(f"{sweep.temp_dir}/optimization_results.yaml") assert isinstance(returns, DictConfig) assert returns["optimizer"] == "ax" assert len(returns) == 2 best_parameters = returns["ax"] assert len(best_parameters) == 2 assert math.isclose(best_parameters["quadratic.x"], -2.0, abs_tol=1e-4) assert math.isclose(best_parameters["quadratic.y"], 2.0, abs_tol=1e-4)
def test_configuration_set_via_cmd_and_default_config( sweep_runner: TSweepRunner, # noqa: F811 ) -> None: sweep = sweep_runner( calling_file=os.path.dirname(os.path.abspath(__file__)), calling_module=None, task_function=quadratic, config_path="tests/config", config_name="default_quadratic.yaml", overrides=[ "hydra/sweeper=ax", "hydra/launcher=basic", "hydra.sweeper.params.ax_config.client.random_seed=1", "hydra.sweeper.params.ax_config.max_trials=2", "hydra.sweeper.params.ax_config.early_stop.max_epochs_without_improvement=2", "quadratic.x=-5:-2", "quadratic.y=-1:1", ], ) with sweep: ax_config = HydraConfig.instance().hydra.sweeper.params.ax_config assert ax_config.max_trials == 2 assert ax_config.early_stop.max_epochs_without_improvement == 2 assert ax_config.experiment.minimize is True assert sweep.returns is None returns = OmegaConf.load(f"{sweep.temp_dir}/optimization_results.yaml") assert isinstance(returns, DictConfig) best_parameters = returns["ax"] assert "quadratic.x" in best_parameters assert "quadratic.y" in best_parameters
def test_sweep_complex_defaults( sweep_runner, calling_file, calling_module # noqa: F811 ): with sweep_runner( calling_file=calling_file, calling_module=calling_module, config_path="conf/config.yaml", overrides=["optimizer=adam,nesterov"], ) as sweep: assert len(sweep.returns[0]) == 2 assert sweep.returns[0][0].overrides == ["optimizer=adam"] assert sweep.returns[0][1].overrides == ["optimizer=nesterov"]
def test_glob_jobs(sweep_runner): # noqa: F811 sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", config_path="configs/compose.yaml", overrides=[ "hydra/sweeper=range", "hydra/launcher=basic", "foo=glob(*)" ], strict=True, ) with sweep: job_ret = sweep.returns[0] print(job_ret)
def test_example_app(sweep_runner: TSweepRunner) -> None: # noqa: F811 with sweep_runner( calling_file="example/my_app.py", calling_module=None, task_function=None, config_path=None, config_name="config", overrides=["task=1,2,3,4"], ) as sweep: overrides = {("task=1",), ("task=2",), ("task=3",), ("task=4",)} assert sweep.returns is not None and len(sweep.returns[0]) == 4 for ret in sweep.returns[0]: assert tuple(ret.overrides) in overrides
def test_launched_jobs(sweep_runner): # noqa: F811 sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", config_path="configs/compose.yaml", overrides=["hydra/sweeper=example", "hydra/launcher=basic", "foo=1,2"], strict=True, ) with sweep: job_ret = sweep.returns[0] assert len(job_ret) == 2 assert job_ret[0].overrides == ["foo=1"] assert job_ret[0].cfg == {"foo": 1, "bar": 100} assert job_ret[1].overrides == ["foo=2"] assert job_ret[1].cfg == {"foo": 2, "bar": 100}
def test_sweeping_example(sweep_runner): # noqa: F811 with sweep_runner( calling_file="examples/tutorial/5_composition/my_app.py", calling_module=None, config_path="conf/config.yaml", overrides=["schema=warehouse,support", "db=mysql,postgresql"], ) as sweep: overrides = { ("schema=warehouse", "db=mysql"), ("schema=warehouse", "db=postgresql"), ("schema=support", "db=mysql"), ("schema=support", "db=postgresql"), } assert len(sweep.returns[0]) == 4 for ret in sweep.returns[0]: assert tuple(ret.overrides) in overrides
def test_launched_jobs( sweep_runner: TSweepRunner) -> None: # noqa: F811 # type: ignore sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", config_path="configs", config_name="compose.yaml", task_function=None, overrides=["hydra/sweeper=example", "hydra/launcher=basic", "foo=1,2"], strict=True, ) with sweep: assert sweep.returns is not None job_ret = sweep.returns[0] assert len(job_ret) == 2 assert job_ret[0].overrides == ["foo=1"] assert job_ret[0].cfg == {"foo": 1, "bar": 100} assert job_ret[1].overrides == ["foo=2"] assert job_ret[1].cfg == {"foo": 2, "bar": 100}
def test_launched_jobs(sweep_runner: TSweepRunner) -> None: # noqa: F811 # type: ignore budget = 8 sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", config_path="configs", config_name="compose.yaml", task_function=None, overrides=[ "hydra/sweeper=nevergrad", "hydra/launcher=basic", f"hydra.sweeper.params.optim.budget={budget}", # small budget to test fast "hydra.sweeper.params.optim.num_workers=3", "foo=1,2", "bar=4:8", ], strict=True, ) with sweep: assert sweep.returns is None