def test_nevergrad_example(with_commandline: bool, tmpdir: Path) -> None:
budget = 32 if with_commandline else 1 # make a full test only once (faster)
cmd = [
"example/my_app.py",
"-m",
"hydra.sweep.dir=" + str(tmpdir),
f"hydra.sweeper.optim.budget={budget}", # small budget to test fast
f"hydra.sweeper.optim.num_workers={min(8, budget)}",
"hydra.sweeper.optim.seed=12", # avoid random failures
]
if with_commandline:
cmd += [
"db=mnist,cifar",
"batch_size=4,8,12,16",
"lr=tag(log, interval(0.001, 1.0))",
"dropout=interval(0,1)",
]
get_run_output(cmd)
returns = OmegaConf.load(f"{tmpdir}/optimization_results.yaml")
assert isinstance(returns, DictConfig)
assert returns.name == "nevergrad"
assert len(returns) == 3
best_parameters = returns.best_evaluated_params
assert not best_parameters.dropout.is_integer()
if budget > 1:
assert best_parameters.batch_size == 4 # this argument should be easy to find
# check that all job folders are created
last_job = max(int(fp.name) for fp in Path(tmpdir).iterdir() if fp.name.isdigit())
assert last_job == budget - 1
def test_optuna_example(with_commandline: bool, tmpdir: Path) -> None:
cmd = [
"example/sphere.py",
"--multirun",
"hydra.sweep.dir=" + str(tmpdir),
"hydra.sweeper.optuna_config.n_trials=20",
"hydra.sweeper.optuna_config.n_jobs=8",
"hydra.sweeper.optuna_config.sampler=random",
"hydra.sweeper.optuna_config.seed=123",
]
if with_commandline:
cmd += [
"x=choice(0, 1, 2)",
"y=0", # Fixed parameter
]
get_run_output(cmd)
returns = OmegaConf.load(f"{tmpdir}/optimization_results.yaml")
assert isinstance(returns, DictConfig)
assert returns.name == "optuna"
assert returns["best_params"]["x"] == 0
if with_commandline:
assert "y" not in returns["best_params"]
else:
assert returns["best_params"]["y"] == 0
assert returns["best_value"] == 0
def test_examples_using_the_config_object(tmpdir: Path) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/3_using_config/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
get_run_output(cmd)
def test_workdir_config(monkeypatch: Any, tmpdir: Path) -> None:
script = str(Path("examples/configure_hydra/workdir/my_app.py").absolute())
monkeypatch.chdir(tmpdir)
result, _err = get_run_output([script])
assert Path(result) == Path(tmpdir) / "run_dir"
result, _err = get_run_output(
[script, "--multirun", "hydra/hydra_logging=disabled"])
assert Path(result) == Path(tmpdir) / "sweep_dir" / "0"
def test_tutorial_config_file_bad_key(tmpdir: Path, args: List[str],
expected: Any) -> None:
""" Similar to the previous test, but also tests exception values"""
with expected:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(args)
get_run_output(cmd)
def build_package(cfg: Config, pkg_path: str) -> None:
try:
prev = os.getcwd()
os.chdir(pkg_path)
log.info(f"Building {get_package_info('.').name}")
shutil.rmtree("dist", ignore_errors=True)
get_run_output(cmd=["setup.py", "build", *cfg.build_targets],
allow_warnings=True)
finally:
os.chdir(prev)
dist = f"{pkg_path}/dist"
for file in os.listdir(dist):
shutil.copy(src=f"{dist}/{file}", dst=cfg.build_dir)
def test_tutorial_config_file_bad_key(tmpdir: Path, args: List[str],
expected: Any) -> None:
""" Similar to the previous test, but also tests exception values"""
cmd = [
"examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(args)
if isinstance(expected, RaisesContext):
with expected:
get_run_output(cmd)
else:
stdout, _stderr = get_run_output(cmd)
assert OmegaConf.create(stdout) == expected
def get_package_info(path: str) -> Package:
try:
prev = os.getcwd()
os.chdir(path)
out, _err = get_run_output(cmd=[f"{path}/setup.py", "--version"])
local_version: Version = parse_version(out)
package_name, _err = get_run_output(cmd=[f"{path}/setup.py", "--name"])
finally:
os.chdir(prev)
remote_metadata = get_metadata(package_name)
latest = get_releases(remote_metadata)[-1]
return Package(name=package_name,
local_version=local_version,
latest_version=latest)
def test_logging(tmpdir: Path) -> None:
cmd = [
"examples/configure_hydra/logging/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = get_run_output(cmd)
assert result == "[INFO] - Info level message"
def test_job_name_with_config_override(tmpdir: Path) -> None:
cmd = [
"examples/configure_hydra/job_name/with_config_file_override.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = get_run_output(cmd)
assert result == "name_from_config_file"
def test_custom_help(tmpdir: Path) -> None:
result, _err = get_run_output([
"examples/configure_hydra/custom_help/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"--help",
])
expected = dedent("""\
== AwesomeApp ==
This is AwesomeApp!
You can choose a db driver by appending
== Configuration groups ==
Compose your configuration from those groups (db=mysql)
db: mysql, postgresql
== Config ==
This is the config generated for this run.
You can override everything, for example:
python my_app.py db.user=foo db.pass=bar
-------
db:
driver: mysql
user: omry
pass: secret
-------
Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
""")
assert_text_same(from_line=expected, to_line=result)
def test_instantiate_schema_recursive(monkeypatch: Any, tmpdir: Path,
overrides: List[str],
expected: str) -> None:
monkeypatch.chdir("examples/instantiate/schema_recursive")
cmd = ["my_app.py", "hydra.run.dir=" + str(tmpdir)] + overrides
result, _err = get_run_output(cmd)
assert_text_same(result, expected)
def test_instantiate_object_recursive(monkeypatch: Any, tmpdir: Path,
overrides: List[str],
output: str) -> None:
monkeypatch.chdir("examples/instantiate/object_recursive")
cmd = ["my_app.py", "hydra.run.dir=" + str(tmpdir)] + overrides
result, _err = get_run_output(cmd)
assert result == output
def test_multi_select(
monkeypatch: Any, tmpdir: Path, overrides: List[str], expected: Any
) -> None:
monkeypatch.chdir("examples/patterns/multi-select")
cmd = ["my_app.py", "hydra.run.dir=" + str(tmpdir)] + overrides
result, _err = get_run_output(cmd)
assert OmegaConf.create(result) == expected
def test_help(tmpdir: Path, script: str, flag: str, overrides: List[str],
expected: Any) -> None:
cmd = [script, "hydra.run.dir=" + str(tmpdir)]
cmd.extend(overrides)
cmd.append(flag)
result, _err = get_run_output(cmd)
assert_text_same(result, expected.format(script=script))
def test_1_basic_override(tmpdir: Path) -> None:
result, _err = get_run_output([
"examples/tutorials/structured_configs/1_minimal/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"port=9090",
])
assert result == "Host: localhost, port: 9090"
def test_tutorial_working_directory(tmpdir: Path) -> None:
cmd = [
"examples/tutorials/basic/running_your_hydra_app/3_working_directory/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = get_run_output(cmd)
assert result == "Working directory : {}".format(tmpdir)
def test_instantiate_structured_config_example(monkeypatch: Any, tmpdir: Path,
overrides: List[str],
output: str) -> None:
monkeypatch.chdir("examples/patterns/instantiate_structured_config/")
cmd = ["my_app.py", "hydra.run.dir=" + str(tmpdir)] + overrides
result, _err = get_run_output(cmd)
assert result == output
def test_structured_with_none_list(monkeypatch: Any, tmpdir: Path) -> None:
monkeypatch.chdir("tests/test_apps/structured_with_none_list")
cmd = [
"my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = get_run_output(cmd)
assert result == "{'list': None}"
def test_short_module_name(tmpdir: Path) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
assert OmegaConf.create(get_run_output(cmd)) == {
"db": {"driver": "mysql", "password": "******", "user": "******"}
}
def test_schema_overrides_hydra(monkeypatch: Any, tmpdir: Path) -> None:
monkeypatch.chdir("tests/test_apps/schema-overrides-hydra")
cmd = [
"my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = get_run_output(cmd)
assert result == "job_name: test, name: James Bond, age: 7, group: a"
def test_run_pass_list(self, cmd_base: List[str], tmpdir: Any) -> None:
cmd = cmd_base + [
"hydra.sweep.dir=" + str(tmpdir),
"+foo=[1,2,3]",
]
expected = {"foo": [1, 2, 3]}
ret, _err = get_run_output(cmd)
assert OmegaConf.create(ret) == OmegaConf.create(expected)
def test_2_static_complex(tmpdir: Path) -> None:
result, _err = get_run_output(
[
"examples/tutorials/structured_configs/2_static_complex/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
)
assert result == "Title=My app, size=1024x768 pixels"
def test_cfg_with_package(tmpdir: Path, flags: List[str], expected: str) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
] + flags
result, _err = get_run_output(cmd)
assert normalize_newlines(result) == expected.rstrip()
def test_tutorial_config_file(tmpdir: Path, args: List[str], output_conf: Any) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(args)
result, _err = get_run_output(cmd)
assert OmegaConf.create(result) == output_conf
def test_tutorial_defaults(tmpdir: Path, args: List[str], expected: DictConfig) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(args)
result, _err = get_run_output(cmd)
assert OmegaConf.create(result) == OmegaConf.create(expected)
def test_advanced_ad_hoc_composition(tmpdir: Path, args: List[str],
expected: Any) -> None:
cmd = [
"examples/advanced/ad_hoc_composition/hydra_compose_example.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = get_run_output(cmd)
assert OmegaConf.create(result) == OmegaConf.create(expected)
def test_3_config_groups(tmpdir: Path, overrides: Any, expected: Any) -> None:
cmd = [
"examples/tutorials/structured_configs/3_config_groups/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(overrides)
result, _err = get_run_output(cmd)
res = OmegaConf.create(result)
assert res == expected
def test_help(tmpdir: Path, script: str, flag: str, overrides: List[str],
expected: Any) -> None:
cmd = [script, "hydra.run.dir=" + str(tmpdir)]
cmd.extend(overrides)
cmd.append(flag)
result, _err = get_run_output(cmd)
# normalize newlines on Windows to make testing easier
assert result == normalize_newlines(
expected.format(script=script)).rstrip()
def test_example_application(monkeypatch: Any, tmpdir: Path):
monkeypatch.chdir("example")
cmd = [
"my_app.py",
f"hydra.run.dir={tmpdir}",
"user.name=Batman",
]
result, _err = get_run_output(cmd)
assert result == "User: name=Batman, age=7"