def test_example_app(tmpdir: Path) -> None:
cmd = [
"example/banana.py",
"-m",
"hydra.run.dir=" + str(tmpdir),
"banana.x=int(interval(-5, 5))",
"banana.y=interval(-5, 10.1)",
"hydra.sweeper.ax_config.max_trials=2",
]
result, _ = run_python_script(cmd)
assert "banana.x: range=[-5, 5]" in result
assert "banana.y: range=[-5.0, 10.1]" in result
def test_tutorial_logging(tmpdir: Path, args: List[str],
expected: List[str]) -> None:
cmd = [
"examples/tutorials/basic/running_your_hydra_app/4_logging/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(args)
result, _err = run_python_script(cmd)
lines = result.splitlines()
assert len(lines) == len(expected)
for i in range(len(lines)):
assert re.findall(re.escape(expected[i]), lines[i])
def test_module_env_override(tmpdir: Path, env_name: str) -> None:
"""
Tests that module name overrides are working.
"""
cmd = [
"examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
modified_env = os.environ.copy()
modified_env[env_name] = "hydra.test_utils.configs.Foo"
result, _err = run_python_script(cmd, env=modified_env)
assert OmegaConf.create(result) == {"normal_yaml_config": True}
def test_cfg_resolve_interpolation(tmpdir: Path, resolve: bool,
expected: str) -> None:
cmd = [
"tests/test_apps/simple_interpolation/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"--cfg=job",
]
if resolve:
cmd.append("--resolve")
result, _err = run_python_script(cmd)
assert_text_same(result, expected)
def test_example_application(monkeypatch: Any, tmpdir: Path):
monkeypatch.chdir("example")
cmd = [
"my_app.py",
f"hydra.run.dir={tmpdir}",
"hydra.job.chdir=True",
"user.name=Batman",
]
result, _err = run_python_script(cmd)
assert result == dedent("""\
User: name=Batman, age=7
Admin: name=Lex Luthor, age=10, private_key=deadbeef""")
def test_experimental_rerun(
tmpdir: Path, warning_msg: str, overrides: List[str]
) -> None:
app_path = "tests/test_apps/app_with_pickle_job_info_callback/my_app.py"
cmd = [
app_path,
"hydra.run.dir=" + str(tmpdir),
"hydra.sweep.dir=" + str(tmpdir),
"hydra.job.chdir=False",
"hydra.hydra_logging.formatters.simple.format='[HYDRA] %(message)s'",
"hydra.job_logging.formatters.simple.format='[JOB] %(message)s'",
]
run_python_script(cmd)
config_file = tmpdir / ".hydra" / "config.pickle"
log_file = tmpdir / "my_app.log"
assert config_file.exists()
assert log_file.exists()
with open(log_file, "r") as file:
logs = file.read().splitlines()
assert "[JOB] Running my_app" in logs
os.remove(str(log_file))
assert not log_file.exists()
# then rerun the application and verify log file is created again
cmd = [
app_path,
"--experimental-rerun",
str(config_file),
]
cmd.extend(overrides)
result, err = run_python_script(cmd, allow_warnings=True)
assert warning_msg in err
with open(log_file, "r") as file:
logs = file.read().splitlines()
assert "[JOB] Running my_app" in logs
def test_optuna_example(with_commandline: bool, tmpdir: Path) -> None:
storage = "sqlite:///" + os.path.join(str(tmpdir), "test.db")
study_name = "test-optuna-example"
cmd = [
"example/sphere.py",
"--multirun",
"hydra.sweep.dir=" + str(tmpdir),
"hydra.job.chdir=True",
"hydra.sweeper.n_trials=20",
"hydra.sweeper.n_jobs=1",
f"hydra.sweeper.storage={storage}",
f"hydra.sweeper.study_name={study_name}",
"hydra/sweeper/sampler=tpe",
"hydra.sweeper.sampler.seed=123",
"~z",
]
if with_commandline:
cmd += [
"x=choice(0, 1, 2)",
"y=0", # Fixed parameter
]
run_python_script(cmd)
returns = OmegaConf.load(f"{tmpdir}/optimization_results.yaml")
study = optuna.load_study(storage=storage, study_name=study_name)
best_trial = study.best_trial
assert isinstance(returns, DictConfig)
assert returns.name == "optuna"
assert returns["best_params"]["x"] == best_trial.params["x"]
if with_commandline:
assert "y" not in returns["best_params"]
assert "y" not in best_trial.params
else:
assert returns["best_params"]["y"] == best_trial.params["y"]
assert returns["best_value"] == best_trial.value
# Check the search performance of the TPE sampler.
# The threshold is the 95th percentile calculated with 1000 different seed values
# to make the test robust against the detailed implementation of the sampler.
# See https://github.com/facebookresearch/hydra/pull/1746#discussion_r681549830.
assert returns["best_value"] <= 2.27
def test_cfg_with_package(
tmpdir: Path, flags: List[str], resolve: bool, expected: str
) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"hydra.job.chdir=True",
] + flags
if resolve:
cmd.append("--resolve")
result, _err = run_python_script(cmd)
assert normalize_newlines(result) == expected.rstrip()
def test_config_search_path(args: List[str], expected: str, tmpdir: Path,
error: Optional[str]) -> None:
cmd = [
"examples/advanced/config_search_path/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
cmd.extend(args)
if error is not None:
ret = run_with_error(cmd)
assert re.search(re.escape(error), ret) is not None
else:
result, _err = run_python_script(cmd)
assert OmegaConf.create(result) == expected
def test_short_module_name(tmpdir: Path) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
out, _err = run_python_script(cmd)
assert OmegaConf.create(out) == {
"db": {
"driver": "mysql",
"password": "******",
"user": "******"
}
}
def test_cfg(tmpdir: Path, flag: str, resolve: bool, expected_keys: List[str]) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"hydra.job.chdir=True",
flag,
]
if resolve:
cmd.append("--resolve")
result, _err = run_python_script(cmd)
conf = OmegaConf.create(result)
for key in expected_keys:
assert key in conf
def test_ax_logging(tmpdir: Path, cmd_arg: str, expected_str: str) -> None:
cmd = [
"tests/apps/polynomial.py",
"-m",
"hydra.run.dir=" + str(tmpdir),
"polynomial.x=interval(-5, -2)",
"polynomial.z=10",
"hydra.sweeper.ax_config.max_trials=2",
] + [cmd_arg]
result, _ = run_python_script(cmd, allow_warnings=True)
assert "polynomial.x: range=[-5.0, -2.0]" in result
assert expected_str in result
assert "polynomial.z: fixed=10" in result
def test_config_name_and_path_overrides(tmpdir: Path, config_path: str,
config_name: str) -> None:
cmd = [
"tests/test_apps/app_with_multiple_config_dirs/my_app.py",
"hydra.run.dir=" + str(tmpdir),
f"--config-name={config_name}",
f"--config-path={config_path}",
]
print(" ".join(cmd))
result, _err = run_python_script(cmd)
# normalize newlines on Windows to make testing easier
result = result.replace("\r\n", "\n")
assert result == f"{config_path}_{config_name}: true"
def test_5_structured_config_schema(tmpdir: Path, path: str) -> None:
cmd = [path, "hydra.run.dir=" + str(tmpdir)]
result, _err = run_python_script(cmd)
assert OmegaConf.create(result) == {
"db": {
"driver": "mysql",
"host": "localhost",
"password": "******",
"port": 3306,
"user": "******",
},
"debug": True,
}
def test_advanced_package_override_simple(tmpdir: Path) -> None:
cmd = [
"examples/advanced/package_overrides/simple.py",
"hydra.run.dir=" + str(tmpdir),
"hydra.job.chdir=True",
]
result, _err = run_python_script(cmd)
assert OmegaConf.create(result) == {
"db": {
"driver": "mysql",
"user": "******",
"pass": "******"
}
}
def test_multirun_structured_conflict(tmpdir: Any, overrides: List[str],
error: bool, expected: Any) -> None:
cmd = [
"tests/test_apps/multirun_structured_conflict/my_app.py",
"hydra.sweep.dir=" + str(tmpdir),
]
cmd.extend(overrides)
if error:
expected = normalize_newlines(expected)
ret = normalize_newlines(run_with_error(cmd))
assert re.search(re.escape(expected), ret) is not None
else:
ret, _err = run_python_script(cmd)
assert ret == expected
def test_cfg_resolve_interpolation(
tmpdir: Path, script: str, resolve: bool, flags: List[str], expected: str
) -> None:
cmd = [
script,
"hydra.run.dir=" + str(tmpdir),
"hydra.job.chdir=True",
"--cfg=job",
] + flags
if resolve:
cmd.append("--resolve")
result, _err = run_python_script(cmd)
assert_text_same(result, expected)
def test_extending_configs(monkeypatch: Any, tmpdir: Path,
overrides: List[str]) -> None:
monkeypatch.chdir("examples/patterns/extending_configs")
cmd = ["my_app.py", "hydra.run.dir=" + str(tmpdir)] + overrides
result, _err = run_python_script(cmd)
assert OmegaConf.create(result) == {
"db": {
"host": "localhost",
"port": 3307,
"user": "******",
"password": "******",
"encoding": "utf8",
}
}
def test_disable_chdir(tmpdir: Path, multirun: bool, expected: List[str]) -> None:
cmd = [
"examples/tutorials/basic/running_your_hydra_app/3_working_directory/my_app.py",
f"hydra.run.dir={tmpdir}",
f"hydra.sweep.dir={tmpdir}",
"hydra.job.chdir=False",
]
if multirun:
cmd += ["-m"]
result, _err = run_python_script(cmd)
assert f"Working directory : {os.getcwd()}" in result
for p in expected:
path = os.path.join(str(tmpdir), p)
assert Path(path).exists()
def test_multirun_defaults_override(self, cmd_base: List[str], tmpdir: Any) -> None:
cmd = cmd_base + [
"hydra.sweep.dir=" + str(tmpdir),
"group1=file1,file2",
"--multirun",
"--config-path=../../../hydra/test_utils/configs",
"--config-name=compose",
]
expected = """foo: 10
bar: 100
foo: 20
bar: 100"""
ret, _err = run_python_script(cmd)
assert normalize_newlines(ret) == normalize_newlines(expected)
def test_4_defaults(tmpdir: Path) -> None:
cmd = [
"examples/tutorials/structured_configs/4_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = run_python_script(cmd)
assert OmegaConf.create(result) == {
"db": {
"driver": "mysql",
"host": "localhost",
"password": "******",
"port": 3306,
"user": "******",
}
}
def test_module_run(tmpdir: Any, directory: str, file: str, module: str,
error: Optional[str]) -> None:
cmd = [
directory + "/" + file,
"hydra.run.dir=" + str(tmpdir),
]
modified_env = os.environ.copy()
modified_env["PYTHONPATH"] = directory
modified_env["HYDRA_MAIN_MODULE"] = module
if error is not None:
ret = run_with_error(cmd, modified_env)
assert re.search(re.escape(error), ret) is not None
else:
result, _err = run_python_script(cmd, env=modified_env)
assert OmegaConf.create(result) == {"x": 10}
def test_hydra_deprecation_warning(
env_deprecation_err: bool, expected: str, tmpdir: Path
) -> None:
cmd = [
"tests/test_apps/deprecation_warning/my_app.py",
f"hydra.run.dir={tmpdir}",
"hydra.job.chdir=True",
]
env = os.environ.copy()
if env_deprecation_err:
env["HYDRA_DEPRECATION_WARNINGS_AS_ERRORS"] = "1"
_, err = run_python_script(
cmd, env=env, allow_warnings=True, print_error=False, raise_exception=False
)
assert_multiline_regex_search(expected, err)
def test_defaults_pkg_with_dot(monkeypatch: Any, tmpdir: Path) -> None:
monkeypatch.chdir("tests/test_apps/defaults_pkg_with_dot")
cmd = [
"my_app.py",
"hydra.run.dir=" + str(tmpdir),
]
result, _err = run_python_script(cmd)
assert OmegaConf.create(result) == {
"dataset": {
"test": {
"name": "imagenet",
"path": "/datasets/imagenet"
}
}
}
def test_multirun_config_overrides_evaluated_lazily(
self, cmd_base: List[str], tmpdir: Any) -> None:
cmd = cmd_base + [
"hydra.sweep.dir=" + str(tmpdir),
"+foo=10,20",
"+bar=${foo}",
"--multirun",
]
expected = """foo: 10
bar: 10
foo: 20
bar: 20"""
ret, _err = run_python_script(cmd)
assert normalize_newlines(ret) == normalize_newlines(expected)
def test_jobs_using_choice_between_dicts(
tmpdir: Path,
cmd_arg: str,
serialized_encoding: str,
best_coefficients: str,
best_value: float,
) -> None:
cmd = [
"tests/apps/polynomial_with_dict_coefficients.py",
"-m",
"hydra.run.dir=" + str(tmpdir),
"hydra.sweeper.ax_config.max_trials=3",
] + [cmd_arg]
result, _ = run_python_script(cmd, allow_warnings=True)
assert f"polynomial.coefficients: {serialized_encoding}" in result
assert f"'+polynomial.coefficients': {best_coefficients}" in result
assert f"New best value: {best_value}" in result
def test_experimental_save_job_info_callback(tmpdir: Path, multirun: bool) -> None:
app_path = "tests/test_apps/app_with_pickle_job_info_callback/my_app.py"
cmd = [
app_path,
"hydra.run.dir=" + str(tmpdir),
"hydra.sweep.dir=" + str(tmpdir),
"hydra.job.chdir=True",
]
if multirun:
cmd.append("-m")
_, _err = run_python_script(cmd)
def load_pickle(path: Path) -> Any:
with open(str(path), "rb") as input:
obj = pickle.load(input) # nosec
return obj
# load pickles from callbacks
callback_output = tmpdir / Path("0") / ".hydra" if multirun else tmpdir / ".hydra"
config_on_job_start = load_pickle(callback_output / "config.pickle")
job_return_on_job_end: JobReturn = load_pickle(
callback_output / "job_return.pickle"
)
task_cfg_from_callback = copy.deepcopy(config_on_job_start)
with read_write(task_cfg_from_callback):
with open_dict(task_cfg_from_callback):
del task_cfg_from_callback["hydra"]
# load pickles generated from the application
app_output_dir = tmpdir / "0" if multirun else tmpdir
task_cfg_from_app = load_pickle(app_output_dir / "task_cfg.pickle")
hydra_cfg_from_app = load_pickle(app_output_dir / "hydra_cfg.pickle")
# verify the cfg pickles are the same on_job_start
assert task_cfg_from_callback == task_cfg_from_app
assert config_on_job_start.hydra == hydra_cfg_from_app
# verify pickled object are the same on_job_end
assert job_return_on_job_end.cfg == task_cfg_from_app
assert job_return_on_job_end.hydra_cfg.hydra == hydra_cfg_from_app # type: ignore
assert job_return_on_job_end.return_value == "hello world"
assert job_return_on_job_end.status == JobStatus.COMPLETED
def test_hydra_main_without_config_path(tmpdir: Path) -> None:
cmd = [
"tests/test_apps/hydra_main_without_config_path/my_app.py",
f"hydra.run.dir={tmpdir}",
]
_, err = run_python_script(cmd, allow_warnings=True)
expected = dedent("""
.*my_app.py:7: UserWarning:
config_path is not specified in @hydra.main().
See https://hydra.cc/docs/next/upgrades/1.0_to_1.1/changes_to_hydra_main_config_path for more information.
@hydra.main()
""")
assert_regex_match(
from_line=expected,
to_line=err,
from_name="Expected error",
to_name="Actual error",
)
def test_multirun_structured_conflict(tmpdir: Any, overrides: List[str],
error: bool, expected: Any) -> None:
cmd = [
"tests/test_apps/multirun_structured_conflict/my_app.py",
"hydra.sweep.dir=" + str(tmpdir),
]
cmd.extend(overrides)
if error:
expected = normalize_newlines(expected)
ret = run_with_error(cmd)
assert_regex_match(
from_line=expected,
to_line=ret,
from_name="Expected output",
to_name="Actual output",
)
else:
ret, _err = run_python_script(cmd)
assert ret == expected
def test_job_chdir_not_specified(tmpdir: Path) -> None:
cmd = [
"tests/test_apps/app_with_no_chdir_override/my_app.py",
f"hydra.run.dir={tmpdir}",
]
_, err = run_python_script(cmd, allow_warnings=True)
expected = dedent(
"""
.*UserWarning: Hydra 1.3 will no longer change working directory at job runtime by default.
See https://hydra.cc/docs/upgrades/1.1_to_1.2/changes_to_job_working_dir for more information..*
.*
"""
)
assert_regex_match(
from_line=expected,
to_line=err,
from_name="Expected error",
to_name="Actual error",
)