def test_multirun_structured_conflict(tmpdir: Any, overrides: List[str],
error: bool, expected: Any) -> None:
cmd = [
"tests/test_apps/multirun_structured_conflict/my_app.py",
"hydra.sweep.dir=" + str(tmpdir),
]
cmd.extend(overrides)
if error:
expected = normalize_newlines(expected)
ret = normalize_newlines(run_with_error(cmd))
assert re.search(re.escape(expected), ret) is not None
else:
ret, _err = get_run_output(cmd)
assert ret == expected
def test_multirun_config_overrides_evaluated_lazily(
self, cmd_base: List[str], tmpdir: Any) -> None:
cmd = cmd_base + [
"hydra.sweep.dir=" + str(tmpdir),
"+foo=10,20",
"+bar=${foo}",
"--multirun",
]
expected = """foo: 10
bar: 10
foo: 20
bar: 20"""
ret, _err = run_python_script(cmd)
assert normalize_newlines(ret) == normalize_newlines(expected)
def test_command_line_interpolations_evaluated_lazily(
self, cmd_base: List[str], tmpdir: Any) -> None:
cmd = cmd_base + [
"hydra.sweep.dir=" + str(tmpdir),
"+foo=10,20",
"+bar=${foo}",
"--multirun",
]
expected = """foo: 10
bar: 10
foo: 20
bar: 20"""
ret, _err = get_run_output(cmd)
assert normalize_newlines(ret) == normalize_newlines(expected)
def test_multirun_defaults_override(self, cmd_base: List[str], tmpdir: Any) -> None:
cmd = cmd_base + [
"hydra.sweep.dir=" + str(tmpdir),
"group1=file1,file2",
"--multirun",
"--config-path=../../../hydra/test_utils/configs",
"--config-name=compose",
]
expected = """foo: 10
bar: 100
foo: 20
bar: 100"""
ret, _err = get_run_output(cmd)
assert normalize_newlines(ret) == normalize_newlines(expected)
def test_cli_error(tmpdir: Any, monkeypatch: Any, override: Any,
expected: str) -> None:
monkeypatch.chdir("tests/test_apps/app_without_config/")
if isinstance(override, str):
override = [override]
cmd = ["my_app.py", "hydra.sweep.dir=" + str(tmpdir)] + override
ret = normalize_newlines(run_with_error(cmd))
assert (re.search("^" + re.escape(normalize_newlines(expected.strip())),
ret) is not None), (f"Result:"
f"\n---"
f"\n{ret}"
f"\n---"
f"\nDid not match expected:"
f"\n---"
f"\n{expected}"
f"\n---")
def test_cfg_with_package(tmpdir: Path, flags: List[str], expected: str) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
] + flags
result, _err = get_run_output(cmd)
assert normalize_newlines(result) == expected.rstrip()
def test_help(tmpdir: Path, script: str, flag: str, overrides: List[str],
expected: Any) -> None:
cmd = [script, "hydra.run.dir=" + str(tmpdir)]
cmd.extend(overrides)
cmd.append(flag)
result, _err = get_run_output(cmd)
# normalize newlines on Windows to make testing easier
assert result == normalize_newlines(
expected.format(script=script)).rstrip()
def test_cfg_with_package(tmpdir: Path, flags: List[str], resolve: bool,
expected: str) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py",
"hydra.run.dir=" + str(tmpdir),
] + flags
if resolve:
cmd.append("--resolve")
result, _err = run_python_script(cmd)
assert normalize_newlines(result) == expected.rstrip()
def test_resolve_flag_without_cfg_flag(tmpdir: Path) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/3_using_config/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"--resolve",
]
err = run_with_error(cmd)
assert normalize_newlines(err).endswith(
"ValueError: The --resolve flag can only be used in conjunction with --cfg"
)
def test_app_with_error_exception_sanitized(tmpdir: Any,
monkeypatch: Any) -> None:
monkeypatch.chdir("tests/test_apps/app_with_runtime_config_error")
cmd = [
"my_app.py",
"hydra.sweep.dir=" + str(tmpdir),
]
expected = """Traceback (most recent call last):
File "my_app.py", line 13, in my_app
foo(cfg)
File "my_app.py", line 8, in foo
cfg.foo = "bar" # does not exist in the config
omegaconf.errors.ConfigAttributeError: Key 'foo' is not in struct
\tfull_key: foo
\treference_type=Optional[Dict[Union[str, Enum], Any]]
\tobject_type=dict
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace."""
ret = run_with_error(cmd)
assert normalize_newlines(expected) == normalize_newlines(ret)
def test_resolve_flag_errmsg(tmpdir: Path, other_flag: Optional[str]) -> None:
cmd = [
"examples/tutorials/basic/your_first_hydra_app/3_using_config/my_app.py",
"hydra.run.dir=" + str(tmpdir),
"--resolve",
]
if other_flag is not None:
cmd.append(other_flag)
err = run_with_error(cmd)
assert normalize_newlines(err).endswith(
"ValueError: The --resolve flag can only be used in conjunction with --cfg or --help"
)
def test_multirun_structured_conflict(tmpdir: Any, overrides: List[str],
error: bool, expected: Any) -> None:
cmd = [
"tests/test_apps/multirun_structured_conflict/my_app.py",
"hydra.sweep.dir=" + str(tmpdir),
]
cmd.extend(overrides)
if error:
expected = normalize_newlines(expected)
ret = run_with_error(cmd)
assert_regex_match(
from_line=expected,
to_line=ret,
from_name="Expected output",
to_name="Actual output",
)
else:
ret, _err = run_python_script(cmd)
assert ret == expected
