def test_save_job_return_callback(tmpdir: Path, multirun: bool) -> None:
app_path = "tests/test_apps/app_with_log_jobreturn_callback/my_app.py"
cmd = [
sys.executable,
app_path,
"hydra.run.dir=" + str(tmpdir),
"hydra.sweep.dir=" + str(tmpdir),
"hydra.job.chdir=True",
]
if multirun:
extra = ["+x=0,1", "-m"]
cmd.extend(extra)
log_msg = "omegaconf.errors.ConfigAttributeError: Key 'divisor' is not in struct\n"
run_process(cmd=cmd, print_error=False, raise_exception=False)
if multirun:
log_paths = [tmpdir / "0" / "my_app.log", tmpdir / "1" / "my_app.log"]
else:
log_paths = [tmpdir / "my_app.log"]
for p in log_paths:
with open(p, "r") as file:
logs = file.readlines()
assert log_msg in logs
def test_partial_failure(
tmpdir: Any,
) -> None:
cmd = [
sys.executable,
"-Werror",
"tests/test_apps/app_can_fail/my_app.py",
"--multirun",
"+divisor=1,0",
"hydra.run.dir=" + str(tmpdir),
"hydra.hydra_logging.formatters.simple.format='[HYDRA] %(message)s'",
]
out, err = run_process(cmd=cmd, print_error=False, raise_exception=False)
expected_out = dedent(
"""\
[HYDRA] Launching 2 jobs locally
[HYDRA] \t#0 : +divisor=1
val=1.0
[HYDRA] \t#1 : +divisor=0"""
)
assert_regex_match(
from_line=expected_out,
to_line=out,
from_name="Expected output",
to_name="Actual output",
)
expected_err = dedent(
"""
Error executing job with overrides: ['+divisor=0']
Traceback (most recent call last):
File ".*my_app.py", line 9, in my_app
val = 1 / cfg.divisor
ZeroDivisionError: division by zero
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
"""
)
assert_regex_match(
from_line=expected_err,
to_line=err,
from_name="Expected error",
to_name="Actual error",
)
def test_failure_rate(max_failure_rate: float, tmpdir: Path) -> None:
cmd = [
sys.executable,
"example/my_app.py",
"-m",
f"hydra.sweep.dir={tmpdir}",
"hydra.sweeper.optim.budget=2", # small budget to test fast
"hydra.sweeper.optim.num_workers=2",
f"hydra.sweeper.optim.max_failure_rate={max_failure_rate}",
"error=true",
]
out, err = run_process(cmd, print_error=False, raise_exception=False)
assert "Returning infinity for failed experiment" in out
error_string = "RuntimeError: cfg.error is True"
if max_failure_rate < 1.0:
assert error_string in err
else:
assert error_string not in err