def interrupt_experiment_on_unhealthy_probe(probe: Probe,
run: Run,
configuration: Configuration,
secrets=Secrets) -> None:
if experiment_finished.is_set():
return
tolerance = probe.get("tolerance")
checked = within_tolerance(tolerance,
run["output"],
configuration=configuration,
secrets=secrets)
if not checked and not guardian.interrupted:
guardian.interrupted = True
if not experiment_finished.is_set():
logger.critical(
"Safeguard '{}' triggered the end of the experiment".format(
probe["name"]))
exit_gracefully()
def _exit_soon():
time.sleep(1.5)
exit_gracefully()
def interrupt_gracefully_in(seconds: int = 1.5):
time.sleep(seconds)
exit_gracefully()