def on_exception(self, runner: "IRunner"):
"""
Expection handler.
Args:
runner: current runner
"""
exception = runner.exception
if not is_exception(exception):
return
if runner.device.type == "xla":
from torch_xla.core.xla_model import save
else:
from torch import save
try:
checkpoint = _pack_runner(runner)
suffix = self._get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
save_checkpoint(
logdir=Path(f"{runner.logdir}/checkpoints/"),
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False,
saver_fn=save,
)
metrics = self.metrics
metrics[suffix] = runner.valid_metrics
self._save_metric(runner.logdir, metrics)
except Exception: # noqa: S110
pass
def on_exception(self, runner: "IRunner"):
"""Notify about raised ``Exception``."""
if self.log_on_exception:
exception = runner.exception
if is_exception(exception) and not isinstance(
exception, KeyboardInterrupt):
text = (f"`{type(exception).__name__}` exception was raised:\n"
f"{exception}")
self._send_text(text)
def on_exception(self, runner: "IRunner"):
"""Called if an Exception was raised."""
exception = runner.exception
if not is_exception(exception):
return
if isinstance(exception, KeyboardInterrupt):
if self.tqdm is not None:
self.tqdm.write("Early exiting")
runner.need_exception_reraise = False
def on_exception(self, runner: "IRunner"):
"""Called if an Exception was raised."""
exception = runner.exception
if not is_exception(exception):
return
if isinstance(exception, KeyboardInterrupt):
if self.tqdm is not None:
self.tqdm.write("Keyboard Interrupt")
self.tqdm.clear()
self.tqdm.close()
self.tqdm = None