def on_exception(self, runner: IRunner):
exception = runner.exception
if not utils.is_exception(exception):
return
if runner.device.type == "xla":
from torch_xla.core.xla_model import save
else:
from torch import save
try:
checkpoint = _pack_runner(runner)
suffix = self.get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
utils.save_checkpoint(
logdir=Path(f"{runner.logdir}/checkpoints/"),
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False,
saver_fn=save,
)
metrics = self.metrics
metrics[suffix] = runner.valid_metrics
self.save_metric(runner.logdir, metrics)
except Exception: # noqa: S110
pass
def on_exception(self, runner: IRunner):
"""@TODO: Docs. Contribution is welcome."""
exception = runner.exception
if not utils.is_exception(exception):
return
if runner.need_exception_reraise:
raise exception
def on_exception(self, state: State):
"""@TODO: Docs. Contribution is welcome."""
exception = state.exception
if not utils.is_exception(exception):
return
if state.need_exception_reraise:
raise exception
def on_exception(self, state: State):
"""Called if an Exception was raised."""
exception = state.exception
if not utils.is_exception(exception):
return
if isinstance(exception, KeyboardInterrupt):
self.tqdm.write("Early exiting")
state.need_exception_reraise = False
def on_exception(self, runner: IRunner):
"""Called if an Exception was raised."""
exception = runner.exception
if not utils.is_exception(exception):
return
if isinstance(exception, KeyboardInterrupt):
if self.tqdm is not None:
self.tqdm.write("Early exiting")
runner.need_exception_reraise = False
def on_exception(self, state: State):
exception = state.exception
if not utils.is_exception(exception):
return
try:
checkpoint = _pack_state(state)
suffix = self.get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
utils.save_checkpoint(
logdir=Path(f"{state.logdir}/checkpoints/"),
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False,
)
metrics = self.metrics
metrics[suffix] = state.valid_metrics
self.save_metric(state.logdir, metrics)
except Exception:
pass
def on_exception(self, runner: IRunner):
exception = runner.exception
if not utils.is_exception(exception):
return
try:
checkpoint = _pack_runner(runner)
suffix = self.get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
utils.save_checkpoint(
logdir=Path(f"{runner.logdir}/checkpoints/"),
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False,
)
metrics = self.metrics
metrics[suffix] = runner.valid_metrics
self.save_metric(runner.logdir, metrics)
except Exception: # noqa: S110
pass
