def on_exception(self, state: RunnerState):
exception = state.exception
if not is_exception(exception):
return
try:
valid_metrics = state.metrics.valid_values
epoch_metrics = state.metrics.epoch_values
checkpoint = utils.pack_checkpoint(
model=state.model,
criterion=state.criterion,
optimizer=state.optimizer,
scheduler=state.scheduler,
epoch_metrics=epoch_metrics,
valid_metrics=valid_metrics,
stage=state.stage,
epoch=state.epoch_log,
checkpoint_data=state.checkpoint_data
)
suffix = self.get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
utils.save_checkpoint(
logdir=f"{state.logdir}/checkpoints/",
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False
)
metrics = self.metrics
metrics[suffix] = valid_metrics
self.save_metric(state.logdir, metrics)
except Exception:
pass
def on_exception(self, state: RunnerState):
exception = state.exception
if not is_exception(exception):
return
if state.need_reraise_exception:
raise exception
def on_exception(self, state: IRunner):
exception = state.exception
if not utils.is_exception(exception):
return
try:
valid_metrics = state.valid_metrics
epoch_metrics = state.epoch_metrics
checkpoint = utils.pack_checkpoint(
model=state.model,
criterion=state.criterion,
optimizer=state.optimizer,
scheduler=state.scheduler,
epoch_metrics=epoch_metrics,
valid_metrics=valid_metrics,
stage=state.stage_name,
epoch=state.epoch,
checkpoint_data=state.checkpoint_data,
)
suffix = self.get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
utils.save_checkpoint(
logdir=Path(f"{state.logdir}/{self.checkpoints_dir}/"),
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False,
)
metrics = self.metrics
metrics[suffix] = valid_metrics
self.save_metric(state.logdir, metrics)
except Exception:
pass
def on_exception(self, state: _State):
exception = state.exception
if not utils.is_exception(exception):
return
if state.need_exception_reraise:
raise exception
def on_exception(self, state: RunnerState):
exception = state.exception
if not is_exception(exception):
return
if isinstance(exception, KeyboardInterrupt):
self.tqdm.write("Early exiting")
state.need_reraise_exception = False
def on_exception(self, state: State):
"""@TODO: Docs. Contribution is welcome."""
exception = state.exception
if not utils.is_exception(exception):
return
if state.need_exception_reraise:
raise exception
def on_exception(self, state: State):
"""Called if an Exception was raised"""
exception = state.exception
if not utils.is_exception(exception):
return
if isinstance(exception, KeyboardInterrupt):
self.tqdm.write("Early exiting")
state.need_exception_reraise = False
def on_exception(self, state: _State):
"""Notify about raised Exception"""
if self.log_on_exception:
exception = state.exception
if utils.is_exception(exception) and not isinstance(
exception, KeyboardInterrupt):
text = (f"`{type(exception).__name__}` exception was raised:\n"
f"{exception}")
self._send_text(text)
def on_exception(self, runner: IRunner):
"""Notify about raised ``Exception``."""
if self.log_on_exception:
exception = runner.exception
if utils.is_exception(exception) and not isinstance(
exception, KeyboardInterrupt):
text = (f"`{type(exception).__name__}` exception was raised:\n"
f"{exception}")
self._send_text(text)
def on_exception(self, state: State):
exception = state.exception
if not utils.is_exception(exception):
return
try:
checkpoint = _pack_state(state)
suffix = self.get_checkpoint_suffix(checkpoint)
suffix = f"{suffix}.exception_{exception.__class__.__name__}"
utils.save_checkpoint(logdir=Path(f"{state.logdir}/checkpoints/"),
checkpoint=checkpoint,
suffix=suffix,
is_best=False,
is_last=False)
metrics = self.metrics
metrics[suffix] = state.valid_metrics
self.save_metric(state.logdir, metrics)
except Exception:
pass
