def on_exception(self, state: RunnerState): exception = state.exception if not is_exception(exception): return try: valid_metrics = state.metrics.valid_values epoch_metrics = state.metrics.epoch_values checkpoint = utils.pack_checkpoint( model=state.model, criterion=state.criterion, optimizer=state.optimizer, scheduler=state.scheduler, epoch_metrics=epoch_metrics, valid_metrics=valid_metrics, stage=state.stage, epoch=state.epoch_log, checkpoint_data=state.checkpoint_data ) suffix = self.get_checkpoint_suffix(checkpoint) suffix = f"{suffix}.exception_{exception.__class__.__name__}" utils.save_checkpoint( logdir=f"{state.logdir}/checkpoints/", checkpoint=checkpoint, suffix=suffix, is_best=False, is_last=False ) metrics = self.metrics metrics[suffix] = valid_metrics self.save_metric(state.logdir, metrics) except Exception: pass
def on_exception(self, state: RunnerState): exception = state.exception if not is_exception(exception): return if state.need_reraise_exception: raise exception
def on_exception(self, state: IRunner): exception = state.exception if not utils.is_exception(exception): return try: valid_metrics = state.valid_metrics epoch_metrics = state.epoch_metrics checkpoint = utils.pack_checkpoint( model=state.model, criterion=state.criterion, optimizer=state.optimizer, scheduler=state.scheduler, epoch_metrics=epoch_metrics, valid_metrics=valid_metrics, stage=state.stage_name, epoch=state.epoch, checkpoint_data=state.checkpoint_data, ) suffix = self.get_checkpoint_suffix(checkpoint) suffix = f"{suffix}.exception_{exception.__class__.__name__}" utils.save_checkpoint( logdir=Path(f"{state.logdir}/{self.checkpoints_dir}/"), checkpoint=checkpoint, suffix=suffix, is_best=False, is_last=False, ) metrics = self.metrics metrics[suffix] = valid_metrics self.save_metric(state.logdir, metrics) except Exception: pass
def on_exception(self, state: _State): exception = state.exception if not utils.is_exception(exception): return if state.need_exception_reraise: raise exception
def on_exception(self, state: RunnerState): exception = state.exception if not is_exception(exception): return if isinstance(exception, KeyboardInterrupt): self.tqdm.write("Early exiting") state.need_reraise_exception = False
def on_exception(self, state: State): """@TODO: Docs. Contribution is welcome.""" exception = state.exception if not utils.is_exception(exception): return if state.need_exception_reraise: raise exception
def on_exception(self, state: State): """Called if an Exception was raised""" exception = state.exception if not utils.is_exception(exception): return if isinstance(exception, KeyboardInterrupt): self.tqdm.write("Early exiting") state.need_exception_reraise = False
def on_exception(self, state: _State): """Notify about raised Exception""" if self.log_on_exception: exception = state.exception if utils.is_exception(exception) and not isinstance( exception, KeyboardInterrupt): text = (f"`{type(exception).__name__}` exception was raised:\n" f"{exception}") self._send_text(text)
def on_exception(self, runner: IRunner): """Notify about raised ``Exception``.""" if self.log_on_exception: exception = runner.exception if utils.is_exception(exception) and not isinstance( exception, KeyboardInterrupt): text = (f"`{type(exception).__name__}` exception was raised:\n" f"{exception}") self._send_text(text)
def on_exception(self, state: State): exception = state.exception if not utils.is_exception(exception): return try: checkpoint = _pack_state(state) suffix = self.get_checkpoint_suffix(checkpoint) suffix = f"{suffix}.exception_{exception.__class__.__name__}" utils.save_checkpoint(logdir=Path(f"{state.logdir}/checkpoints/"), checkpoint=checkpoint, suffix=suffix, is_best=False, is_last=False) metrics = self.metrics metrics[suffix] = state.valid_metrics self.save_metric(state.logdir, metrics) except Exception: pass