def _default_iteration_print(self, engine: Engine) -> None: """ Execute iteration log operation based on Ignite `engine.state.output` data. Print the values from `self.output_transform(engine.state.output)`. Since `engine.state.output` is a decollated list and we replicated the loss value for every item of the decollated list, the default behavior is to print the loss from `output[0]`. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. """ loss = self.output_transform(engine.state.output) if loss is None: return # no printing if the output is empty out_str = "" if isinstance(loss, dict): # print dictionary items for name in sorted(loss): value = loss[name] if not is_scalar(value): warnings.warn( "ignoring non-scalar output in StatsHandler," " make sure `output_transform(engine.state.output)` returns" " a scalar or dictionary of key and scalar pairs to avoid this warning." " {}:{}".format(name, type(value)) ) continue # not printing multi dimensional output out_str += self.key_var_format.format(name, value.item() if isinstance(value, torch.Tensor) else value) elif is_scalar(loss): # not printing multi dimensional output out_str += self.key_var_format.format( self.tag_name, loss.item() if isinstance(loss, torch.Tensor) else loss ) else: warnings.warn( "ignoring non-scalar output in StatsHandler," " make sure `output_transform(engine.state.output)` returns" " a scalar or a dictionary of key and scalar pairs to avoid this warning." " {}".format(type(loss)) ) if not out_str: return # no value to print num_iterations = engine.state.epoch_length current_iteration = engine.state.iteration if num_iterations is not None: current_iteration = (current_iteration - 1) % num_iterations + 1 current_epoch = engine.state.epoch num_epochs = engine.state.max_epochs base_str = f"Epoch: {current_epoch}/{num_epochs}, Iter: {current_iteration}/{num_iterations} --" self.logger.info(" ".join([base_str, out_str]))
def _default_iteration_writer(self, engine: Engine, writer: SummaryWriter) -> None: """ Execute iteration level event write operation based on Ignite `engine.state.output` data. Extract the values from `self.output_transform(engine.state.output)`. Since `engine.state.output` is a decollated list and we replicated the loss value for every item of the decollated list, the default behavior is to track the loss from `output[0]`. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. writer: TensorBoard or TensorBoardX writer, passed or created in TensorBoardHandler. """ loss = self.output_transform(engine.state.output) if loss is None: return # do nothing if output is empty if isinstance(loss, dict): for name in sorted(loss): value = loss[name] if not is_scalar(value): warnings.warn( "ignoring non-scalar output in TensorBoardStatsHandler," " make sure `output_transform(engine.state.output)` returns" " a scalar or dictionary of key and scalar pairs to avoid this warning." " {}:{}".format(name, type(value))) continue # not plot multi dimensional output self._write_scalar( _engine=engine, writer=writer, tag=name, value=value.item() if isinstance(value, torch.Tensor) else value, step=engine.state.iteration, ) elif is_scalar(loss): # not printing multi dimensional output self._write_scalar( _engine=engine, writer=writer, tag=self.tag_name, value=loss.item() if isinstance(loss, torch.Tensor) else loss, step=engine.state.iteration, ) else: warnings.warn( "ignoring non-scalar output in TensorBoardStatsHandler," " make sure `output_transform(engine.state.output)` returns" " a scalar or a dictionary of key and scalar pairs to avoid this warning." " {}".format(type(loss))) writer.flush()
def _default_epoch_print(self, engine: Engine) -> None: """ Execute epoch level log operation. Default to print the values from Ignite `engine.state.metrics` dict and print the values of specified attributes of `engine.state`. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. """ current_epoch = self.global_epoch_transform(engine.state.epoch) prints_dict = engine.state.metrics if prints_dict is not None and len(prints_dict) > 0: out_str = f"Epoch[{current_epoch}] Metrics -- " for name in sorted(prints_dict): value = prints_dict[name] out_str += self.key_var_format.format(name, value) if is_scalar(value) else f"{name}: {str(value)}" self.logger.info(out_str) if ( hasattr(engine.state, "key_metric_name") and hasattr(engine.state, "best_metric") and hasattr(engine.state, "best_metric_epoch") ): out_str = f"Key metric: {engine.state.key_metric_name} " # type: ignore out_str += f"best value: {engine.state.best_metric} " # type: ignore out_str += f"at epoch: {engine.state.best_metric_epoch}" # type: ignore self.logger.info(out_str) if self.state_attributes is not None and len(self.state_attributes) > 0: out_str = "State values: " for attr in self.state_attributes: out_str += f"{attr}: {getattr(engine.state, attr, None)} " self.logger.info(out_str)
def _default_iteration_writer( self, engine: "ignite.engine.Engine", writer: "torch.utils.tensorboard.SummaryWriter") -> None: """ Execute iteration level event write operation based on Ignite engine.state data. Default is to write the loss value of current iteration. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. writer: TensorBoard writer, created in TensorBoardHandler. """ loss = self.output_transform(engine.state.output) if loss is None: return # do nothing if output is empty if isinstance(loss, dict): for name in sorted(loss): value = loss[name] if not is_scalar(value): warnings.warn( "ignoring non-scalar output in TensorBoardStatsHandler," " make sure `output_transform(engine.state.output)` returns" " a scalar or dictionary of key and scalar pairs to avoid this warning." " {}:{}".format(name, type(value))) continue # not plot multi dimensional output writer.add_scalar( name, value.item() if torch.is_tensor(value) else value, engine.state.iteration) elif is_scalar(loss): # not printing multi dimensional output writer.add_scalar(self.tag_name, loss.item() if torch.is_tensor(loss) else loss, engine.state.iteration) else: warnings.warn( "ignoring non-scalar output in TensorBoardStatsHandler," " make sure `output_transform(engine.state.output)` returns" " a scalar or a dictionary of key and scalar pairs to avoid this warning." " {}".format(type(loss))) writer.flush()
def _compare_metrics(engine: Workflow) -> None: key_metric_name = engine.state.key_metric_name if key_metric_name is not None: current_val_metric = engine.state.metrics[key_metric_name] if not is_scalar(current_val_metric): warnings.warn( "key metric is not a scalar value, skip the metric comparison with the current best metric." "please set other metrics as the key metric, or change the `reduction` mode to 'mean'." ) return if self.metric_cmp_fn(current_val_metric, engine.state.best_metric): self.logger.info(f"Got new best metric of {key_metric_name}: {current_val_metric}") engine.state.best_metric = current_val_metric engine.state.best_metric_epoch = engine.state.epoch
def _score_func(engine: Engine): if isinstance(key_metric_name, str): metric_name = key_metric_name elif hasattr(engine.state, "key_metric_name"): metric_name = engine.state.key_metric_name # type: ignore else: raise ValueError( f"Incompatible values: save_key_metric=True and key_metric_name={key_metric_name}." ) metric = engine.state.metrics[metric_name] if not is_scalar(metric): warnings.warn( "key metric is not a scalar value, skip metric comaprison and don't save a model." "please use other metrics as key metric, or change the `reduction` mode to 'mean'." ) return -1 return (-1 if key_metric_negative_sign else 1) * metric
def _default_epoch_writer(self, engine: Engine, writer) -> None: """ Execute epoch level event write operation. Default to write the values from Ignite `engine.state.metrics` dict and write the values of specified attributes of `engine.state`. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. writer: TensorBoard or TensorBoardX writer, passed or created in TensorBoardHandler. """ current_epoch = self.global_epoch_transform(engine.state.epoch) summary_dict = engine.state.metrics for name, value in summary_dict.items(): if is_scalar(value): self._write_scalar(engine, writer, name, value, current_epoch) if self.state_attributes is not None: for attr in self.state_attributes: self._write_scalar(engine, writer, attr, getattr(engine.state, attr, None), current_epoch) writer.flush()