def register_model_for_best_epoch( self, checkpoint_handler: CheckpointHandler, test_metrics: Optional[InferenceMetricsForSegmentation], val_metrics: Optional[InferenceMetricsForSegmentation], model_proc: ModelProcessing) -> int: if val_metrics is not None: best_epoch = val_metrics.get_best_epoch() num_epochs = len(val_metrics.epochs) model_description = f"Epoch {best_epoch} has best validation set metrics (out of {num_epochs} epochs " \ f"available). Validation set Dice: {val_metrics.epochs[best_epoch]}. " if test_metrics: model_description += f"Test set Dice: {test_metrics.epochs[best_epoch]}." else: model_description += "Test set metrics not available." elif test_metrics is not None: # We should normally not get here. We presently always run inference on both validation and test set # together. best_epoch = test_metrics.get_best_epoch() num_epochs = len(test_metrics.epochs) model_description = f"Epoch {best_epoch} has best test set metrics (out of {num_epochs} epochs " \ f"available). Test set Dice: {test_metrics.epochs[best_epoch]}" else: best_epoch = self.model_config.get_test_epochs()[-1] model_description = f"Model for epoch {best_epoch}. No validation or test set metrics were available." checkpoint_paths = checkpoint_handler.get_checkpoint_paths_from_epoch_or_fail( best_epoch) self.register_model_for_epoch(checkpoint_paths, model_description, model_proc) return best_epoch
def run_inference_and_register_model( self, checkpoint_handler: CheckpointHandler, model_proc: ModelProcessing) -> Optional[int]: """ Run inference as required, and register the model, but not necessarily in that order: if we can identify the epoch to register at without running inference, we register first. :param checkpoint_handler: Checkpoint handler object to find checkpoint paths for model initialization :param model_proc: whether we are running an ensemble model from within a child run with index 0. If we are, then outputs will be written to OTHER_RUNS/ENSEMBLE under the main outputs directory. """ registration_epoch = self.decide_registration_epoch_without_evaluating( ) if registration_epoch is not None: model_description = f"Registering model for epoch {registration_epoch} without considering metrics." checkpoint_paths = checkpoint_handler.get_checkpoint_paths_from_epoch_or_fail( registration_epoch) self.register_model_for_epoch(checkpoint_paths, model_description, model_proc) if self.azure_config.register_model_only_for_epoch is not None: return self.azure_config.register_model_only_for_epoch # run full image inference on existing or newly trained model on the training, and testing set test_metrics, val_metrics, _ = self.model_inference_train_and_test( checkpoint_handler=checkpoint_handler, model_proc=model_proc) # register the generated model from the run if we haven't already done so if self.model_config.is_segmentation_model and ( not self.model_config.is_offline_run): if registration_epoch is None: if self.should_register_model(): assert test_metrics is None or isinstance( test_metrics, InferenceMetricsForSegmentation) assert val_metrics is None or isinstance( val_metrics, InferenceMetricsForSegmentation) registration_epoch = self.register_model_for_best_epoch( checkpoint_handler, test_metrics, val_metrics, model_proc) self.try_compare_scores_against_baselines(model_proc) else: logging.warning("Couldn't register model in offline mode") return registration_epoch