def segmentation_model_test(
config: SegmentationModelBase,
data_split: ModelExecutionMode,
checkpoint_handler: CheckpointHandler,
model_proc: ModelProcessing = ModelProcessing.DEFAULT
) -> InferenceMetricsForSegmentation:
"""
The main testing loop for segmentation models.
It loads the model and datasets, then proceeds to test the model for all requested checkpoints.
:param config: The arguments object which has a valid random seed attribute.
:param data_split: Indicates which of the 3 sets (training, test, or validation) is being processed.
:param checkpoint_handler: Checkpoint handler object to find checkpoint paths for model initialization
:param model_proc: whether we are testing an ensemble or single model
:return: InferenceMetric object that contains metrics related for all of the checkpoint epochs.
"""
results: Dict[int, float] = {}
checkpoints_to_test = checkpoint_handler.get_checkpoints_to_test()
if not checkpoints_to_test:
raise ValueError(
"There were no checkpoints available for model testing.")
for checkpoint_paths_and_epoch in checkpoints_to_test:
epoch = checkpoint_paths_and_epoch.epoch
epoch_results_folder = config.outputs_folder / get_epoch_results_path(
epoch, data_split, model_proc)
# save the datasets.csv used
config.write_dataset_files(root=epoch_results_folder)
epoch_and_split = "epoch {} {} set".format(epoch, data_split.value)
epoch_dice_per_image = segmentation_model_test_epoch(
config=copy.deepcopy(config),
data_split=data_split,
checkpoint_paths=checkpoint_paths_and_epoch.checkpoint_paths,
results_folder=epoch_results_folder,
epoch_and_split=epoch_and_split)
if epoch_dice_per_image is None:
logging.warning(
"There is no checkpoint file for epoch {}".format(epoch))
else:
epoch_average_dice: float = np.mean(
epoch_dice_per_image) if len(epoch_dice_per_image) > 0 else 0
results[epoch] = epoch_average_dice
logging.info("Epoch: {:3} | Mean Dice: {:4f}".format(
epoch, epoch_average_dice))
if model_proc == ModelProcessing.ENSEMBLE_CREATION:
# For the upload, we want the path without the "OTHER_RUNS/ENSEMBLE" prefix.
name = str(
get_epoch_results_path(epoch, data_split,
ModelProcessing.DEFAULT))
PARENT_RUN_CONTEXT.upload_folder(
name=name, path=str(epoch_results_folder))
if len(results) == 0:
raise ValueError(
"There was no single checkpoint file available for model testing.")
return InferenceMetricsForSegmentation(data_split=data_split,
epochs=results)
def segmentation_model_test(
config: SegmentationModelBase,
execution_mode: ModelExecutionMode,
checkpoint_paths: List[Path],
model_proc: ModelProcessing = ModelProcessing.DEFAULT
) -> InferenceMetricsForSegmentation:
"""
The main testing loop for segmentation models.
It loads the model and datasets, then proceeds to test the model for all requested checkpoints.
:param config: The arguments object which has a valid random seed attribute.
:param execution_mode: Indicates which of the 3 sets (training, test, or validation) is being processed.
:param checkpoint_handler: Checkpoint handler object to find checkpoint paths for model initialization.
:param model_proc: Whether we are testing an ensemble or single model.
:param patient_id: String which contains subject identifier.
:return: InferenceMetric object that contains metrics related for all of the checkpoint epochs.
"""
epoch_results_folder = config.outputs_folder / get_best_epoch_results_path(
execution_mode, model_proc)
# save the datasets.csv used
config.write_dataset_files(root=epoch_results_folder)
epoch_and_split = f"{execution_mode.value} set"
epoch_dice_per_image = segmentation_model_test_epoch(
config=copy.deepcopy(config),
execution_mode=execution_mode,
checkpoint_paths=checkpoint_paths,
results_folder=epoch_results_folder,
epoch_and_split=epoch_and_split)
if epoch_dice_per_image is None:
raise ValueError(
"There was no single checkpoint file available for model testing.")
else:
epoch_average_dice: float = np.mean(
epoch_dice_per_image) if len(epoch_dice_per_image) > 0 else 0
result = epoch_average_dice
logging.info(f"Mean Dice: {epoch_average_dice:4f}")
if model_proc == ModelProcessing.ENSEMBLE_CREATION:
# For the upload, we want the path without the "OTHER_RUNS/ENSEMBLE" prefix.
name = str(
get_best_epoch_results_path(execution_mode,
ModelProcessing.DEFAULT))
PARENT_RUN_CONTEXT.upload_folder(name=name,
path=str(epoch_results_folder))
return InferenceMetricsForSegmentation(execution_mode=execution_mode,
metrics=result)