def test_aggregate_results() -> None: """ Test to make sure inference results are aggregated as expected """ torch.manual_seed(1) num_models = 3 # set expected posteriors model_results = [] # create results for each model for x in range(num_models): posteriors = torch.nn.functional.softmax(torch.rand(3, 3, 3, 3), dim=0).numpy() model_results.append( InferencePipeline.Result( epoch=0, patient_id=0, posteriors=posteriors, segmentation=posteriors_to_segmentation(posteriors), voxel_spacing_mm=(1, 1, 1))) # We calculate expected_posteriors before aggregating, as aggregation modifies model_results. expected_posteriors = np.mean([x.posteriors for x in model_results], axis=0) ensemble_result = EnsemblePipeline.aggregate_results( model_results, aggregation_type=EnsembleAggregationType.Average) assert ensemble_result.epoch == model_results[0].epoch assert ensemble_result.patient_id == model_results[0].patient_id assert np.array_equal(ensemble_result.posteriors, expected_posteriors) assert np.array_equal(ensemble_result.segmentation, posteriors_to_segmentation(expected_posteriors))
def test_store_inference_results( test_output_dirs: TestOutputDirectories) -> None: np.random.seed(0) num_classes = 2 posterior = torch.nn.functional.softmax(torch.from_numpy( np.random.random_sample((num_classes, dim_z, dim_y, dim_x))), dim=0).numpy() segmentation = np.argmax(posterior, axis=0) assert segmentation.shape == (dim_z, dim_y, dim_x) posterior0 = to_unique_bytes(posterior[0], (0, 1)) posterior1 = to_unique_bytes(posterior[1], (0, 1)) spacing = (2.0, 2.0, 2.0) header = ImageHeader(origin=(0, 0, 0), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=spacing) inference_result = InferencePipeline.Result(epoch=1, patient_id=12, posteriors=posterior, segmentation=segmentation, voxel_spacing_mm=(1, 1, 1)) test_config = _create_config_with_folders(test_output_dirs) assert test_config.class_and_index_with_background() == { "background": 0, "region": 1 } results_folder = test_output_dirs.root_dir store_inference_results(inference_result, test_config, Path(results_folder), header) assert_nifti_content( os.path.join(results_folder, "012", "posterior_background.nii.gz"), segmentation.shape, header, list(posterior0), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", "posterior_region.nii.gz"), segmentation.shape, header, list(posterior1), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", "background.nii.gz"), segmentation.shape, header, list([0, 1]), np.ubyte) assert_nifti_content(os.path.join(results_folder, "012", "region.nii.gz"), segmentation.shape, header, list([0, 1]), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", DEFAULT_RESULT_IMAGE_NAME), segmentation.shape, header, list(np.unique(segmentation)), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", "uncertainty.nii.gz"), inference_result.uncertainty.shape, header, list([248, 249, 253, 254]), np.ubyte)
def test_check_inference_result(segmentation: Any, posteriors: Any, voxel_spacing_mm: Any) -> None: """ Tests to make sure correct checks are made when creating results. :return: """ with pytest.raises(Exception): InferencePipeline.Result( epoch=0, patient_id=0, segmentation=segmentation, posteriors=posteriors, voxel_spacing_mm=voxel_spacing_mm )
def test_evaluate_model_predictions() -> None: """ Creates an 'InferencePipeline.Result' object using pre-defined volumes, stores results and evaluates metrics. """ # Patients 3, 4, and 5 are in test dataset such that: # Patient 3 has one missing ground truth channel: "region" # Patient 4 has all missing ground truth channels: "region", "region_1" # Patient 5 has no missing ground truth channels. input_list = [ ["1", "train_and_test_data/id1_channel1.nii.gz", "channel1"], ["1", "train_and_test_data/id1_channel1.nii.gz", "channel2"], ["1", "train_and_test_data/id1_mask.nii.gz", "mask"], ["1", "train_and_test_data/id1_region.nii.gz", "region"], ["1", "train_and_test_data/id1_region.nii.gz", "region_1"], ["2", "train_and_test_data/id2_channel1.nii.gz", "channel1"], ["2", "train_and_test_data/id2_channel1.nii.gz", "channel2"], ["2", "train_and_test_data/id2_mask.nii.gz", "mask"], ["2", "train_and_test_data/id2_region.nii.gz", "region"], ["2", "train_and_test_data/id2_region.nii.gz", "region_1"], ["3", "train_and_test_data/id2_channel1.nii.gz", "channel1"], ["3", "train_and_test_data/id2_channel1.nii.gz", "channel2"], ["3", "train_and_test_data/id2_mask.nii.gz", "mask"], # ["3", "train_and_test_data/id2_region.nii.gz", "region"], # commented on purpose ["3", "train_and_test_data/id2_region.nii.gz", "region_1"], ["4", "train_and_test_data/id2_channel1.nii.gz", "channel1"], ["4", "train_and_test_data/id2_channel1.nii.gz", "channel2"], ["4", "train_and_test_data/id2_mask.nii.gz", "mask"], # ["4", "train_and_test_data/id2_region.nii.gz", "region"], # commented on purpose # ["4", "train_and_test_data/id2_region.nii.gz", "region_1"], # commented on purpose ["5", "train_and_test_data/id2_channel1.nii.gz", "channel1"], ["5", "train_and_test_data/id2_channel1.nii.gz", "channel2"], ["5", "train_and_test_data/id2_mask.nii.gz", "mask"], ["5", "train_and_test_data/id2_region.nii.gz", "region"], ["5", "train_and_test_data/id2_region.nii.gz", "region_1"] ] config = create_config_from_dataset(input_list, train=['1'], val=['2'], test=['3', '4', '5']) config.allow_incomplete_labels = True ds = config.get_torch_dataset_for_inference(ModelExecutionMode.TEST) results_folder = config.outputs_folder if not results_folder.is_dir(): results_folder.mkdir() model_prediction_evaluations: List[Tuple[PatientMetadata, MetricsDict]] = [] for sample_index, sample in enumerate(ds, 1): sample = Sample.from_dict(sample=sample) posteriors = np.zeros((3, ) + sample.mask.shape, 'float32') posteriors[0][:] = 0.2 posteriors[1][:] = 0.6 posteriors[2][:] = 0.2 assert config.dataset_expected_spacing_xyz is not None inference_result = InferencePipeline.Result( patient_id=sample.patient_id, posteriors=posteriors, segmentation=np.argmax(posteriors, 0), voxel_spacing_mm=config.dataset_expected_spacing_xyz) store_inference_results(inference_result=inference_result, config=config, results_folder=results_folder, image_header=sample.metadata.image_header) metadata, metrics_per_class = evaluate_model_predictions( sample_index - 1, config=config, dataset=ds, results_folder=results_folder) model_prediction_evaluations.append((metadata, metrics_per_class)) # Patient 3 has one missing ground truth channel: "region" if sample.metadata.patient_id == '3': assert 'Dice' in metrics_per_class.values('region_1').keys() assert 'HausdorffDistance_millimeters' in metrics_per_class.values( 'region_1').keys() assert 'MeanSurfaceDistance_millimeters' in metrics_per_class.values( 'region_1').keys() for hue_name in ['region', 'Default']: for metric_type in metrics_per_class.values(hue_name).keys(): assert np.isnan( metrics_per_class.values(hue_name)[metric_type]).all() # Patient 4 has all missing ground truth channels: "region", "region_1" if sample.metadata.patient_id == '4': for hue_name in ['region_1', 'region', 'Default']: for metric_type in metrics_per_class.values(hue_name).keys(): assert np.isnan( metrics_per_class.values(hue_name)[metric_type]).all() # Patient 5 has no missing ground truth channels if sample.metadata.patient_id == '5': for metric_type in metrics_per_class.values('Default').keys(): assert np.isnan( metrics_per_class.values('Default')[metric_type]).all() for hue_name in ['region_1', 'region']: assert 'Dice' in metrics_per_class.values(hue_name).keys() assert 'HausdorffDistance_millimeters' in metrics_per_class.values( hue_name).keys() assert 'MeanSurfaceDistance_millimeters' in metrics_per_class.values( hue_name).keys() metrics_writer, average_dice = populate_metrics_writer( model_prediction_evaluations, config) # Patient 3 has only one missing ground truth channel assert not np.isnan(average_dice[0]) assert np.isnan(float(metrics_writer.columns["Dice"][0])) assert not np.isnan(float(metrics_writer.columns["Dice"][1])) assert np.isnan(float(metrics_writer.columns["HausdorffDistance_mm"][0])) assert not np.isnan( float(metrics_writer.columns["HausdorffDistance_mm"][1])) assert np.isnan(float(metrics_writer.columns["MeanDistance_mm"][0])) assert not np.isnan(float(metrics_writer.columns["MeanDistance_mm"][1])) # Patient 4 has all missing ground truth channels assert np.isnan(average_dice[1]) assert np.isnan(float(metrics_writer.columns["Dice"][2])) assert np.isnan(float(metrics_writer.columns["Dice"][3])) assert np.isnan(float(metrics_writer.columns["HausdorffDistance_mm"][2])) assert np.isnan(float(metrics_writer.columns["HausdorffDistance_mm"][3])) assert np.isnan(float(metrics_writer.columns["MeanDistance_mm"][2])) assert np.isnan(float(metrics_writer.columns["MeanDistance_mm"][3])) # Patient 5 has no missing ground truth channels. assert average_dice[2] > 0 assert float(metrics_writer.columns["Dice"][4]) >= 0 assert float(metrics_writer.columns["Dice"][5]) >= 0 assert float(metrics_writer.columns["HausdorffDistance_mm"][4]) >= 0 assert float(metrics_writer.columns["HausdorffDistance_mm"][5]) >= 0 assert float(metrics_writer.columns["MeanDistance_mm"][4]) >= 0 assert float(metrics_writer.columns["MeanDistance_mm"][5]) >= 0