def test_store_as_scaled_ubyte_nifti_fail(test_output_dirs: TestOutputDirectories, input_range: Any) -> None: image = np.random.random_sample((dim_z, dim_y, dim_x)) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 2, 4)) with pytest.raises(Exception): io_util.store_as_scaled_ubyte_nifti(image, header, test_output_dirs.create_file_or_folder_path(default_image_name), input_range)
def test_store_image_as_short_nifti(test_output_dirs: TestOutputDirectories, norm_method: PhotometricNormalizationMethod, image_range: Any, window_level: Any) -> None: window, level = window_level if window_level else (400, 0) image = np.random.random_sample((1, 2, 3)) image_shape = image.shape args = SegmentationModelBase(norm_method=norm_method, window=window, level=level, should_validate=False) # Get integer values that are in the image range image1 = LinearTransform.transform(data=image, input_range=(0, 1), output_range=args.output_range) image = image1.astype(np.short) # type: ignore header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 1, 1)) nifti_name = test_output_dirs.create_file_or_folder_path(default_image_name) io_util.store_image_as_short_nifti(image, header, nifti_name, args) if norm_method == PhotometricNormalizationMethod.CtWindow: output_range = get_range_for_window_level(args.level, args.window) image = LinearTransform.transform(data=image, input_range=args.output_range, output_range=output_range) image = image.astype(np.short) else: image = image * 1000 t = np.unique(image) assert_nifti_content(nifti_name, image_shape, header, list(t), np.short)
def test_store_as_binary_nifti(test_output_dirs: TestOutputDirectories, image: Any) -> None: image = np.array(image) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 2, 4)) io_util.store_binary_mask_as_nifti(image, header, test_output_dirs.create_file_or_folder_path(default_image_name)) t = np.unique(image) assert_nifti_content(test_output_dirs.create_file_or_folder_path(default_image_name), image.shape, header, list(t), np.ubyte)
def test_store_posteriors_nifti_invalid_entries(test_output_dirs: TestOutputDirectories) -> None: image = np.array([0, 1, 2.71, np.nan]) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 1, 0, 0, 1, 0, 0), spacing=(1, 1, 1)) with pytest.raises(ValueError) as ex: io_util.store_posteriors_as_nifti(image, header, test_output_dirs.create_file_or_folder_path(default_image_name)) assert "invalid values" in ex.value.args[0] assert "2.71" in ex.value.args[0] assert "nan" in ex.value.args[0]
def test_store_as_ubyte_nifti(test_output_dirs: TestOutputDirectories) -> None: image = np.random.random_sample((dim_z, dim_y, dim_x)) # get values in [0, 255] range image = np.array((image + 1) * 255).astype(int) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 2, 4)) io_util.store_as_ubyte_nifti(image, header, test_output_dirs.create_file_or_folder_path(default_image_name)) t = np.unique(image).astype(np.ubyte) assert_nifti_content(test_output_dirs.create_file_or_folder_path(default_image_name), image.shape, header, list(t), np.ubyte)
def test_store_inference_results( test_output_dirs: TestOutputDirectories) -> None: np.random.seed(0) num_classes = 2 posterior = torch.nn.functional.softmax(torch.from_numpy( np.random.random_sample((num_classes, dim_z, dim_y, dim_x))), dim=0).numpy() segmentation = np.argmax(posterior, axis=0) assert segmentation.shape == (dim_z, dim_y, dim_x) posterior0 = to_unique_bytes(posterior[0], (0, 1)) posterior1 = to_unique_bytes(posterior[1], (0, 1)) spacing = (2.0, 2.0, 2.0) header = ImageHeader(origin=(0, 0, 0), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=spacing) inference_result = InferencePipeline.Result(epoch=1, patient_id=12, posteriors=posterior, segmentation=segmentation, voxel_spacing_mm=(1, 1, 1)) test_config = _create_config_with_folders(test_output_dirs) assert test_config.class_and_index_with_background() == { "background": 0, "region": 1 } results_folder = test_output_dirs.root_dir store_inference_results(inference_result, test_config, Path(results_folder), header) assert_nifti_content( os.path.join(results_folder, "012", "posterior_background.nii.gz"), segmentation.shape, header, list(posterior0), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", "posterior_region.nii.gz"), segmentation.shape, header, list(posterior1), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", "background.nii.gz"), segmentation.shape, header, list([0, 1]), np.ubyte) assert_nifti_content(os.path.join(results_folder, "012", "region.nii.gz"), segmentation.shape, header, list([0, 1]), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", DEFAULT_RESULT_IMAGE_NAME), segmentation.shape, header, list(np.unique(segmentation)), np.ubyte) assert_nifti_content( os.path.join(results_folder, "012", "uncertainty.nii.gz"), inference_result.uncertainty.shape, header, list([248, 249, 253, 254]), np.ubyte)
def test_store_posteriors_nifti_fail(test_output_dirs: TestOutputDirectories, image: Any) -> None: image = np.array(image) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 1, 0, 0, 1, 0, 0), spacing=(1, 1, 1)) with pytest.raises(Exception): io_util.store_posteriors_as_nifti( image, header, test_output_dirs.create_file_or_folder_path(default_image_name))
def test_store_as_scaled_ubyte_nifti(test_output_dirs: TestOutputDirectories, input_range: Any) -> None: image = np.random.random_sample((dim_z, dim_y, dim_x)) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 2, 4)) io_util.store_as_scaled_ubyte_nifti(image, header, test_output_dirs.create_file_or_folder_path(default_image_name), input_range) image = LinearTransform.transform(data=image, input_range=input_range, output_range=(0, 255)) t = np.unique(image.astype(np.ubyte)) assert_nifti_content(test_output_dirs.create_file_or_folder_path(default_image_name), image.shape, header, list(t), np.ubyte)
def test_store_as_binary_nifti_fail(test_output_dirs: OutputFolderForTests, image: Any) -> None: image = np.array(image) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 1, 0, 0, 1, 0, 0), spacing=(1, 2, 4)) with pytest.raises(Exception): io_util.store_binary_mask_as_nifti( image, header, test_output_dirs.create_file_or_folder_path(default_image_name))
def test_store_posteriors_nifti(test_output_dirs: TestOutputDirectories, image: Any, expected: Any) -> None: image = np.array(image) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 1, 1)) io_util.store_posteriors_as_nifti( image, header, test_output_dirs.create_file_or_folder_path(default_image_name)) assert_nifti_content( test_output_dirs.create_file_or_folder_path(default_image_name), image.shape, header, list(expected), np.ubyte)
def test_store_as_nifti_fail(test_output_dirs: OutputFolderForTests, image_type: Any, scale: Any, input_range: Any, output_range: Any) \ -> None: header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 1, 0, 0, 1, 0, 0), spacing=(1, 2, 4)) image = np.random.random_sample((dim_z, dim_y, dim_x)) with pytest.raises(Exception): io_util.store_as_nifti( image, header, test_output_dirs.create_file_or_folder_path(default_image_name), image_type, scale, input_range, output_range)
def test_save_dataset_example(test_output_dirs: OutputFolderForTests) -> None: """ Test if the example dataset can be saved as expected. """ image_size = (10, 20, 30) label_size = (2, ) + image_size spacing = (1, 2, 3) np.random.seed(0) # Image should look similar to what a photonormalized image looks like: Centered around 0 image = np.random.rand(*image_size) * 2 - 1 # Labels are expected in one-hot encoding, predictions as class index labels = np.zeros(label_size, dtype=int) labels[0] = 1 labels[0, 5:6, 10:11, 15:16] = 0 labels[1, 5:6, 10:11, 15:16] = 1 prediction = np.zeros(image_size, dtype=int) prediction[4:7, 9:12, 14:17] = 1 dataset_sample = DatasetExample(epoch=1, patient_id=2, header=ImageHeader(origin=(0, 1, 0), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=spacing), image=image, prediction=prediction, labels=labels) images_folder = test_output_dirs.root_dir config = SegmentationModelBase( should_validate=False, norm_method=PhotometricNormalizationMethod.Unchanged) config.set_output_to(images_folder) store_and_upload_example(dataset_sample, config) image_from_disk = io_util.load_nifti_image( os.path.join(config.example_images_folder, "p2_e_1_image.nii.gz")) labels_from_disk = io_util.load_nifti_image( os.path.join(config.example_images_folder, "p2_e_1_label.nii.gz")) prediction_from_disk = io_util.load_nifti_image( os.path.join(config.example_images_folder, "p2_e_1_prediction.nii.gz")) assert image_from_disk.header.spacing == spacing # When no photometric normalization is provided when saving, image is multiplied by 1000. # It is then rounded to int64, but converted back to float when read back in. expected_from_disk = (image * 1000).astype(np.int16).astype(np.float64) assert np.array_equal(image_from_disk.image, expected_from_disk) assert labels_from_disk.header.spacing == spacing assert np.array_equal(labels_from_disk.image, np.argmax(labels, axis=0)) assert prediction_from_disk.header.spacing == spacing assert np.array_equal(prediction_from_disk.image, prediction)
def test_scale_and_unscale_image( test_output_dirs: TestOutputDirectories) -> None: """ Test if an image in the CT value range can be recovered when we save dataset examples (undoing the effects of CT Windowing) """ image_size = (5, 5, 5) spacing = (1, 2, 3) header = ImageHeader(origin=(0, 1, 0), direction=(-1, 0, 0, 0, -1, 0, 0, 0, -1), spacing=spacing) np.random.seed(0) # Random image values with mean -100, std 100. This will cover a range # from -400 to +200 HU image = np.random.normal(-100, 100, size=image_size) window = 200 level = -100 # Lower and upper bounds of the interval of raw CT values that will be retained. lower = level - window / 2 upper = level + window / 2 # Create a copy of the image with all values outside of the (Window, Level) range set to the boundaries. # When saving and loading back in, we will not be able to recover any values that fell outside those boundaries. image_restricted = image.copy() image_restricted[image < lower] = lower image_restricted[image > upper] = upper # The image will be saved with voxel type short image_restricted = image_restricted.astype(int) # Apply window and level, mapping to the usual CNN input value range cnn_input_range = (-1, +1) image_windowed = LinearTransform.transform(data=image, input_range=(lower, upper), output_range=cnn_input_range) args = SegmentationModelBase( norm_method=PhotometricNormalizationMethod.CtWindow, output_range=cnn_input_range, window=window, level=level, should_validate=False) file_name = test_output_dirs.create_file_or_folder_path( "scale_and_unscale_image.nii.gz") io_util.store_image_as_short_nifti(image_windowed, header, file_name, args) image_from_disk = io_util.load_nifti_image(file_name) # noinspection PyTypeChecker assert_nifti_content(file_name, image_size, header, np.unique(image_restricted).tolist(), np.short) assert np.array_equal(image_from_disk.image, image_restricted)
def test_store_as_nifti(test_output_dirs: TestOutputDirectories, image_type: Any, scale: Any, input_range: Any, output_range: Any) \ -> None: image = np.random.random_sample((dim_z, dim_y, dim_x)) spacingzyx = (1, 2, 3) path_image = test_output_dirs.create_file_or_folder_path(default_image_name) header = ImageHeader(origin=(1, 1, 1), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=spacingzyx) io_util.store_as_nifti(image, header, path_image, image_type, scale, input_range, output_range) if scale: linear_transform = LinearTransform.transform(data=image, input_range=input_range, output_range=output_range) image = linear_transform.astype(image_type) # type: ignore assert_nifti_content(test_output_dirs.create_file_or_folder_path(default_image_name), image.shape, header, list(np.unique(image.astype(image_type))), image_type) loaded_image = io_util.load_nifti_image(path_image, image_type) assert loaded_image.header.spacing == spacingzyx
def test_register_and_score_model( is_ensemble: bool, dataset_expected_spacing_xyz: Any, model_outside_package: bool, test_output_dirs: TestOutputDirectories) -> None: """ End-to-end test which ensures the scoring pipeline is functioning as expected by performing the following: 1) Registering a pre-trained model to AML 2) Checking that a model zip from the registered model can be created successfully 3) Calling the scoring pipeline to check inference can be run from the published model successfully """ ws = get_default_workspace() # Get an existing config as template loader = get_model_loader( "Tests.ML.configs" if model_outside_package else None) config: SegmentationModelBase = loader.create_model_config_from_name( model_name="BasicModel2EpochsOutsidePackage" if model_outside_package else "BasicModel2Epochs") config.dataset_expected_spacing_xyz = dataset_expected_spacing_xyz config.set_output_to(test_output_dirs.root_dir) # copy checkpoints into the outputs (simulating a run) stored_checkpoints = full_ml_test_data_path( os.path.join("train_and_test_data", "checkpoints")) shutil.copytree(str(stored_checkpoints), config.checkpoint_folder) paths = [Path(config.checkpoint_folder) / "1_checkpoint.pth.tar"] checkpoints = paths * 2 if is_ensemble else paths model = None model_path = None # Mocking to get the source from the current directory # the score.py and python_wrapper.py cannot be moved inside the InnerEye package, which will be the # only code running (if these tests are run on the package). with mock.patch('InnerEye.Common.fixed_paths.repository_root_directory', return_value=tests_root_directory().parent): try: tags = {"model_name": config.model_name} azure_config = get_default_azure_config() if model_outside_package: azure_config.extra_code_directory = "Tests" # contains DummyModel deployment_hook = lambda cfg, azure_cfg, mdl, is_ens: (Path( cfg.model_name), azure_cfg.docker_shm_size) ml_runner = MLRunner(config, azure_config, model_deployment_hook=deployment_hook) model, deployment_path, deployment_details = ml_runner.register_segmentation_model( workspace=ws, tags=tags, best_epoch=0, best_epoch_dice=0, checkpoint_paths=checkpoints, model_proc=ModelProcessing.DEFAULT) assert model is not None model_path = Path( model.get_model_path(model.name, model.version, ws)) assert (model_path / fixed_paths.ENVIRONMENT_YAML_FILE_NAME).exists() assert (model_path / Path("InnerEye/ML/runner.py")).exists() assert deployment_path == Path(config.model_name) assert deployment_details == azure_config.docker_shm_size # move test data into the data folder to simulate an actual run train_and_test_data_dir = full_ml_test_data_path( "train_and_test_data") img_channel_1_name = "id1_channel1.nii.gz" img_channel_1_path = train_and_test_data_dir / img_channel_1_name img_channel_2_name = "id1_channel2.nii.gz" img_channel_2_path = train_and_test_data_dir / img_channel_2_name # download the registered model and test that we can run the score pipeline on it model_root = Path(model.download(str(test_output_dirs.root_dir))) # create a dummy datastore to store model checkpoints and image data # this simulates the code shapshot being executed in a real run test_datastore = Path(test_output_dirs.root_dir) / "test_datastore" shutil.move(str(model_root / "test_outputs"), str(test_datastore / RELATIVE_TEST_OUTPUTS_PATH)) data_root = test_datastore / DEFAULT_DATA_FOLDER os.makedirs(data_root) shutil.copy(str(img_channel_1_path), data_root) shutil.copy(str(img_channel_2_path), data_root) # run score pipeline as a separate process using the python_wrapper.py code to simulate a real run return_code = SubprocessConfig( process="python", args=[ str(model_root / "python_wrapper.py"), "--spawnprocess=python", str(model_root / "score.py"), f"--data-folder={str(test_datastore)}", f"--test_image_channels={img_channel_1_name},{img_channel_2_name}", "--use_gpu=False" ]).spawn_and_monitor_subprocess() # check that the process completed as expected assert return_code == 0 expected_segmentation_path = Path( model_root) / DEFAULT_RESULT_IMAGE_NAME assert expected_segmentation_path.exists() # sanity check the resulting segmentation expected_shape = get_nifti_shape(img_channel_1_path) image_header = ImageHeader(origin=(0, 0, 0), direction=(1, 0, 0, 0, 1, 0, 0, 0, 1), spacing=(1, 1, 1)) assert_nifti_content(str(expected_segmentation_path), expected_shape, image_header, [0], np.ubyte) finally: # delete the registered model, and any downloaded artifacts shutil.rmtree(test_output_dirs.root_dir) if model and model_path: model.delete() shutil.rmtree(model_path)