def create_torch_datasets( self, dataset_splits: DatasetSplits) -> Dict[ModelExecutionMode, Any]: from InnerEye.ML.dataset.sequence_dataset import SequenceDataset sample_transform = self.get_scalar_item_transform() assert sample_transform.train is not None # for mypy assert sample_transform.val is not None # for mypy assert sample_transform.test is not None # for mypy train = SequenceDataset(self, dataset_splits.train, name="training", sample_transform=sample_transform.train) val = SequenceDataset(self, dataset_splits.val, feature_statistics=train.feature_statistics, name="validation", sample_transform=sample_transform.val) test = SequenceDataset(self, dataset_splits.test, feature_statistics=train.feature_statistics, name="test", sample_transform=sample_transform.test) return { ModelExecutionMode.TRAIN: train, ModelExecutionMode.VAL: val, ModelExecutionMode.TEST: test }
def create_torch_datasets( self, dataset_splits: DatasetSplits) -> Dict[ModelExecutionMode, Any]: from InnerEye.ML.dataset.sequence_dataset import SequenceDataset sample_transforms = self.get_image_sample_transforms() train = SequenceDataset( self, dataset_splits.train, name="training", sample_transforms=sample_transforms.train) # type: ignore val = SequenceDataset( self, dataset_splits.val, feature_statistics=train.feature_statistics, name="validation", sample_transforms=sample_transforms.val) # type: ignore test = SequenceDataset( self, dataset_splits.test, feature_statistics=train.feature_statistics, name="test", sample_transforms=sample_transforms.test) # type: ignore return { ModelExecutionMode.TRAIN: train, ModelExecutionMode.VAL: val, ModelExecutionMode.TEST: test }
def test_visualization_for_different_target_weeks(test_output_dirs: TestOutputDirectories) -> None: """ Tests that the visualizations are differentiated depending on the target week for which we visualize it. """ config = ToyMultiLabelSequenceModel(should_validate=False) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() model = create_model_with_temperature_scaling(config) dataloader = SequenceDataset(config, data_frame=config.dataset_data_frame).as_data_loader(shuffle=False, batch_size=2) batch = next(iter(dataloader)) model_inputs_and_labels = get_scalar_model_inputs_and_labels(config, model, batch) # type: ignore visualizer = VisualizationMaps(model, config) # Pseudo-grad cam explaining the prediction at target sequence 2 _, _, pseudo_cam_non_img_3, probas_3 = visualizer.generate(model_inputs_and_labels.model_inputs, target_position=2, target_label_index=2) # Pseudo-grad cam explaining the prediction at target sequence 0 _, _, pseudo_cam_non_img_1, probas_1 = visualizer.generate(model_inputs_and_labels.model_inputs, target_position=0, target_label_index=0) assert pseudo_cam_non_img_1.shape[1] == 1 assert pseudo_cam_non_img_3.shape[1] == 3 # Both visualizations should not be equal assert np.any(pseudo_cam_non_img_1 != pseudo_cam_non_img_3) assert np.any(probas_3 != probas_1)
def test_visualization_with_sequence_model( use_combined_model: bool, imaging_feature_type: ImagingFeatureType, test_output_dirs: OutputFolderForTests) -> None: config = ToySequenceModel(use_combined_model, imaging_feature_type, should_validate=False) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_mock_sequence_dataset() config.num_epochs = 1 model = config.create_model() if config.use_gpu: model = model.cuda() dataloader = SequenceDataset( config, data_frame=config.dataset_data_frame).as_data_loader(shuffle=False, batch_size=2) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray]( images=np.random.uniform(0, 1, SCAN_SIZE), segmentations=np.random.randint(0, 2, SCAN_SIZE)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): batch = next(iter(dataloader)) if config.use_gpu: batch = transfer_batch_to_device(batch, torch.device(0)) model_inputs_and_labels = get_scalar_model_inputs_and_labels( model, target_indices=config.get_target_indices(), sample=batch) # type: ignore number_sequences = model_inputs_and_labels.model_inputs[0].shape[1] number_subjects = len(model_inputs_and_labels.subject_ids) visualizer = VisualizationMaps(model, config) guided_grad_cams, grad_cams, pseudo_cam_non_img, probas = visualizer.generate( model_inputs_and_labels.model_inputs) if use_combined_model: if imaging_feature_type == ImagingFeatureType.ImageAndSegmentation: assert guided_grad_cams.shape[:2] == (number_subjects, number_sequences * 2) assert grad_cams.shape[:2] == (number_subjects, number_sequences * 2) else: assert guided_grad_cams.shape[:2] == (number_subjects, number_sequences) assert grad_cams.shape[:2] == (number_subjects, number_sequences) else: assert guided_grad_cams is None assert grad_cams is None assert pseudo_cam_non_img.shape[:2] == (number_subjects, number_sequences) assert probas.shape[0] == number_subjects non_image_features = config.numerical_columns + config.categorical_columns non_imaging_plot_labels = visualizer._get_non_imaging_plot_labels( model_inputs_and_labels.data_item, non_image_features, index=0, target_position=3) assert non_imaging_plot_labels == [ 'numerical1_0', 'numerical2_0', 'cat1_0', 'numerical1_1', 'numerical2_1', 'cat1_1', 'numerical1_2', 'numerical2_2', 'cat1_2', 'numerical1_3', 'numerical2_3', 'cat1_3' ]
def test_sequence_dataloader() -> None: """ Test if we can create a data loader from the dataset, and recover the items as expected in batched form. Including instances where not all elements of the sequence have labels. """ csv_string = StringIO("""subject,seq,path,value,scalar1,scalar2,META S1,0,foo.nii,,0,0,M1 S1,1,,True,1.1,1.2,M2 S2,0,bar.nii,False,2.1,2.2,M3 S2,1,,False,2.0,2.0,M4 """) df = pd.read_csv(csv_string, sep=",", dtype=str) config = SequenceModelBase(image_file_column=None, label_value_column="value", numerical_columns=["scalar1"], sequence_target_positions=[1], sequence_column="seq", local_dataset=Path.cwd(), should_validate=False) dataset = SequenceDataset(config, data_frame=df) assert len(dataset) == 2 data_loader = dataset.as_data_loader(shuffle=False, batch_size=2, num_dataload_workers=0) # We have 2 subjects, with a batch size of 2 those should be turned into 1 batch data_loader_output = list(i for i in data_loader) assert len(data_loader_output) == 1 loaded = list(ClassificationItemSequence(**i) for i in data_loader_output) assert loaded[0].id == ["S1", "S2"] assert isinstance(loaded[0].items[0][0], ScalarItem) assert loaded[0].items[0][0].metadata.id == "S1" assert loaded[0].items[0][1].metadata.id == "S1" assert loaded[0].items[1][0].metadata.id == "S2" assert loaded[0].items[1][1].metadata.id == "S2" # The batched sequence data are awkward to work with. Check if we can un-roll them correctly via # from_minibatch un_batched = ClassificationItemSequence.from_minibatch( data_loader_output[0]) assert len(un_batched) == 2 for i in range(2): assert un_batched[i].id == dataset.items[i].id assert len(un_batched[i].items) == len(dataset.items[i].items) for j in range(len(un_batched[i].items)): assert un_batched[i].items[j].metadata.id == dataset.items[ i].items[j].metadata.id
def test_seq_dataset_loader() -> None: dummy_dataset = full_ml_test_data_path( ) / "sequence_data_for_classification" / "dataset.csv" df = pd.read_csv(dummy_dataset, sep=",", dtype=str) dataset = SequenceDataset(args=SequenceModelBase( image_file_column="IMG", label_value_column="Label", numerical_columns=["NUM1", "NUM2", "NUM3", "NUM4"], sequence_target_positions=[8], sequence_column="Position", local_dataset=Path(), should_validate=False), data_frame=df) assert len(dataset) == 2 # Patch the load_images function that well be called once we access a dataset item with mock.patch('InnerEye.ML.dataset.scalar_sample.load_images_and_stack', return_value=ImageAndSegmentations[torch.Tensor]( images=torch.ones(1), segmentations=torch.empty(0))): item0 = ClassificationItemSequence(**dataset[0]) item1 = ClassificationItemSequence(**dataset[1]) assert item0.id == "2627.00001" len_2627 = 3 assert len(item0.items) == len_2627 assert item1.id == "3250.00005" len_3250 = 9 assert len(item1.items) == len_3250 # Data loaders use a customized collate function, that must work with the sequences too. collated = collate_with_metadata([dataset[0], dataset[1]]) assert collated["id"] == ["2627.00001", "3250.00005"] # All subject sequences should be turned into lists of lists. assert isinstance(collated["items"], list) assert len(collated["items"]) == 2 assert isinstance(collated["items"][0], list) assert isinstance(collated["items"][1], list) assert len(collated["items"][0]) == len_2627 assert len(collated["items"][1]) == len_3250 back_to_items = ClassificationItemSequence(**collated) assert back_to_items.id == ["2627.00001", "3250.00005"]