def test_visualization_with_sequence_model( use_combined_model: bool, imaging_feature_type: ImagingFeatureType, test_output_dirs: OutputFolderForTests) -> None: config = ToySequenceModel(use_combined_model, imaging_feature_type, should_validate=False) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_mock_sequence_dataset() config.num_epochs = 1 model = config.create_model() if config.use_gpu: model = model.cuda() dataloader = SequenceDataset( config, data_frame=config.dataset_data_frame).as_data_loader(shuffle=False, batch_size=2) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray]( images=np.random.uniform(0, 1, SCAN_SIZE), segmentations=np.random.randint(0, 2, SCAN_SIZE)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): batch = next(iter(dataloader)) if config.use_gpu: batch = transfer_batch_to_device(batch, torch.device(0)) model_inputs_and_labels = get_scalar_model_inputs_and_labels( model, target_indices=config.get_target_indices(), sample=batch) # type: ignore number_sequences = model_inputs_and_labels.model_inputs[0].shape[1] number_subjects = len(model_inputs_and_labels.subject_ids) visualizer = VisualizationMaps(model, config) guided_grad_cams, grad_cams, pseudo_cam_non_img, probas = visualizer.generate( model_inputs_and_labels.model_inputs) if use_combined_model: if imaging_feature_type == ImagingFeatureType.ImageAndSegmentation: assert guided_grad_cams.shape[:2] == (number_subjects, number_sequences * 2) assert grad_cams.shape[:2] == (number_subjects, number_sequences * 2) else: assert guided_grad_cams.shape[:2] == (number_subjects, number_sequences) assert grad_cams.shape[:2] == (number_subjects, number_sequences) else: assert guided_grad_cams is None assert grad_cams is None assert pseudo_cam_non_img.shape[:2] == (number_subjects, number_sequences) assert probas.shape[0] == number_subjects non_image_features = config.numerical_columns + config.categorical_columns non_imaging_plot_labels = visualizer._get_non_imaging_plot_labels( model_inputs_and_labels.data_item, non_image_features, index=0, target_position=3) assert non_imaging_plot_labels == [ 'numerical1_0', 'numerical2_0', 'cat1_0', 'numerical1_1', 'numerical2_1', 'cat1_1', 'numerical1_2', 'numerical2_2', 'cat1_2', 'numerical1_3', 'numerical2_3', 'cat1_3' ]
def training_or_validation_step(self, sample: Dict[str, Any], batch_index: int, is_training: bool) -> torch.Tensor: """ Runs training for a single minibatch of training or validation data, and computes all metrics. :param is_training: If true, the method is called from `training_step`, otherwise it is called from `validation_step`. :param sample: The batched sample on which the model should be trained. :param batch_index: The index of the present batch (supplied only for diagnostics). Runs a minibatch of training or validation data through the model. """ model_inputs_and_labels = get_scalar_model_inputs_and_labels(self.model, self.target_indices, sample) labels = model_inputs_and_labels.labels if is_training: logits = self.model(*model_inputs_and_labels.model_inputs) else: with torch.no_grad(): logits = self.model(*model_inputs_and_labels.model_inputs) subject_ids = model_inputs_and_labels.subject_ids loss = self.loss_fn(logits, labels) self.write_loss(is_training, loss) self.compute_and_log_metrics(logits, labels, subject_ids, is_training) self.log_on_epoch(name=MetricType.SUBJECT_COUNT, value=len(model_inputs_and_labels.subject_ids), is_training=is_training, reduce_fx=sum) return loss
def predict(self, sample: Dict[str, Any]) -> ScalarInferencePipelineBase.Result: """ Runs the forward pass on a single batch. :param sample: Single batch of input data. In the form of a dict containing at least the fields: metadata, label, images, numerical_non_image_features, categorical_non_image_features and segmentations. :return: Returns ScalarInferencePipelineBase.Result with the subject ids, ground truth labels and predictions. """ assert isinstance(self.model_config, ScalarModelBase) model_inputs_and_labels = get_scalar_model_inputs_and_labels(self.model.model, sample=sample) model_inputs_and_labels.move_to_device(self.model.device) with torch.no_grad(): # This already gives the model outputs converted to posteriors posteriors: torch.Tensor = self.model.forward(*model_inputs_and_labels.model_inputs) return ScalarInferencePipelineBase.Result(subject_ids=model_inputs_and_labels.subject_ids, labels=model_inputs_and_labels.labels, posteriors=posteriors)
def test_visualization_for_different_target_weeks( test_output_dirs: OutputFolderForTests) -> None: """ Tests that the visualizations are differentiated depending on the target week for which we visualize it. """ config = ToyMultiLabelSequenceModel(should_validate=False) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() model = create_model_with_temperature_scaling(config) dataloader = SequenceDataset( config, data_frame=config.dataset_data_frame).as_data_loader(shuffle=False, batch_size=2) batch = next(iter(dataloader)) model_inputs_and_labels = get_scalar_model_inputs_and_labels( model, target_indices=config.get_target_indices(), sample=batch) visualizer = VisualizationMaps(model, config) if config.use_gpu: device = visualizer.grad_cam.device batch = transfer_batch_to_device(batch, device) model = model.to(device) # Pseudo-grad cam explaining the prediction at target sequence 2 _, _, pseudo_cam_non_img_3, probas_3 = visualizer.generate( model_inputs_and_labels.model_inputs, target_position=2, target_label_index=2) # Pseudo-grad cam explaining the prediction at target sequence 0 _, _, pseudo_cam_non_img_1, probas_1 = visualizer.generate( model_inputs_and_labels.model_inputs, target_position=0, target_label_index=0) assert pseudo_cam_non_img_1.shape[1] == 1 assert pseudo_cam_non_img_3.shape[1] == 3 # Both visualizations should not be equal assert np.any(pseudo_cam_non_img_1 != pseudo_cam_non_img_3) assert np.any(probas_3 != probas_1)
def test_visualization_with_scalar_model(use_non_imaging_features: bool, imaging_feature_type: ImagingFeatureType, encode_channels_jointly: bool, test_output_dirs: OutputFolderForTests) -> None: dataset_contents = """subject,channel,path,label,numerical1,numerical2,categorical1,categorical2 S1,week0,scan1.npy,,1,10,Male,Val1 S1,week1,scan2.npy,True,2,20,Female,Val2 S2,week0,scan3.npy,,3,30,Female,Val3 S2,week1,scan4.npy,False,4,40,Female,Val1 S3,week0,scan1.npy,,5,50,Male,Val2 S3,week1,scan3.npy,True,6,60,Male,Val2 """ dataset_dataframe = pd.read_csv(StringIO(dataset_contents), dtype=str) numerical_columns = ["numerical1", "numerical2"] if use_non_imaging_features else [] categorical_columns = ["categorical1", "categorical2"] if use_non_imaging_features else [] non_image_feature_channels = get_non_image_features_dict(default_channels=["week1", "week0"], specific_channels={"categorical2": ["week1"]}) \ if use_non_imaging_features else {} config = ImageEncoder( local_dataset=Path(), encode_channels_jointly=encode_channels_jointly, should_validate=False, numerical_columns=numerical_columns, categorical_columns=categorical_columns, imaging_feature_type=imaging_feature_type, non_image_feature_channels=non_image_feature_channels, categorical_feature_encoder=CategoricalToOneHotEncoder.create_from_dataframe( dataframe=dataset_dataframe, columns=categorical_columns) ) dataloader = ScalarDataset(config, data_frame=dataset_dataframe) \ .as_data_loader(shuffle=False, batch_size=2) config.set_output_to(test_output_dirs.root_dir) config.num_epochs = 1 model = create_model_with_temperature_scaling(config) visualizer = VisualizationMaps(model, config) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray](images=np.random.uniform(0, 1, (6, 64, 60)), segmentations=np.random.randint(0, 2, (6, 64, 60))) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): batch = next(iter(dataloader)) if config.use_gpu: device = visualizer.grad_cam.device batch = transfer_batch_to_device(batch, device) visualizer.grad_cam.model = visualizer.grad_cam.model.to(device) model_inputs_and_labels = get_scalar_model_inputs_and_labels(model, target_indices=[], sample=batch) number_channels = len(config.image_channels) number_subjects = len(model_inputs_and_labels.subject_ids) guided_grad_cams, grad_cams, pseudo_cam_non_img, probas = visualizer.generate( model_inputs_and_labels.model_inputs) if imaging_feature_type == ImagingFeatureType.ImageAndSegmentation: assert guided_grad_cams.shape[:2] == (number_subjects, number_channels * 2) else: assert guided_grad_cams.shape[:2] == (number_subjects, number_channels) assert grad_cams.shape[:2] == (number_subjects, 1) if encode_channels_jointly \ else (number_subjects, number_channels) if use_non_imaging_features: non_image_features = config.numerical_columns + config.categorical_columns non_imaging_plot_labels = visualizer._get_non_imaging_plot_labels(model_inputs_and_labels.data_item, non_image_features, index=0) assert non_imaging_plot_labels == ['numerical1_week1', 'numerical1_week0', 'numerical2_week1', 'numerical2_week0', 'categorical1_week1', 'categorical1_week0', 'categorical2_week1'] assert pseudo_cam_non_img.shape == (number_subjects, 1, len(non_imaging_plot_labels))