def test_visualization_for_different_target_weeks(test_output_dirs: TestOutputDirectories) -> None: """ Tests that the visualizations are differentiated depending on the target week for which we visualize it. """ config = ToyMultiLabelSequenceModel(should_validate=False) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() model = create_model_with_temperature_scaling(config) dataloader = SequenceDataset(config, data_frame=config.dataset_data_frame).as_data_loader(shuffle=False, batch_size=2) batch = next(iter(dataloader)) model_inputs_and_labels = get_scalar_model_inputs_and_labels(config, model, batch) # type: ignore visualizer = VisualizationMaps(model, config) # Pseudo-grad cam explaining the prediction at target sequence 2 _, _, pseudo_cam_non_img_3, probas_3 = visualizer.generate(model_inputs_and_labels.model_inputs, target_position=2, target_label_index=2) # Pseudo-grad cam explaining the prediction at target sequence 0 _, _, pseudo_cam_non_img_1, probas_1 = visualizer.generate(model_inputs_and_labels.model_inputs, target_position=0, target_label_index=0) assert pseudo_cam_non_img_1.shape[1] == 1 assert pseudo_cam_non_img_3.shape[1] == 3 # Both visualizations should not be equal assert np.any(pseudo_cam_non_img_1 != pseudo_cam_non_img_3) assert np.any(probas_3 != probas_1)
def predict(self, sample: Dict[str, Any]) -> ScalarInferencePipelineBase.Result: """ Runs the forward pass on a single batch. :param sample: Single batch of input data. In the form of a dict containing at least the fields: metadata, label, images, numerical_non_image_features, categorical_non_image_features and segmentations. :return: Returns ScalarInferencePipelineBase.Result with the subject ids, ground truth labels and predictions. """ assert isinstance(self.model_config, ScalarModelBase) model_inputs_and_labels = get_scalar_model_inputs_and_labels( self.model_config, self.model, sample) subject_ids = model_inputs_and_labels.subject_ids labels = self.model_config.get_gpu_tensor_if_possible( model_inputs_and_labels.labels) model_output: torch.Tensor = self.model.forward( *model_inputs_and_labels.model_inputs) if isinstance(model_output, list): # Model output is a list if we are using data parallel. Here, this will be a degenerate list with # only 1 element model_output = torch.nn.parallel.gather(model_output, target_device=0) # Apply any post loss normalization to logits model_output = self.model_config.get_post_loss_logits_normalization_function( )(model_output) # Cast labels and model outputs back to float32, if the model had been run in mixed precision return ScalarInferencePipelineBase.Result(subject_ids, labels.float(), model_output.float())
def generate_and_print_model_summary(config: ModelConfigBase, model: DeviceAwareModule) -> None: """ Writes a human readable summary of the present model to logging.info, and logs the number of trainable parameters to AzureML. :param config: The configuration for the model. :param model: The instantiated Pytorch model. """ random_state = RandomStateSnapshot.snapshot_random_state() # There appears to be a bug in apex, where previous use (in training for example) causes problems # when another model is later built on the CPU (for example, before loading from a checkpoint) # https://github.com/NVIDIA/apex/issues/694 # Hence, move the model to the GPU before doing model summary. if config.use_gpu: model = model.cuda() if isinstance(config, ScalarModelBase): # To generate the model summary, read the first item of the dataset. Then use the model's own # get_model_input function to convert the dataset item to input tensors, and feed them through the model. train_dataset = config.get_torch_dataset_for_inference(ModelExecutionMode.TRAIN) train_item_0 = next(iter(train_dataset.as_data_loader(shuffle=False, batch_size=1, num_dataload_workers=0))) model_inputs = get_scalar_model_inputs_and_labels(config, model, train_item_0).model_inputs # The model inputs may already be converted to float16, assuming that we would do mixed precision. # However, the model is not yet converted to float16 when this function is called, hence convert back to float32 summary = ModelSummary(model) summary.generate_summary(input_tensors=model_inputs, log_summaries_to_files=config.log_summaries_to_files) elif config.is_segmentation_model: summary_for_segmentation_models(config, model) assert model.summarizer summary = model.summarizer # type: ignore else: raise ValueError("Don't know how to generate a summary for this type of model?") RUN_CONTEXT.log(LoggingColumns.NumTrainableParameters, summary.n_trainable_params) random_state.restore_random_state()
def test_amp_and_parallel_for_scalar_models( test_output_dirs: TestOutputDirectories, execution_mode: ModelExecutionMode, use_mixed_precision: bool) -> None: """ Tests the mix precision flag and data parallel for scalar models. """ assert machine_has_gpu, "This test must be executed on a GPU machine." assert torch.cuda.device_count( ) > 1, "This test must be executed on a multi-GPU machine" config = ClassificationModelForTesting() config.use_mixed_precision = use_mixed_precision model = DummyScalarModel( expected_image_size_zyx=config.expected_image_size_zyx, activation=Identity()) model.use_mixed_precision = use_mixed_precision model_and_info = ModelAndInfo(model=model, model_execution_mode=execution_mode) # This is the same logic spelt out in update_model_for_multiple_gpu # execution_mode == ModelExecutionMode.TRAIN or (not use_model_parallel), which is always True in our case use_data_parallel = True model_and_info = model_util.update_model_for_multiple_gpus( model_and_info, config) if use_data_parallel: assert isinstance(model_and_info.model, DataParallelModel) data_loaders = config.create_data_loaders() gradient_scaler = GradScaler() if use_mixed_precision else None train_val_parameters: TrainValidateParameters = TrainValidateParameters( model=model_and_info.model, data_loader=data_loaders[execution_mode], in_training_mode=execution_mode == ModelExecutionMode.TRAIN, gradient_scaler=gradient_scaler, dataframe_loggers=MetricsDataframeLoggers( Path(test_output_dirs.root_dir)), summary_writers=SummaryWriters(train=None, val=None) # type: ignore ) training_steps = ModelTrainingStepsForScalarModel(config, train_val_parameters) sample = list(data_loaders[execution_mode])[0] model_input = get_scalar_model_inputs_and_labels(config, model, sample) logits, posteriors, loss = training_steps._compute_model_output_and_loss( model_input) # When using DataParallel, we expect to get a list of tensors back, one per GPU. if use_data_parallel: assert isinstance(logits, list) first_logit = logits[0] else: first_logit = logits if use_mixed_precision: assert first_logit.dtype == torch.float16 assert posteriors.dtype == torch.float16 # BCEWithLogitsLoss outputs float32, even with float16 args assert loss.dtype == torch.float32 else: assert first_logit.dtype == torch.float32 assert posteriors.dtype == torch.float32 assert loss.dtype == torch.float32 # Verify that forward pass does not throw. It would for example if it fails to gather tensors or not convert # float16 to float32 _, _, _ = training_steps._compute_model_output_and_loss(model_input)
def test_visualization_with_sequence_model(use_combined_model: bool, imaging_feature_type: ImagingFeatureType, test_output_dirs: TestOutputDirectories) -> None: config = ToySequenceModel(use_combined_model, imaging_feature_type, should_validate=False) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_mock_sequence_dataset() config.num_epochs = 1 model_and_info = ModelAndInfo(config=config, model_execution_mode=ModelExecutionMode.TEST, is_mean_teacher=False, checkpoint_path=None) model_loaded = model_and_info.try_create_model_load_from_checkpoint_and_adjust() assert model_loaded model = model_and_info.model dataloader = SequenceDataset(config, data_frame=config.dataset_data_frame).as_data_loader(shuffle=False, batch_size=2) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray](images=np.random.uniform(0, 1, SCAN_SIZE), segmentations=np.random.randint(0, 2, SCAN_SIZE)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): batch = next(iter(dataloader)) model_inputs_and_labels = get_scalar_model_inputs_and_labels(config, model, batch) # type: ignore number_sequences = model_inputs_and_labels.model_inputs[0].shape[1] number_subjects = len(model_inputs_and_labels.subject_ids) visualizer = VisualizationMaps(model, config) guided_grad_cams, grad_cams, pseudo_cam_non_img, probas = visualizer.generate( model_inputs_and_labels.model_inputs) if use_combined_model: if imaging_feature_type == ImagingFeatureType.ImageAndSegmentation: assert guided_grad_cams.shape[:2] == (number_subjects, number_sequences * 2) assert grad_cams.shape[:2] == (number_subjects, number_sequences * 2) else: assert guided_grad_cams.shape[:2] == (number_subjects, number_sequences) assert grad_cams.shape[:2] == (number_subjects, number_sequences) else: assert guided_grad_cams is None assert grad_cams is None assert pseudo_cam_non_img.shape[:2] == (number_subjects, number_sequences) assert probas.shape[0] == number_subjects non_image_features = config.numerical_columns + config.categorical_columns non_imaging_plot_labels = visualizer._get_non_imaging_plot_labels(model_inputs_and_labels.data_item, non_image_features, index=0, target_position=3) assert non_imaging_plot_labels == ['numerical1_0', 'numerical2_0', 'cat1_0', 'numerical1_1', 'numerical2_1', 'cat1_1', 'numerical1_2', 'numerical2_2', 'cat1_2', 'numerical1_3', 'numerical2_3', 'cat1_3']
def test_visualization_with_scalar_model(use_non_imaging_features: bool, imaging_feature_type: ImagingFeatureType, encode_channels_jointly: bool, test_output_dirs: TestOutputDirectories) -> None: dataset_contents = """subject,channel,path,label,numerical1,numerical2,categorical1,categorical2 S1,week0,scan1.npy,,1,10,Male,Val1 S1,week1,scan2.npy,True,2,20,Female,Val2 S2,week0,scan3.npy,,3,30,Female,Val3 S2,week1,scan4.npy,False,4,40,Female,Val1 S3,week0,scan1.npy,,5,50,Male,Val2 S3,week1,scan3.npy,True,6,60,Male,Val2 """ dataset_dataframe = pd.read_csv(StringIO(dataset_contents), dtype=str) numerical_columns = ["numerical1", "numerical2"] if use_non_imaging_features else [] categorical_columns = ["categorical1", "categorical2"] if use_non_imaging_features else [] non_image_feature_channels = get_non_image_features_dict(default_channels=["week1", "week0"], specific_channels={"categorical2": ["week1"]}) \ if use_non_imaging_features else {} config = ImageEncoder( local_dataset=Path(), encode_channels_jointly=encode_channels_jointly, should_validate=False, numerical_columns=numerical_columns, categorical_columns=categorical_columns, imaging_feature_type=imaging_feature_type, non_image_feature_channels=non_image_feature_channels, categorical_feature_encoder=CategoricalToOneHotEncoder.create_from_dataframe( dataframe=dataset_dataframe, columns=categorical_columns) ) dataloader = ScalarDataset(config, data_frame=dataset_dataframe) \ .as_data_loader(shuffle=False, batch_size=2) config.set_output_to(test_output_dirs.root_dir) config.num_epochs = 1 model = create_model_with_temperature_scaling(config) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray](images=np.random.uniform(0, 1, (6, 64, 60)), segmentations=np.random.randint(0, 2, (6, 64, 60))) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): batch = next(iter(dataloader)) model_inputs_and_labels = get_scalar_model_inputs_and_labels(config, model, batch) number_channels = len(config.image_channels) number_subjects = len(model_inputs_and_labels.subject_ids) visualizer = VisualizationMaps(model, config) guided_grad_cams, grad_cams, pseudo_cam_non_img, probas = visualizer.generate( model_inputs_and_labels.model_inputs) if imaging_feature_type == ImagingFeatureType.ImageAndSegmentation: assert guided_grad_cams.shape[:2] == (number_subjects, number_channels * 2) else: assert guided_grad_cams.shape[:2] == (number_subjects, number_channels) assert grad_cams.shape[:2] == (number_subjects, 1) if encode_channels_jointly \ else (number_subjects, number_channels) if use_non_imaging_features: non_image_features = config.numerical_columns + config.categorical_columns non_imaging_plot_labels = visualizer._get_non_imaging_plot_labels(model_inputs_and_labels.data_item, non_image_features, index=0) assert non_imaging_plot_labels == ['numerical1_week1', 'numerical1_week0', 'numerical2_week1', 'numerical2_week0', 'categorical1_week1', 'categorical1_week0', 'categorical2_week1'] assert pseudo_cam_non_img.shape == (number_subjects, 1, len(non_imaging_plot_labels))