def get_epoch_path(mode: ModelExecutionMode) -> Path: p = get_epoch_results_path(mode=mode) return config.outputs_folder / p / SUBJECT_METRICS_FILE_NAME
def test_train_classification_model( class_name: str, test_output_dirs: OutputFolderForTests) -> None: """ Test training and testing of classification models, asserting on the individual results from training and testing. Expected test results are stored for GPU with and without mixed precision. """ logging_to_stdout(logging.DEBUG) config = ClassificationModelForTesting() config.class_names = [class_name] config.set_output_to(test_output_dirs.root_dir) checkpoint_handler = get_default_checkpoint_handler( model_config=config, project_root=Path(test_output_dirs.root_dir)) # Train for 4 epochs, checkpoints at epochs 2 and 4 config.num_epochs = 4 model_training_result = model_training.model_train( config, checkpoint_handler=checkpoint_handler) assert model_training_result is not None expected_learning_rates = [0.0001, 9.99971e-05, 9.99930e-05, 9.99861e-05] expected_train_loss = [0.686614, 0.686465, 0.686316, 0.686167] expected_val_loss = [0.737061, 0.736691, 0.736321, 0.735952] # Ensure that all metrics are computed on both training and validation set assert len( model_training_result.train_results_per_epoch) == config.num_epochs assert len( model_training_result.val_results_per_epoch) == config.num_epochs assert len(model_training_result.train_results_per_epoch[0]) >= 11 assert len(model_training_result.val_results_per_epoch[0]) >= 11 for metric in [ MetricType.ACCURACY_AT_THRESHOLD_05, MetricType.ACCURACY_AT_OPTIMAL_THRESHOLD, MetricType.AREA_UNDER_PR_CURVE, MetricType.AREA_UNDER_ROC_CURVE, MetricType.CROSS_ENTROPY, MetricType.LOSS, MetricType.SECONDS_PER_BATCH, MetricType.SECONDS_PER_EPOCH, MetricType.SUBJECT_COUNT ]: assert metric.value in model_training_result.train_results_per_epoch[0], \ f"{metric.value} not in training" assert metric.value in model_training_result.val_results_per_epoch[0], \ f"{metric.value} not in validation" actual_train_loss = model_training_result.get_metric( is_training=True, metric_type=MetricType.LOSS.value) actual_val_loss = model_training_result.get_metric( is_training=False, metric_type=MetricType.LOSS.value) actual_lr = model_training_result.get_metric( is_training=True, metric_type=MetricType.LEARNING_RATE.value) assert actual_train_loss == pytest.approx(expected_train_loss, abs=1e-6), "Training loss" assert actual_val_loss == pytest.approx(expected_val_loss, abs=1e-6), "Validation loss" assert actual_lr == pytest.approx(expected_learning_rates, rel=1e-5), "Learning rates" test_results = model_testing.model_test( config, ModelExecutionMode.TRAIN, checkpoint_handler=checkpoint_handler) assert isinstance(test_results, InferenceMetricsForClassification) expected_metrics = [0.636085, 0.735952] assert test_results.metrics.values(class_name)[MetricType.CROSS_ENTROPY.value] == \ pytest.approx(expected_metrics, abs=1e-5) # Run detailed logs file check only on CPU, it will contain slightly different metrics on GPU, but here # we want to mostly assert that the files look reasonable if machine_has_gpu: return # Check epoch_metrics.csv epoch_metrics_path = config.outputs_folder / ModelExecutionMode.TRAIN.value / EPOCH_METRICS_FILE_NAME # Auto-format will break the long header line, hence the strange way of writing it! expected_epoch_metrics = \ f"{LoggingColumns.Loss.value},{LoggingColumns.CrossEntropy.value}," \ f"{LoggingColumns.AccuracyAtThreshold05.value},{LoggingColumns.LearningRate.value}," + \ f"{LoggingColumns.AreaUnderRocCurve.value}," \ f"{LoggingColumns.AreaUnderPRCurve.value}," \ f"{LoggingColumns.AccuracyAtOptimalThreshold.value}," \ f"{LoggingColumns.FalsePositiveRateAtOptimalThreshold.value}," \ f"{LoggingColumns.FalseNegativeRateAtOptimalThreshold.value}," \ f"{LoggingColumns.OptimalThreshold.value}," \ f"{LoggingColumns.SubjectCount.value},{LoggingColumns.Epoch.value}," \ f"{LoggingColumns.CrossValidationSplitIndex.value}\n" + \ """0.6866141557693481,0.6866141557693481,0.5,0.0001,1.0,1.0,0.5,0.0,0.0,0.529514,2.0,0,-1 0.6864652633666992,0.6864652633666992,0.5,9.999712322065557e-05,1.0,1.0,0.5,0.0,0.0,0.529475,2.0,1,-1 0.6863163113594055,0.6863162517547607,0.5,9.999306876841536e-05,1.0,1.0,0.5,0.0,0.0,0.529437,2.0,2,-1 0.6861673593521118,0.6861673593521118,0.5,9.998613801725043e-05,1.0,1.0,0.5,0.0,0.0,0.529399,2.0,3,-1 """ check_log_file(epoch_metrics_path, expected_epoch_metrics, ignore_columns=[]) # Check metrics.csv: This contains the per-subject per-epoch model outputs # Randomization comes out slightly different on Windows, hence only execute the test on Linux if common_util.is_windows(): return metrics_path = config.outputs_folder / ModelExecutionMode.TRAIN.value / SUBJECT_METRICS_FILE_NAME metrics_expected = \ f"""epoch,subject,prediction_target,model_output,label,data_split,cross_validation_split_index 0,S2,{class_name},0.529514,1,Train,-1 0,S4,{class_name},0.521659,0,Train,-1 1,S4,{class_name},0.521482,0,Train,-1 1,S2,{class_name},0.529475,1,Train,-1 2,S4,{class_name},0.521305,0,Train,-1 2,S2,{class_name},0.529437,1,Train,-1 3,S2,{class_name},0.529399,1,Train,-1 3,S4,{class_name},0.521128,0,Train,-1 """ check_log_file(metrics_path, metrics_expected, ignore_columns=[]) # Check log METRICS_FILE_NAME inside of the folder epoch_004/Train, which is written when we run model_test. # Normally, we would run it on the Test and Val splits, but for convenience we test on the train split here. inference_metrics_path = config.outputs_folder / get_epoch_results_path(ModelExecutionMode.TRAIN) / \ SUBJECT_METRICS_FILE_NAME inference_metrics_expected = \ f"""prediction_target,subject,model_output,label,cross_validation_split_index,data_split {class_name},S2,0.5293986201286316,1.0,-1,Train {class_name},S4,0.5211275815963745,0.0,-1,Train """ check_log_file(inference_metrics_path, inference_metrics_expected, ignore_columns=[])
def test_model_test(test_output_dirs: OutputFolderForTests) -> None: train_and_test_data_dir = full_ml_test_data_path("train_and_test_data") config = DummyModel() config.set_output_to(test_output_dirs.root_dir) epoch = 1 config.num_epochs = epoch assert config.get_test_epochs() == [epoch] placeholder_dataset_id = "place_holder_dataset_id" config.azure_dataset_id = placeholder_dataset_id transform = config.get_full_image_sample_transforms().test df = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME)) df = df[df.subject.isin([1, 2])] # noinspection PyTypeHints config._datasets_for_inference = \ {ModelExecutionMode.TEST: FullImageDataset(config, df, full_image_sample_transforms=transform)} # type: ignore execution_mode = ModelExecutionMode.TEST checkpoint_handler = get_default_checkpoint_handler(model_config=config, project_root=test_output_dirs.root_dir) # Mimic the behaviour that checkpoints are downloaded from blob storage into the checkpoints folder. stored_checkpoints = full_ml_test_data_path("checkpoints") shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder)) checkpoint_handler.additional_training_done() inference_results = model_testing.segmentation_model_test(config, data_split=execution_mode, checkpoint_handler=checkpoint_handler) epoch_dir = config.outputs_folder / get_epoch_results_path(epoch, execution_mode) assert inference_results.epochs[epoch] == pytest.approx(0.66606902, abs=1e-6) assert config.outputs_folder.is_dir() assert epoch_dir.is_dir() patient1 = io_util.load_nifti_image(train_and_test_data_dir / "id1_channel1.nii.gz") patient2 = io_util.load_nifti_image(train_and_test_data_dir / "id2_channel1.nii.gz") assert_file_contains_string(epoch_dir / DATASET_ID_FILE, placeholder_dataset_id) assert_file_contains_string(epoch_dir / GROUND_TRUTH_IDS_FILE, "region") assert_text_files_match(epoch_dir / model_testing.METRICS_FILE_NAME, train_and_test_data_dir / model_testing.METRICS_FILE_NAME) assert_text_files_match(epoch_dir / model_testing.METRICS_AGGREGATES_FILE, train_and_test_data_dir / model_testing.METRICS_AGGREGATES_FILE) # Plotting results vary between platforms. Can only check if the file is generated, but not its contents. assert (epoch_dir / model_testing.BOXPLOT_FILE).exists() assert_nifti_content(epoch_dir / "001" / "posterior_region.nii.gz", get_image_shape(patient1), patient1.header, [136], np.ubyte) assert_nifti_content(epoch_dir / "002" / "posterior_region.nii.gz", get_image_shape(patient2), patient2.header, [136], np.ubyte) assert_nifti_content(epoch_dir / "001" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient1), patient1.header, [1], np.ubyte) assert_nifti_content(epoch_dir / "002" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient2), patient2.header, [1], np.ubyte) assert_nifti_content(epoch_dir / "001" / "posterior_background.nii.gz", get_image_shape(patient1), patient1.header, [118], np.ubyte) assert_nifti_content(epoch_dir / "002" / "posterior_background.nii.gz", get_image_shape(patient2), patient2.header, [118], np.ubyte) thumbnails_folder = epoch_dir / model_testing.THUMBNAILS_FOLDER assert thumbnails_folder.is_dir() png_files = list(thumbnails_folder.glob("*.png")) overlays = [f for f in png_files if "_region_slice_" in str(f)] assert len(overlays) == len(df.subject.unique()), "There should be one overlay/contour file per subject" # Writing dataset.csv normally happens at the beginning of training, # but this test reads off a saved checkpoint file. # Dataset.csv must be present for plot_cross_validation. config.write_dataset_files() # Test if the metrics files can be picked up correctly by the cross validation code config_and_files = get_config_and_results_for_offline_runs(config) result_files = config_and_files.files assert len(result_files) == 1 for file in result_files: assert file.execution_mode == execution_mode assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_train_classification_model( test_output_dirs: OutputFolderForTests) -> None: """ Test training and testing of classification models, asserting on the individual results from training and testing. Expected test results are stored for GPU with and without mixed precision. """ logging_to_stdout(logging.DEBUG) config = ClassificationModelForTesting() config.set_output_to(test_output_dirs.root_dir) checkpoint_handler = get_default_checkpoint_handler( model_config=config, project_root=Path(test_output_dirs.root_dir)) # Train for 4 epochs, checkpoints at epochs 2 and 4 config.num_epochs = 4 model_training_result = model_training.model_train( config, checkpoint_handler=checkpoint_handler) assert model_training_result is not None expected_learning_rates = [0.0001, 9.99971e-05, 9.99930e-05, 9.99861e-05] expected_train_loss = [0.686614, 0.686465, 0.686316, 0.686167] expected_val_loss = [0.737061, 0.736691, 0.736321, 0.735952] # Ensure that all metrics are computed on both training and validation set assert len( model_training_result.train_results_per_epoch) == config.num_epochs assert len( model_training_result.val_results_per_epoch) == config.num_epochs assert len(model_training_result.train_results_per_epoch[0]) >= 11 assert len(model_training_result.val_results_per_epoch[0]) >= 11 for metric in [ MetricType.ACCURACY_AT_THRESHOLD_05, MetricType.ACCURACY_AT_OPTIMAL_THRESHOLD, MetricType.AREA_UNDER_PR_CURVE, MetricType.AREA_UNDER_ROC_CURVE, MetricType.CROSS_ENTROPY, MetricType.LOSS, # For unknown reasons, we don't get seconds_per_batch for the training data. # MetricType.SECONDS_PER_BATCH, MetricType.SECONDS_PER_EPOCH, MetricType.SUBJECT_COUNT, ]: assert metric.value in model_training_result.train_results_per_epoch[ 0], f"{metric.value} not in training" assert metric.value in model_training_result.val_results_per_epoch[ 0], f"{metric.value} not in validation" actual_train_loss = model_training_result.get_metric( is_training=True, metric_type=MetricType.LOSS.value) actual_val_loss = model_training_result.get_metric( is_training=False, metric_type=MetricType.LOSS.value) actual_lr = model_training_result.get_metric( is_training=True, metric_type=MetricType.LEARNING_RATE.value) assert actual_train_loss == pytest.approx(expected_train_loss, abs=1e-6), "Training loss" assert actual_val_loss == pytest.approx(expected_val_loss, abs=1e-6), "Validation loss" assert actual_lr == pytest.approx(expected_learning_rates, rel=1e-5), "Learning rates" test_results = model_testing.model_test( config, ModelExecutionMode.TRAIN, checkpoint_handler=checkpoint_handler) assert isinstance(test_results, InferenceMetricsForClassification) expected_metrics = [0.636085, 0.735952] assert test_results.metrics.values()[MetricType.CROSS_ENTROPY.value] == \ pytest.approx(expected_metrics, abs=1e-5) # Run detailed logs file check only on CPU, it will contain slightly different metrics on GPU, but here # we want to mostly assert that the files look reasonable if machine_has_gpu: return # Check epoch_metrics.csv epoch_metrics_path = config.outputs_folder / ModelExecutionMode.TRAIN.value / EPOCH_METRICS_FILE_NAME # Auto-format will break the long header line, hence the strange way of writing it! expected_epoch_metrics = \ "loss,cross_entropy,accuracy_at_threshold_05,seconds_per_epoch,learning_rate," + \ "area_under_roc_curve,area_under_pr_curve,accuracy_at_optimal_threshold," \ "false_positive_rate_at_optimal_threshold,false_negative_rate_at_optimal_threshold," \ "optimal_threshold,subject_count,epoch,cross_validation_split_index\n" + \ """0.6866141557693481,0.6866141557693481,0.5,0,0.0001,1.0,1.0,0.5,0.0,0.0,0.529514,2.0,0,-1 0.6864652633666992,0.6864652633666992,0.5,0,9.999712322065557e-05,1.0,1.0,0.5,0.0,0.0,0.529475,2.0,1,-1 0.6863163113594055,0.6863162517547607,0.5,0,9.999306876841536e-05,1.0,1.0,0.5,0.0,0.0,0.529437,2.0,2,-1 0.6861673593521118,0.6861673593521118,0.5,0,9.998613801725043e-05,1.0,1.0,0.5,0.0,0.0,0.529399,2.0,3,-1 """ # We cannot compare columns like "seconds_per_epoch" because timing will obviously vary between machines. # Column must still be present, though. check_log_file(epoch_metrics_path, expected_epoch_metrics, ignore_columns=[LoggingColumns.SecondsPerEpoch.value]) # Check metrics.csv: This contains the per-subject per-epoch model outputs # Randomization comes out slightly different on Windows, hence only execute the test on Linux if common_util.is_windows(): return metrics_path = config.outputs_folder / ModelExecutionMode.TRAIN.value / SUBJECT_METRICS_FILE_NAME metrics_expected = \ """prediction_target,epoch,subject,model_output,label,cross_validation_split_index,data_split Default,0,S2,0.5295137763023376,1.0,-1,Train Default,0,S4,0.5216594338417053,0.0,-1,Train Default,1,S4,0.5214819312095642,0.0,-1,Train Default,1,S2,0.5294750332832336,1.0,-1,Train Default,2,S2,0.5294366478919983,1.0,-1,Train Default,2,S4,0.5213046073913574,0.0,-1,Train Default,3,S2,0.5293986201286316,1.0,-1,Train Default,3,S4,0.5211275815963745,0.0,-1,Train """ check_log_file(metrics_path, metrics_expected, ignore_columns=[]) # Check log METRICS_FILE_NAME inside of the folder epoch_004/Train, which is written when we run model_test. # Normally, we would run it on the Test and Val splits, but for convenience we test on the train split here. inference_metrics_path = config.outputs_folder / get_epoch_results_path(ModelExecutionMode.TRAIN) / \ SUBJECT_METRICS_FILE_NAME inference_metrics_expected = \ """prediction_target,subject,model_output,label,cross_validation_split_index,data_split Default,S2,0.5293986201286316,1.0,-1,Train Default,S4,0.5211275815963745,0.0,-1,Train """ check_log_file(inference_metrics_path, inference_metrics_expected, ignore_columns=[])
def classification_model_test(config: ScalarModelBase, data_split: ModelExecutionMode, checkpoint_handler: CheckpointHandler, model_proc: ModelProcessing) -> InferenceMetricsForClassification: """ The main testing loop for classification models. It runs a loop over all epochs for which testing should be done. It loads the model and datasets, then proceeds to test the model for all requested checkpoints. :param config: The model configuration. :param data_split: The name of the folder to store the results inside each epoch folder in the outputs_dir, used mainly in model evaluation using different dataset splits. :param checkpoint_handler: Checkpoint handler object to find checkpoint paths for model initialization :param model_proc: whether we are testing an ensemble or single model :return: InferenceMetricsForClassification object that contains metrics related for all of the checkpoint epochs. """ def test_epoch(checkpoint_paths: List[Path]) -> Optional[MetricsDict]: pipeline = create_inference_pipeline(config=config, checkpoint_paths=checkpoint_paths) if pipeline is None: return None # for mypy assert isinstance(pipeline, ScalarInferencePipelineBase) ml_util.set_random_seed(config.get_effective_random_seed(), "Model Testing") ds = config.get_torch_dataset_for_inference(data_split).as_data_loader( shuffle=False, batch_size=1, num_dataload_workers=0 ) logging.info(f"Starting to evaluate model on {data_split.value} set.") metrics_dict = create_metrics_dict_for_scalar_models(config) for sample in ds: result = pipeline.predict(sample) model_output = result.posteriors label = result.labels.to(device=model_output.device) sample_id = result.subject_ids[0] compute_scalar_metrics(metrics_dict, subject_ids=[sample_id], model_output=model_output, labels=label, loss_type=config.loss_type) logging.debug(f"Example {sample_id}: {metrics_dict.to_string()}") average = metrics_dict.average(across_hues=False) logging.info(average.to_string()) return metrics_dict checkpoints_to_test = checkpoint_handler.get_checkpoints_to_test() if not checkpoints_to_test: raise ValueError("There were no checkpoints available for model testing.") result = test_epoch(checkpoint_paths=checkpoints_to_test) if result is None: raise ValueError("There was no single checkpoint file available for model testing.") else: if isinstance(result, ScalarMetricsDict): results_folder = config.outputs_folder / get_epoch_results_path(data_split, model_proc) csv_file = results_folder / SUBJECT_METRICS_FILE_NAME logging.info(f"Writing {data_split.value} metrics to file {str(csv_file)}") # If we are running inference after a training run, the validation set metrics may have been written # during train time. If this is not the case, or we are running on the test set, create the metrics # file. if not csv_file.exists(): os.makedirs(str(results_folder), exist_ok=False) df_logger = DataframeLogger(csv_file) # cross validation split index not relevant during test time result.store_metrics_per_subject(df_logger=df_logger, mode=data_split) # write to disk df_logger.flush() return InferenceMetricsForClassification(metrics=result)
def get_epoch_path(mode: ModelExecutionMode) -> Path: p = get_epoch_results_path(best_epoch, mode=mode, model_proc=model_proc) return config.outputs_folder / p / METRICS_FILE_NAME
def classification_model_test( config: ScalarModelBase, data_split: ModelExecutionMode, run_recovery: Optional[RunRecovery], model_proc: ModelProcessing) -> InferenceMetricsForClassification: """ The main testing loop for classification models. It runs a loop over all epochs for which testing should be done. It loads the model and datasets, then proceeds to test the model for all requested checkpoints. :param config: The model configuration. :param data_split: The name of the folder to store the results inside each epoch folder in the outputs_dir, used mainly in model evaluation using different dataset splits. :param run_recovery: RunRecovery data if applicable :param model_proc: whether we are testing an ensemble or single model :return: InferenceMetricsForClassification object that contains metrics related for all of the checkpoint epochs. """ def test_epoch( test_epoch: int, run_recovery: Optional[RunRecovery]) -> Optional[MetricsDict]: pipeline = create_inference_pipeline(config, test_epoch, run_recovery) if pipeline is None: return None # for mypy assert isinstance(pipeline, ScalarInferencePipelineBase) ml_util.set_random_seed(config.get_effective_random_seed(), "Model Testing") ds = config.get_torch_dataset_for_inference(data_split).as_data_loader( shuffle=False, batch_size=1, num_dataload_workers=0) logging.info( f"Starting to evaluate model from epoch {test_epoch} on {data_split.value} set." ) metrics_dict = create_metrics_dict_from_config(config) for sample in ds: result = pipeline.predict(sample) # Since batch size is 1, we only have 1 item in each of the fields in result sample_id, label_gpu, model_output = result.subject_ids[ 0], result.labels, result.model_outputs compute_scalar_metrics(metrics_dict, [sample_id], model_output, label_gpu, config.loss_type) logging.debug(f"Example {sample_id}: {metrics_dict.to_string()}") average = metrics_dict.average(across_hues=False) logging.info(average.to_string()) return metrics_dict results: Dict[int, MetricsDict] = {} for epoch in config.get_test_epochs(): epoch_result = test_epoch(test_epoch=epoch, run_recovery=run_recovery) if epoch_result is None: logging.warning( "There is no checkpoint file for epoch {}".format(epoch)) else: results[epoch] = epoch_result if isinstance(epoch_result, ScalarMetricsDict): epoch_folder = config.outputs_folder / get_epoch_results_path( epoch, data_split, model_proc) csv_file = epoch_folder / METRICS_FILE_NAME logging.info( f"Writing {data_split.value} metrics to file {str(csv_file)}" ) # If we are running inference after a training run, the validation set metrics may have been written # during train time. If this is not the case, or we are running on the test set, create the metrics # file. if not csv_file.exists(): os.makedirs(str(epoch_folder), exist_ok=False) df_logger = DataframeLogger(csv_file) # cross validation split index not relevant during test time epoch_result.store_metrics_per_subject(epoch=epoch, df_logger=df_logger, mode=data_split) # write to disk df_logger.flush() if len(results) == 0: raise ValueError( "There was no single checkpoint file available for model testing.") return InferenceMetricsForClassification(epochs=results)