def score_and_postprocess_model_based_on_rank_then_aggregate(): """ Similarly to BraTS 2017 - BraTS 2019, each participant will be ranked for each of the X test cases. Each case includes 3 regions of evaluation, and the metrics used to produce the rankings will be the Dice Similarity Coefficient and the 95% Hausdorff distance. Thus, for X number of cases included in the BraTS 2020, each participant ends up having X*3*2 rankings. The final ranking score is the average of all these rankings normalized by the number of teams. https://zenodo.org/record/3718904 -> let's optimize for this. Important: the outcome very much depends on the competing models. We need some references. We only got our own, so let's hope this still works :return: """ base = "/media/fabian/Results/nnUNet/3d_fullres/Task082_BraTS2020" replace_with = 2 num_processes = 24 expected_num_cases_val = 125 # use a separate output folder from the previous experiments to ensure we are not messing things up output_base_here = join(base, 'use_brats_ranking') maybe_mkdir_p(output_base_here) # collect cv niftis and compute metrics with evaluate_BraTS_folder to ensure we work with the same metrics as brats out = join(output_base_here, 'cv_results') experiments = subfolders(base, join=False, prefix='nnUNetTrainer') gt_dir = join(base, 'gt_niftis') experiments_with_full_cv = [] for e in experiments: print(e) o = join(out, e) maybe_mkdir_p(o) try: collect_cv_niftis(join(base, e), o) if not isfile(join(o, 'results.csv')): evaluate_BraTS_folder(o, gt_dir, num_processes, strict=True) experiments_with_full_cv.append(e) except Exception as ex: print("\nERROR\n", e, ex, "\n") if isfile(join(o, 'results.csv')): os.remove(join(o, 'results.csv')) # rank the non-postprocessed models tmp = np.loadtxt(join(out, experiments_with_full_cv[0], 'results.csv'), dtype='str', delimiter=',') num_cases = len(tmp) - 1 data_for_ranking = np.zeros((6, len(experiments_with_full_cv), num_cases)) for i, e in enumerate(experiments_with_full_cv): scores = load_csv_for_ranking(join(out, e, 'results.csv')) for metric in range(6): data_for_ranking[metric, i] = scores[:, metric] final_ranks, average_rank, ranks = rank_algorithms(data_for_ranking) for t in np.argsort(final_ranks): print(final_ranks[t], average_rank[t], experiments_with_full_cv[t]) # for each model, create output directories with different thresholds. evaluate ALL OF THEM (might take a while lol) thresholds = np.arange(25, 751, 25) output_pp_tmp = join(output_base_here, 'cv_determine_pp_thresholds') for e in experiments_with_full_cv: input_folder = join(out, e) for t in thresholds: output_directory = join(output_pp_tmp, e, str(t)) maybe_mkdir_p(output_directory) if not isfile(join(output_directory, 'results.csv')): apply_threshold_to_folder(input_folder, output_directory, t, replace_with, processes=16) evaluate_BraTS_folder(output_directory, gt_dir, num_processes) # load ALL the results! results = [] experiment_names = [] for e in experiments_with_full_cv: for t in thresholds: output_directory = join(output_pp_tmp, e, str(t)) expected_file = join(output_directory, 'results.csv') if not isfile(expected_file): print(e, 'does not have a results file for threshold', t) continue results.append(load_csv_for_ranking(expected_file)) experiment_names.append("%s___%d" % (e, t)) all_results = np.concatenate([i[None] for i in results], 0).transpose((2, 0, 1)) # concatenate with non postprocessed models all_results = np.concatenate((data_for_ranking, all_results), 1) experiment_names += experiments_with_full_cv final_ranks, average_rank, ranks = rank_algorithms(all_results) for t in np.argsort(final_ranks): print(final_ranks[t], average_rank[t], experiment_names[t]) # for each model, print the non postprocessed model as well as the best postprocessed model. If there are # validation set predictions, apply the best threshold to the validation set pred_val_base = join(base, 'predVal_PP_rank') has_val_pred = [] for e in experiments_with_full_cv: rank_nonpp = final_ranks[experiment_names.index(e)] avg_rank_nonpp = average_rank[experiment_names.index(e)] print(e, avg_rank_nonpp, rank_nonpp) predicted_val = join(base, 'predVal', e) pp_models = [j for j, i in enumerate(experiment_names) if i.split("___")[0] == e and i != e] if len(pp_models) > 0: ranks = [final_ranks[i] for i in pp_models] best_idx = np.argmin(ranks) best = experiment_names[pp_models[best_idx]] best_avg_rank = average_rank[pp_models[best_idx]] print(best, best_avg_rank, min(ranks)) print('') # apply threshold to validation set best_threshold = int(best.split('___')[-1]) if not isdir(predicted_val): print(e, 'has not valset predictions') else: files = subfiles(predicted_val, suffix='.nii.gz') if len(files) != expected_num_cases_val: print(e, 'has missing val cases. found: %d expected: %d' % (len(files), expected_num_cases_val)) else: apply_threshold_to_folder(predicted_val, join(pred_val_base, e), best_threshold, replace_with, num_processes) has_val_pred.append(e) else: print(e, 'not found in ranking') # apply nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5 to nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold e = 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5' pp_models = [j for j, i in enumerate(experiment_names) if i.split("___")[0] == e and i != e] ranks = [final_ranks[i] for i in pp_models] best_idx = np.argmin(ranks) best = experiment_names[pp_models[best_idx]] best_avg_rank = average_rank[pp_models[best_idx]] best_threshold = int(best.split('___')[-1]) predicted_val = join(base, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold') apply_threshold_to_folder(predicted_val, join(pred_val_base, 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold'), best_threshold, replace_with, num_processes) has_val_pred.append('nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold') # apply nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5 to nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold e = 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5' pp_models = [j for j, i in enumerate(experiment_names) if i.split("___")[0] == e and i != e] ranks = [final_ranks[i] for i in pp_models] best_idx = np.argmin(ranks) best = experiment_names[pp_models[best_idx]] best_avg_rank = average_rank[pp_models[best_idx]] best_threshold = int(best.split('___')[-1]) predicted_val = join(base, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold') apply_threshold_to_folder(predicted_val, join(pred_val_base, 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold'), best_threshold, replace_with, num_processes) has_val_pred.append('nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold') # convert valsets output_converted = join(base, 'converted_valSet') for e in has_val_pred: expected_source_folder = join(base, 'predVal_PP_rank', e) if not isdir(expected_source_folder): print(e, 'has no predVal_PP_rank') raise RuntimeError() files = subfiles(expected_source_folder, suffix='.nii.gz', join=False) if len(files) != expected_num_cases_val: print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases_val)) continue target_folder = join(output_converted, 'predVal_PP_rank', e) maybe_mkdir_p(target_folder) convert_labels_back_to_BraTS_2018_2019_convention(expected_source_folder, target_folder) # now load all the csvs for the validation set (obtained from evaluation platform) and rank our models on the # validation set flds = subdirs(output_converted, join=False) results_valset = [] names_valset = [] for f in flds: curr = join(output_converted, f) experiments = subdirs(curr, join=False) for e in experiments: currr = join(curr, e) expected_file = join(currr, 'Stats_Validation_final.csv') if not isfile(expected_file): print(f, e, "has not been evaluated yet!") else: res = load_csv_for_ranking(expected_file)[:-5] assert res.shape[0] == expected_num_cases_val results_valset.append(res[None]) names_valset.append("%s___%s" % (f, e)) results_valset = np.concatenate(results_valset, 0) # experiments x cases x metrics # convert to metrics x experiments x cases results_valset = results_valset.transpose((2, 0, 1)) final_ranks, average_rank, ranks = rank_algorithms(results_valset) for t in np.argsort(final_ranks): print(final_ranks[t], average_rank[t], names_valset[t])
def main(): import argparse parser = argparse.ArgumentParser( usage= "This is intended to identify the best model based on the five fold " "cross-validation. Running this script requires all models to have been run " "already. This script will summarize the results of the five folds of all " "models in one json each for easy interpretability") parser.add_argument( "-m", '--models', nargs="+", required=False, default=['2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres']) parser.add_argument("-t", '--task_ids', nargs="+", required=True) parser.add_argument("-tr", type=str, required=False, default=default_trainer, help="nnUNetTrainer class. Default: %s" % default_trainer) parser.add_argument( "-ctr", type=str, required=False, default=default_cascade_trainer, help="nnUNetTrainer class for cascade model. Default: %s" % default_cascade_trainer) parser.add_argument("-pl", type=str, required=False, default=default_plans_identifier, help="plans name, Default: %s" % default_plans_identifier) parser.add_argument('-f', '--folds', nargs='+', default=(0, 1, 2, 3, 4), help="Use this if you have non-standard " "folds. Experienced users only.") parser.add_argument( '--disable_ensembling', required=False, default=False, action='store_true', help= 'Set this flag to disable the use of ensembling. This will find the best single ' 'configuration for each task.') parser.add_argument( "--disable_postprocessing", required=False, default=False, action="store_true", help="Set this flag if you want to disable the use of postprocessing") args = parser.parse_args() tasks = [int(i) for i in args.task_ids] models = args.models tr = args.tr trc = args.ctr pl = args.pl disable_ensembling = args.disable_ensembling disable_postprocessing = args.disable_postprocessing folds = tuple(int(i) for i in args.folds) validation_folder = "validation_raw" # this script now acts independently from the summary jsons. That was unnecessary id_task_mapping = {} for t in tasks: # first collect pure model performance results = {} all_results = {} valid_models = [] for m in models: if m == "3d_cascade_fullres": trainer = trc else: trainer = tr if t not in id_task_mapping.keys(): task_name = find_task_name(get_output_folder_name(m), t) id_task_mapping[t] = task_name output_folder = get_output_folder_name(m, id_task_mapping[t], trainer, pl) if not isdir(output_folder): raise RuntimeError( "Output folder for model %s is missing, expected: %s" % (m, output_folder)) if disable_postprocessing: # we need to collect the predicted niftis from the 5-fold cv and evaluate them against the ground truth cv_niftis_folder = join(output_folder, 'cv_niftis_raw') if not isfile(join(cv_niftis_folder, 'summary.json')): print(t, m, ': collecting niftis from 5-fold cv') if isdir(cv_niftis_folder): shutil.rmtree(cv_niftis_folder) collect_cv_niftis(output_folder, cv_niftis_folder, validation_folder, folds) niftis_gt = subfiles(join(output_folder, "gt_niftis"), suffix='.nii.gz', join=False) niftis_cv = subfiles(cv_niftis_folder, suffix='.nii.gz', join=False) if not all([i in niftis_gt for i in niftis_cv]): raise AssertionError("It does not seem like you trained all the folds! Train " \ "all folds first! There are %d gt niftis in %s but only " \ "%d predicted niftis in %s" % (len(niftis_gt), niftis_gt, len(niftis_cv), niftis_cv)) # load a summary file so that we can know what class labels to expect summary_fold0 = load_json( join(output_folder, "fold_%d" % folds[0], validation_folder, "summary.json"))['results']['mean'] # read classes from summary.json classes = tuple((int(i) for i in summary_fold0.keys())) # evaluate the cv niftis print(t, m, ': evaluating 5-fold cv results') evaluate_folder(join(output_folder, "gt_niftis"), cv_niftis_folder, classes) else: postprocessing_json = join(output_folder, "postprocessing.json") cv_niftis_folder = join(output_folder, "cv_niftis_raw") # we need cv_niftis_postprocessed to know the single model performance. And we need the # postprocessing_json. If either of those is missing, rerun consolidate_folds if not isfile(postprocessing_json) or not isdir( cv_niftis_folder): print( "running missing postprocessing for %s and model %s" % (id_task_mapping[t], m)) consolidate_folds(output_folder, folds=folds) assert isfile( postprocessing_json ), "Postprocessing json missing, expected: %s" % postprocessing_json assert isdir( cv_niftis_folder ), "Folder with niftis from CV missing, expected: %s" % cv_niftis_folder # obtain mean foreground dice summary_file = join(cv_niftis_folder, "summary.json") results[m] = get_mean_foreground_dice(summary_file) foreground_mean(summary_file) all_results[m] = load_json(summary_file)['results']['mean'] valid_models.append(m) if not disable_ensembling: # now run ensembling and add ensembling to results print( "\nI will now ensemble combinations of the following models:\n", valid_models) if len(valid_models) > 1: for m1, m2 in combinations(valid_models, 2): trainer_m1 = trc if m1 == "3d_cascade_fullres" else tr trainer_m2 = trc if m2 == "3d_cascade_fullres" else tr ensemble_name = "ensemble_" + m1 + "__" + trainer_m1 + "__" + pl + "--" + m2 + "__" + trainer_m2 + "__" + pl output_folder_base = join(network_training_output_dir, "ensembles", id_task_mapping[t], ensemble_name) os.makedirs(output_folder_base, exist_ok=True) network1_folder = get_output_folder_name( m1, id_task_mapping[t], trainer_m1, pl) network2_folder = get_output_folder_name( m2, id_task_mapping[t], trainer_m2, pl) print("ensembling", network1_folder, network2_folder) ensemble(network1_folder, network2_folder, output_folder_base, id_task_mapping[t], validation_folder, folds, allow_ensembling=not disable_postprocessing) # ensembling will automatically do postprocessingget_foreground_mean # now get result of ensemble results[ensemble_name] = get_mean_foreground_dice( join(output_folder_base, "ensembled_raw", "summary.json")) summary_file = join(output_folder_base, "ensembled_raw", "summary.json") foreground_mean(summary_file) all_results[ensemble_name] = load_json( summary_file)['results']['mean'] # now print all mean foreground dice and highlight the best foreground_dices = list(results.values()) best = np.max(foreground_dices) for k, v in results.items(): print(k, v) predict_str = "" best_model = None for k, v in results.items(): if v == best: print("%s submit model %s" % (id_task_mapping[t], k), v) best_model = k print( "\nHere is how you should predict test cases. Run in sequential order and replace all input and output folder names with your personalized ones\n" ) if k.startswith("ensemble"): tmp = k[len("ensemble_"):] model1, model2 = tmp.split("--") m1, t1, pl1 = model1.split("__") m2, t2, pl2 = model2.split("__") predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + m1 + " -p " + pl + " -t " + \ id_task_mapping[t] + "\n" predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL2 -tr " + tr + " -ctr " + trc + " -m " + m2 + " -p " + pl + " -t " + \ id_task_mapping[t] + "\n" if not disable_postprocessing: predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER -pp " + join( network_training_output_dir, "ensembles", id_task_mapping[t], k, "postprocessing.json") + "\n" else: predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER\n" else: predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + k + " -p " + pl + " -t " + \ id_task_mapping[t] + "\n" print(predict_str) summary_folder = join(network_training_output_dir, "ensembles", id_task_mapping[t]) os.makedirs(summary_folder, exist_ok=True) with open(join(summary_folder, "prediction_commands.txt"), 'w') as f: f.write(predict_str) num_classes = len([ i for i in all_results[best_model].keys() if i != 'mean' and i != '0' ]) with open(join(summary_folder, "summary.csv"), 'w') as f: f.write("model") for c in range(1, num_classes + 1): f.write(",class%d" % c) f.write(",average") f.write("\n") for m in all_results.keys(): f.write(m) for c in range(1, num_classes + 1): f.write(",%01.4f" % all_results[m][str(c)]["Dice"]) f.write(",%01.4f" % all_results[m]['mean']["Dice"]) f.write("\n")
def collect_and_prepare(base_dir, num_processes = 12, clean=False): """ collect all cv_niftis, compute brats metrics, compute enh tumor thresholds and summarize in csv :param base_dir: :return: """ out = join(base_dir, 'cv_results') out_pp = join(base_dir, 'cv_results_pp') experiments = subfolders(base_dir, join=False, prefix='nnUNetTrainer') regions = get_brats_regions() gt_dir = join(base_dir, 'gt_niftis') replace_with = 2 failed = [] successful = [] for e in experiments: print(e) try: o = join(out, e) o_p = join(out_pp, e) maybe_mkdir_p(o) maybe_mkdir_p(o_p) collect_cv_niftis(join(base_dir, e), o) if clean or not isfile(join(o, 'summary.csv')): evaluate_regions(o, gt_dir, regions, num_processes) if clean or not isfile(join(o_p, 'threshold.pkl')): determine_brats_postprocessing(o, gt_dir, o_p, num_processes, thresholds=list(np.arange(0, 760, 10)), replace_with=replace_with) if clean or not isfile(join(o_p, 'summary.csv')): evaluate_regions(o_p, gt_dir, regions, num_processes) successful.append(e) except Exception as ex: print("\nERROR\n", e, ex, "\n") failed.append(e) # we are interested in the mean (nan is 1) column with open(join(base_dir, 'cv_summary.csv'), 'w') as f: f.write('name,whole,core,enh,mean\n') for e in successful: expected_nopp = join(out, e, 'summary.csv') expected_pp = join(out, out_pp, e, 'summary.csv') if isfile(expected_nopp): res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_noPP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) if isfile(expected_pp): res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_PP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) # this just crawls the folders and evaluates what it finds with open(join(base_dir, 'cv_summary2.csv'), 'w') as f: for folder in ['cv_results', 'cv_results_pp']: for ex in subdirs(join(base_dir, folder), join=False): print(folder, ex) expected = join(base_dir, folder, ex, 'summary.csv') if clean or not isfile(expected): evaluate_regions(join(base_dir, folder, ex), gt_dir, regions, num_processes) if isfile(expected): res = np.loadtxt(expected, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write('%s__%s,' % (folder, ex)) f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) f.write('name,whole,core,enh,mean\n') for e in successful: expected_nopp = join(out, e, 'summary.csv') expected_pp = join(out, out_pp, e, 'summary.csv') if isfile(expected_nopp): res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_noPP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) if isfile(expected_pp): res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_PP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) # apply threshold to val set expected_num_cases = 125 missing_valset = [] has_val_pred = [] for e in successful: if isdir(join(base_dir, 'predVal', e)): currdir = join(base_dir, 'predVal', e) files = subfiles(currdir, suffix='.nii.gz', join=False) if len(files) != expected_num_cases: print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases)) continue output_folder = join(base_dir, 'predVal_PP', e) maybe_mkdir_p(output_folder) threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2] if threshold > 1000: threshold = 750 # don't make it too big! apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes) has_val_pred.append(e) else: print(e, 'has no valset predictions') missing_valset.append(e) # 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment e = 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5' currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold') output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold') maybe_mkdir_p(output_folder) threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2] if threshold > 1000: threshold = 750 # don't make it too big! apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes) # 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment e = 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5' currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold') output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold') maybe_mkdir_p(output_folder) threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2] if threshold > 1000: threshold = 750 # don't make it too big! apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes) # convert val set to brats labels for submission output_converted = join(base_dir, 'converted_valSet') for source in ['predVal', 'predVal_PP']: for e in has_val_pred + ['nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold']: expected_source_folder = join(base_dir, source, e) if not isdir(expected_source_folder): print(e, 'has no', source) raise RuntimeError() files = subfiles(expected_source_folder, suffix='.nii.gz', join=False) if len(files) != expected_num_cases: print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases)) continue target_folder = join(output_converted, source, e) maybe_mkdir_p(target_folder) convert_labels_back_to_BraTS_2018_2019_convention(expected_source_folder, target_folder) summarize_validation_set_predictions(output_converted)
f.write(",%02.4f" % dc) all_results[r].append(dc) f.write("\n") f.write('mean') for r in region_names: f.write(",%02.4f" % np.nanmean(all_results[r])) f.write("\n") f.write('median') for r in region_names: f.write(",%02.4f" % np.nanmedian(all_results[r])) f.write("\n") f.write('mean (nan is 1)') for r in region_names: tmp = np.array(all_results[r]) tmp[np.isnan(tmp)] = 1 f.write(",%02.4f" % np.mean(tmp)) f.write("\n") f.write('median (nan is 1)') for r in region_names: tmp = np.array(all_results[r]) tmp[np.isnan(tmp)] = 1 f.write(",%02.4f" % np.median(tmp)) f.write("\n") if __name__ == '__main__': collect_cv_niftis('./', './cv_niftis') evaluate_regions('./cv_niftis/', './gt_niftis/', get_brats_regions())