def score_and_postprocess_model_based_on_rank_then_aggregate():
"""
Similarly to BraTS 2017 - BraTS 2019, each participant will be ranked for each of the X test cases. Each case
includes 3 regions of evaluation, and the metrics used to produce the rankings will be the Dice Similarity
Coefficient and the 95% Hausdorff distance. Thus, for X number of cases included in the BraTS 2020, each
participant ends up having X*3*2 rankings. The final ranking score is the average of all these rankings normalized
by the number of teams.
https://zenodo.org/record/3718904
-> let's optimize for this.
Important: the outcome very much depends on the competing models. We need some references. We only got our own,
so let's hope this still works
:return:
"""
base = "/media/fabian/Results/nnUNet/3d_fullres/Task082_BraTS2020"
replace_with = 2
num_processes = 24
expected_num_cases_val = 125
# use a separate output folder from the previous experiments to ensure we are not messing things up
output_base_here = join(base, 'use_brats_ranking')
maybe_mkdir_p(output_base_here)
# collect cv niftis and compute metrics with evaluate_BraTS_folder to ensure we work with the same metrics as brats
out = join(output_base_here, 'cv_results')
experiments = subfolders(base, join=False, prefix='nnUNetTrainer')
gt_dir = join(base, 'gt_niftis')
experiments_with_full_cv = []
for e in experiments:
print(e)
o = join(out, e)
maybe_mkdir_p(o)
try:
collect_cv_niftis(join(base, e), o)
if not isfile(join(o, 'results.csv')):
evaluate_BraTS_folder(o, gt_dir, num_processes, strict=True)
experiments_with_full_cv.append(e)
except Exception as ex:
print("\nERROR\n", e, ex, "\n")
if isfile(join(o, 'results.csv')):
os.remove(join(o, 'results.csv'))
# rank the non-postprocessed models
tmp = np.loadtxt(join(out, experiments_with_full_cv[0], 'results.csv'), dtype='str', delimiter=',')
num_cases = len(tmp) - 1
data_for_ranking = np.zeros((6, len(experiments_with_full_cv), num_cases))
for i, e in enumerate(experiments_with_full_cv):
scores = load_csv_for_ranking(join(out, e, 'results.csv'))
for metric in range(6):
data_for_ranking[metric, i] = scores[:, metric]
final_ranks, average_rank, ranks = rank_algorithms(data_for_ranking)
for t in np.argsort(final_ranks):
print(final_ranks[t], average_rank[t], experiments_with_full_cv[t])
# for each model, create output directories with different thresholds. evaluate ALL OF THEM (might take a while lol)
thresholds = np.arange(25, 751, 25)
output_pp_tmp = join(output_base_here, 'cv_determine_pp_thresholds')
for e in experiments_with_full_cv:
input_folder = join(out, e)
for t in thresholds:
output_directory = join(output_pp_tmp, e, str(t))
maybe_mkdir_p(output_directory)
if not isfile(join(output_directory, 'results.csv')):
apply_threshold_to_folder(input_folder, output_directory, t, replace_with, processes=16)
evaluate_BraTS_folder(output_directory, gt_dir, num_processes)
# load ALL the results!
results = []
experiment_names = []
for e in experiments_with_full_cv:
for t in thresholds:
output_directory = join(output_pp_tmp, e, str(t))
expected_file = join(output_directory, 'results.csv')
if not isfile(expected_file):
print(e, 'does not have a results file for threshold', t)
continue
results.append(load_csv_for_ranking(expected_file))
experiment_names.append("%s___%d" % (e, t))
all_results = np.concatenate([i[None] for i in results], 0).transpose((2, 0, 1))
# concatenate with non postprocessed models
all_results = np.concatenate((data_for_ranking, all_results), 1)
experiment_names += experiments_with_full_cv
final_ranks, average_rank, ranks = rank_algorithms(all_results)
for t in np.argsort(final_ranks):
print(final_ranks[t], average_rank[t], experiment_names[t])
# for each model, print the non postprocessed model as well as the best postprocessed model. If there are
# validation set predictions, apply the best threshold to the validation set
pred_val_base = join(base, 'predVal_PP_rank')
has_val_pred = []
for e in experiments_with_full_cv:
rank_nonpp = final_ranks[experiment_names.index(e)]
avg_rank_nonpp = average_rank[experiment_names.index(e)]
print(e, avg_rank_nonpp, rank_nonpp)
predicted_val = join(base, 'predVal', e)
pp_models = [j for j, i in enumerate(experiment_names) if i.split("___")[0] == e and i != e]
if len(pp_models) > 0:
ranks = [final_ranks[i] for i in pp_models]
best_idx = np.argmin(ranks)
best = experiment_names[pp_models[best_idx]]
best_avg_rank = average_rank[pp_models[best_idx]]
print(best, best_avg_rank, min(ranks))
print('')
# apply threshold to validation set
best_threshold = int(best.split('___')[-1])
if not isdir(predicted_val):
print(e, 'has not valset predictions')
else:
files = subfiles(predicted_val, suffix='.nii.gz')
if len(files) != expected_num_cases_val:
print(e, 'has missing val cases. found: %d expected: %d' % (len(files), expected_num_cases_val))
else:
apply_threshold_to_folder(predicted_val, join(pred_val_base, e), best_threshold, replace_with, num_processes)
has_val_pred.append(e)
else:
print(e, 'not found in ranking')
# apply nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5 to nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold
e = 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5'
pp_models = [j for j, i in enumerate(experiment_names) if i.split("___")[0] == e and i != e]
ranks = [final_ranks[i] for i in pp_models]
best_idx = np.argmin(ranks)
best = experiment_names[pp_models[best_idx]]
best_avg_rank = average_rank[pp_models[best_idx]]
best_threshold = int(best.split('___')[-1])
predicted_val = join(base, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold')
apply_threshold_to_folder(predicted_val, join(pred_val_base, 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold'), best_threshold, replace_with, num_processes)
has_val_pred.append('nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold')
# apply nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5 to nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold
e = 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5'
pp_models = [j for j, i in enumerate(experiment_names) if i.split("___")[0] == e and i != e]
ranks = [final_ranks[i] for i in pp_models]
best_idx = np.argmin(ranks)
best = experiment_names[pp_models[best_idx]]
best_avg_rank = average_rank[pp_models[best_idx]]
best_threshold = int(best.split('___')[-1])
predicted_val = join(base, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold')
apply_threshold_to_folder(predicted_val, join(pred_val_base, 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold'), best_threshold, replace_with, num_processes)
has_val_pred.append('nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold')
# convert valsets
output_converted = join(base, 'converted_valSet')
for e in has_val_pred:
expected_source_folder = join(base, 'predVal_PP_rank', e)
if not isdir(expected_source_folder):
print(e, 'has no predVal_PP_rank')
raise RuntimeError()
files = subfiles(expected_source_folder, suffix='.nii.gz', join=False)
if len(files) != expected_num_cases_val:
print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases_val))
continue
target_folder = join(output_converted, 'predVal_PP_rank', e)
maybe_mkdir_p(target_folder)
convert_labels_back_to_BraTS_2018_2019_convention(expected_source_folder, target_folder)
# now load all the csvs for the validation set (obtained from evaluation platform) and rank our models on the
# validation set
flds = subdirs(output_converted, join=False)
results_valset = []
names_valset = []
for f in flds:
curr = join(output_converted, f)
experiments = subdirs(curr, join=False)
for e in experiments:
currr = join(curr, e)
expected_file = join(currr, 'Stats_Validation_final.csv')
if not isfile(expected_file):
print(f, e, "has not been evaluated yet!")
else:
res = load_csv_for_ranking(expected_file)[:-5]
assert res.shape[0] == expected_num_cases_val
results_valset.append(res[None])
names_valset.append("%s___%s" % (f, e))
results_valset = np.concatenate(results_valset, 0) # experiments x cases x metrics
# convert to metrics x experiments x cases
results_valset = results_valset.transpose((2, 0, 1))
final_ranks, average_rank, ranks = rank_algorithms(results_valset)
for t in np.argsort(final_ranks):
print(final_ranks[t], average_rank[t], names_valset[t])
def main():
import argparse
parser = argparse.ArgumentParser(
usage=
"This is intended to identify the best model based on the five fold "
"cross-validation. Running this script requires all models to have been run "
"already. This script will summarize the results of the five folds of all "
"models in one json each for easy interpretability")
parser.add_argument(
"-m",
'--models',
nargs="+",
required=False,
default=['2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'])
parser.add_argument("-t", '--task_ids', nargs="+", required=True)
parser.add_argument("-tr",
type=str,
required=False,
default=default_trainer,
help="nnUNetTrainer class. Default: %s" %
default_trainer)
parser.add_argument(
"-ctr",
type=str,
required=False,
default=default_cascade_trainer,
help="nnUNetTrainer class for cascade model. Default: %s" %
default_cascade_trainer)
parser.add_argument("-pl",
type=str,
required=False,
default=default_plans_identifier,
help="plans name, Default: %s" %
default_plans_identifier)
parser.add_argument('-f',
'--folds',
nargs='+',
default=(0, 1, 2, 3, 4),
help="Use this if you have non-standard "
"folds. Experienced users only.")
parser.add_argument(
'--disable_ensembling',
required=False,
default=False,
action='store_true',
help=
'Set this flag to disable the use of ensembling. This will find the best single '
'configuration for each task.')
parser.add_argument(
"--disable_postprocessing",
required=False,
default=False,
action="store_true",
help="Set this flag if you want to disable the use of postprocessing")
args = parser.parse_args()
tasks = [int(i) for i in args.task_ids]
models = args.models
tr = args.tr
trc = args.ctr
pl = args.pl
disable_ensembling = args.disable_ensembling
disable_postprocessing = args.disable_postprocessing
folds = tuple(int(i) for i in args.folds)
validation_folder = "validation_raw"
# this script now acts independently from the summary jsons. That was unnecessary
id_task_mapping = {}
for t in tasks:
# first collect pure model performance
results = {}
all_results = {}
valid_models = []
for m in models:
if m == "3d_cascade_fullres":
trainer = trc
else:
trainer = tr
if t not in id_task_mapping.keys():
task_name = find_task_name(get_output_folder_name(m), t)
id_task_mapping[t] = task_name
output_folder = get_output_folder_name(m, id_task_mapping[t],
trainer, pl)
if not isdir(output_folder):
raise RuntimeError(
"Output folder for model %s is missing, expected: %s" %
(m, output_folder))
if disable_postprocessing:
# we need to collect the predicted niftis from the 5-fold cv and evaluate them against the ground truth
cv_niftis_folder = join(output_folder, 'cv_niftis_raw')
if not isfile(join(cv_niftis_folder, 'summary.json')):
print(t, m, ': collecting niftis from 5-fold cv')
if isdir(cv_niftis_folder):
shutil.rmtree(cv_niftis_folder)
collect_cv_niftis(output_folder, cv_niftis_folder,
validation_folder, folds)
niftis_gt = subfiles(join(output_folder, "gt_niftis"),
suffix='.nii.gz',
join=False)
niftis_cv = subfiles(cv_niftis_folder,
suffix='.nii.gz',
join=False)
if not all([i in niftis_gt for i in niftis_cv]):
raise AssertionError("It does not seem like you trained all the folds! Train " \
"all folds first! There are %d gt niftis in %s but only " \
"%d predicted niftis in %s" % (len(niftis_gt), niftis_gt,
len(niftis_cv), niftis_cv))
# load a summary file so that we can know what class labels to expect
summary_fold0 = load_json(
join(output_folder, "fold_%d" % folds[0],
validation_folder,
"summary.json"))['results']['mean']
# read classes from summary.json
classes = tuple((int(i) for i in summary_fold0.keys()))
# evaluate the cv niftis
print(t, m, ': evaluating 5-fold cv results')
evaluate_folder(join(output_folder, "gt_niftis"),
cv_niftis_folder, classes)
else:
postprocessing_json = join(output_folder,
"postprocessing.json")
cv_niftis_folder = join(output_folder, "cv_niftis_raw")
# we need cv_niftis_postprocessed to know the single model performance. And we need the
# postprocessing_json. If either of those is missing, rerun consolidate_folds
if not isfile(postprocessing_json) or not isdir(
cv_niftis_folder):
print(
"running missing postprocessing for %s and model %s" %
(id_task_mapping[t], m))
consolidate_folds(output_folder, folds=folds)
assert isfile(
postprocessing_json
), "Postprocessing json missing, expected: %s" % postprocessing_json
assert isdir(
cv_niftis_folder
), "Folder with niftis from CV missing, expected: %s" % cv_niftis_folder
# obtain mean foreground dice
summary_file = join(cv_niftis_folder, "summary.json")
results[m] = get_mean_foreground_dice(summary_file)
foreground_mean(summary_file)
all_results[m] = load_json(summary_file)['results']['mean']
valid_models.append(m)
if not disable_ensembling:
# now run ensembling and add ensembling to results
print(
"\nI will now ensemble combinations of the following models:\n",
valid_models)
if len(valid_models) > 1:
for m1, m2 in combinations(valid_models, 2):
trainer_m1 = trc if m1 == "3d_cascade_fullres" else tr
trainer_m2 = trc if m2 == "3d_cascade_fullres" else tr
ensemble_name = "ensemble_" + m1 + "__" + trainer_m1 + "__" + pl + "--" + m2 + "__" + trainer_m2 + "__" + pl
output_folder_base = join(network_training_output_dir,
"ensembles", id_task_mapping[t],
ensemble_name)
os.makedirs(output_folder_base, exist_ok=True)
network1_folder = get_output_folder_name(
m1, id_task_mapping[t], trainer_m1, pl)
network2_folder = get_output_folder_name(
m2, id_task_mapping[t], trainer_m2, pl)
print("ensembling", network1_folder, network2_folder)
ensemble(network1_folder,
network2_folder,
output_folder_base,
id_task_mapping[t],
validation_folder,
folds,
allow_ensembling=not disable_postprocessing)
# ensembling will automatically do postprocessingget_foreground_mean
# now get result of ensemble
results[ensemble_name] = get_mean_foreground_dice(
join(output_folder_base, "ensembled_raw",
"summary.json"))
summary_file = join(output_folder_base, "ensembled_raw",
"summary.json")
foreground_mean(summary_file)
all_results[ensemble_name] = load_json(
summary_file)['results']['mean']
# now print all mean foreground dice and highlight the best
foreground_dices = list(results.values())
best = np.max(foreground_dices)
for k, v in results.items():
print(k, v)
predict_str = ""
best_model = None
for k, v in results.items():
if v == best:
print("%s submit model %s" % (id_task_mapping[t], k), v)
best_model = k
print(
"\nHere is how you should predict test cases. Run in sequential order and replace all input and output folder names with your personalized ones\n"
)
if k.startswith("ensemble"):
tmp = k[len("ensemble_"):]
model1, model2 = tmp.split("--")
m1, t1, pl1 = model1.split("__")
m2, t2, pl2 = model2.split("__")
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + m1 + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL2 -tr " + tr + " -ctr " + trc + " -m " + m2 + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
if not disable_postprocessing:
predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER -pp " + join(
network_training_output_dir, "ensembles",
id_task_mapping[t], k,
"postprocessing.json") + "\n"
else:
predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER\n"
else:
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + k + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
print(predict_str)
summary_folder = join(network_training_output_dir, "ensembles",
id_task_mapping[t])
os.makedirs(summary_folder, exist_ok=True)
with open(join(summary_folder, "prediction_commands.txt"), 'w') as f:
f.write(predict_str)
num_classes = len([
i for i in all_results[best_model].keys()
if i != 'mean' and i != '0'
])
with open(join(summary_folder, "summary.csv"), 'w') as f:
f.write("model")
for c in range(1, num_classes + 1):
f.write(",class%d" % c)
f.write(",average")
f.write("\n")
for m in all_results.keys():
f.write(m)
for c in range(1, num_classes + 1):
f.write(",%01.4f" % all_results[m][str(c)]["Dice"])
f.write(",%01.4f" % all_results[m]['mean']["Dice"])
f.write("\n")
def collect_and_prepare(base_dir, num_processes = 12, clean=False):
"""
collect all cv_niftis, compute brats metrics, compute enh tumor thresholds and summarize in csv
:param base_dir:
:return:
"""
out = join(base_dir, 'cv_results')
out_pp = join(base_dir, 'cv_results_pp')
experiments = subfolders(base_dir, join=False, prefix='nnUNetTrainer')
regions = get_brats_regions()
gt_dir = join(base_dir, 'gt_niftis')
replace_with = 2
failed = []
successful = []
for e in experiments:
print(e)
try:
o = join(out, e)
o_p = join(out_pp, e)
maybe_mkdir_p(o)
maybe_mkdir_p(o_p)
collect_cv_niftis(join(base_dir, e), o)
if clean or not isfile(join(o, 'summary.csv')):
evaluate_regions(o, gt_dir, regions, num_processes)
if clean or not isfile(join(o_p, 'threshold.pkl')):
determine_brats_postprocessing(o, gt_dir, o_p, num_processes, thresholds=list(np.arange(0, 760, 10)), replace_with=replace_with)
if clean or not isfile(join(o_p, 'summary.csv')):
evaluate_regions(o_p, gt_dir, regions, num_processes)
successful.append(e)
except Exception as ex:
print("\nERROR\n", e, ex, "\n")
failed.append(e)
# we are interested in the mean (nan is 1) column
with open(join(base_dir, 'cv_summary.csv'), 'w') as f:
f.write('name,whole,core,enh,mean\n')
for e in successful:
expected_nopp = join(out, e, 'summary.csv')
expected_pp = join(out, out_pp, e, 'summary.csv')
if isfile(expected_nopp):
res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_noPP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
if isfile(expected_pp):
res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_PP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
# this just crawls the folders and evaluates what it finds
with open(join(base_dir, 'cv_summary2.csv'), 'w') as f:
for folder in ['cv_results', 'cv_results_pp']:
for ex in subdirs(join(base_dir, folder), join=False):
print(folder, ex)
expected = join(base_dir, folder, ex, 'summary.csv')
if clean or not isfile(expected):
evaluate_regions(join(base_dir, folder, ex), gt_dir, regions, num_processes)
if isfile(expected):
res = np.loadtxt(expected, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write('%s__%s,' % (folder, ex))
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
f.write('name,whole,core,enh,mean\n')
for e in successful:
expected_nopp = join(out, e, 'summary.csv')
expected_pp = join(out, out_pp, e, 'summary.csv')
if isfile(expected_nopp):
res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_noPP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
if isfile(expected_pp):
res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_PP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
# apply threshold to val set
expected_num_cases = 125
missing_valset = []
has_val_pred = []
for e in successful:
if isdir(join(base_dir, 'predVal', e)):
currdir = join(base_dir, 'predVal', e)
files = subfiles(currdir, suffix='.nii.gz', join=False)
if len(files) != expected_num_cases:
print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases))
continue
output_folder = join(base_dir, 'predVal_PP', e)
maybe_mkdir_p(output_folder)
threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2]
if threshold > 1000: threshold = 750 # don't make it too big!
apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes)
has_val_pred.append(e)
else:
print(e, 'has no valset predictions')
missing_valset.append(e)
# 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment
e = 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5'
currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold')
output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold')
maybe_mkdir_p(output_folder)
threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2]
if threshold > 1000: threshold = 750 # don't make it too big!
apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes)
# 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment
e = 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5'
currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold')
output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold')
maybe_mkdir_p(output_folder)
threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2]
if threshold > 1000: threshold = 750 # don't make it too big!
apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes)
# convert val set to brats labels for submission
output_converted = join(base_dir, 'converted_valSet')
for source in ['predVal', 'predVal_PP']:
for e in has_val_pred + ['nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold']:
expected_source_folder = join(base_dir, source, e)
if not isdir(expected_source_folder):
print(e, 'has no', source)
raise RuntimeError()
files = subfiles(expected_source_folder, suffix='.nii.gz', join=False)
if len(files) != expected_num_cases:
print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases))
continue
target_folder = join(output_converted, source, e)
maybe_mkdir_p(target_folder)
convert_labels_back_to_BraTS_2018_2019_convention(expected_source_folder, target_folder)
summarize_validation_set_predictions(output_converted)
f.write(",%02.4f" % dc)
all_results[r].append(dc)
f.write("\n")
f.write('mean')
for r in region_names:
f.write(",%02.4f" % np.nanmean(all_results[r]))
f.write("\n")
f.write('median')
for r in region_names:
f.write(",%02.4f" % np.nanmedian(all_results[r]))
f.write("\n")
f.write('mean (nan is 1)')
for r in region_names:
tmp = np.array(all_results[r])
tmp[np.isnan(tmp)] = 1
f.write(",%02.4f" % np.mean(tmp))
f.write("\n")
f.write('median (nan is 1)')
for r in region_names:
tmp = np.array(all_results[r])
tmp[np.isnan(tmp)] = 1
f.write(",%02.4f" % np.median(tmp))
f.write("\n")
if __name__ == '__main__':
collect_cv_niftis('./', './cv_niftis')
evaluate_regions('./cv_niftis/', './gt_niftis/', get_brats_regions())
