def main():
argparser = argparse.ArgumentParser(usage="Used to determine the postprocessing for a trained model. Useful for "
"when the best configuration (2d, 3d_fullres etc) as selected manually.")
argparser.add_argument("-m", type=str, required=True, help="U-Net model (2d, 3d_lowres, 3d_fullres or "
"3d_cascade_fullres)")
argparser.add_argument("-t", type=str, required=True, help="Task name or id")
argparser.add_argument("-tr", type=str, required=False, default=None,
help="nnUNetTrainer class. Default: %s, unless 3d_cascade_fullres "
"(then it's %s)" % (default_trainer, default_cascade_trainer))
argparser.add_argument("-pl", type=str, required=False, default=default_plans_identifier,
help="Plans name, Default=%s" % default_plans_identifier)
argparser.add_argument("-val", type=str, required=False, default="validation_raw",
help="Validation folder name. Default: validation_raw")
args = argparser.parse_args()
model = args.m
task = args.t
trainer = args.tr
plans = args.pl
val = args.val
if not task.startswith("Task"):
task_id = int(task)
task = convert_id_to_task_name(task_id)
if trainer is None:
if model == "3d_cascade_fullres":
trainer = "nnUNetTrainerV2CascadeFullRes"
else:
trainer = "nnUNetTrainerV2"
folder = get_output_folder_name(model, task, trainer, plans, None)
consolidate_folds(folder, val)
def main():
import argparse
parser = argparse.ArgumentParser(
usage=
"This is intended to identify the best model based on the five fold "
"cross-validation. Running this script requires all models to have been run "
"already. This script will summarize the results of the five folds of all "
"models in one json each for easy interpretability")
parser.add_argument(
"-m",
'--models',
nargs="+",
required=False,
default=['2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'])
parser.add_argument("-t", '--task_ids', nargs="+", required=True)
parser.add_argument("-tr",
type=str,
required=False,
default=default_trainer,
help="nnUNetTrainer class. Default: %s" %
default_trainer)
parser.add_argument(
"-ctr",
type=str,
required=False,
default=default_cascade_trainer,
help="nnUNetTrainer class for cascade model. Default: %s" %
default_cascade_trainer)
parser.add_argument("-pl",
type=str,
required=False,
default=default_plans_identifier,
help="plans name, Default: %s" %
default_plans_identifier)
parser.add_argument('-f',
'--folds',
nargs='+',
default=(0, 1, 2, 3, 4),
help="use this if you have non-standard folds")
parser.add_argument(
"--strict",
required=False,
default=False,
action="store_true",
help=
"set this flag if you want this script to crash of one of the models is missing"
)
args = parser.parse_args()
tasks = [int(i) for i in args.task_ids]
models = args.models
tr = args.tr
trc = args.ctr
strict = args.strict
pl = args.pl
folds = tuple(int(i) for i in args.folds)
validation_folder = "validation_raw"
# this script now acts independently from the summary jsons. That was unnecessary
id_task_mapping = {}
# for each task, run ensembling using all combinations of two models
for t in tasks:
# first collect pure model performance (postprocessed)
results = {}
all_results = {}
valid_models = []
for m in models:
try:
if m == "3d_cascade_fullres":
trainer = trc
else:
trainer = tr
if t not in id_task_mapping.keys():
task_name = find_task_name(get_output_folder_name(m), t)
id_task_mapping[t] = task_name
output_folder = get_output_folder_name(m, id_task_mapping[t],
trainer, pl)
assert isdir(
output_folder
), "Output folder for model %s is missing, expected: %s" % (
m, output_folder)
# we need a postprocessing_json for inference, so that must be present
postprocessing_json = join(output_folder,
"postprocessing.json")
# we need cv_niftis_postprocessed to know the single model performance
cv_niftis_folder = join(output_folder, "cv_niftis_raw")
if not isfile(postprocessing_json) or not isdir(
cv_niftis_folder):
print(
"running missing postprocessing for %s and model %s" %
(id_task_mapping[t], m))
consolidate_folds(output_folder, folds=folds)
assert isfile(
postprocessing_json
), "Postprocessing json missing, expected: %s" % postprocessing_json
assert isdir(
cv_niftis_folder
), "Folder with niftis from CV missing, expected: %s" % cv_niftis_folder
# obtain mean foreground dice
summary_file = join(cv_niftis_folder, "summary.json")
results[m] = get_mean_foreground_dice(summary_file)
foreground_mean(summary_file)
all_results[m] = load_json(summary_file)['results']['mean']
valid_models.append(m)
except Exception as e:
if strict:
raise e
else:
print("WARNING!")
print(e)
# now run ensembling and add ensembling to results
print("\nFound the following valid models:\n", valid_models)
if len(valid_models) > 1:
# for m1, m2 in combinations(valid_models, 2):
#
# trainer_m1 = trc if m1 == "3d_cascade_fullres" else tr
# trainer_m2 = trc if m2 == "3d_cascade_fullres" else tr
#
# ensemble_name = "ensemble_" + m1 + "__" + trainer_m1 + "__" + pl + "--" + m2 + "__" + trainer_m2 + "__" + pl
# output_folder_base = join(network_training_output_dir, "ensembles", id_task_mapping[t], ensemble_name)
# maybe_mkdir_p(output_folder_base)
#
# network1_folder = get_output_folder_name(m1, id_task_mapping[t], trainer_m1, pl)
# network2_folder = get_output_folder_name(m2, id_task_mapping[t], trainer_m2, pl)
#
# print("ensembling", network1_folder, network2_folder)
# ensemble2(network1_folder, network2_folder, output_folder_base, id_task_mapping[t], validation_folder, folds)
# # ensembling will automatically do postprocessingget_foreground_mean
#
# # now get result of ensemble
# results[ensemble_name] = get_mean_foreground_dice(join(output_folder_base, "ensembled_raw", "summary.json"))
# summary_file = join(output_folder_base, "ensembled_raw", "summary.json")
# foreground_mean(summary_file)
# all_results[ensemble_name] = load_json(summary_file)['results']['mean']
# for m1, m2, m3 in combinations(valid_models, 3):
#
# trainer_m1 = trc if m1 == "3d_cascade_fullres" else tr
# trainer_m2 = trc if m2 == "3d_cascade_fullres" else tr
# trainer_m3 = trc if m3 == "3d_cascade_fullres" else tr
#
# ensemble_name = "ensemble_" + m1 + "__" + trainer_m1 + "__" + pl + "--" + m2 + "__" + trainer_m2 + "__" + pl + "--" + m3 + "__" + trainer_m3 + "__" + pl
# output_folder_base = join(network_training_output_dir, "ensembles", id_task_mapping[t], ensemble_name)
# maybe_mkdir_p(output_folder_base)
#
# network1_folder = get_output_folder_name(m1, id_task_mapping[t], trainer_m1, pl)
# network2_folder = get_output_folder_name(m2, id_task_mapping[t], trainer_m2, pl)
# network3_folder = get_output_folder_name(m3, id_task_mapping[t], trainer_m3, pl)
#
# print("ensembling", network1_folder, network2_folder, network3_folder)
# ensemble3(network1_folder, network2_folder, network3_folder, output_folder_base, id_task_mapping[t], validation_folder, folds)
# # ensembling will automatically do postprocessingget_foreground_mean
#
# # now get result of ensemble
# results[ensemble_name] = get_mean_foreground_dice(join(output_folder_base, "ensembled_raw", "summary.json"))
# summary_file = join(output_folder_base, "ensembled_raw", "summary.json")
# foreground_mean(summary_file)
# all_results[ensemble_name] = load_json(summary_file)['results']['mean']
for m1, m2, m3, m4 in combinations(valid_models, 4):
trainer_m1 = trc if m1 == "3d_cascade_fullres" else tr
trainer_m2 = trc if m2 == "3d_cascade_fullres" else tr
trainer_m3 = trc if m3 == "3d_cascade_fullres" else tr
trainer_m4 = trc if m4 == "3d_cascade_fullres" else tr
ensemble_name = "ensemble_" + m1 + "__" + trainer_m1 + "__" + pl + "--" + m2 + "__" + trainer_m2 + "__" + pl + "--" + m3 + "__" + trainer_m3 + "__" + pl + m4 + "__" + trainer_m4 + "__" + pl
output_folder_base = join(network_training_output_dir,
"ensembles", id_task_mapping[t],
ensemble_name)
maybe_mkdir_p(output_folder_base)
network1_folder = get_output_folder_name(
m1, id_task_mapping[t], trainer_m1, pl)
network2_folder = get_output_folder_name(
m2, id_task_mapping[t], trainer_m2, pl)
network3_folder = get_output_folder_name(
m3, id_task_mapping[t], trainer_m3, pl)
network4_folder = get_output_folder_name(
m4, id_task_mapping[t], trainer_m4, pl)
print("ensembling", network1_folder, network2_folder,
network3_folder, network4_folder)
ensemble4(network1_folder, network2_folder, network3_folder,
network4_folder, output_folder_base,
id_task_mapping[t], validation_folder, folds)
# ensembling will automatically do postprocessingget_foreground_mean
# now get result of ensemble
results[ensemble_name] = get_mean_foreground_dice(
join(output_folder_base, "ensembled_raw", "summary.json"))
summary_file = join(output_folder_base, "ensembled_raw",
"summary.json")
foreground_mean(summary_file)
all_results[ensemble_name] = load_json(
summary_file)['results']['mean']
# now print all mean foreground dice and highlight the best
foreground_dices = list(results.values())
best = np.max(foreground_dices)
for k, v in results.items():
print(k, v)
predict_str = ""
best_model = None
for k, v in results.items():
if v == best:
print("%s submit model %s" % (id_task_mapping[t], k), v)
best_model = k
print(
"\nHere is how you should predict test cases. Run in sequential order and replace all input and output folder names with your personalized ones\n"
)
if k.startswith("ensemble"):
tmp = k[len("ensemble_"):]
model1, model2 = tmp.split("--")
m1, t1, pl1 = model1.split("__")
m2, t2, pl2 = model2.split("__")
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + m1 + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL2 -tr " + tr + " -ctr " + trc + " -m " + m2 + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER -pp " + join(
network_training_output_dir, "ensembles",
id_task_mapping[t], k, "postprocessing.json") + "\n"
else:
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + k + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
print(predict_str)
summary_folder = join(network_training_output_dir, "ensembles",
id_task_mapping[t])
maybe_mkdir_p(summary_folder)
with open(join(summary_folder, "prediction_commands.txt"), 'w') as f:
f.write(predict_str)
num_classes = len(
[i for i in all_results[best_model].keys() if i != 'mean'])
with open(join(summary_folder, "summary.csv"), 'w') as f:
f.write("model")
for c in range(1, num_classes):
f.write(",class%d" % c)
f.write(",average")
f.write("\n")
for m in all_results.keys():
f.write(m)
for c in range(1, num_classes):
f.write(",%01.4f" % all_results[m][str(c)]["Dice"])
f.write(",%01.4f" % all_results[m]['mean']["Dice"])
f.write("\n")
def main():
import argparse
parser = argparse.ArgumentParser(
usage=
"This is intended to identify the best model based on the five fold "
"cross-validation. Running this script requires all models to have been run "
"already. This script will summarize the results of the five folds of all "
"models in one json each for easy interpretability")
parser.add_argument(
"-m",
'--models',
nargs="+",
required=False,
default=['2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'])
parser.add_argument("-t", '--task_ids', nargs="+", required=True)
parser.add_argument("-tr",
type=str,
required=False,
default=default_trainer,
help="nnUNetTrainer class. Default: %s" %
default_trainer)
parser.add_argument(
"-ctr",
type=str,
required=False,
default=default_cascade_trainer,
help="nnUNetTrainer class for cascade model. Default: %s" %
default_cascade_trainer)
parser.add_argument("-pl",
type=str,
required=False,
default=default_plans_identifier,
help="plans name, Default: %s" %
default_plans_identifier)
parser.add_argument('-f',
'--folds',
nargs='+',
default=(0, 1, 2, 3, 4),
help="Use this if you have non-standard "
"folds. Experienced users only.")
parser.add_argument(
'--disable_ensembling',
required=False,
default=False,
action='store_true',
help=
'Set this flag to disable the use of ensembling. This will find the best single '
'configuration for each task.')
parser.add_argument(
"--disable_postprocessing",
required=False,
default=False,
action="store_true",
help="Set this flag if you want to disable the use of postprocessing")
args = parser.parse_args()
tasks = [int(i) for i in args.task_ids]
models = args.models
tr = args.tr
trc = args.ctr
pl = args.pl
disable_ensembling = args.disable_ensembling
disable_postprocessing = args.disable_postprocessing
folds = tuple(int(i) for i in args.folds)
validation_folder = "validation_raw"
# this script now acts independently from the summary jsons. That was unnecessary
id_task_mapping = {}
for t in tasks:
# first collect pure model performance
results = {}
all_results = {}
valid_models = []
for m in models:
if m == "3d_cascade_fullres":
trainer = trc
else:
trainer = tr
if t not in id_task_mapping.keys():
task_name = find_task_name(get_output_folder_name(m), t)
id_task_mapping[t] = task_name
output_folder = get_output_folder_name(m, id_task_mapping[t],
trainer, pl)
if not isdir(output_folder):
raise RuntimeError(
"Output folder for model %s is missing, expected: %s" %
(m, output_folder))
if disable_postprocessing:
# we need to collect the predicted niftis from the 5-fold cv and evaluate them against the ground truth
cv_niftis_folder = join(output_folder, 'cv_niftis_raw')
if not isfile(join(cv_niftis_folder, 'summary.json')):
print(t, m, ': collecting niftis from 5-fold cv')
if isdir(cv_niftis_folder):
shutil.rmtree(cv_niftis_folder)
collect_cv_niftis(output_folder, cv_niftis_folder,
validation_folder, folds)
niftis_gt = subfiles(join(output_folder, "gt_niftis"),
suffix='.nii.gz',
join=False)
niftis_cv = subfiles(cv_niftis_folder,
suffix='.nii.gz',
join=False)
if not all([i in niftis_gt for i in niftis_cv]):
raise AssertionError("It does not seem like you trained all the folds! Train " \
"all folds first! There are %d gt niftis in %s but only " \
"%d predicted niftis in %s" % (len(niftis_gt), niftis_gt,
len(niftis_cv), niftis_cv))
# load a summary file so that we can know what class labels to expect
summary_fold0 = load_json(
join(output_folder, "fold_%d" % folds[0],
validation_folder,
"summary.json"))['results']['mean']
# read classes from summary.json
classes = tuple((int(i) for i in summary_fold0.keys()))
# evaluate the cv niftis
print(t, m, ': evaluating 5-fold cv results')
evaluate_folder(join(output_folder, "gt_niftis"),
cv_niftis_folder, classes)
else:
postprocessing_json = join(output_folder,
"postprocessing.json")
cv_niftis_folder = join(output_folder, "cv_niftis_raw")
# we need cv_niftis_postprocessed to know the single model performance. And we need the
# postprocessing_json. If either of those is missing, rerun consolidate_folds
if not isfile(postprocessing_json) or not isdir(
cv_niftis_folder):
print(
"running missing postprocessing for %s and model %s" %
(id_task_mapping[t], m))
consolidate_folds(output_folder, folds=folds)
assert isfile(
postprocessing_json
), "Postprocessing json missing, expected: %s" % postprocessing_json
assert isdir(
cv_niftis_folder
), "Folder with niftis from CV missing, expected: %s" % cv_niftis_folder
# obtain mean foreground dice
summary_file = join(cv_niftis_folder, "summary.json")
results[m] = get_mean_foreground_dice(summary_file)
foreground_mean(summary_file)
all_results[m] = load_json(summary_file)['results']['mean']
valid_models.append(m)
if not disable_ensembling:
# now run ensembling and add ensembling to results
print(
"\nI will now ensemble combinations of the following models:\n",
valid_models)
if len(valid_models) > 1:
for m1, m2 in combinations(valid_models, 2):
trainer_m1 = trc if m1 == "3d_cascade_fullres" else tr
trainer_m2 = trc if m2 == "3d_cascade_fullres" else tr
ensemble_name = "ensemble_" + m1 + "__" + trainer_m1 + "__" + pl + "--" + m2 + "__" + trainer_m2 + "__" + pl
output_folder_base = join(network_training_output_dir,
"ensembles", id_task_mapping[t],
ensemble_name)
os.makedirs(output_folder_base, exist_ok=True)
network1_folder = get_output_folder_name(
m1, id_task_mapping[t], trainer_m1, pl)
network2_folder = get_output_folder_name(
m2, id_task_mapping[t], trainer_m2, pl)
print("ensembling", network1_folder, network2_folder)
ensemble(network1_folder,
network2_folder,
output_folder_base,
id_task_mapping[t],
validation_folder,
folds,
allow_ensembling=not disable_postprocessing)
# ensembling will automatically do postprocessingget_foreground_mean
# now get result of ensemble
results[ensemble_name] = get_mean_foreground_dice(
join(output_folder_base, "ensembled_raw",
"summary.json"))
summary_file = join(output_folder_base, "ensembled_raw",
"summary.json")
foreground_mean(summary_file)
all_results[ensemble_name] = load_json(
summary_file)['results']['mean']
# now print all mean foreground dice and highlight the best
foreground_dices = list(results.values())
best = np.max(foreground_dices)
for k, v in results.items():
print(k, v)
predict_str = ""
best_model = None
for k, v in results.items():
if v == best:
print("%s submit model %s" % (id_task_mapping[t], k), v)
best_model = k
print(
"\nHere is how you should predict test cases. Run in sequential order and replace all input and output folder names with your personalized ones\n"
)
if k.startswith("ensemble"):
tmp = k[len("ensemble_"):]
model1, model2 = tmp.split("--")
m1, t1, pl1 = model1.split("__")
m2, t2, pl2 = model2.split("__")
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + m1 + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL2 -tr " + tr + " -ctr " + trc + " -m " + m2 + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
if not disable_postprocessing:
predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER -pp " + join(
network_training_output_dir, "ensembles",
id_task_mapping[t], k,
"postprocessing.json") + "\n"
else:
predict_str += "nnUNet_ensemble -f OUTPUT_FOLDER_MODEL1 OUTPUT_FOLDER_MODEL2 -o OUTPUT_FOLDER\n"
else:
predict_str += "nnUNet_predict -i FOLDER_WITH_TEST_CASES -o OUTPUT_FOLDER_MODEL1 -tr " + tr + " -ctr " + trc + " -m " + k + " -p " + pl + " -t " + \
id_task_mapping[t] + "\n"
print(predict_str)
summary_folder = join(network_training_output_dir, "ensembles",
id_task_mapping[t])
os.makedirs(summary_folder, exist_ok=True)
with open(join(summary_folder, "prediction_commands.txt"), 'w') as f:
f.write(predict_str)
num_classes = len([
i for i in all_results[best_model].keys()
if i != 'mean' and i != '0'
])
with open(join(summary_folder, "summary.csv"), 'w') as f:
f.write("model")
for c in range(1, num_classes + 1):
f.write(",class%d" % c)
f.write(",average")
f.write("\n")
for m in all_results.keys():
f.write(m)
for c in range(1, num_classes + 1):
f.write(",%01.4f" % all_results[m][str(c)]["Dice"])
f.write(",%01.4f" % all_results[m]['mean']["Dice"])
f.write("\n")