def merge(folders,
output_folder,
threads,
override=True,
postprocessing_file=None,
store_npz=False):
maybe_mkdir_p(output_folder)
if postprocessing_file is not None:
output_folder_orig = deepcopy(output_folder)
output_folder = join(output_folder, 'not_postprocessed')
maybe_mkdir_p(output_folder)
else:
output_folder_orig = None
patient_ids = [subfiles(i, suffix=".npz", join=False) for i in folders]
patient_ids = [i for j in patient_ids for i in j]
patient_ids = [i[:-4] for i in patient_ids]
patient_ids = np.unique(patient_ids)
for f in folders:
assert all([isfile(join(f, i + ".npz")) for i in patient_ids]), "Not all patient npz are available in " \
"all folders"
assert all([isfile(join(f, i + ".pkl")) for i in patient_ids]), "Not all patient pkl are available in " \
"all folders"
files = []
property_files = []
out_files = []
for p in patient_ids:
files.append([join(f, p + ".npz") for f in folders])
property_files.append(join(folders[0], p + ".pkl"))
out_files.append(join(output_folder, p + ".nii.gz"))
plans = load_pickle(join(folders[0], "plans.pkl"))
only_keep_largest_connected_component, min_region_size_per_class = plans['keep_only_largest_region'], \
plans['min_region_size_per_class']
p = Pool(threads)
p.map(
merge_files,
zip(files, property_files, out_files,
[only_keep_largest_connected_component] * len(out_files),
[min_region_size_per_class] * len(out_files),
[override] * len(out_files), [store_npz] * len(out_files)))
p.close()
p.join()
if postprocessing_file is not None:
for_which_classes, min_valid_obj_size = load_postprocessing(
postprocessing_file)
print('Postprocessing...')
apply_postprocessing_to_folder(output_folder, output_folder_orig,
for_which_classes, min_valid_obj_size,
threads)
shutil.copy(postprocessing_file, output_folder_orig)
def merge(folders,
output_folder,
threads,
override=True,
postprocessing_file=None,
store_npz=False):
os.makedirs(output_folder, exist_ok=True)
if postprocessing_file is not None:
output_folder_orig = deepcopy(output_folder)
output_folder = join(output_folder, 'not_postprocessed')
os.makedirs(output_folder, exist_ok=True)
else:
output_folder_orig = None
patient_ids = [subfiles(i, suffix=".npz", join=False) for i in folders]
patient_ids = [i for j in patient_ids for i in j]
patient_ids = [i[:-4] for i in patient_ids]
patient_ids = np.unique(patient_ids)
for f in folders:
assert all([isfile(join(f, i + ".npz")) for i in patient_ids]), "Not all patient npz are available in " \
"all folders"
assert all([isfile(join(f, i + ".pkl")) for i in patient_ids]), "Not all patient pkl are available in " \
"all folders"
files = []
property_files = []
out_files = []
for p in patient_ids:
files.append([join(f, p + ".npz") for f in folders])
property_files.append([join(f, p + ".pkl") for f in folders])
out_files.append(join(output_folder, p + ".nii.gz"))
p = Pool(threads)
p.starmap(
merge_files,
zip(files, property_files, out_files, [override] * len(out_files),
[store_npz] * len(out_files)))
p.close()
p.join()
if postprocessing_file is not None:
for_which_classes, min_valid_obj_size = load_postprocessing(
postprocessing_file)
print('Postprocessing...')
apply_postprocessing_to_folder(output_folder, output_folder_orig,
for_which_classes, min_valid_obj_size,
threads)
shutil.copy(postprocessing_file, output_folder_orig)
def convert_variant2_predicted_test_to_submission_format(
folder_with_predictions,
output_folder="/home/fabian/drives/datasets/results/nnUNet/test_sets/Task038_CHAOS_Task_3_5_Variant2/ready_to_submit",
postprocessing_file="/home/fabian/drives/datasets/results/nnUNet/ensembles/Task038_CHAOS_Task_3_5_Variant2/ensemble_2d__nnUNetTrainerV2__nnUNetPlansv2.1--3d_fullres__nnUNetTrainerV2__nnUNetPlansv2.1/postprocessing.json"
):
"""
output_folder is where the extracted template is
:param folder_with_predictions:
:param output_folder:
:return:
"""
postprocessing_file = "/media/fabian/Results/nnUNet/3d_fullres/Task039_CHAOS_Task_3_5_Variant2_highres/" \
"nnUNetTrainerV2__nnUNetPlansfixed/postprocessing.json"
# variant 2 treats in and out phase as two training examples, so we need to ensemble these two again
final_predictions_folder = join(output_folder, "final")
maybe_mkdir_p(final_predictions_folder)
t1_patient_names = [
i.split("_")[-1][:-7] for i in subfiles(
folder_with_predictions, prefix="T1", suffix=".nii.gz", join=False)
]
folder_for_ensembing0 = join(output_folder, "ens0")
folder_for_ensembing1 = join(output_folder, "ens1")
maybe_mkdir_p(folder_for_ensembing0)
maybe_mkdir_p(folder_for_ensembing1)
# now copy all t1 out phases in ens0 and all in phases in ens1. Name them the same.
for t1 in t1_patient_names:
shutil.copy(join(folder_with_predictions, "T1_in_%s.npz" % t1),
join(folder_for_ensembing1, "T1_%s.npz" % t1))
shutil.copy(join(folder_with_predictions, "T1_in_%s.pkl" % t1),
join(folder_for_ensembing1, "T1_%s.pkl" % t1))
shutil.copy(join(folder_with_predictions, "T1_out_%s.npz" % t1),
join(folder_for_ensembing0, "T1_%s.npz" % t1))
shutil.copy(join(folder_with_predictions, "T1_out_%s.pkl" % t1),
join(folder_for_ensembing0, "T1_%s.pkl" % t1))
shutil.copy(join(folder_with_predictions, "plans.pkl"),
join(folder_for_ensembing0, "plans.pkl"))
shutil.copy(join(folder_with_predictions, "plans.pkl"),
join(folder_for_ensembing1, "plans.pkl"))
# there is a problem with T1_35 that I need to correct manually (different crop size, will not negatively impact results)
#ens0_softmax = np.load(join(folder_for_ensembing0, "T1_35.npz"))['softmax']
ens1_softmax = np.load(join(folder_for_ensembing1, "T1_35.npz"))['softmax']
#ens0_props = load_pickle(join(folder_for_ensembing0, "T1_35.pkl"))
#ens1_props = load_pickle(join(folder_for_ensembing1, "T1_35.pkl"))
ens1_softmax = ens1_softmax[:, :, :-1, :]
np.savez_compressed(join(folder_for_ensembing1, "T1_35.npz"),
softmax=ens1_softmax)
shutil.copy(join(folder_for_ensembing0, "T1_35.pkl"),
join(folder_for_ensembing1, "T1_35.pkl"))
# now call my ensemble function
merge((folder_for_ensembing0, folder_for_ensembing1),
final_predictions_folder,
8,
True,
postprocessing_file=postprocessing_file)
# copy t2 files to final_predictions_folder as well
t2_files = subfiles(folder_with_predictions,
prefix="T2",
suffix=".nii.gz",
join=False)
for t2 in t2_files:
shutil.copy(join(folder_with_predictions, t2),
join(final_predictions_folder, t2))
# apply postprocessing
from nnunet.postprocessing.connected_components import apply_postprocessing_to_folder, load_postprocessing
postprocessed_folder = join(output_folder, "final_postprocessed")
for_which_classes, min_valid_obj_size = load_postprocessing(
postprocessing_file)
apply_postprocessing_to_folder(final_predictions_folder,
postprocessed_folder, for_which_classes,
min_valid_obj_size, 8)
# now export the niftis in the weird png format
# task 3
output_dir = join(output_folder, "CHAOS_submission_template_new", "Task3",
"MR")
for t1 in t1_patient_names:
output_folder_here = join(output_dir, t1, "T1DUAL", "Results")
nifti_file = join(postprocessed_folder, "T1_%s.nii.gz" % t1)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task3)
for t2 in t2_files:
patname = t2.split("_")[-1][:-7]
output_folder_here = join(output_dir, patname, "T2SPIR", "Results")
nifti_file = join(postprocessed_folder, "T2_%s.nii.gz" % patname)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task3)
# task 5
output_dir = join(output_folder, "CHAOS_submission_template_new", "Task5",
"MR")
for t1 in t1_patient_names:
output_folder_here = join(output_dir, t1, "T1DUAL", "Results")
nifti_file = join(postprocessed_folder, "T1_%s.nii.gz" % t1)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task5)
for t2 in t2_files:
patname = t2.split("_")[-1][:-7]
output_folder_here = join(output_dir, patname, "T2SPIR", "Results")
nifti_file = join(postprocessed_folder, "T2_%s.nii.gz" % patname)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task5)