def prepare_submission(fld= "/home/fabian/drives/datasets/results/nnUNet/test_sets/Task048_KiTS_clean/predicted_ens_3d_fullres_3d_cascade_fullres_postprocessed", # '/home/fabian/datasets_fabian/predicted_KiTS_nnUNetTrainerNewCandidate23_FabiansResNet',
out='/home/fabian/drives/datasets/results/nnUNet/test_sets/Task048_KiTS_clean/submission'):
nii = subfiles(fld, join=False, suffix='.nii.gz')
maybe_mkdir_p(out)
for n in nii:
outfname = n.replace('case', 'prediction')
shutil_sol.copyfile(join(fld, n), join(out, outfname))
def split_4d_nifti(filename, output_folder):
img_itk = sitk.ReadImage(filename)
dim = img_itk.GetDimension()
file_base = filename.split("/")[-1]
if dim == 3:
shutil_sol.copyfile(
filename, join(output_folder, file_base[:-7] + "_0000.nii.gz"))
return
elif dim != 4:
raise RuntimeError(
"Unexpected dimensionality: %d of file %s, cannot split" %
(dim, filename))
else:
img_npy = sitk.GetArrayFromImage(img_itk)
spacing = img_itk.GetSpacing()
origin = img_itk.GetOrigin()
direction = np.array(img_itk.GetDirection()).reshape(4, 4)
# now modify these to remove the fourth dimension
spacing = tuple(list(spacing[:-1]))
origin = tuple(list(origin[:-1]))
direction = tuple(direction[:-1, :-1].reshape(-1))
for i, t in enumerate(range(img_npy.shape[0])):
img = img_npy[t]
img_itk_new = sitk.GetImageFromArray(img)
img_itk_new.SetSpacing(spacing)
img_itk_new.SetOrigin(origin)
img_itk_new.SetDirection(direction)
sitk.WriteImage(
img_itk_new,
join(output_folder, file_base[:-7] + "_%04.0d.nii.gz" % i))
def crawl_and_copy(current_folder,
out_folder,
prefix="fabian_",
suffix="ummary.json"):
"""
This script will run recursively through all subfolders of current_folder and copy all files that end with
suffix with some automatically generated prefix into out_folder
:param current_folder:
:param out_folder:
:param prefix:
:return:
"""
s = subdirs(current_folder, join=False)
f = subfiles(current_folder, join=False)
f = [i for i in f if i.endswith(suffix)]
if current_folder.find("fold0") != -1:
for fl in f:
shutil_sol.copyfile(os.path.join(current_folder, fl),
os.path.join(out_folder, prefix + fl))
for su in s:
if prefix == "":
add = su
else:
add = "__" + su
crawl_and_copy(os.path.join(current_folder, su),
out_folder,
prefix=prefix + add)
def prepare_submission(folder_in, folder_out):
nii = subfiles(folder_in, suffix='.gz', join=False)
maybe_mkdir_p(folder_out)
for n in nii:
i = n.split('-')[-1][:-10]
shutil_sol.copyfile(join(folder_in, n), join(folder_out,
i + '.nii.gz'))
def split_4d(input_folder,
num_processes=default_num_threads,
overwrite_task_output_id=None):
assert isdir(join(input_folder, "imagesTr")) and isdir(join(input_folder, "labelsTr")) and \
isfile(join(input_folder, "dataset.json")), \
"The input folder must be a valid Task folder from the Medical Segmentation Decathlon with at least the " \
"imagesTr and labelsTr subfolders and the dataset.json file"
while input_folder.endswith("/"):
input_folder = input_folder[:-1]
full_task_name = input_folder.split("/")[-1]
assert full_task_name.startswith(
"Task"
), "The input folder must point to a folder that starts with TaskXX_"
first_underscore = full_task_name.find("_")
assert first_underscore == 6, "Input folder start with TaskXX with XX being a 3-digit id: 00, 01, 02 etc"
input_task_id = int(full_task_name[4:6])
if overwrite_task_output_id is None:
overwrite_task_output_id = input_task_id
task_name = full_task_name[7:]
output_folder = join(nnUNet_raw_data,
"Task%03.0d_" % overwrite_task_output_id + task_name)
if isdir(output_folder):
shutil.rmtree(output_folder)
files = []
output_dirs = []
maybe_mkdir_p(output_folder)
for subdir in ["imagesTr", "imagesTs"]:
curr_out_dir = join(output_folder, subdir)
if not isdir(curr_out_dir):
os.mkdir(curr_out_dir)
curr_dir = join(input_folder, subdir)
nii_files = [
join(curr_dir, i) for i in os.listdir(curr_dir)
if i.endswith(".nii.gz")
]
nii_files.sort()
for n in nii_files:
files.append(n)
output_dirs.append(curr_out_dir)
shutil_sol.copytree(join(input_folder, "labelsTr"),
join(output_folder, "labelsTr"))
p = Pool(num_processes)
p.starmap(split_4d_nifti, zip(files, output_dirs))
p.close()
p.join()
shutil_sol.copyfile(join(input_folder, "dataset.json"), output_folder)
def plan_and_preprocess(task_string,
processes_lowres=default_num_threads,
processes_fullres=3,
no_preprocessing=False):
from nnunet.experiment_planning.experiment_planner_baseline_2DUNet import ExperimentPlanner2D
from nnunet.experiment_planning.experiment_planner_baseline_3DUNet import ExperimentPlanner
preprocessing_output_dir_this_task_train = join(preprocessing_output_dir,
task_string)
cropped_out_dir = join(nnUNet_cropped_data, task_string)
maybe_mkdir_p(preprocessing_output_dir_this_task_train)
shutil_sol.copyfile(join(cropped_out_dir, "dataset_properties.pkl"),
preprocessing_output_dir_this_task_train)
shutil_sol.copyfile(join(nnUNet_raw_data, task_string, "dataset.json"),
preprocessing_output_dir_this_task_train)
exp_planner = ExperimentPlanner(cropped_out_dir,
preprocessing_output_dir_this_task_train)
exp_planner.plan_experiment()
if not no_preprocessing:
exp_planner.run_preprocessing((processes_lowres, processes_fullres))
exp_planner = ExperimentPlanner2D(
cropped_out_dir, preprocessing_output_dir_this_task_train)
exp_planner.plan_experiment()
if not no_preprocessing:
exp_planner.run_preprocessing(processes_fullres)
# write which class is in which slice to all training cases (required to speed up 2D Dataloader)
# This is done for all data so that if we wanted to use them with 2D we could do so
if not no_preprocessing:
p = Pool(default_num_threads)
# if there is more than one my_data_identifier (different brnaches) then this code will run for all of them if
# they start with the same string. not problematic, but not pretty
stages = [
i for i in subdirs(
preprocessing_output_dir_this_task_train, join=True, sort=True)
if i.split("/")[-1].find("stage") != -1
]
for s in stages:
print(s.split("/")[-1])
list_of_npz_files = subfiles(s, True, None, ".npz", True)
list_of_pkl_files = [i[:-4] + ".pkl" for i in list_of_npz_files]
all_classes = []
for pk in list_of_pkl_files:
with open(pk, 'rb') as f:
props = pickle.load(f)
all_classes_tmp = np.array(props['classes'])
all_classes.append(all_classes_tmp[all_classes_tmp >= 0])
p.map(add_classes_in_slice_info,
zip(list_of_npz_files, list_of_pkl_files, all_classes))
p.close()
p.join()
def copy_fold(in_folder: str, out_folder: str):
shutil_sol.copyfile(join(in_folder, "debug.json"),
join(out_folder, "debug.json"))
shutil_sol.copyfile(join(in_folder, "model_final_checkpoint.model"),
join(out_folder, "model_final_checkpoint.model"))
shutil_sol.copyfile(join(in_folder, "model_final_checkpoint.model.pkl"),
join(out_folder, "model_final_checkpoint.model.pkl"))
shutil_sol.copyfile(join(in_folder, "progress.png"),
join(out_folder, "progress.png"))
if isfile(join(in_folder, "network_architecture.pdf")):
shutil_sol.copyfile(join(in_folder, "network_architecture.pdf"),
join(out_folder, "network_architecture.pdf"))
def select_annotated_frames_mms(
data_folder,
out_folder,
add_zeros=False,
mode='mnms',
df_path="/media/full/tera2/data/challenges/mms/Training-corrected_original/M&Ms Dataset Information.xlsx"
):
table = pd.read_excel(df_path, index_col='External code')
for idx in table.index:
ed = table.loc[idx, 'ED']
es = table.loc[idx, 'ES']
vendor = table.loc[idx, 'Vendor']
centre = table.loc[idx, 'Centre']
if vendor != "C":
# generate old filename (w/o vendor and centre)
filename_ed_original = generate_filename_for_nnunet(
pat_id=idx,
ts=ed,
pat_folder=data_folder,
vendor=None,
centre=None,
add_zeros=False)
filename_es_original = generate_filename_for_nnunet(
pat_id=idx,
ts=es,
pat_folder=data_folder,
vendor=None,
centre=None,
add_zeros=False)
# generate new filename with vendor and centre
filename_ed = generate_filename_for_nnunet(pat_id=idx,
ts=ed,
pat_folder=out_folder,
vendor=vendor,
centre=centre,
add_zeros=add_zeros,
mode=mode)
filename_es = generate_filename_for_nnunet(pat_id=idx,
ts=es,
pat_folder=out_folder,
vendor=vendor,
centre=centre,
add_zeros=add_zeros,
mode=mode)
shutil_sol.copyfile(filename_ed_original, filename_ed)
shutil_sol.copyfile(filename_es_original, filename_es)
def merge(folders,
output_folder,
threads,
override=True,
postprocessing_file=None,
store_npz=False):
maybe_mkdir_p(output_folder)
if postprocessing_file is not None:
output_folder_orig = deepcopy(output_folder)
output_folder = join(output_folder, 'not_postprocessed')
maybe_mkdir_p(output_folder)
else:
output_folder_orig = None
patient_ids = [subfiles(i, suffix=".npz", join=False) for i in folders]
patient_ids = [i for j in patient_ids for i in j]
patient_ids = [i[:-4] for i in patient_ids]
patient_ids = np.unique(patient_ids)
for f in folders:
assert all([isfile(join(f, i + ".npz")) for i in patient_ids]), "Not all patient npz are available in " \
"all folders"
assert all([isfile(join(f, i + ".pkl")) for i in patient_ids]), "Not all patient pkl are available in " \
"all folders"
files = []
property_files = []
out_files = []
for p in patient_ids:
files.append([join(f, p + ".npz") for f in folders])
property_files.append([join(f, p + ".pkl") for f in folders])
out_files.append(join(output_folder, p + ".nii.gz"))
p = Pool(threads)
p.starmap(
merge_files,
zip(files, property_files, out_files, [override] * len(out_files),
[store_npz] * len(out_files)))
p.close()
p.join()
if postprocessing_file is not None:
for_which_classes, min_valid_obj_size = load_postprocessing(
postprocessing_file)
print('Postprocessing...')
apply_postprocessing_to_folder(output_folder, output_folder_orig,
for_which_classes, min_valid_obj_size,
threads)
shutil_sol.copyfile(postprocessing_file, output_folder_orig)
def save_debug_information(self):
# saving some debug information
dct = OrderedDict()
for k in self.__dir__():
if not k.startswith("__"):
if not callable(getattr(self, k)):
dct[k] = str(getattr(self, k))
del dct['plans']
del dct['intensity_properties']
del dct['dataset']
del dct['dataset_tr']
del dct['dataset_val']
save_json(dct, join(self.output_folder, "debug.json"))
shutil_sol.copyfile(self.plans_file, join(self.output_folder_base, "plans.pkl"))
def crop(task_string, override=False, num_threads=default_num_threads):
cropped_out_dir = join(nnUNet_cropped_data, task_string)
maybe_mkdir_p(cropped_out_dir)
if override and isdir(cropped_out_dir):
shutil.rmtree(cropped_out_dir)
maybe_mkdir_p(cropped_out_dir)
splitted_4d_output_dir_task = join(nnUNet_raw_data, task_string)
lists, _ = create_lists_from_splitted_dataset(splitted_4d_output_dir_task)
imgcrop = ImageCropper(num_threads, cropped_out_dir)
imgcrop.run_cropping(lists, overwrite_existing=override)
shutil_sol.copyfile(join(nnUNet_raw_data, task_string, "dataset.json"),
cropped_out_dir)
def convert_to_submission(source_dir, target_dir):
niftis = subfiles(source_dir, join=False, suffix=".nii.gz")
patientids = np.unique([i[:10] for i in niftis])
maybe_mkdir_p(target_dir)
for p in patientids:
files_of_that_patient = subfiles(source_dir,
prefix=p,
suffix=".nii.gz",
join=False)
assert len(files_of_that_patient)
files_of_that_patient.sort()
# first is ED, second is ES
shutil_sol.copyfile(join(source_dir, files_of_that_patient[0]),
join(target_dir, p + "_ED.nii.gz"))
shutil_sol.copyfile(join(source_dir, files_of_that_patient[1]),
join(target_dir, p + "_ES.nii.gz"))
def collect_cv_niftis(cv_folder: str,
output_folder: str,
validation_folder_name: str = 'validation_raw',
folds: tuple = (0, 1, 2, 3, 4)):
validation_raw_folders = [
join(cv_folder, "fold_%d" % i, validation_folder_name) for i in folds
]
exist = [isdir(i) for i in validation_raw_folders]
if not all(exist):
raise RuntimeError(
"some folds are missing. Please run the full 5-fold cross-validation. "
"The following folds seem to be missing: %s" %
[i for j, i in enumerate(folds) if not exist[j]])
# now copy all raw niftis into cv_niftis_raw
maybe_mkdir_p(output_folder)
for f in folds:
niftis = subfiles(validation_raw_folders[f], suffix=".nii.gz")
for n in niftis:
shutil_sol.copyfile(n, join(output_folder))
def remove_all_but_the_two_largest_conn_comp(img_itk_file: str, file_out: str):
"""
This was not used. I was just curious because others used this. Turns out this is not necessary for my networks
"""
img_itk = sitk.ReadImage(img_itk_file)
img_npy = sitk.GetArrayFromImage(img_itk)
labelmap, num_labels = label((img_npy > 0).astype(int))
if num_labels > 2:
label_sizes = []
for i in range(1, num_labels + 1):
label_sizes.append(np.sum(labelmap == i))
argsrt = np.argsort(label_sizes)[::-1] # two largest are now argsrt[0] and argsrt[1]
keep_mask = (labelmap == argsrt[0] + 1) | (labelmap == argsrt[1] + 1)
img_npy[~keep_mask] = 0
new = sitk.GetImageFromArray(img_npy)
new.CopyInformation(img_itk)
sitk.WriteImage(new, file_out)
print(os.path.basename(img_itk_file), num_labels, label_sizes)
else:
shutil_sol.copyfile(img_itk_file, file_out)
def copy_ensembles(taskname,
output_folder,
valid_models=('2d', '3d_fullres', '3d_lowres',
'3d_cascade_fullres'),
valid_trainers=(default_trainer, default_cascade_trainer),
valid_plans=(default_plans_identifier, )):
ensemble_dir = join(network_training_output_dir, 'ensembles', taskname)
if not isdir(ensemble_dir):
print("No ensemble directory found for task", taskname)
return
subd = subdirs(ensemble_dir, join=False)
valid = []
for s in subd:
v = check_if_valid(s, valid_models, valid_trainers, valid_plans)
if v:
valid.append(s)
output_ensemble = join(output_folder, 'ensembles', taskname)
maybe_mkdir_p(output_ensemble)
for v in valid:
this_output = join(output_ensemble, v)
maybe_mkdir_p(this_output)
shutil_sol.copyfile(join(ensemble_dir, v, 'postprocessing.json'),
this_output)
def copy_model(directory: str, output_directory: str):
"""
:param directory: must have the 5 fold_X subfolders as well as a postprocessing.json and plans.pkl
:param output_directory:
:return:
"""
expected_folders = ["fold_%d" % i for i in range(5)]
assert all([isdir(join(directory, i))
for i in expected_folders]), "not all folds present"
assert isfile(join(directory, "plans.pkl")), "plans.pkl missing"
assert isfile(join(directory,
"postprocessing.json")), "postprocessing.json missing"
for e in expected_folders:
maybe_mkdir_p(join(output_directory, e))
copy_fold(join(directory, e), join(output_directory, e))
shutil_sol.copyfile(join(directory, "plans.pkl"),
join(output_directory, "plans.pkl"))
shutil_sol.copyfile(join(directory, "postprocessing.json"),
join(output_directory, "postprocessing.json"))
def copy_npz_fom_valsets():
'''
this is preparation for ensembling
:return:
'''
base = join(network_training_output_dir, "3d_lowres/Task048_KiTS_clean")
folders = ['nnUNetTrainerNewCandidate23_FabiansPreActResNet__nnUNetPlans',
'nnUNetTrainerNewCandidate23_FabiansResNet__nnUNetPlans',
'nnUNetTrainerNewCandidate23__nnUNetPlans']
for f in folders:
out = join(base, f, 'crossval_npz')
maybe_mkdir_p(out)
shutil_sol.copyfile(join(base, f, 'plans.pkl'), out)
for fold in range(5):
cur = join(base, f, 'fold_%d' % fold, 'validation_raw')
npz_files = subfiles(cur, suffix='.npz', join=False)
pkl_files = [i[:-3] + 'pkl' for i in npz_files]
assert all([isfile(join(cur, i)) for i in pkl_files])
for n in npz_files:
corresponding_pkl = n[:-3] + 'pkl'
shutil_sol.copyfile(join(cur, n), out)
shutil_sol.copyfile(join(cur, corresponding_pkl), out)
def validate(self, do_mirroring: bool = True, use_sliding_window: bool = True,
step_size: float = 0.5, save_softmax: bool = True, use_gaussian: bool = True, overwrite: bool = True,
validation_folder_name: str = 'validation_raw', debug: bool = False, all_in_gpu: bool = False,
segmentation_export_kwargs: dict = None, run_postprocessing_on_folds: bool = True):
if isinstance(self.network, DDP):
net = self.network.module
else:
net = self.network
ds = net.do_ds
net.do_ds = False
current_mode = self.network.training
self.network.eval()
assert self.was_initialized, "must initialize, ideally with checkpoint (or train first)"
if self.dataset_val is None:
self.load_dataset()
self.do_split()
if segmentation_export_kwargs is None:
if 'segmentation_export_params' in self.plans.keys():
force_separate_z = self.plans['segmentation_export_params']['force_separate_z']
interpolation_order = self.plans['segmentation_export_params']['interpolation_order']
interpolation_order_z = self.plans['segmentation_export_params']['interpolation_order_z']
else:
force_separate_z = None
interpolation_order = 1
interpolation_order_z = 0
else:
force_separate_z = segmentation_export_kwargs['force_separate_z']
interpolation_order = segmentation_export_kwargs['interpolation_order']
interpolation_order_z = segmentation_export_kwargs['interpolation_order_z']
# predictions as they come from the network go here
output_folder = join(self.output_folder, validation_folder_name)
maybe_mkdir_p(output_folder)
# this is for debug purposes
my_input_args = {'do_mirroring': do_mirroring,
'use_sliding_window': use_sliding_window,
'step_size': step_size,
'save_softmax': save_softmax,
'use_gaussian': use_gaussian,
'overwrite': overwrite,
'validation_folder_name': validation_folder_name,
'debug': debug,
'all_in_gpu': all_in_gpu,
'segmentation_export_kwargs': segmentation_export_kwargs,
}
save_json(my_input_args, join(output_folder, "validation_args.json"))
if do_mirroring:
if not self.data_aug_params['do_mirror']:
raise RuntimeError(
"We did not train with mirroring so you cannot do inference with mirroring enabled")
mirror_axes = self.data_aug_params['mirror_axes']
else:
mirror_axes = ()
pred_gt_tuples = []
export_pool = Pool(default_num_threads)
results = []
all_keys = list(self.dataset_val.keys())
my_keys = all_keys[self.local_rank::dist.get_world_size()]
# we cannot simply iterate over all_keys because we need to know pred_gt_tuples and valid_labels of all cases
# for evaluation (which is done by local rank 0)
for k in my_keys:
properties = load_pickle(self.dataset[k]['properties_file'])
fname = properties['list_of_data_files'][0].split("/")[-1][:-12]
pred_gt_tuples.append([join(output_folder, fname + ".nii.gz"),
join(self.gt_niftis_folder, fname + ".nii.gz")])
if k in my_keys:
if overwrite or (not isfile(join(output_folder, fname + ".nii.gz"))) or \
(save_softmax and not isfile(join(output_folder, fname + ".npz"))):
data = np.load(self.dataset[k]['data_file'])['data']
print(k, data.shape)
data[-1][data[-1] == -1] = 0
softmax_pred = self.predict_preprocessed_data_return_seg_and_softmax(data[:-1],
do_mirroring=do_mirroring,
mirror_axes=mirror_axes,
use_sliding_window=use_sliding_window,
step_size=step_size,
use_gaussian=use_gaussian,
all_in_gpu=all_in_gpu,
mixed_precision=self.fp16)[1]
softmax_pred = softmax_pred.transpose([0] + [i + 1 for i in self.transpose_backward])
if save_softmax:
softmax_fname = join(output_folder, fname + ".npz")
else:
softmax_fname = None
"""There is a problem with python process communication that prevents us from communicating obejcts
larger than 2 GB between processes (basically when the length of the pickle string that will be sent is
communicated by the multiprocessing.Pipe object then the placeholder (\%i I think) does not allow for long
enough strings (lol). This could be fixed by changing i to l (for long) but that would require manually
patching system python code. We circumvent that problem here by saving softmax_pred to a npy file that will
then be read (and finally deleted) by the Process. save_segmentation_nifti_from_softmax can take either
filename or np.ndarray and will handle this automatically"""
if np.prod(softmax_pred.shape) > (2e9 / 4 * 0.85): # *0.85 just to be save
np.save(join(output_folder, fname + ".npy"), softmax_pred)
softmax_pred = join(output_folder, fname + ".npy")
results.append(export_pool.starmap_async(save_segmentation_nifti_from_softmax,
((softmax_pred, join(output_folder, fname + ".nii.gz"),
properties, interpolation_order,
self.regions_class_order,
None, None,
softmax_fname, None, force_separate_z,
interpolation_order_z),
)
)
)
_ = [i.get() for i in results]
self.print_to_log_file("finished prediction")
distributed.barrier()
if self.local_rank == 0:
# evaluate raw predictions
self.print_to_log_file("evaluation of raw predictions")
task = self.dataset_directory.split("/")[-1]
job_name = self.experiment_name
_ = aggregate_scores(pred_gt_tuples, labels=list(range(self.num_classes)),
json_output_file=join(output_folder, "summary.json"),
json_name=job_name + " val tiled %s" % (str(use_sliding_window)),
json_author="Fabian",
json_task=task, num_threads=default_num_threads)
if run_postprocessing_on_folds:
# in the old nnunet we would stop here. Now we add a postprocessing. This postprocessing can remove everything
# except the largest connected component for each class. To see if this improves results, we do this for all
# classes and then rerun the evaluation. Those classes for which this resulted in an improved dice score will
# have this applied during inference as well
self.print_to_log_file("determining postprocessing")
determine_postprocessing(self.output_folder, self.gt_niftis_folder, validation_folder_name,
final_subf_name=validation_folder_name + "_postprocessed", debug=debug)
# after this the final predictions for the vlaidation set can be found in validation_folder_name_base + "_postprocessed"
# They are always in that folder, even if no postprocessing as applied!
# detemining postprocesing on a per-fold basis may be OK for this fold but what if another fold finds another
# postprocesing to be better? In this case we need to consolidate. At the time the consolidation is going to be
# done we won't know what self.gt_niftis_folder was, so now we copy all the niftis into a separate folder to
# be used later
gt_nifti_folder = join(self.output_folder_base, "gt_niftis")
maybe_mkdir_p(gt_nifti_folder)
for f in subfiles(self.gt_niftis_folder, suffix=".nii.gz"):
success = False
attempts = 0
e = None
while not success and attempts < 10:
try:
shutil_sol.copyfile(f, gt_nifti_folder)
success = True
except OSError as e:
attempts += 1
sleep(1)
if not success:
print("Could not copy gt nifti file %s into folder %s" % (f, gt_nifti_folder))
if e is not None:
raise e
self.network.train(current_mode)
net.do_ds = ds
all_train_files = []
patient_dirs_train = subfolders(folder, prefix="patient")
for p in patient_dirs_train:
current_dir = p
data_files_train = [
i for i in subfiles(current_dir, suffix=".nii.gz")
if i.find("_gt") == -1 and i.find("_4d") == -1
]
corresponding_seg_files = [
i[:-7] + "_gt.nii.gz" for i in data_files_train
]
for d, s in zip(data_files_train, corresponding_seg_files):
patient_identifier = d.split("/")[-1][:-7]
all_train_files.append(patient_identifier + "_0000.nii.gz")
shutil_sol.copyfile(
d,
join(out_folder, "imagesTr",
patient_identifier + "_0000.nii.gz"))
shutil_sol.copyfile(
s, join(out_folder, "labelsTr",
patient_identifier + ".nii.gz"))
# test
all_test_files = []
patient_dirs_test = subfolders(folder_test, prefix="patient")
for p in patient_dirs_test:
current_dir = p
data_files_test = [
i for i in subfiles(current_dir, suffix=".nii.gz")
if i.find("_gt") == -1 and i.find("_4d") == -1
]
for d in data_files_test:
# the niftis are 3d, but they are just stacks of 2d slices from different patients. So no 3d U-Net, please
# the training stack has 100 slices, so we split it into 5 equally sized parts (20 slices each) for cross-validation
training_data = sitk.GetArrayFromImage(sitk.ReadImage(join(download_dir, 'tr_im.nii.gz')))
training_labels = sitk.GetArrayFromImage(sitk.ReadImage(join(download_dir, 'tr_mask.nii.gz')))
for f in range(5):
this_name = 'part_%d' % f
data = training_data[f::5]
labels = training_labels[f::5]
sitk.WriteImage(sitk.GetImageFromArray(data), join(imagestr, this_name + '_0000.nii.gz'))
sitk.WriteImage(sitk.GetImageFromArray(labels), join(labelstr, this_name + '.nii.gz'))
train_patient_names.append(this_name)
shutil_sol.copyfile(join(download_dir, 'val_im.nii.gz'), join(imagests, 'val_im.nii.gz'))
test_patient_names.append('val_im')
json_dict = {}
json_dict['name'] = task_name
json_dict['description'] = ""
json_dict['tensorImageSize'] = "4D"
json_dict['reference'] = ""
json_dict['licence'] = ""
json_dict['release'] = "0.0"
json_dict['modality'] = {
"0": "nonct",
}
json_dict['labels'] = {
"0": "background",
patient_names.append(patient_name)
t1 = join(patdir, p + "_t1.nii.gz")
t1c = join(patdir, p + "_t1ce.nii.gz")
t2 = join(patdir, p + "_t2.nii.gz")
flair = join(patdir, p + "_flair.nii.gz")
seg = join(patdir, p + "_seg.nii.gz")
assert all([
isfile(t1),
isfile(t1c),
isfile(t2),
isfile(flair),
isfile(seg)
]), "%s" % patient_name
shutil_sol.copyfile(
t1, join(target_imagesTr, patient_name + "_0000.nii.gz"))
shutil_sol.copyfile(
t1c, join(target_imagesTr, patient_name + "_0001.nii.gz"))
shutil_sol.copyfile(
t2, join(target_imagesTr, patient_name + "_0002.nii.gz"))
shutil_sol.copyfile(
flair, join(target_imagesTr, patient_name + "_0003.nii.gz"))
copy_BraTS_segmentation_and_convert_labels(
seg, join(target_labelsTr, patient_name + ".nii.gz"))
json_dict = OrderedDict()
json_dict['name'] = "BraTS2018"
json_dict['description'] = "nothing"
json_dict['tensorImageSize'] = "4D"
json_dict['reference'] = "see BraTS2018"
out_base = join(nnUNet_raw_data, foldername)
imagestr = join(out_base, "imagesTr")
imagests = join(out_base, "imagesTs")
labelstr = join(out_base, "labelsTr")
maybe_mkdir_p(imagestr)
maybe_mkdir_p(imagests)
maybe_mkdir_p(labelstr)
train_patient_names = []
test_patient_names = []
train_patients = subfolders(join(base, "train"), join=False)
for p in train_patients:
curr = join(base, "train", p)
label_file = join(curr, "GT.nii.gz")
image_file = join(curr, p + ".nii.gz")
shutil_sol.copyfile(image_file, join(imagestr, p + "_0000.nii.gz"))
shutil_sol.copyfile(label_file, join(labelstr, p + ".nii.gz"))
train_patient_names.append(p)
test_patients = subfiles(join(base, "test"), join=False, suffix=".nii.gz")
for p in test_patients:
p = p[:-7]
curr = join(base, "test")
image_file = join(curr, p + ".nii.gz")
shutil_sol.copyfile(image_file, join(imagests, p + "_0000.nii.gz"))
test_patient_names.append(p)
json_dict = OrderedDict()
json_dict['name'] = "SegTHOR"
json_dict['description'] = "SegTHOR"
json_dict['tensorImageSize'] = "4D"
# now we have to deal with the training masks, we do it the quick and dirty way here by just creating copies of the
# training data
train_folder = '/media/fabian/My Book/MedicalDecathlon/Task035_ISBILesionSegmentation/imagesTr'
for patientid in range(1, 6):
for t in range(1, 6):
fnames_original = subfiles(train_folder,
prefix="case__%02.0d__%02.0d" %
(patientid, t),
suffix=".nii.gz",
sort=True)
for f in fnames_original:
for mask in [1, 2]:
fname_target = f[:-12] + "__mask%d" % mask + f[-12:]
shutil_sol.copyfile(f, fname_target)
os.remove(f)
labels_folder = '/media/fabian/My Book/MedicalDecathlon/Task035_ISBILesionSegmentation/labelsTr'
for patientid in range(1, 6):
for t in range(1, 6):
for mask in [1, 2]:
f = join(
labels_folder,
"training%02d_%02d_mask%d.nii.gz" % (patientid, t, mask))
if isfile(f):
os.rename(
f,
join(
labels_folder,
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"-t",
"--task_ids",
nargs="+",
help="List of integers belonging to the task ids you wish to run"
" experiment planning and preprocessing for. Each of these "
"ids must, have a matching folder 'TaskXXX_' in the raw "
"data folder")
parser.add_argument(
"-pl3d",
"--planner3d",
type=str,
default="ExperimentPlanner3D_v21",
help=
"Name of the ExperimentPlanner class for the full resolution 3D U-Net and U-Net cascade. "
"Default is ExperimentPlanner3D_v21. Can be 'None', in which case these U-Nets will not be "
"configured")
parser.add_argument(
"-pl2d",
"--planner2d",
type=str,
default="ExperimentPlanner2D_v21",
help=
"Name of the ExperimentPlanner class for the 2D U-Net. Default is ExperimentPlanner2D_v21. "
"Can be 'None', in which case this U-Net will not be configured")
parser.add_argument(
"-no_pp",
action="store_true",
help=
"Set this flag if you dont want to run the preprocessing. If this is set then this script "
"will only run the experiment planning and create the plans file")
parser.add_argument(
"-tl",
type=int,
required=False,
default=8,
help=
"Number of processes used for preprocessing the low resolution data for the 3D low "
"resolution U-Net. This can be larger than -tf. Don't overdo it or you will run out of "
"RAM")
parser.add_argument(
"-tf",
type=int,
required=False,
default=8,
help=
"Number of processes used for preprocessing the full resolution data of the 2D U-Net and "
"3D U-Net. Don't overdo it or you will run out of RAM")
parser.add_argument(
"--verify_dataset_integrity",
required=False,
default=False,
action="store_true",
help=
"set this flag to check the dataset integrity. This is useful and should be done once for "
"each dataset!")
parser.add_argument(
"-overwrite_plans",
type=str,
default=None,
required=False,
help=
"Use this to specify a plans file that should be used instead of whatever nnU-Net would "
"configure automatically. This will overwrite everything: intensity normalization, "
"network architecture, target spacing etc. Using this is useful for using pretrained "
"model weights as this will guarantee that the network architecture on the target "
"dataset is the same as on the source dataset and the weights can therefore be transferred.\n"
"Pro tip: If you want to pretrain on Hepaticvessel and apply the result to LiTS then use "
"the LiTS plans to run the preprocessing of the HepaticVessel task.\n"
"Make sure to only use plans files that were "
"generated with the same number of modalities as the target dataset (LiTS -> BCV or "
"LiTS -> Task008_HepaticVessel is OK. BraTS -> LiTS is not (BraTS has 4 input modalities, "
"LiTS has just one)). Also only do things that make sense. This functionality is beta with"
"no support given.\n"
"Note that this will first print the old plans (which are going to be overwritten) and "
"then the new ones (provided that -no_pp was NOT set).")
parser.add_argument(
"-overwrite_plans_identifier",
type=str,
default=None,
required=False,
help=
"If you set overwrite_plans you need to provide a unique identifier so that nnUNet knows "
"where to look for the correct plans and data. Assume your identifier is called "
"IDENTIFIER, the correct training command would be:\n"
"'nnUNet_train CONFIG TRAINER TASKID FOLD -p nnUNetPlans_pretrained_IDENTIFIER "
"-pretrained_weights FILENAME'")
args = parser.parse_args()
task_ids = args.task_ids
dont_run_preprocessing = args.no_pp
tl = args.tl
tf = args.tf
planner_name3d = args.planner3d
planner_name2d = args.planner2d
if planner_name3d == "None":
planner_name3d = None
if planner_name2d == "None":
planner_name2d = None
if args.overwrite_plans is not None:
if planner_name2d is not None:
print(
"Overwriting plans only works for the 3d planner. I am setting '--planner2d' to None. This will "
"skip 2d planning and preprocessing.")
assert planner_name3d == 'ExperimentPlanner3D_v21_Pretrained', "When using --overwrite_plans you need to use " \
"'-pl3d ExperimentPlanner3D_v21_Pretrained'"
# we need raw data
tasks = []
for i in task_ids:
i = int(i)
task_name = convert_id_to_task_name(i)
if args.verify_dataset_integrity:
verify_dataset_integrity(join(nnUNet_raw_data, task_name))
crop(task_name, False, tf)
tasks.append(task_name)
search_in = join(nnunet.__path__[0], "experiment_planning")
if planner_name3d is not None:
planner_3d = recursive_find_python_class(
[search_in],
planner_name3d,
current_module="nnunet.experiment_planning")
if planner_3d is None:
raise RuntimeError(
"Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name3d)
else:
planner_3d = None
if planner_name2d is not None:
planner_2d = recursive_find_python_class(
[search_in],
planner_name2d,
current_module="nnunet.experiment_planning")
if planner_2d is None:
raise RuntimeError(
"Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name2d)
else:
planner_2d = None
for t in tasks:
print("\n\n\n", t)
cropped_out_dir = os.path.join(nnUNet_cropped_data, t)
preprocessing_output_dir_this_task = os.path.join(
preprocessing_output_dir, t)
#splitted_4d_output_dir_task = os.path.join(nnUNet_raw_data, t)
#lists, modalities = create_lists_from_splitted_dataset(splitted_4d_output_dir_task)
# we need to figure out if we need the intensity propoerties. We collect them only if one of the modalities is CT
dataset_json = load_json(join(cropped_out_dir, 'dataset.json'))
modalities = list(dataset_json["modality"].values())
collect_intensityproperties = True if (("CT" in modalities) or
("ct" in modalities)) else False
dataset_analyzer = DatasetAnalyzer(
cropped_out_dir, overwrite=False,
num_processes=tf) # this class creates the fingerprint
_ = dataset_analyzer.analyze_dataset(
collect_intensityproperties
) # this will write output files that will be used by the ExperimentPlanner
maybe_mkdir_p(preprocessing_output_dir_this_task)
shutil_sol.copyfile(join(cropped_out_dir, "dataset_properties.pkl"),
preprocessing_output_dir_this_task)
shutil_sol.copyfile(join(nnUNet_raw_data, t, "dataset.json"),
preprocessing_output_dir_this_task)
threads = (tl, tf)
print("number of threads: ", threads, "\n")
if planner_3d is not None:
if args.overwrite_plans is not None:
assert args.overwrite_plans_identifier is not None, "You need to specify -overwrite_plans_identifier"
exp_planner = planner_3d(cropped_out_dir,
preprocessing_output_dir_this_task,
args.overwrite_plans,
args.overwrite_plans_identifier)
else:
exp_planner = planner_3d(cropped_out_dir,
preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
if planner_2d is not None:
exp_planner = planner_2d(cropped_out_dir,
preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
maybe_mkdir_p(imagests)
maybe_mkdir_p(labelstr)
train_folder = join(base, "Training/img")
label_folder = join(base, "Training/label")
test_folder = join(base, "Test/img")
train_patient_names = []
test_patient_names = []
train_patients = subfiles(train_folder, join=False, suffix='nii.gz')
for p in train_patients:
serial_number = int(p[3:7])
train_patient_name = f'{prefix}_{serial_number:03d}.nii.gz'
label_file = join(label_folder, f'label{p[3:]}')
image_file = join(train_folder, p)
shutil_sol.copyfile(
image_file, join(imagestr,
f'{train_patient_name[:7]}_0000.nii.gz'))
shutil_sol.copyfile(label_file, join(labelstr, train_patient_name))
train_patient_names.append(train_patient_name)
test_patients = subfiles(test_folder, join=False, suffix=".nii.gz")
for p in test_patients:
p = p[:-7]
image_file = join(test_folder, p + ".nii.gz")
serial_number = int(p[3:7])
test_patient_name = f'{prefix}_{serial_number:03d}.nii.gz'
shutil_sol.copyfile(
image_file, join(imagests, f'{test_patient_name[:7]}_0000.nii.gz'))
test_patient_names.append(test_patient_name)
json_dict = OrderedDict()
def validate(self,
do_mirroring: bool = True,
use_sliding_window: bool = True,
step_size: float = 0.5,
save_softmax: bool = True,
use_gaussian: bool = True,
overwrite: bool = True,
validation_folder_name: str = 'validation_raw',
debug: bool = False,
all_in_gpu: bool = False,
segmentation_export_kwargs: dict = None,
run_postprocessing_on_folds: bool = True):
current_mode = self.network.training
self.network.eval()
assert self.was_initialized, "must initialize, ideally with checkpoint (or train first)"
if self.dataset_val is None:
self.load_dataset()
self.do_split()
if segmentation_export_kwargs is None:
if 'segmentation_export_params' in self.plans.keys():
force_separate_z = self.plans['segmentation_export_params'][
'force_separate_z']
interpolation_order = self.plans['segmentation_export_params'][
'interpolation_order']
interpolation_order_z = self.plans[
'segmentation_export_params']['interpolation_order_z']
else:
force_separate_z = None
interpolation_order = 1
interpolation_order_z = 0
else:
force_separate_z = segmentation_export_kwargs['force_separate_z']
interpolation_order = segmentation_export_kwargs[
'interpolation_order']
interpolation_order_z = segmentation_export_kwargs[
'interpolation_order_z']
output_folder = join(self.output_folder, validation_folder_name)
maybe_mkdir_p(output_folder)
if do_mirroring:
mirror_axes = self.data_aug_params['mirror_axes']
else:
mirror_axes = ()
pred_gt_tuples = []
export_pool = Pool(2)
results = []
transpose_backward = self.plans.get('transpose_backward')
for k in self.dataset_val.keys():
properties = load_pickle(self.dataset[k]['properties_file'])
data = np.load(self.dataset[k]['data_file'])['data']
# concat segmentation of previous step
seg_from_prev_stage = np.load(
join(self.folder_with_segs_from_prev_stage,
k + "_segFromPrevStage.npz"))['data'][None]
print(data.shape)
data[-1][data[-1] == -1] = 0
data_for_net = np.concatenate(
(data[:-1],
to_one_hot(seg_from_prev_stage[0], range(1,
self.num_classes))))
softmax_pred = self.predict_preprocessed_data_return_seg_and_softmax(
data_for_net,
do_mirroring=do_mirroring,
mirror_axes=mirror_axes,
use_sliding_window=use_sliding_window,
step_size=step_size,
use_gaussian=use_gaussian,
all_in_gpu=all_in_gpu,
mixed_precision=self.fp16)[1]
if transpose_backward is not None:
transpose_backward = self.plans.get('transpose_backward')
softmax_pred = softmax_pred.transpose(
[0] + [i + 1 for i in transpose_backward])
fname = properties['list_of_data_files'][0].split("/")[-1][:-12]
if save_softmax:
softmax_fname = join(output_folder, fname + ".npz")
else:
softmax_fname = None
"""There is a problem with python process communication that prevents us from communicating obejcts
larger than 2 GB between processes (basically when the length of the pickle string that will be sent is
communicated by the multiprocessing.Pipe object then the placeholder (\%i I think) does not allow for long
enough strings (lol). This could be fixed by changing i to l (for long) but that would require manually
patching system python code. We circumvent that problem here by saving softmax_pred to a npy file that will
then be read (and finally deleted) by the Process. save_segmentation_nifti_from_softmax can take either
filename or np.ndarray and will handle this automatically"""
if np.prod(softmax_pred.shape) > (2e9 / 4 *
0.85): # *0.85 just to be save
np.save(fname + ".npy", softmax_pred)
softmax_pred = fname + ".npy"
results.append(
export_pool.starmap_async(
save_segmentation_nifti_from_softmax,
((softmax_pred, join(output_folder, fname + ".nii.gz"),
properties, interpolation_order,
self.regions_class_order, None, None, softmax_fname,
None, force_separate_z, interpolation_order_z), )))
pred_gt_tuples.append([
join(output_folder, fname + ".nii.gz"),
join(self.gt_niftis_folder, fname + ".nii.gz")
])
_ = [i.get() for i in results]
task = self.dataset_directory.split("/")[-1]
job_name = self.experiment_name
_ = aggregate_scores(pred_gt_tuples,
labels=list(range(self.num_classes)),
json_output_file=join(output_folder,
"summary.json"),
json_name=job_name,
json_author="Fabian",
json_description="",
json_task=task)
if run_postprocessing_on_folds:
# in the old nnunet we would stop here. Now we add a postprocessing. This postprocessing can remove everything
# except the largest connected component for each class. To see if this improves results, we do this for all
# classes and then rerun the evaluation. Those classes for which this resulted in an improved dice score will
# have this applied during inference as well
self.print_to_log_file("determining postprocessing")
determine_postprocessing(self.output_folder,
self.gt_niftis_folder,
validation_folder_name,
final_subf_name=validation_folder_name +
"_postprocessed",
debug=debug)
# after this the final predictions for the vlaidation set can be found in validation_folder_name_base + "_postprocessed"
# They are always in that folder, even if no postprocessing as applied!
# detemining postprocesing on a per-fold basis may be OK for this fold but what if another fold finds another
# postprocesing to be better? In this case we need to consolidate. At the time the consolidation is going to be
# done we won't know what self.gt_niftis_folder was, so now we copy all the niftis into a separate folder to
# be used later
gt_nifti_folder = join(self.output_folder_base, "gt_niftis")
maybe_mkdir_p(gt_nifti_folder)
for f in subfiles(self.gt_niftis_folder, suffix=".nii.gz"):
success = False
attempts = 0
while not success and attempts < 10:
try:
shutil_sol.copyfile(f, gt_nifti_folder)
success = True
except OSError:
attempts += 1
sleep(1)
self.network.train(current_mode)
export_pool.close()
export_pool.join()
patient_ids_test.append(patient_name)
# Process T2 train
d = join(root, "MR")
patients = subdirs(d, join=False)
for p in patients:
patient_name = "T2_" + p
gt_dir = join(d, p, "T2SPIR", "Ground")
seg = convert_MR_seg(load_png_stack(gt_dir)[::-1])
img_dir = join(d, p, "T2SPIR", "DICOM_anon")
img_outfile = join(output_images, patient_name + "_0000.nii.gz")
_ = dicom2nifti.convert_dicom.dicom_series_to_nifti(
img_dir, img_outfile, reorient_nifti=False)
shutil_sol.copyfile(join(output_images, patient_name + "_0000.nii.gz"),
join(output_images, patient_name + "_0001.nii.gz"))
img_sitk = sitk.ReadImage(img_outfile)
img_sitk_npy = sitk.GetArrayFromImage(img_sitk)
seg_itk = sitk.GetImageFromArray(seg.astype(np.uint8))
seg_itk = copy_geometry(seg_itk, img_sitk)
sitk.WriteImage(seg_itk, join(output_labels, patient_name + ".nii.gz"))
patient_ids.append(patient_name)
# Process T2 test
d = join(root_test, "MR")
patients = subdirs(d, join=False)
for p in patients:
patient_name = "T2_" + p
gt_dir = join(d, p, "T2SPIR", "Ground")
def convert_variant2_predicted_test_to_submission_format(
folder_with_predictions,
output_folder="/home/fabian/drives/datasets/results/nnUNet/test_sets/Task038_CHAOS_Task_3_5_Variant2/ready_to_submit",
postprocessing_file="/home/fabian/drives/datasets/results/nnUNet/ensembles/Task038_CHAOS_Task_3_5_Variant2/ensemble_2d__nnUNetTrainerV2__nnUNetPlansv2.1--3d_fullres__nnUNetTrainerV2__nnUNetPlansv2.1/postprocessing.json"
):
"""
output_folder is where the extracted template is
:param folder_with_predictions:
:param output_folder:
:return:
"""
postprocessing_file = "/media/fabian/Results/nnUNet/3d_fullres/Task039_CHAOS_Task_3_5_Variant2_highres/" \
"nnUNetTrainerV2__nnUNetPlansfixed/postprocessing.json"
# variant 2 treats in and out phase as two training examples, so we need to ensemble these two again
final_predictions_folder = join(output_folder, "final")
maybe_mkdir_p(final_predictions_folder)
t1_patient_names = [
i.split("_")[-1][:-7] for i in subfiles(
folder_with_predictions, prefix="T1", suffix=".nii.gz", join=False)
]
folder_for_ensembing0 = join(output_folder, "ens0")
folder_for_ensembing1 = join(output_folder, "ens1")
maybe_mkdir_p(folder_for_ensembing0)
maybe_mkdir_p(folder_for_ensembing1)
# now copy all t1 out phases in ens0 and all in phases in ens1. Name them the same.
for t1 in t1_patient_names:
shutil_sol.copyfile(join(folder_with_predictions, "T1_in_%s.npz" % t1),
join(folder_for_ensembing1, "T1_%s.npz" % t1))
shutil_sol.copyfile(join(folder_with_predictions, "T1_in_%s.pkl" % t1),
join(folder_for_ensembing1, "T1_%s.pkl" % t1))
shutil_sol.copyfile(
join(folder_with_predictions, "T1_out_%s.npz" % t1),
join(folder_for_ensembing0, "T1_%s.npz" % t1))
shutil_sol.copyfile(
join(folder_with_predictions, "T1_out_%s.pkl" % t1),
join(folder_for_ensembing0, "T1_%s.pkl" % t1))
shutil_sol.copyfile(join(folder_with_predictions, "plans.pkl"),
join(folder_for_ensembing0, "plans.pkl"))
shutil_sol.copyfile(join(folder_with_predictions, "plans.pkl"),
join(folder_for_ensembing1, "plans.pkl"))
# there is a problem with T1_35 that I need to correct manually (different crop size, will not negatively impact results)
#ens0_softmax = np.load(join(folder_for_ensembing0, "T1_35.npz"))['softmax']
ens1_softmax = np.load(join(folder_for_ensembing1, "T1_35.npz"))['softmax']
#ens0_props = load_pickle(join(folder_for_ensembing0, "T1_35.pkl"))
#ens1_props = load_pickle(join(folder_for_ensembing1, "T1_35.pkl"))
ens1_softmax = ens1_softmax[:, :, :-1, :]
np.savez_compressed(join(folder_for_ensembing1, "T1_35.npz"),
softmax=ens1_softmax)
shutil_sol.copyfile(join(folder_for_ensembing0, "T1_35.pkl"),
join(folder_for_ensembing1, "T1_35.pkl"))
# now call my ensemble function
merge((folder_for_ensembing0, folder_for_ensembing1),
final_predictions_folder,
8,
True,
postprocessing_file=postprocessing_file)
# copy t2 files to final_predictions_folder as well
t2_files = subfiles(folder_with_predictions,
prefix="T2",
suffix=".nii.gz",
join=False)
for t2 in t2_files:
shutil_sol.copyfile(join(folder_with_predictions, t2),
join(final_predictions_folder, t2))
# apply postprocessing
from nnunet.postprocessing.connected_components import apply_postprocessing_to_folder, load_postprocessing
postprocessed_folder = join(output_folder, "final_postprocessed")
for_which_classes, min_valid_obj_size = load_postprocessing(
postprocessing_file)
apply_postprocessing_to_folder(final_predictions_folder,
postprocessed_folder, for_which_classes,
min_valid_obj_size, 8)
# now export the niftis in the weird png format
# task 3
output_dir = join(output_folder, "CHAOS_submission_template_new", "Task3",
"MR")
for t1 in t1_patient_names:
output_folder_here = join(output_dir, t1, "T1DUAL", "Results")
nifti_file = join(postprocessed_folder, "T1_%s.nii.gz" % t1)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task3)
for t2 in t2_files:
patname = t2.split("_")[-1][:-7]
output_folder_here = join(output_dir, patname, "T2SPIR", "Results")
nifti_file = join(postprocessed_folder, "T2_%s.nii.gz" % patname)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task3)
# task 5
output_dir = join(output_folder, "CHAOS_submission_template_new", "Task5",
"MR")
for t1 in t1_patient_names:
output_folder_here = join(output_dir, t1, "T1DUAL", "Results")
nifti_file = join(postprocessed_folder, "T1_%s.nii.gz" % t1)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task5)
for t2 in t2_files:
patname = t2.split("_")[-1][:-7]
output_folder_here = join(output_dir, patname, "T2SPIR", "Results")
nifti_file = join(postprocessed_folder, "T2_%s.nii.gz" % patname)
write_pngs_from_nifti(nifti_file,
output_folder_here,
converter=convert_seg_to_intensity_task5)
maybe_mkdir_p(labelstr)
maybe_mkdir_p(labelste)
img_tr_itk = sitk.GetImageFromArray(train_volume.astype(np.float32))
lab_tr_itk = sitk.GetImageFromArray(train_labels.astype(np.uint8))
img_te_itk = sitk.GetImageFromArray(test_volume.astype(np.float32))
lab_te_itk = sitk.GetImageFromArray(test_labels.astype(np.uint8))
img_tr_itk.SetSpacing((5, 5, 5))
lab_tr_itk.SetSpacing((5, 5, 5))
img_te_itk.SetSpacing((5, 5, 5))
lab_te_itk.SetSpacing((5, 5, 5))
# 5 copies, otherwise we cannot run nnunet (5 fold cv needs that)
sitk.WriteImage(img_tr_itk, join(imagestr, "training0_0000.nii.gz"))
shutil_sol.copyfile(join(imagestr, "training0_0000.nii.gz"),
join(imagestr, "training1_0000.nii.gz"))
shutil_sol.copyfile(join(imagestr, "training0_0000.nii.gz"),
join(imagestr, "training2_0000.nii.gz"))
shutil_sol.copyfile(join(imagestr, "training0_0000.nii.gz"),
join(imagestr, "training3_0000.nii.gz"))
shutil_sol.copyfile(join(imagestr, "training0_0000.nii.gz"),
join(imagestr, "training4_0000.nii.gz"))
sitk.WriteImage(lab_tr_itk, join(labelstr, "training0.nii.gz"))
shutil_sol.copyfile(join(labelstr, "training0.nii.gz"),
join(labelstr, "training1.nii.gz"))
shutil_sol.copyfile(join(labelstr, "training0.nii.gz"),
join(labelstr, "training2.nii.gz"))
shutil_sol.copyfile(join(labelstr, "training0.nii.gz"),
join(labelstr, "training3.nii.gz"))
shutil_sol.copyfile(join(labelstr, "training0.nii.gz"),
print("\n\n\n", t)
cropped_out_dir = os.path.join(nnUNet_cropped_data, t)
preprocessing_output_dir_this_task = os.path.join(
preprocessing_output_dir, t)
splitted_4d_output_dir_task = os.path.join(nnUNet_raw_data, t)
lists, modalities = create_lists_from_splitted_dataset(
splitted_4d_output_dir_task)
dataset_analyzer = DatasetAnalyzer(cropped_out_dir,
overwrite=False)
_ = dataset_analyzer.analyze_dataset(
) # this will write output files that will be used by the ExperimentPlanner
maybe_mkdir_p(preprocessing_output_dir_this_task)
shutil_sol.copyfile(
join(cropped_out_dir, "dataset_properties.pkl"),
preprocessing_output_dir_this_task)
shutil_sol.copyfile(join(nnUNet_raw_data, t, "dataset.json"),
preprocessing_output_dir_this_task)
threads = (tl, tf)
print("number of threads: ", threads, "\n")
exp_planner = ExperimentPlanner(
cropped_out_dir, preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if run_preprocessing:
exp_planner.run_preprocessing(threads)
except Exception as e:
print(e)
