def run_preprocessing(self, num_threads):
if os.path.isdir(
join(self.preprocessed_output_folder, "gt_segmentations")):
shutil.rmtree(
join(self.preprocessed_output_folder, "gt_segmentations"))
shutil.copytree(
join(self.folder_with_cropped_data, "gt_segmentations"),
join(self.preprocessed_output_folder, "gt_segmentations"))
normalization_schemes = self.plans['normalization_schemes']
use_nonzero_mask_for_normalization = self.plans['use_mask_for_norm']
intensityproperties = self.plans['dataset_properties'][
'intensityproperties']
preprocessor_class = recursive_find_python_class(
[join(nnunet.__path__[0], "preprocessing")],
self.preprocessor_name,
current_module="nnunet.preprocessing")
assert preprocessor_class is not None
preprocessor = preprocessor_class(normalization_schemes,
use_nonzero_mask_for_normalization,
self.transpose_forward,
intensityproperties)
target_spacings = [
i["current_spacing"] for i in self.plans_per_stage.values()
]
if self.plans['num_stages'] > 1 and not isinstance(
num_threads, (list, tuple)):
num_threads = (default_num_threads, num_threads)
elif self.plans['num_stages'] == 1 and isinstance(
num_threads, (list, tuple)):
num_threads = num_threads[-1]
preprocessor.run(target_spacings, self.folder_with_cropped_data,
self.preprocessed_output_folder,
self.plans['data_identifier'], num_threads)
def preprocess_patient(self, input_files):
"""
Used to predict new unseen data. Not used for the preprocessing of the training/test data
:param input_files:
:return:
"""
from nnunet.training.model_restore import recursive_find_python_class
preprocessor_name = self.plans.get('preprocessor_name')
if preprocessor_name is None:
if self.threeD:
preprocessor_name = "GenericPreprocessor"
else:
preprocessor_name = "PreprocessorFor2D"
print("using preprocessor", preprocessor_name)
preprocessor_class = recursive_find_python_class([join(nnunet.__path__[0], "preprocessing")],
preprocessor_name,
current_module="nnunet.preprocessing")
assert preprocessor_class is not None, "Could not find preprocessor %s in nnunet.preprocessing" % \
preprocessor_name
preprocessor = preprocessor_class(self.normalization_schemes, self.use_mask_for_norm,
self.transpose_forward, self.intensity_properties)
d, s, properties = preprocessor.preprocess_test_case(input_files,
self.plans['plans_per_stage'][self.stage][
'current_spacing'])
return d, s, properties
def get_default_configuration(network,
task,
network_trainer,
plans_identifier=default_plans_identifier,
search_in=(nnunet.__path__[0], "training",
"network_training"),
base_module='nnunet.training.network_training'):
assert network in ['2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'], \
"network can only be one of the following: \'3d\', \'3d_lowres\', \'3d_fullres\', \'3d_cascade_fullres\'"
dataset_directory = join(preprocessing_output_dir, task)
os.makedirs(dataset_directory, exist_ok=True)
if network == '2d':
plans_file = join(preprocessing_output_dir, task,
plans_identifier + "_plans_2D.pkl")
else:
plans_file = join(preprocessing_output_dir, task,
plans_identifier + "_plans_3D.pkl")
plans = load_pickle(plans_file)
possible_stages = list(plans['plans_per_stage'].keys())
if (network == '3d_cascade_fullres'
or network == "3d_lowres") and len(possible_stages) == 1:
raise RuntimeError(
"3d_lowres/3d_cascade_fullres only applies if there is more than one stage. This task does "
"not require the cascade. Run 3d_fullres instead")
if network == '2d' or network == "3d_lowres":
stage = 0
else:
stage = possible_stages[-1]
trainer_class = recursive_find_python_class([join(*search_in)],
network_trainer,
current_module=base_module)
output_folder_name = join(network_training_output_dir, network, task,
network_trainer + "__" + plans_identifier)
print("###############################################")
print("I am running the following nnUNet: %s" % network)
print("My trainer class is: ", trainer_class)
print("For that I will be using the following configuration:")
summarize_plans(plans_file)
print("I am using stage %d from these plans" % stage)
if (network == '2d'
or len(possible_stages) > 1) and not network == '3d_lowres':
batch_dice = True
print("I am using batch dice + CE loss")
else:
batch_dice = False
print("I am using sample dice + CE loss")
print("\nI am using data from this folder: ",
join(dataset_directory, plans['data_identifier']))
print("###############################################")
return plans_file, output_folder_name, dataset_directory, batch_dice, stage, trainer_class
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"-t",
"--task_ids",
nargs="+",
help="List of integers belonging to the task ids you wish to run"
" experiment planning and preprocessing for. Each of these "
"ids must, have a matching folder 'TaskXXX_' in the raw "
"data folder")
parser.add_argument(
"-pl3d",
"--planner3d",
type=str,
default="ExperimentPlanner3D_v21",
help=
"Name of the ExperimentPlanner class for the full resolution 3D U-Net and U-Net cascade. "
"Default is ExperimentPlanner3D_v21. Can be 'None', in which case these U-Nets will not be "
"configured")
parser.add_argument(
"-pl2d",
"--planner2d",
type=str,
default="ExperimentPlanner2D_v21",
help=
"Name of the ExperimentPlanner class for the 2D U-Net. Default is ExperimentPlanner2D_v21. "
"Can be 'None', in which case this U-Net will not be configured")
parser.add_argument(
"-no_pp",
action="store_true",
help=
"Set this flag if you dont want to run the preprocessing. If this is set then this script "
"will only run the experiment planning and create the plans file")
parser.add_argument(
"-tl",
type=int,
required=False,
default=8,
help=
"Number of processes used for preprocessing the low resolution data for the 3D low "
"resolution U-Net. This can be larger than -tf. Don't overdo it or you will run out of "
"RAM")
parser.add_argument(
"-tf",
type=int,
required=False,
default=8,
help=
"Number of processes used for preprocessing the full resolution data of the 2D U-Net and "
"3D U-Net. Don't overdo it or you will run out of RAM")
parser.add_argument(
"--verify_dataset_integrity",
required=False,
default=False,
action="store_true",
help=
"set this flag to check the dataset integrity. This is useful and should be done once for "
"each dataset!")
args = parser.parse_args()
task_ids = args.task_ids
task_ids = [6]
dont_run_preprocessing = args.no_pp
tl = args.tl
tf = args.tf
planner_name3d = args.planner3d
planner_name2d = args.planner2d
if planner_name3d == "None":
planner_name3d = None
if planner_name2d == "None":
planner_name2d = None
# we need raw data
tasks = []
for i in task_ids:
i = int(i)
task_name = convert_id_to_task_name(i)
#task_name = "Breast"
if args.verify_dataset_integrity:
verify_dataset_integrity(join(nnUNet_raw_data, task_name))
crop(task_name, False, tf)
tasks.append(task_name)
search_in = join(nnunet.__path__[0], "experiment_planning")
if planner_name3d is not None:
planner_3d = recursive_find_python_class(
[search_in],
planner_name3d,
current_module="nnunet.experiment_planning")
if planner_3d is None:
raise RuntimeError(
"Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name3d)
else:
planner_3d = None
if planner_name2d is not None:
planner_2d = recursive_find_python_class(
[search_in],
planner_name2d,
current_module="nnunet.experiment_planning")
if planner_2d is None:
raise RuntimeError(
"Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name2d)
else:
planner_2d = None
for t in tasks:
print("\n\n\n", t)
cropped_out_dir = os.path.join(nnUNet_cropped_data, t)
preprocessing_output_dir_this_task = os.path.join(
preprocessing_output_dir, t)
#splitted_4d_output_dir_task = os.path.join(nnUNet_raw_data, t)
#lists, modalities = create_lists_from_splitted_dataset(splitted_4d_output_dir_task)
# we need to figure out if we need the intensity propoerties. We collect them only if one of the modalities is CT
dataset_json = load_json(join(cropped_out_dir, 'dataset.json'))
modalities = list(dataset_json["modality"].values())
collect_intensityproperties = True if (("CT" in modalities) or
("ct" in modalities)) else False
dataset_analyzer = DatasetAnalyzer(
cropped_out_dir, overwrite=False,
num_processes=tf) # this class creates the fingerprint
_ = dataset_analyzer.analyze_dataset(
collect_intensityproperties
) # this will write output files that will be used by the ExperimentPlanner
#maybe_mkdir_p(preprocessing_output_dir_this_task)
if not os.path.isdir(preprocessing_output_dir_this_task):
os.makedirs(preprocessing_output_dir_this_task)
shutil.copy(join(cropped_out_dir, "dataset_properties.pkl"),
preprocessing_output_dir_this_task)
shutil.copy(join(nnUNet_raw_data, t, "dataset.json"),
preprocessing_output_dir_this_task)
threads = (tl, tf)
print("number of threads: ", threads, "\n")
if planner_3d is not None:
exp_planner = planner_3d(cropped_out_dir,
preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
if planner_2d is not None:
exp_planner = planner_2d(cropped_out_dir,
preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"-t",
"--task_ids",
nargs="+",
help="List of integers belonging to the task ids you wish to run"
" experiment planning and preprocessing for. Each of these "
"ids must, have a matching folder 'TaskXXX_' in the raw "
"data folder")
parser.add_argument(
"-pl3d",
"--planner3d",
type=str,
default="ExperimentPlanner3D_v21",
help=
"Name of the ExperimentPlanner class for the full resolution 3D U-Net and U-Net cascade. "
"Default is ExperimentPlanner3D_v21. Can be 'None', in which case these U-Nets will not be "
"configured")
parser.add_argument(
"-pl2d",
"--planner2d",
type=str,
default="ExperimentPlanner2D_v21",
help=
"Name of the ExperimentPlanner class for the 2D U-Net. Default is ExperimentPlanner2D_v21. "
"Can be 'None', in which case this U-Net will not be configured")
parser.add_argument(
"-no_pp",
action="store_true",
help=
"Set this flag if you dont want to run the preprocessing. If this is set then this script "
"will only run the experiment planning and create the plans file")
parser.add_argument(
"-tl",
type=int,
required=False,
default=8,
help=
"Number of processes used for preprocessing the low resolution data for the 3D low "
"resolution U-Net. This can be larger than -tf. Don't overdo it or you will run out of "
"RAM")
parser.add_argument(
"-tf",
type=int,
required=False,
default=8,
help=
"Number of processes used for preprocessing the full resolution data of the 2D U-Net and "
"3D U-Net. Don't overdo it or you will run out of RAM")
parser.add_argument(
"--verify_dataset_integrity",
required=False,
default=False,
action="store_true",
help=
"set this flag to check the dataset integrity. This is useful and should be done once for "
"each dataset!")
parser.add_argument(
"-overwrite_plans",
type=str,
default=None,
required=False,
help=
"Use this to specify a plans file that should be used instead of whatever nnU-Net would "
"configure automatically. This will overwrite everything: intensity normalization, "
"network architecture, target spacing etc. Using this is useful for using pretrained "
"model weights as this will guarantee that the network architecture on the target "
"dataset is the same as on the source dataset and the weights can therefore be transferred.\n"
"Pro tip: If you want to pretrain on Hepaticvessel and apply the result to LiTS then use "
"the LiTS plans to run the preprocessing of the HepaticVessel task.\n"
"Make sure to only use plans files that were "
"generated with the same number of modalities as the target dataset (LiTS -> BCV or "
"LiTS -> Task008_HepaticVessel is OK. BraTS -> LiTS is not (BraTS has 4 input modalities, "
"LiTS has just one)). Also only do things that make sense. This functionality is beta with"
"no support given.\n"
"Note that this will first print the old plans (which are going to be overwritten) and "
"then the new ones (provided that -no_pp was NOT set).")
parser.add_argument(
"-overwrite_plans_identifier",
type=str,
default=None,
required=False,
help=
"If you set overwrite_plans you need to provide a unique identifier so that nnUNet knows "
"where to look for the correct plans and data. Assume your identifier is called "
"IDENTIFIER, the correct training command would be:\n"
"'nnUNet_train CONFIG TRAINER TASKID FOLD -p nnUNetPlans_pretrained_IDENTIFIER "
"-pretrained_weights FILENAME'")
args = parser.parse_args()
task_ids = args.task_ids
dont_run_preprocessing = args.no_pp
tl = args.tl
tf = args.tf
planner_name3d = args.planner3d
planner_name2d = args.planner2d
if planner_name3d == "None":
planner_name3d = None
if planner_name2d == "None":
planner_name2d = None
if args.overwrite_plans is not None:
if planner_name2d is not None:
print(
"Overwriting plans only works for the 3d planner. I am setting '--planner2d' to None. This will "
"skip 2d planning and preprocessing.")
assert planner_name3d == 'ExperimentPlanner3D_v21_Pretrained', "When using --overwrite_plans you need to use " \
"'-pl3d ExperimentPlanner3D_v21_Pretrained'"
# we need raw data
tasks = []
for i in task_ids:
i = int(i)
task_name = convert_id_to_task_name(i)
if args.verify_dataset_integrity:
verify_dataset_integrity(join(nnUNet_raw_data, task_name))
crop(task_name, False, tf)
tasks.append(task_name)
search_in = join(nnunet.__path__[0], "experiment_planning")
if planner_name3d is not None:
planner_3d = recursive_find_python_class(
[search_in],
planner_name3d,
current_module="nnunet.experiment_planning")
if planner_3d is None:
raise RuntimeError(
"Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name3d)
else:
planner_3d = None
if planner_name2d is not None:
planner_2d = recursive_find_python_class(
[search_in],
planner_name2d,
current_module="nnunet.experiment_planning")
if planner_2d is None:
raise RuntimeError(
"Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name2d)
else:
planner_2d = None
for t in tasks:
print("\n\n\n", t)
cropped_out_dir = os.path.join(nnUNet_cropped_data, t)
preprocessing_output_dir_this_task = os.path.join(
preprocessing_output_dir, t)
#splitted_4d_output_dir_task = os.path.join(nnUNet_raw_data, t)
#lists, modalities = create_lists_from_splitted_dataset(splitted_4d_output_dir_task)
# we need to figure out if we need the intensity propoerties. We collect them only if one of the modalities is CT
dataset_json = load_json(join(cropped_out_dir, 'dataset.json'))
modalities = list(dataset_json["modality"].values())
collect_intensityproperties = True if (("CT" in modalities) or
("ct" in modalities)) else False
dataset_analyzer = DatasetAnalyzer(
cropped_out_dir, overwrite=False,
num_processes=tf) # this class creates the fingerprint
_ = dataset_analyzer.analyze_dataset(
collect_intensityproperties
) # this will write output files that will be used by the ExperimentPlanner
maybe_mkdir_p(preprocessing_output_dir_this_task)
shutil.copy(join(cropped_out_dir, "dataset_properties.pkl"),
preprocessing_output_dir_this_task)
shutil.copy(join(nnUNet_raw_data, t, "dataset.json"),
preprocessing_output_dir_this_task)
threads = (tl, tf)
print("number of threads: ", threads, "\n")
if planner_3d is not None:
if args.overwrite_plans is not None:
assert args.overwrite_plans_identifier is not None, "You need to specify -overwrite_plans_identifier"
exp_planner = planner_3d(cropped_out_dir,
preprocessing_output_dir_this_task,
args.overwrite_plans,
args.overwrite_plans_identifier)
else:
exp_planner = planner_3d(cropped_out_dir,
preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
if planner_2d is not None:
exp_planner = planner_2d(cropped_out_dir,
preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
parser = argparse.ArgumentParser()
parser.add_argument("network_trainer")
parser.add_argument("task")
parser.add_argument("fold", type=int)
args = parser.parse_args()
trainerclass = args.network_trainer
task = args.task
fold = args.fold
plans_file, folder_with_preprocessed_data, output_folder_name, dataset_directory, batch_dice, stage = \
get_default_configuration("3d_lowres", task)
trainer_class = recursive_find_python_class(
[join(nnunet.__path__[0], "training", "network_training")],
trainerclass, "nnunet.training.network_training")
if trainer_class is None:
raise RuntimeError(
"Could not find trainer class in nnunet.training.network_training")
else:
assert issubclass(
trainer_class, nnUNetTrainer
), "network_trainer was found but is not derived from nnUNetTrainer"
trainer = trainer_class(plans_file,
fold,
folder_with_preprocessed_data,
output_folder=output_folder_name,
dataset_directory=dataset_directory,
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-t", "--task_ids", nargs="+", default=["4"])
parser.add_argument("-pl3d", "--planner3d", type=str, default="ExperimentPlanner3D_v21")
parser.add_argument("-pl2d", "--planner2d", type=str, default="ExperimentPlanner2D_v21")
parser.add_argument("-no_pp", action="store_true", default=False)
parser.add_argument("-tl", type=int, required=False, default=8)
parser.add_argument("-tf", type=int, required=False, default=8)
parser.add_argument("--verify_dataset_integrity", required=False, default=False, action="store_true")
args = parser.parse_args()
task_ids = args.task_ids
dont_run_preprocessing = args.no_pp
tl = args.tl
tf = args.tf
planner_name3d = args.planner3d
planner_name2d = args.planner2d
if planner_name3d == "None":
planner_name3d = None
if planner_name2d == "None":
planner_name2d = None
# we need raw data
tasks = []
for i in task_ids:
i = int(i)
task_name = convert_id_to_task_name(i)
if args.verify_dataset_integrity:
verify_dataset_integrity(join(nnUNet_raw_data, task_name))
crop(task_name, False, tf)
tasks.append(task_name)
search_in = join(nnunet.__path__[0], "experiment_planning")
if planner_name3d is not None:
planner_3d = recursive_find_python_class([search_in], planner_name3d, current_module="nnunet.experiment_planning")
if planner_3d is None:
raise RuntimeError("Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name3d)
else:
planner_3d = None
if planner_name2d is not None:
planner_2d = recursive_find_python_class([search_in], planner_name2d, current_module="nnunet.experiment_planning")
if planner_2d is None:
raise RuntimeError("Could not find the Planner class %s. Make sure it is located somewhere in "
"nnunet.experiment_planning" % planner_name2d)
else:
planner_2d = None
for t in tasks:
print("\n\n\n", t)
cropped_out_dir = os.path.join(nnUNet_cropped_data, t)
preprocessing_output_dir_this_task = os.path.join(preprocessing_output_dir, t)
#splitted_4d_output_dir_task = os.path.join(nnUNet_raw_data, t)
#lists, modalities = create_lists_from_splitted_dataset(splitted_4d_output_dir_task)
dataset_analyzer = DatasetAnalyzer(cropped_out_dir, overwrite=False) # this class creates the fingerprint
_ = dataset_analyzer.analyze_dataset() # this will write output files that will be used by the ExperimentPlanner
maybe_mkdir_p(preprocessing_output_dir_this_task)
shutil.copy(join(cropped_out_dir, "dataset_properties.pkl"), preprocessing_output_dir_this_task)
shutil.copy(join(nnUNet_raw_data, t, "dataset.json"), preprocessing_output_dir_this_task)
threads = (tl, tf)
print("number of threads: ", threads, "\n")
if planner_3d is not None:
exp_planner = planner_3d(cropped_out_dir, preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
if planner_2d is not None:
exp_planner = planner_2d(cropped_out_dir, preprocessing_output_dir_this_task)
exp_planner.plan_experiment()
if not dont_run_preprocessing: # double negative, yooo
exp_planner.run_preprocessing(threads)
