def main():
argparser = argparse.ArgumentParser(
usage=
"Used to determine the postprocessing for a trained model. Useful for "
"when the best configuration (2d, 3d_fullres etc) as selected manually."
)
argparser.add_argument("-m",
type=str,
required=True,
help="U-Net model (2d, 3d_lowres, 3d_fullres or "
"3d_cascade_fullres)")
argparser.add_argument("-t",
type=str,
required=True,
help="Task name or id")
argparser.add_argument(
"-tr",
type=str,
required=False,
default=None,
help="tuframeworkTrainer class. Default: %s, unless 3d_cascade_fullres "
"(then it's %s)" % (default_trainer, default_cascade_trainer))
argparser.add_argument("-pl",
type=str,
required=False,
default=default_plans_identifier,
help="Plans name, Default=%s" %
default_plans_identifier)
argparser.add_argument(
"-val",
type=str,
required=False,
default="validation_raw",
help="Validation folder name. Default: validation_raw")
args = argparser.parse_args()
model = args.m
task = args.t
trainer = args.tr
plans = args.pl
val = args.val
if not task.startswith("Task"):
task_id = int(task)
task = convert_id_to_task_name(task_id)
if trainer is None:
if model == "3d_cascade_fullres":
trainer = "tuframeworkTrainerV2CascadeFullRes"
else:
trainer = "tuframeworkTrainerV2"
folder = get_output_folder_name(model, task, trainer, plans, None)
consolidate_folds(folder, val)
def export_for_paper():
output_base = "/media/fabian/DeepLearningData/tuframework_trained_models"
task_ids = [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 24, 27, 29, 35, 48, 55, 61, 38
]
for t in task_ids:
if t == 61:
models = ("3d_fullres", )
else:
models = ("2d", "3d_lowres", "3d_fullres", "3d_cascade_fullres")
taskname = convert_id_to_task_name(t)
print(taskname)
output_folder = output_base + "/" + taskname
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
copy_pretrained_models_for_task(taskname, output_folder, models)
copy_ensembles(taskname, output_folder)
compress_everything(output_base, 8)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("network")
parser.add_argument("network_trainer")
parser.add_argument("task", help="can be task name or task id")
parser.add_argument("fold", help='0, 1, ..., 5 or \'all\'')
parser.add_argument("-val",
"--validation_only",
help="use this if you want to only run the validation",
action="store_true")
parser.add_argument("-c",
"--continue_training",
help="use this if you want to continue a training",
action="store_true")
parser.add_argument(
"-p",
help=
"plans identifier. Only change this if you created a custom experiment planner",
default=default_plans_identifier,
required=False)
parser.add_argument(
"--use_compressed_data",
default=False,
action="store_true",
help=
"If you set use_compressed_data, the training cases will not be decompressed. Reading compressed data "
"is much more CPU and RAM intensive and should only be used if you know what you are "
"doing",
required=False)
parser.add_argument(
"--deterministic",
help=
"Makes training deterministic, but reduces training speed substantially. I (Fabian) think "
"this is not necessary. Deterministic training will make you overfit to some random seed. "
"Don't use that.",
required=False,
default=False,
action="store_true")
parser.add_argument("--npz",
required=False,
default=False,
action="store_true",
help="if set then tuframework will "
"export npz files of "
"predicted segmentations "
"in the validation as well. "
"This is needed to run the "
"ensembling step so unless "
"you are developing tuframework "
"you should enable this")
parser.add_argument("--find_lr",
required=False,
default=False,
action="store_true",
help="not used here, just for fun")
parser.add_argument("--valbest",
required=False,
default=False,
action="store_true",
help="hands off. This is not intended to be used")
parser.add_argument(
"--fp32",
required=False,
default=False,
action="store_true",
help="disable mixed precision training and run old school fp32")
parser.add_argument(
"--val_folder",
required=False,
default="validation_raw",
help=
"name of the validation folder. No need to use this for most people")
parser.add_argument(
"--disable_saving",
required=False,
action='store_true',
help=
"If set nnU-Net will not save any parameter files. Useful for development when you are "
"only interested in the results and want to save some disk space")
parser.add_argument(
"--disable_postprocessing_on_folds",
required=False,
action='store_true',
help=
"Running postprocessing on each fold only makes sense when developing with nnU-Net and "
"closely observing the model performance on specific configurations. You do not need it "
"when applying nnU-Net because the postprocessing for this will be determined only once "
"all five folds have been trained and tuframework_find_best_configuration is called. Usually "
"running postprocessing on each fold is computationally cheap, but some users have "
"reported issues with very large images. If your images are large (>600x600x600 voxels) "
"you should consider setting this flag.")
# parser.add_argument("--interp_order", required=False, default=3, type=int,
# help="order of interpolation for segmentations. Testing purpose only. Hands off")
# parser.add_argument("--interp_order_z", required=False, default=0, type=int,
# help="order of interpolation along z if z is resampled separately. Testing purpose only. "
# "Hands off")
# parser.add_argument("--force_separate_z", required=False, default="None", type=str,
# help="force_separate_z resampling. Can be None, True or False. Testing purpose only. Hands off")
sys.argv = ['run_training.py', 'byol', 'byolTrainerV2', '6', 'all']
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
task = args.task
fold = args.fold
network = args.network
network_trainer = args.network_trainer
validation_only = args.validation_only
plans_identifier = args.p
find_lr = args.find_lr
disable_postprocessing_on_folds = args.disable_postprocessing_on_folds
use_compressed_data = args.use_compressed_data
decompress_data = not use_compressed_data
deterministic = args.deterministic
valbest = args.valbest
fp32 = args.fp32
run_mixed_precision = not fp32
val_folder = args.val_folder
# interp_order = args.interp_order
# interp_order_z = args.interp_order_z
# force_separate_z = args.force_separate_z
if not task.startswith("Task"):
task_id = int(task)
task = convert_id_to_task_name(task_id)
if fold == 'all':
pass
else:
fold = int(fold)
# if force_separate_z == "None":
# force_separate_z = None
# elif force_separate_z == "False":
# force_separate_z = False
# elif force_separate_z == "True":
# force_separate_z = True
# else:
# raise ValueError("force_separate_z must be None, True or False. Given: %s" % force_separate_z)
plans_file, output_folder_name, dataset_directory, batch_dice, stage, \
trainer_class = get_default_configuration(network, task, network_trainer, plans_identifier)
if trainer_class is None:
raise RuntimeError(
"Could not find trainer class in tuframework.training.network_training"
)
if network == "3d_cascade_fullres":
assert issubclass(trainer_class, (tuframeworkTrainerCascadeFullRes, tuframeworkTrainerV2CascadeFullRes)), \
"If running 3d_cascade_fullres then your " \
"trainer class must be derived from " \
"tuframeworkTrainerCascadeFullRes"
else:
assert issubclass(
trainer_class, tuframeworkTrainer
), "network_trainer was found but is not derived from tuframeworkTrainer"
trainer = trainer_class(plans_file,
fold,
output_folder=output_folder_name,
dataset_directory=dataset_directory,
batch_dice=batch_dice,
stage=stage,
unpack_data=decompress_data,
deterministic=deterministic,
fp16=run_mixed_precision)
if args.disable_saving:
trainer.save_latest_only = False # if false it will not store/overwrite _latest but separate files each
trainer.save_intermediate_checkpoints = False # whether or not to save checkpoint_latest
trainer.save_best_checkpoint = False # whether or not to save the best checkpoint according to self.best_val_eval_criterion_MA
trainer.save_final_checkpoint = False # whether or not to save the final checkpoint
trainer.initialize(not validation_only)
if find_lr:
trainer.find_lr()
else:
if not validation_only:
if args.continue_training:
trainer.load_latest_checkpoint()
trainer.run_training()
else:
if valbest:
trainer.load_best_checkpoint(train=False)
else:
trainer.load_final_checkpoint(train=False)
trainer.network.eval()
# predict validation
#trainer.validate(save_softmax=args.npz, validation_folder_name=val_folder,
#run_postprocessing_on_folds=not disable_postprocessing_on_folds)
if network == '3d_lowres':
print("predicting segmentations for the next stage of the cascade")
predict_next_stage(
trainer, dataset_directory + "/" +
trainer.plans['data_identifier'] + "_stage%d" % 1)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-i",
'--input_folder',
help="Must contain all modalities for each patient in the correct"
" order (same as training). Files must be named "
"CASENAME_XXXX.nii.gz where XXXX is the modality "
"identifier (0000, 0001, etc)",
required=True)
parser.add_argument('-o',
"--output_folder",
required=True,
help="folder for saving predictions")
parser.add_argument('-t',
'--task_name',
help='task name or task ID, required.',
default=default_plans_identifier,
required=True)
parser.add_argument(
'-tr',
'--trainer_class_name',
help=
'Name of the tuframeworkTrainer used for 2D U-Net, full resolution 3D U-Net and low resolution '
'U-Net. The default is %s. If you are running inference with the cascade and the folder '
'pointed to by --lowres_segmentations does not contain the segmentation maps generated by '
'the low resolution U-Net then the low resolution segmentation maps will be automatically '
'generated. For this case, make sure to set the trainer class here that matches your '
'--cascade_trainer_class_name (this part can be ignored if defaults are used).'
% default_trainer,
required=False,
default=default_trainer)
parser.add_argument(
'-ctr',
'--cascade_trainer_class_name',
help=
"Trainer class name used for predicting the 3D full resolution U-Net part of the cascade."
"Default is %s" % default_cascade_trainer,
required=False,
default=default_cascade_trainer)
parser.add_argument(
'-m',
'--model',
help=
"2d, 3d_lowres, 3d_fullres or 3d_cascade_fullres. Default: 3d_fullres",
default="3d_fullres",
required=False)
parser.add_argument(
'-p',
'--plans_identifier',
help='do not touch this unless you know what you are doing',
default=default_plans_identifier,
required=False)
parser.add_argument(
'-f',
'--folds',
nargs='+',
default='None',
help=
"folds to use for prediction. Default is None which means that folds will be detected "
"automatically in the model output folder")
parser.add_argument(
'-z',
'--save_npz',
required=False,
action='store_true',
help=
"use this if you want to ensemble these predictions with those of other models. Softmax "
"probabilities will be saved as compressed numpy arrays in output_folder and can be "
"merged between output_folders with tuframework_ensemble_predictions")
parser.add_argument(
'-l',
'--lowres_segmentations',
required=False,
default='None',
help=
"if model is the highres stage of the cascade then you can use this folder to provide "
"predictions from the low resolution 3D U-Net. If this is left at default, the "
"predictions will be generated automatically (provided that the 3D low resolution U-Net "
"network weights are present")
parser.add_argument("--part_id",
type=int,
required=False,
default=0,
help="Used to parallelize the prediction of "
"the folder over several GPUs. If you "
"want to use n GPUs to predict this "
"folder you need to run this command "
"n times with --part_id=0, ... n-1 and "
"--num_parts=n (each with a different "
"GPU (for example via "
"CUDA_VISIBLE_DEVICES=X)")
parser.add_argument("--num_parts",
type=int,
required=False,
default=1,
help="Used to parallelize the prediction of "
"the folder over several GPUs. If you "
"want to use n GPUs to predict this "
"folder you need to run this command "
"n times with --part_id=0, ... n-1 and "
"--num_parts=n (each with a different "
"GPU (via "
"CUDA_VISIBLE_DEVICES=X)")
parser.add_argument(
"--num_threads_preprocessing",
required=False,
default=6,
type=int,
help=
"Determines many background processes will be used for data preprocessing. Reduce this if you "
"run into out of memory (RAM) problems. Default: 6")
parser.add_argument(
"--num_threads_nifti_save",
required=False,
default=2,
type=int,
help=
"Determines many background processes will be used for segmentation export. Reduce this if you "
"run into out of memory (RAM) problems. Default: 2")
parser.add_argument(
"--disable_tta",
required=False,
default=False,
action="store_true",
help=
"set this flag to disable test time data augmentation via mirroring. Speeds up inference "
"by roughly factor 4 (2D) or 8 (3D)")
parser.add_argument(
"--overwrite_existing",
required=False,
default=False,
action="store_true",
help=
"Set this flag if the target folder contains predictions that you would like to overwrite"
)
parser.add_argument("--mode",
type=str,
default="normal",
required=False,
help="Hands off!")
parser.add_argument("--all_in_gpu",
type=str,
default="None",
required=False,
help="can be None, False or True. "
"Do not touch.")
parser.add_argument("--step_size",
type=float,
default=0.5,
required=False,
help="don't touch")
# parser.add_argument("--interp_order", required=False, default=3, type=int,
# help="order of interpolation for segmentations, has no effect if mode=fastest. Do not touch this.")
# parser.add_argument("--interp_order_z", required=False, default=0, type=int,
# help="order of interpolation along z is z is done differently. Do not touch this.")
# parser.add_argument("--force_separate_z", required=False, default="None", type=str,
# help="force_separate_z resampling. Can be None, True or False, has no effect if mode=fastest. "
# "Do not touch this.")
parser.add_argument(
'-chk',
help='checkpoint name, default: model_final_checkpoint',
required=False,
default='model_final_checkpoint')
parser.add_argument(
'--disable_mixed_precision',
default=False,
action='store_true',
required=False,
help=
'Predictions are done with mixed precision by default. This improves speed and reduces '
'the required vram. If you want to disable mixed precision you can set this flag. Note '
'that yhis is not recommended (mixed precision is ~2x faster!)')
args = parser.parse_args()
input_folder = args.input_folder
output_folder = args.output_folder
part_id = args.part_id
num_parts = args.num_parts
folds = args.folds
save_npz = args.save_npz
lowres_segmentations = args.lowres_segmentations
num_threads_preprocessing = args.num_threads_preprocessing
num_threads_nifti_save = args.num_threads_nifti_save
disable_tta = args.disable_tta
step_size = args.step_size
# interp_order = args.interp_order
# interp_order_z = args.interp_order_z
# force_separate_z = args.force_separate_z
overwrite_existing = args.overwrite_existing
mode = args.mode
all_in_gpu = args.all_in_gpu
model = args.model
trainer_class_name = args.trainer_class_name
cascade_trainer_class_name = args.cascade_trainer_class_name
task_name = args.task_name
if not task_name.startswith("Task"):
task_id = int(task_name)
task_name = convert_id_to_task_name(task_id)
assert model in ["2d", "3d_lowres", "3d_fullres", "3d_cascade_fullres"], "-m must be 2d, 3d_lowres, 3d_fullres or " \
"3d_cascade_fullres"
# if force_separate_z == "None":
# force_separate_z = None
# elif force_separate_z == "False":
# force_separate_z = False
# elif force_separate_z == "True":
# force_separate_z = True
# else:
# raise ValueError("force_separate_z must be None, True or False. Given: %s" % force_separate_z)
if lowres_segmentations == "None":
lowres_segmentations = None
if isinstance(folds, list):
if folds[0] == 'all' and len(folds) == 1:
pass
else:
folds = [int(i) for i in folds]
elif folds == "None":
folds = None
else:
raise ValueError("Unexpected value for argument folds")
assert all_in_gpu in ['None', 'False', 'True']
if all_in_gpu == "None":
all_in_gpu = None
elif all_in_gpu == "True":
all_in_gpu = True
elif all_in_gpu == "False":
all_in_gpu = False
# we need to catch the case where model is 3d cascade fullres and the low resolution folder has not been set.
# In that case we need to try and predict with 3d low res first
if model == "3d_cascade_fullres" and lowres_segmentations is None:
print(
"lowres_segmentations is None. Attempting to predict 3d_lowres first..."
)
assert part_id == 0 and num_parts == 1, "if you don't specify a --lowres_segmentations folder for the " \
"inference of the cascade, custom values for part_id and num_parts " \
"are not supported. If you wish to have multiple parts, please " \
"run the 3d_lowres inference first (separately)"
model_folder_name = network_training_output_dir + "/" + "3d_lowres" + "/" + task_name + "/" + trainer_class_name + "__" + args.plans_identifier
assert isdir(
model_folder_name
), "model output folder not found. Expected: %s" % model_folder_name
lowres_output_folder = output_folder + "/" + "3d_lowres_predictions"
predict_from_folder(model_folder_name,
input_folder,
lowres_output_folder,
folds,
False,
num_threads_preprocessing,
num_threads_nifti_save,
None,
part_id,
num_parts,
not disable_tta,
overwrite_existing=overwrite_existing,
mode=mode,
overwrite_all_in_gpu=all_in_gpu,
mixed_precision=not args.disable_mixed_precision,
step_size=step_size)
lowres_segmentations = lowres_output_folder
torch.cuda.empty_cache()
print("3d_lowres done")
if model == "3d_cascade_fullres":
trainer = cascade_trainer_class_name
else:
trainer = trainer_class_name
model_folder_name = network_training_output_dir + "/" + model + "/" + task_name + "/" + trainer + "__" + args.plans_identifier
print("using model stored in ", model_folder_name)
assert isdir(
model_folder_name
), "model output folder not found. Expected: %s" % model_folder_name
predict_from_folder(model_folder_name,
input_folder,
output_folder,
folds,
save_npz,
num_threads_preprocessing,
num_threads_nifti_save,
lowres_segmentations,
part_id,
num_parts,
not disable_tta,
overwrite_existing=overwrite_existing,
mode=mode,
overwrite_all_in_gpu=all_in_gpu,
mixed_precision=not args.disable_mixed_precision,
step_size=step_size,
checkpoint_name=args.chk)
def export_entry_point():
import argparse
parser = argparse.ArgumentParser(
description=
"Use this script to export models to a zip file for sharing with "
"others. You can upload the zip file and then either share the url "
"for usage with tuframework_download_pretrained_model_by_url, or share the "
"zip for usage with tuframework_install_pretrained_model_from_zip")
parser.add_argument('-t', type=str, help='task name or task id')
parser.add_argument('-o',
type=str,
help='output file name. Should end with .zip')
parser.add_argument(
'-m',
nargs='+',
help=
'list of model configurations. Default: 2d 3d_lowres 3d_fullres 3d_cascade_fullres. Must '
'be adapted to fit the available models of a task',
default=("2d", "3d_lowres", "3d_fullres", "3d_cascade_fullres"),
required=False)
parser.add_argument(
'-tr',
type=str,
help='trainer class used for 2d 3d_lowres and 3d_fullres. '
'Default: %s' % default_trainer,
required=False,
default=default_trainer)
parser.add_argument('-trc',
type=str,
help='trainer class used for 3d_cascade_fullres. '
'Default: %s' % default_cascade_trainer,
required=False,
default=default_cascade_trainer)
parser.add_argument('-pl',
type=str,
help='tuframework plans identifier. Default: %s' %
default_plans_identifier,
required=False,
default=default_plans_identifier)
parser.add_argument('--disable_strict',
action='store_true',
help='set this if you want to allow skipping '
'missing things',
required=False)
parser.add_argument('-f',
nargs='+',
help='Folds. Default: 0 1 2 3 4',
required=False,
default=[0, 1, 2, 3, 4])
args = parser.parse_args()
folds = args.f
folds = [int(i) if i != 'all' else i for i in folds]
taskname = args.t
if taskname.startswith("Task"):
pass
else:
try:
taskid = int(taskname)
except Exception as e:
print(
'-t must be either a Task name (TaskXXX_YYY) or a task id (integer)'
)
raise e
taskname = convert_id_to_task_name(taskid)
export_pretrained_model(taskname,
args.o,
args.m,
args.tr,
args.trc,
args.pl,
strict=not args.disable_strict,
folds=folds)