def main(): argparser = argparse.ArgumentParser( usage= "Used to determine the postprocessing for a trained model. Useful for " "when the best configuration (2d, 3d_fullres etc) as selected manually." ) argparser.add_argument("-m", type=str, required=True, help="U-Net model (2d, 3d_lowres, 3d_fullres or " "3d_cascade_fullres)") argparser.add_argument("-t", type=str, required=True, help="Task name or id") argparser.add_argument( "-tr", type=str, required=False, default=None, help="tuframeworkTrainer class. Default: %s, unless 3d_cascade_fullres " "(then it's %s)" % (default_trainer, default_cascade_trainer)) argparser.add_argument("-pl", type=str, required=False, default=default_plans_identifier, help="Plans name, Default=%s" % default_plans_identifier) argparser.add_argument( "-val", type=str, required=False, default="validation_raw", help="Validation folder name. Default: validation_raw") args = argparser.parse_args() model = args.m task = args.t trainer = args.tr plans = args.pl val = args.val if not task.startswith("Task"): task_id = int(task) task = convert_id_to_task_name(task_id) if trainer is None: if model == "3d_cascade_fullres": trainer = "tuframeworkTrainerV2CascadeFullRes" else: trainer = "tuframeworkTrainerV2" folder = get_output_folder_name(model, task, trainer, plans, None) consolidate_folds(folder, val)
def export_for_paper(): output_base = "/media/fabian/DeepLearningData/tuframework_trained_models" task_ids = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 24, 27, 29, 35, 48, 55, 61, 38 ] for t in task_ids: if t == 61: models = ("3d_fullres", ) else: models = ("2d", "3d_lowres", "3d_fullres", "3d_cascade_fullres") taskname = convert_id_to_task_name(t) print(taskname) output_folder = output_base + "/" + taskname if not os.path.isdir(output_folder): os.makedirs(output_folder) copy_pretrained_models_for_task(taskname, output_folder, models) copy_ensembles(taskname, output_folder) compress_everything(output_base, 8)
def main(): parser = argparse.ArgumentParser() parser.add_argument("network") parser.add_argument("network_trainer") parser.add_argument("task", help="can be task name or task id") parser.add_argument("fold", help='0, 1, ..., 5 or \'all\'') parser.add_argument("-val", "--validation_only", help="use this if you want to only run the validation", action="store_true") parser.add_argument("-c", "--continue_training", help="use this if you want to continue a training", action="store_true") parser.add_argument( "-p", help= "plans identifier. Only change this if you created a custom experiment planner", default=default_plans_identifier, required=False) parser.add_argument( "--use_compressed_data", default=False, action="store_true", help= "If you set use_compressed_data, the training cases will not be decompressed. Reading compressed data " "is much more CPU and RAM intensive and should only be used if you know what you are " "doing", required=False) parser.add_argument( "--deterministic", help= "Makes training deterministic, but reduces training speed substantially. I (Fabian) think " "this is not necessary. Deterministic training will make you overfit to some random seed. " "Don't use that.", required=False, default=False, action="store_true") parser.add_argument("--npz", required=False, default=False, action="store_true", help="if set then tuframework will " "export npz files of " "predicted segmentations " "in the validation as well. " "This is needed to run the " "ensembling step so unless " "you are developing tuframework " "you should enable this") parser.add_argument("--find_lr", required=False, default=False, action="store_true", help="not used here, just for fun") parser.add_argument("--valbest", required=False, default=False, action="store_true", help="hands off. This is not intended to be used") parser.add_argument( "--fp32", required=False, default=False, action="store_true", help="disable mixed precision training and run old school fp32") parser.add_argument( "--val_folder", required=False, default="validation_raw", help= "name of the validation folder. No need to use this for most people") parser.add_argument( "--disable_saving", required=False, action='store_true', help= "If set nnU-Net will not save any parameter files. Useful for development when you are " "only interested in the results and want to save some disk space") parser.add_argument( "--disable_postprocessing_on_folds", required=False, action='store_true', help= "Running postprocessing on each fold only makes sense when developing with nnU-Net and " "closely observing the model performance on specific configurations. You do not need it " "when applying nnU-Net because the postprocessing for this will be determined only once " "all five folds have been trained and tuframework_find_best_configuration is called. Usually " "running postprocessing on each fold is computationally cheap, but some users have " "reported issues with very large images. If your images are large (>600x600x600 voxels) " "you should consider setting this flag.") # parser.add_argument("--interp_order", required=False, default=3, type=int, # help="order of interpolation for segmentations. Testing purpose only. Hands off") # parser.add_argument("--interp_order_z", required=False, default=0, type=int, # help="order of interpolation along z if z is resampled separately. Testing purpose only. " # "Hands off") # parser.add_argument("--force_separate_z", required=False, default="None", type=str, # help="force_separate_z resampling. Can be None, True or False. Testing purpose only. Hands off") sys.argv = ['run_training.py', 'byol', 'byolTrainerV2', '6', 'all'] args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = "1" task = args.task fold = args.fold network = args.network network_trainer = args.network_trainer validation_only = args.validation_only plans_identifier = args.p find_lr = args.find_lr disable_postprocessing_on_folds = args.disable_postprocessing_on_folds use_compressed_data = args.use_compressed_data decompress_data = not use_compressed_data deterministic = args.deterministic valbest = args.valbest fp32 = args.fp32 run_mixed_precision = not fp32 val_folder = args.val_folder # interp_order = args.interp_order # interp_order_z = args.interp_order_z # force_separate_z = args.force_separate_z if not task.startswith("Task"): task_id = int(task) task = convert_id_to_task_name(task_id) if fold == 'all': pass else: fold = int(fold) # if force_separate_z == "None": # force_separate_z = None # elif force_separate_z == "False": # force_separate_z = False # elif force_separate_z == "True": # force_separate_z = True # else: # raise ValueError("force_separate_z must be None, True or False. Given: %s" % force_separate_z) plans_file, output_folder_name, dataset_directory, batch_dice, stage, \ trainer_class = get_default_configuration(network, task, network_trainer, plans_identifier) if trainer_class is None: raise RuntimeError( "Could not find trainer class in tuframework.training.network_training" ) if network == "3d_cascade_fullres": assert issubclass(trainer_class, (tuframeworkTrainerCascadeFullRes, tuframeworkTrainerV2CascadeFullRes)), \ "If running 3d_cascade_fullres then your " \ "trainer class must be derived from " \ "tuframeworkTrainerCascadeFullRes" else: assert issubclass( trainer_class, tuframeworkTrainer ), "network_trainer was found but is not derived from tuframeworkTrainer" trainer = trainer_class(plans_file, fold, output_folder=output_folder_name, dataset_directory=dataset_directory, batch_dice=batch_dice, stage=stage, unpack_data=decompress_data, deterministic=deterministic, fp16=run_mixed_precision) if args.disable_saving: trainer.save_latest_only = False # if false it will not store/overwrite _latest but separate files each trainer.save_intermediate_checkpoints = False # whether or not to save checkpoint_latest trainer.save_best_checkpoint = False # whether or not to save the best checkpoint according to self.best_val_eval_criterion_MA trainer.save_final_checkpoint = False # whether or not to save the final checkpoint trainer.initialize(not validation_only) if find_lr: trainer.find_lr() else: if not validation_only: if args.continue_training: trainer.load_latest_checkpoint() trainer.run_training() else: if valbest: trainer.load_best_checkpoint(train=False) else: trainer.load_final_checkpoint(train=False) trainer.network.eval() # predict validation #trainer.validate(save_softmax=args.npz, validation_folder_name=val_folder, #run_postprocessing_on_folds=not disable_postprocessing_on_folds) if network == '3d_lowres': print("predicting segmentations for the next stage of the cascade") predict_next_stage( trainer, dataset_directory + "/" + trainer.plans['data_identifier'] + "_stage%d" % 1)
def main(): parser = argparse.ArgumentParser() parser.add_argument( "-i", '--input_folder', help="Must contain all modalities for each patient in the correct" " order (same as training). Files must be named " "CASENAME_XXXX.nii.gz where XXXX is the modality " "identifier (0000, 0001, etc)", required=True) parser.add_argument('-o', "--output_folder", required=True, help="folder for saving predictions") parser.add_argument('-t', '--task_name', help='task name or task ID, required.', default=default_plans_identifier, required=True) parser.add_argument( '-tr', '--trainer_class_name', help= 'Name of the tuframeworkTrainer used for 2D U-Net, full resolution 3D U-Net and low resolution ' 'U-Net. The default is %s. If you are running inference with the cascade and the folder ' 'pointed to by --lowres_segmentations does not contain the segmentation maps generated by ' 'the low resolution U-Net then the low resolution segmentation maps will be automatically ' 'generated. For this case, make sure to set the trainer class here that matches your ' '--cascade_trainer_class_name (this part can be ignored if defaults are used).' % default_trainer, required=False, default=default_trainer) parser.add_argument( '-ctr', '--cascade_trainer_class_name', help= "Trainer class name used for predicting the 3D full resolution U-Net part of the cascade." "Default is %s" % default_cascade_trainer, required=False, default=default_cascade_trainer) parser.add_argument( '-m', '--model', help= "2d, 3d_lowres, 3d_fullres or 3d_cascade_fullres. Default: 3d_fullres", default="3d_fullres", required=False) parser.add_argument( '-p', '--plans_identifier', help='do not touch this unless you know what you are doing', default=default_plans_identifier, required=False) parser.add_argument( '-f', '--folds', nargs='+', default='None', help= "folds to use for prediction. Default is None which means that folds will be detected " "automatically in the model output folder") parser.add_argument( '-z', '--save_npz', required=False, action='store_true', help= "use this if you want to ensemble these predictions with those of other models. Softmax " "probabilities will be saved as compressed numpy arrays in output_folder and can be " "merged between output_folders with tuframework_ensemble_predictions") parser.add_argument( '-l', '--lowres_segmentations', required=False, default='None', help= "if model is the highres stage of the cascade then you can use this folder to provide " "predictions from the low resolution 3D U-Net. If this is left at default, the " "predictions will be generated automatically (provided that the 3D low resolution U-Net " "network weights are present") parser.add_argument("--part_id", type=int, required=False, default=0, help="Used to parallelize the prediction of " "the folder over several GPUs. If you " "want to use n GPUs to predict this " "folder you need to run this command " "n times with --part_id=0, ... n-1 and " "--num_parts=n (each with a different " "GPU (for example via " "CUDA_VISIBLE_DEVICES=X)") parser.add_argument("--num_parts", type=int, required=False, default=1, help="Used to parallelize the prediction of " "the folder over several GPUs. If you " "want to use n GPUs to predict this " "folder you need to run this command " "n times with --part_id=0, ... n-1 and " "--num_parts=n (each with a different " "GPU (via " "CUDA_VISIBLE_DEVICES=X)") parser.add_argument( "--num_threads_preprocessing", required=False, default=6, type=int, help= "Determines many background processes will be used for data preprocessing. Reduce this if you " "run into out of memory (RAM) problems. Default: 6") parser.add_argument( "--num_threads_nifti_save", required=False, default=2, type=int, help= "Determines many background processes will be used for segmentation export. Reduce this if you " "run into out of memory (RAM) problems. Default: 2") parser.add_argument( "--disable_tta", required=False, default=False, action="store_true", help= "set this flag to disable test time data augmentation via mirroring. Speeds up inference " "by roughly factor 4 (2D) or 8 (3D)") parser.add_argument( "--overwrite_existing", required=False, default=False, action="store_true", help= "Set this flag if the target folder contains predictions that you would like to overwrite" ) parser.add_argument("--mode", type=str, default="normal", required=False, help="Hands off!") parser.add_argument("--all_in_gpu", type=str, default="None", required=False, help="can be None, False or True. " "Do not touch.") parser.add_argument("--step_size", type=float, default=0.5, required=False, help="don't touch") # parser.add_argument("--interp_order", required=False, default=3, type=int, # help="order of interpolation for segmentations, has no effect if mode=fastest. Do not touch this.") # parser.add_argument("--interp_order_z", required=False, default=0, type=int, # help="order of interpolation along z is z is done differently. Do not touch this.") # parser.add_argument("--force_separate_z", required=False, default="None", type=str, # help="force_separate_z resampling. Can be None, True or False, has no effect if mode=fastest. " # "Do not touch this.") parser.add_argument( '-chk', help='checkpoint name, default: model_final_checkpoint', required=False, default='model_final_checkpoint') parser.add_argument( '--disable_mixed_precision', default=False, action='store_true', required=False, help= 'Predictions are done with mixed precision by default. This improves speed and reduces ' 'the required vram. If you want to disable mixed precision you can set this flag. Note ' 'that yhis is not recommended (mixed precision is ~2x faster!)') args = parser.parse_args() input_folder = args.input_folder output_folder = args.output_folder part_id = args.part_id num_parts = args.num_parts folds = args.folds save_npz = args.save_npz lowres_segmentations = args.lowres_segmentations num_threads_preprocessing = args.num_threads_preprocessing num_threads_nifti_save = args.num_threads_nifti_save disable_tta = args.disable_tta step_size = args.step_size # interp_order = args.interp_order # interp_order_z = args.interp_order_z # force_separate_z = args.force_separate_z overwrite_existing = args.overwrite_existing mode = args.mode all_in_gpu = args.all_in_gpu model = args.model trainer_class_name = args.trainer_class_name cascade_trainer_class_name = args.cascade_trainer_class_name task_name = args.task_name if not task_name.startswith("Task"): task_id = int(task_name) task_name = convert_id_to_task_name(task_id) assert model in ["2d", "3d_lowres", "3d_fullres", "3d_cascade_fullres"], "-m must be 2d, 3d_lowres, 3d_fullres or " \ "3d_cascade_fullres" # if force_separate_z == "None": # force_separate_z = None # elif force_separate_z == "False": # force_separate_z = False # elif force_separate_z == "True": # force_separate_z = True # else: # raise ValueError("force_separate_z must be None, True or False. Given: %s" % force_separate_z) if lowres_segmentations == "None": lowres_segmentations = None if isinstance(folds, list): if folds[0] == 'all' and len(folds) == 1: pass else: folds = [int(i) for i in folds] elif folds == "None": folds = None else: raise ValueError("Unexpected value for argument folds") assert all_in_gpu in ['None', 'False', 'True'] if all_in_gpu == "None": all_in_gpu = None elif all_in_gpu == "True": all_in_gpu = True elif all_in_gpu == "False": all_in_gpu = False # we need to catch the case where model is 3d cascade fullres and the low resolution folder has not been set. # In that case we need to try and predict with 3d low res first if model == "3d_cascade_fullres" and lowres_segmentations is None: print( "lowres_segmentations is None. Attempting to predict 3d_lowres first..." ) assert part_id == 0 and num_parts == 1, "if you don't specify a --lowres_segmentations folder for the " \ "inference of the cascade, custom values for part_id and num_parts " \ "are not supported. If you wish to have multiple parts, please " \ "run the 3d_lowres inference first (separately)" model_folder_name = network_training_output_dir + "/" + "3d_lowres" + "/" + task_name + "/" + trainer_class_name + "__" + args.plans_identifier assert isdir( model_folder_name ), "model output folder not found. Expected: %s" % model_folder_name lowres_output_folder = output_folder + "/" + "3d_lowres_predictions" predict_from_folder(model_folder_name, input_folder, lowres_output_folder, folds, False, num_threads_preprocessing, num_threads_nifti_save, None, part_id, num_parts, not disable_tta, overwrite_existing=overwrite_existing, mode=mode, overwrite_all_in_gpu=all_in_gpu, mixed_precision=not args.disable_mixed_precision, step_size=step_size) lowres_segmentations = lowres_output_folder torch.cuda.empty_cache() print("3d_lowres done") if model == "3d_cascade_fullres": trainer = cascade_trainer_class_name else: trainer = trainer_class_name model_folder_name = network_training_output_dir + "/" + model + "/" + task_name + "/" + trainer + "__" + args.plans_identifier print("using model stored in ", model_folder_name) assert isdir( model_folder_name ), "model output folder not found. Expected: %s" % model_folder_name predict_from_folder(model_folder_name, input_folder, output_folder, folds, save_npz, num_threads_preprocessing, num_threads_nifti_save, lowres_segmentations, part_id, num_parts, not disable_tta, overwrite_existing=overwrite_existing, mode=mode, overwrite_all_in_gpu=all_in_gpu, mixed_precision=not args.disable_mixed_precision, step_size=step_size, checkpoint_name=args.chk)
def export_entry_point(): import argparse parser = argparse.ArgumentParser( description= "Use this script to export models to a zip file for sharing with " "others. You can upload the zip file and then either share the url " "for usage with tuframework_download_pretrained_model_by_url, or share the " "zip for usage with tuframework_install_pretrained_model_from_zip") parser.add_argument('-t', type=str, help='task name or task id') parser.add_argument('-o', type=str, help='output file name. Should end with .zip') parser.add_argument( '-m', nargs='+', help= 'list of model configurations. Default: 2d 3d_lowres 3d_fullres 3d_cascade_fullres. Must ' 'be adapted to fit the available models of a task', default=("2d", "3d_lowres", "3d_fullres", "3d_cascade_fullres"), required=False) parser.add_argument( '-tr', type=str, help='trainer class used for 2d 3d_lowres and 3d_fullres. ' 'Default: %s' % default_trainer, required=False, default=default_trainer) parser.add_argument('-trc', type=str, help='trainer class used for 3d_cascade_fullres. ' 'Default: %s' % default_cascade_trainer, required=False, default=default_cascade_trainer) parser.add_argument('-pl', type=str, help='tuframework plans identifier. Default: %s' % default_plans_identifier, required=False, default=default_plans_identifier) parser.add_argument('--disable_strict', action='store_true', help='set this if you want to allow skipping ' 'missing things', required=False) parser.add_argument('-f', nargs='+', help='Folds. Default: 0 1 2 3 4', required=False, default=[0, 1, 2, 3, 4]) args = parser.parse_args() folds = args.f folds = [int(i) if i != 'all' else i for i in folds] taskname = args.t if taskname.startswith("Task"): pass else: try: taskid = int(taskname) except Exception as e: print( '-t must be either a Task name (TaskXXX_YYY) or a task id (integer)' ) raise e taskname = convert_id_to_task_name(taskid) export_pretrained_model(taskname, args.o, args.m, args.tr, args.trc, args.pl, strict=not args.disable_strict, folds=folds)