def crawl_and_copy(current_folder, out_folder, prefix="fabian_", suffix="ummary.json"): """ This script will run recursively through all subfolders of current_folder and copy all files that end with suffix with some automatically generated prefix into out_folder :param current_folder: :param out_folder: :param prefix: :return: """ s = subdirs(current_folder, join=False) f = subfiles(current_folder, join=False) f = [i for i in f if i.endswith(suffix)] if current_folder.find("fold0") != -1: for fl in f: shutil.copy(os.path.join(current_folder, fl), os.path.join(out_folder, prefix + fl)) for su in s: if prefix == "": add = su else: add = "__" + su crawl_and_copy(os.path.join(current_folder, su), out_folder, prefix=prefix + add)
def plan_and_preprocess(task_string, processes_lowres=default_num_threads, processes_fullres=3, no_preprocessing=False): from tuframework.experiment_planning.experiment_planner_baseline_2DUNet import ExperimentPlanner2D from tuframework.experiment_planning.experiment_planner_baseline_3DUNet import ExperimentPlanner preprocessing_output_dir_this_task_train = preprocessing_output_dir + "/" + task_string cropped_out_dir = tuFramework_cropped_data + "/" + task_string if not os.path.isdir(preprocessing_output_dir_this_task_train): os.makedirs(preprocessing_output_dir_this_task_train) shutil.copy(cropped_out_dir + "/" + "dataset_properties.pkl", preprocessing_output_dir_this_task_train) shutil.copy( tuFramework_raw_data + "/" + task_string + "/" + "dataset.json", preprocessing_output_dir_this_task_train) exp_planner = ExperimentPlanner(cropped_out_dir, preprocessing_output_dir_this_task_train) exp_planner.plan_experiment() if not no_preprocessing: exp_planner.run_preprocessing((processes_lowres, processes_fullres)) exp_planner = ExperimentPlanner2D( cropped_out_dir, preprocessing_output_dir_this_task_train) exp_planner.plan_experiment() if not no_preprocessing: exp_planner.run_preprocessing(processes_fullres) # write which class is in which slice to all training cases (required to speed up 2D Dataloader) # This is done for all data so that if we wanted to use them with 2D we could do so if not no_preprocessing: p = Pool(default_num_threads) # if there is more than one my_data_identifier (different brnaches) then this code will run for all of them if # they start with the same string. not problematic, but not pretty stages = [ i for i in subdirs( preprocessing_output_dir_this_task_train, join=True, sort=True) if i.split("/")[-1].find("stage") != -1 ] for s in stages: print(s.split("/")[-1]) list_of_npz_files = subfiles(s, False, None, ".npz", True) list_of_pkl_files = [i[:-4] + ".pkl" for i in list_of_npz_files] all_classes = [] for pk in list_of_pkl_files: with open(pk, 'rb') as f: props = pickle.load(f) all_classes_tmp = np.array(props['classes']) all_classes.append(all_classes_tmp[all_classes_tmp >= 0]) p.map(add_classes_in_slice_info, zip(list_of_npz_files, list_of_pkl_files, all_classes)) p.close() p.join()