def init_task_option(self, setting_path='../settings/task_settings.json'): """ load task settings from the task setting json some additional settings need to be done if the data manager is not initialized by the data processing json :param setting_path: the path of task setting json :return: ParameterDict, task settings """ par_dataset = pars.ParameterDict() if self.task_root_path: par_dataset.load_JSON(os.path.join(self.task_root_path, 'cur_data_setting.json')) self.task_opt = pars.ParameterDict() self.task_opt.load_JSON(setting_path) #self.task_opt['tsk_set']['extra_info'] = self.get_info_dic() self.task_name = self.task_opt['tsk_set']['task_name'] if self.task_root_path is None: self.task_root_path = self.task_opt['tsk_set']['output_root_path'] par_dataset= self.task_opt['dataset'] task_type = par_dataset[('task_type','reg',"task type can be 'reg' or 'seg'")] self.data_manager.set_task_type(task_type) self.data_manager.set_reg_option(par_dataset) self.data_manager.set_seg_option(par_dataset) self.data_manager.manual_set_task_root_path(self.task_root_path) else: self.task_opt['dataset'] = par_dataset return self.task_opt
def init_aug_env(reg_pair_list_txt, reg_name_list_txt, task_output_path, setting_folder_path): aug_output_path = os.path.join(task_output_path, "aug") os.makedirs(aug_output_path, exist_ok=True) aug_setting_path = os.path.join(setting_folder_path, "data_aug_setting.json") aug_setting = pars.ParameterDict() aug_setting.load_JSON(aug_setting_path) fluid_mode = aug_setting["data_aug"]["fluid_aug"]["fluid_mode"] reg_res_folder_path = os.path.join(task_output_path, "reg/res/records") aug_input_txt = os.path.join(aug_output_path, "aug_input_path.txt") aug_name_txt = os.path.join(aug_output_path, "aug_input_name.txt") affine_path = None if fluid_mode == "aug_with_nonaffined_data": affine_path = reg_res_folder_path if fluid_mode == "aug_with_atlas": aug_setting["data_aug"]["fluid_aug"][ "to_atlas_folder"] = reg_res_folder_path aug_setting["data_aug"]["fluid_aug"][ "atlas_to_folder"] = reg_res_folder_path aug_setting.write_JSON(aug_setting_path) generate_moving_momentum_txt(reg_pair_list_txt, reg_res_folder_path, aug_input_txt, aug_name_txt, reg_name_list_txt, affine_path) return aug_input_txt, aug_name_txt, aug_output_path
def backup_settings(args): """ The settings saved in setting_folder_path/task_name/cur_data_setting.json and setting_folder_path/task_name/cur_task_setting.json :param args: :return: None """ setting_folder_path = args.setting_folder_path dm_json_path = os.path.join(setting_folder_path, 'cur_data_setting.json') tsm_json_path = os.path.join(setting_folder_path, 'cur_task_setting.json') dm = DataTask('task_reg', dm_json_path) if os.path.isfile(dm_json_path) else None tsm = ModelTask('task_reg', tsm_json_path) task_name = args.task_name_record setting_backup = os.path.join(setting_folder_path, task_name + '_backup') os.makedirs(setting_backup, exist_ok=True) dm_backup_json_path = os.path.join(setting_backup, 'cur_data_setting.json') tsm_backup_json_path = os.path.join(setting_backup, 'cur_task_setting.json') if tsm.task_par['tsk_set'][ 'model'] == 'reg_net' and 'mermaid' in tsm.task_par['tsk_set'][ 'method_name']: mermaid_backup_json_path = os.path.join(setting_backup, 'mermaid_nonp_settings.json') mermaid_setting_json = tsm.task_par['tsk_set']['reg']['mermaid_net'][ 'mermaid_net_json_pth'] if len(mermaid_setting_json) == 0: mermaid_setting_json = os.path.join(setting_folder_path, 'mermaid_nonp_settings.json') mermaid_setting = pars.ParameterDict() mermaid_setting.load_JSON(mermaid_setting_json) mermaid_setting.write_ext_JSON(mermaid_backup_json_path) tsm.save(tsm_backup_json_path) if dm is not None: dm.save(dm_backup_json_path)
def get_info_dic(self): """ the info.json include the shared info about the dataset, like label density will be removed in the future release :return: """ data_info = pars.ParameterDict() data_info.load_JSON(os.path.join(self.task_root_path, 'info.json')) return data_info['info']
def __initialize_data_manager(self, setting_path='../settings/data_settings.json'): """ if the data processing file is given , then data manager will be set according to the settings, the data manager prepares the data related environment including preprocessing the data into different sets ( train, val...) assigning dataloaders for different phases( train,val.....) :param setting_path: the path of the settings of the data preprocessing json :return: None """ par_dataset = pars.ParameterDict() par_dataset.load_JSON(setting_path) task_type = par_dataset['datapro'][('task_type','reg',"task type can be 'reg' or 'seg'")] # switch to exist task switch_to_exist_task = par_dataset['datapro']['switch']['switch_to_exist_task'] task_root_path = par_dataset['datapro']['switch']['task_root_path'] # work on current task dataset_name = par_dataset['datapro']['dataset']['dataset_name'] data_pro_task_name = par_dataset['datapro']['dataset']['task_name'] prepare_data = par_dataset['datapro']['dataset']['prepare_data'] data_path = par_dataset['datapro']['dataset']['data_path'] label_path = par_dataset['datapro']['dataset']['label_path'] output_path = par_dataset['datapro']['dataset']['output_path'] data_path = data_path if data_path.ext else None label_path = label_path if label_path.ext else None divided_ratio = par_dataset['datapro']['dataset']['divided_ratio'] # settings for reg sched = par_dataset['datapro']['reg']['sched'] reg_option = par_dataset['datapro']['reg'] self.data_manager = DataManager(task_name=data_pro_task_name, dataset_name=dataset_name) self.data_manager.set_task_type(task_type) if switch_to_exist_task: self.data_manager.manual_set_task_root_path(task_root_path) else: self.data_manager.set_sched(sched) self.data_manager.set_data_path(data_path) self.data_manager.set_output_path(output_path) self.data_manager.set_label_path(label_path) self.data_manager.set_divided_ratio(divided_ratio) # reg self.data_manager.set_reg_option(reg_option) self.data_manager.generate_saving_path(auto=False) self.data_manager.generate_task_path() if prepare_data: self.data_manager.init_dataset() self.data_manager.prepare_data() par_dataset.load_JSON(setting_path) par_dataset.write_ext_JSON(os.path.join(self.data_manager.get_task_root_path(), 'data_settings.json')) task_root_path = self.data_manager.get_task_root_path() self.task_root_path = task_root_path self.data_pro_task_name = self.data_manager.get_full_task_name()
def do_augmentation(input_txt, input_name_txt, setting_folder_path, aug_output_path, gpu_list): aug_setting_path = os.path.join(setting_folder_path, "data_aug_setting.json") mermaid_setting_path = os.path.join(setting_folder_path, "mermaid_nonp_settings.json") assert os.path.isfile( aug_setting_path), "the aug setting json {} is not found".format( aug_setting_path) aug_setting = pars.ParameterDict() aug_setting.load_JSON(aug_setting_path) task_type = aug_setting["data_aug"]["fluid_aug"]["task_type"] num_process = len(gpu_list) if task_type == "rand_sampl": max_aug_num = aug_setting["data_aug"]["max_aug_num"] max_aug_num_per_process = round(max_aug_num / num_process) aug_setting["data_aug"]["max_aug_num"] = max_aug_num_per_process aug_setting_mp_path = os.path.join( setting_folder_path, "data_aug_setting_mutli_process.json") aug_setting.write_ext_JSON(aug_setting_mp_path) processes = [] for i in range(num_process): cmd = "python gen_aug_samples.py " cmd += "-t={} -n={} -as={} -ms={} -o={} -g={}\n".format( input_txt, input_name_txt, aug_setting_mp_path, mermaid_setting_path, aug_output_path, int(gpu_list[i])) p = subprocess.Popen(cmd, shell=True) processes.append(p) time.sleep(1) else: num_process = split_txt(input_txt, num_process, aug_output_path, "aug_p") split_txt(input_name_txt, num_process, aug_output_path, "aug_np") sub_input_txt_list = [ os.path.join(aug_output_path, 'aug_p{}.txt'.format(i)) for i in range(num_process) ] sub_input_name_txt_list = [ os.path.join(aug_output_path, 'aug_np{}.txt'.format(i)) for i in range(num_process) ] processes = [] for i in range(num_process): cmd = "python gen_aug_samples.py " cmd += "-t={} -n={} -as={} -ms={} -o={} -g={}\n".format( sub_input_txt_list[i], sub_input_name_txt_list[i], aug_setting_path, mermaid_setting_path, aug_output_path, int(gpu_list[i])) p = subprocess.Popen(cmd, shell=True) processes.append(p) time.sleep(1) exit_codes = [p.wait() for p in processes]
def save_sz_sp_to_json(info, output_path): """ save img size and img spacing info into json :param info: :param output_path: :return: """ par = pars.ParameterDict() par[('info', {}, 'shared information of data')] par['info'][('img_sz', info['img_size'], 'size of image')] par['info'][('spacing', info['spacing'].tolist(), 'size of image')] if 'num_label' in info: par['info'][('num_label', info['num_label'], 'num of label')] if 'standard_label_index' in info: par['info'][('standard_label_index', info['standard_label_index'], 'standard_label_index')] par.write_JSON(os.path.join(output_path, 'info.json'))
def __init__(self,name,path='../settings/base_task_settings.json'): super(ModelTask,self).__init__(name) self.task_par = pars.ParameterDict() self.task_par.load_JSON(path)
def __init__(self,name,path='../settings/base_data_settings.json'): super(DataTask,self).__init__(name) self.data_par = pars.ParameterDict() self.data_par.load_JSON(path)