def init_task_option(self, setting_path='../settings/task_settings.json'):
"""
load task settings from the task setting json
some additional settings need to be done if the data manager is not initialized by the data processing json
:param setting_path: the path of task setting json
:return: ParameterDict, task settings
"""
par_dataset = pars.ParameterDict()
if self.task_root_path:
par_dataset.load_JSON(os.path.join(self.task_root_path, 'cur_data_setting.json'))
self.task_opt = pars.ParameterDict()
self.task_opt.load_JSON(setting_path)
#self.task_opt['tsk_set']['extra_info'] = self.get_info_dic()
self.task_name = self.task_opt['tsk_set']['task_name']
if self.task_root_path is None:
self.task_root_path = self.task_opt['tsk_set']['output_root_path']
par_dataset= self.task_opt['dataset']
task_type = par_dataset[('task_type','reg',"task type can be 'reg' or 'seg'")]
self.data_manager.set_task_type(task_type)
self.data_manager.set_reg_option(par_dataset)
self.data_manager.set_seg_option(par_dataset)
self.data_manager.manual_set_task_root_path(self.task_root_path)
else:
self.task_opt['dataset'] = par_dataset
return self.task_opt
def init_aug_env(reg_pair_list_txt, reg_name_list_txt, task_output_path,
setting_folder_path):
aug_output_path = os.path.join(task_output_path, "aug")
os.makedirs(aug_output_path, exist_ok=True)
aug_setting_path = os.path.join(setting_folder_path,
"data_aug_setting.json")
aug_setting = pars.ParameterDict()
aug_setting.load_JSON(aug_setting_path)
fluid_mode = aug_setting["data_aug"]["fluid_aug"]["fluid_mode"]
reg_res_folder_path = os.path.join(task_output_path, "reg/res/records")
aug_input_txt = os.path.join(aug_output_path, "aug_input_path.txt")
aug_name_txt = os.path.join(aug_output_path, "aug_input_name.txt")
affine_path = None
if fluid_mode == "aug_with_nonaffined_data":
affine_path = reg_res_folder_path
if fluid_mode == "aug_with_atlas":
aug_setting["data_aug"]["fluid_aug"][
"to_atlas_folder"] = reg_res_folder_path
aug_setting["data_aug"]["fluid_aug"][
"atlas_to_folder"] = reg_res_folder_path
aug_setting.write_JSON(aug_setting_path)
generate_moving_momentum_txt(reg_pair_list_txt, reg_res_folder_path,
aug_input_txt, aug_name_txt,
reg_name_list_txt, affine_path)
return aug_input_txt, aug_name_txt, aug_output_path
def backup_settings(args):
"""
The settings saved in setting_folder_path/task_name/cur_data_setting.json and setting_folder_path/task_name/cur_task_setting.json
:param args:
:return: None
"""
setting_folder_path = args.setting_folder_path
dm_json_path = os.path.join(setting_folder_path, 'cur_data_setting.json')
tsm_json_path = os.path.join(setting_folder_path, 'cur_task_setting.json')
dm = DataTask('task_reg',
dm_json_path) if os.path.isfile(dm_json_path) else None
tsm = ModelTask('task_reg', tsm_json_path)
task_name = args.task_name_record
setting_backup = os.path.join(setting_folder_path, task_name + '_backup')
os.makedirs(setting_backup, exist_ok=True)
dm_backup_json_path = os.path.join(setting_backup, 'cur_data_setting.json')
tsm_backup_json_path = os.path.join(setting_backup,
'cur_task_setting.json')
if tsm.task_par['tsk_set'][
'model'] == 'reg_net' and 'mermaid' in tsm.task_par['tsk_set'][
'method_name']:
mermaid_backup_json_path = os.path.join(setting_backup,
'mermaid_nonp_settings.json')
mermaid_setting_json = tsm.task_par['tsk_set']['reg']['mermaid_net'][
'mermaid_net_json_pth']
if len(mermaid_setting_json) == 0:
mermaid_setting_json = os.path.join(setting_folder_path,
'mermaid_nonp_settings.json')
mermaid_setting = pars.ParameterDict()
mermaid_setting.load_JSON(mermaid_setting_json)
mermaid_setting.write_ext_JSON(mermaid_backup_json_path)
tsm.save(tsm_backup_json_path)
if dm is not None:
dm.save(dm_backup_json_path)
def get_info_dic(self):
"""
the info.json include the shared info about the dataset, like label density
will be removed in the future release
:return:
"""
data_info = pars.ParameterDict()
data_info.load_JSON(os.path.join(self.task_root_path, 'info.json'))
return data_info['info']
def __initialize_data_manager(self, setting_path='../settings/data_settings.json'):
"""
if the data processing file is given , then data manager will be set according to the settings,
the data manager prepares the data related environment including preprocessing the data into different sets ( train, val...)
assigning dataloaders for different phases( train,val.....)
:param setting_path: the path of the settings of the data preprocessing json
:return: None
"""
par_dataset = pars.ParameterDict()
par_dataset.load_JSON(setting_path)
task_type = par_dataset['datapro'][('task_type','reg',"task type can be 'reg' or 'seg'")]
# switch to exist task
switch_to_exist_task = par_dataset['datapro']['switch']['switch_to_exist_task']
task_root_path = par_dataset['datapro']['switch']['task_root_path']
# work on current task
dataset_name = par_dataset['datapro']['dataset']['dataset_name']
data_pro_task_name = par_dataset['datapro']['dataset']['task_name']
prepare_data = par_dataset['datapro']['dataset']['prepare_data']
data_path = par_dataset['datapro']['dataset']['data_path']
label_path = par_dataset['datapro']['dataset']['label_path']
output_path = par_dataset['datapro']['dataset']['output_path']
data_path = data_path if data_path.ext else None
label_path = label_path if label_path.ext else None
divided_ratio = par_dataset['datapro']['dataset']['divided_ratio']
# settings for reg
sched = par_dataset['datapro']['reg']['sched']
reg_option = par_dataset['datapro']['reg']
self.data_manager = DataManager(task_name=data_pro_task_name, dataset_name=dataset_name)
self.data_manager.set_task_type(task_type)
if switch_to_exist_task:
self.data_manager.manual_set_task_root_path(task_root_path)
else:
self.data_manager.set_sched(sched)
self.data_manager.set_data_path(data_path)
self.data_manager.set_output_path(output_path)
self.data_manager.set_label_path(label_path)
self.data_manager.set_divided_ratio(divided_ratio)
# reg
self.data_manager.set_reg_option(reg_option)
self.data_manager.generate_saving_path(auto=False)
self.data_manager.generate_task_path()
if prepare_data:
self.data_manager.init_dataset()
self.data_manager.prepare_data()
par_dataset.load_JSON(setting_path)
par_dataset.write_ext_JSON(os.path.join(self.data_manager.get_task_root_path(), 'data_settings.json'))
task_root_path = self.data_manager.get_task_root_path()
self.task_root_path = task_root_path
self.data_pro_task_name = self.data_manager.get_full_task_name()
def do_augmentation(input_txt, input_name_txt, setting_folder_path,
aug_output_path, gpu_list):
aug_setting_path = os.path.join(setting_folder_path,
"data_aug_setting.json")
mermaid_setting_path = os.path.join(setting_folder_path,
"mermaid_nonp_settings.json")
assert os.path.isfile(
aug_setting_path), "the aug setting json {} is not found".format(
aug_setting_path)
aug_setting = pars.ParameterDict()
aug_setting.load_JSON(aug_setting_path)
task_type = aug_setting["data_aug"]["fluid_aug"]["task_type"]
num_process = len(gpu_list)
if task_type == "rand_sampl":
max_aug_num = aug_setting["data_aug"]["max_aug_num"]
max_aug_num_per_process = round(max_aug_num / num_process)
aug_setting["data_aug"]["max_aug_num"] = max_aug_num_per_process
aug_setting_mp_path = os.path.join(
setting_folder_path, "data_aug_setting_mutli_process.json")
aug_setting.write_ext_JSON(aug_setting_mp_path)
processes = []
for i in range(num_process):
cmd = "python gen_aug_samples.py "
cmd += "-t={} -n={} -as={} -ms={} -o={} -g={}\n".format(
input_txt, input_name_txt, aug_setting_mp_path,
mermaid_setting_path, aug_output_path, int(gpu_list[i]))
p = subprocess.Popen(cmd, shell=True)
processes.append(p)
time.sleep(1)
else:
num_process = split_txt(input_txt, num_process, aug_output_path,
"aug_p")
split_txt(input_name_txt, num_process, aug_output_path, "aug_np")
sub_input_txt_list = [
os.path.join(aug_output_path, 'aug_p{}.txt'.format(i))
for i in range(num_process)
]
sub_input_name_txt_list = [
os.path.join(aug_output_path, 'aug_np{}.txt'.format(i))
for i in range(num_process)
]
processes = []
for i in range(num_process):
cmd = "python gen_aug_samples.py "
cmd += "-t={} -n={} -as={} -ms={} -o={} -g={}\n".format(
sub_input_txt_list[i], sub_input_name_txt_list[i],
aug_setting_path, mermaid_setting_path, aug_output_path,
int(gpu_list[i]))
p = subprocess.Popen(cmd, shell=True)
processes.append(p)
time.sleep(1)
exit_codes = [p.wait() for p in processes]
def save_sz_sp_to_json(info, output_path):
"""
save img size and img spacing info into json
:param info:
:param output_path:
:return:
"""
par = pars.ParameterDict()
par[('info', {}, 'shared information of data')]
par['info'][('img_sz', info['img_size'], 'size of image')]
par['info'][('spacing', info['spacing'].tolist(), 'size of image')]
if 'num_label' in info:
par['info'][('num_label', info['num_label'], 'num of label')]
if 'standard_label_index' in info:
par['info'][('standard_label_index', info['standard_label_index'], 'standard_label_index')]
par.write_JSON(os.path.join(output_path, 'info.json'))
def __init__(self,name,path='../settings/base_task_settings.json'):
super(ModelTask,self).__init__(name)
self.task_par = pars.ParameterDict()
self.task_par.load_JSON(path)
def __init__(self,name,path='../settings/base_data_settings.json'):
super(DataTask,self).__init__(name)
self.data_par = pars.ParameterDict()
self.data_par.load_JSON(path)
