def main(): parser = argparse.ArgumentParser(description='Load a saved pca object') parser.add_argument('--load-pca-bin-path', type=str, help='Location of the pca bin') parser.add_argument('--in-folder', type=str) parser.add_argument('--out-folder', type=str) parser.add_argument('--mask-img', type=str) parser.add_argument('--num-mode', type=int) parser.add_argument('--file-list-txt', type=str) parser.add_argument('--num-process', type=int, default=10) args = parser.parse_args() in_folder_obj = DataFolder(args.in_folder, args.file_list_txt) out_folder_obj = DataFolder(args.out_folder, args.file_list_txt) mask_img_obj = ScanWrapper(args.mask_img) paral_obj = ParalGetResidual( args.load_pca_bin_path, in_folder_obj, out_folder_obj, mask_img_obj, args.num_mode, args.num_process ) paral_obj.run_parallel()
def __init__(self, config, in_jac_folder, in_affine_mat_folder, out_folder, ref_img, file_list_txt): super().__init__(config, in_jac_folder, file_list_txt) self._ref_img = ScanWrapper(ref_img) self._in_affine_mat_folder = DataFolder.get_data_folder_obj( config, in_affine_mat_folder, data_list_txt=file_list_txt) self._out_data_folder = DataFolder.get_data_folder_obj( config, out_folder, data_list_txt=file_list_txt)
def __init__(self, config, in_folder=None, data_file_txt=None, in_data_folder_obj=None): self._data_folder = None if in_data_folder_obj is None: self._data_folder = DataFolder(in_folder, data_file_txt) else: self._data_folder = in_data_folder_obj self._standard_ref = ScanWrapper(self._data_folder.get_first_path()) self._num_processes = config['num_processes']
def __init__(self, config, in_ori_folder, in_jac_folder, batch_size, file_list_txt=None): super().__init__(config, in_ori_folder, file_list_txt) self._in_jac_folder = DataFolder(in_jac_folder, file_list_txt) self._ref_ori = ScanWrapper(self._in_data_folder.get_file_path(0)) self._ref_jac = ScanWrapper(self._in_jac_folder.get_file_path(0)) self._chunk_list = self._in_data_folder.get_chunks_list_batch_size( batch_size) self._data_matrix = [] self._cur_idx = 0
def get_csv(args): file_list = read_file_contents_list(args.file_list_txt) in_ori_folder_obj = DataFolder(args.in_ori_folder, file_list) in_mask_folder_obj = DataFolder(args.in_mask_folder, file_list) exe_obj = MeanIntensityMask(in_ori_folder_obj, in_mask_folder_obj, [2, 4], 20) result_dict_list = exe_obj.run_parallel() result_df = pd.DataFrame(result_dict_list) result_df = result_df.set_index('file_name') print(f'Output csv to {args.out_csv}') result_df.to_csv(args.out_csv)
def _get_slices_dim_list(pc_folder, file_list_txt): center_slices_x = [] center_slices_y = [] center_slices_z = [] data_folder = DataFolder(pc_folder, file_list_txt) for file_idx in range(show_pc_number): data_folder.print_idx(file_idx) file_path = data_folder.get_file_path(file_idx) scan = ScanWrapper(file_path) slice_x, slice_y, slice_z = scan.get_center_slices() center_slices_x.append(slice_x) center_slices_y.append(slice_y) center_slices_z.append(slice_z) return center_slices_x, center_slices_y, center_slices_z
def __init__(self, config, in_dat_folder, out_folder, ref_img, file_list_txt): super().__init__(config, in_dat_folder, file_list_txt) self._ref_img = ScanWrapper(ref_img) self._out_data_folder = DataFolder.get_data_folder_obj( config, out_folder, data_list_txt=file_list_txt) self._jacobian_tool = config['deedsBCV_jacobian']
def main(): parser = argparse.ArgumentParser(description='ThoraxPCA') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--save-path', type=str, help='Output location for union average', required=True) parser.add_argument('--data-file-list', type=str, help='Data file list') args = parser.parse_args() config = load_config(args.config) average_tool = None if args.data_file_list: data_folder_obj = DataFolder(args.in_folder, args.data_file_list) average_tool = AverageScans(config, in_data_folder_obj=data_folder_obj) else: average_tool = AverageScans(config, args.in_folder) average_tool.get_average_image_union(args.save_path)
def __init__(self, config, in_folder, mask_img, out_png_folder, file_list_txt): super().__init__(config, in_folder, file_list_txt) self._mask_img = ScanWrapper(mask_img) self._out_png_folder = DataFolder.get_data_folder_obj(config, out_png_folder, data_list_txt=file_list_txt) self._out_png_folder.change_suffix('.png') self._vmax = 500 self._vmin = -1000
def __init__(self, config, in_folder, out_folder, ref_img, order=3): super().__init__(config, in_folder) self._out_data_folder = DataFolder.get_data_folder_obj( config, out_folder) self._ref_img = ScanWrapper(ref_img) self._order = order self._reg_resample = config['niftyreg_resample'] self._reg_resmaple_ref = config['reg_resample_ref_img']
def __init__(self, config, in_folder, out_folder, average_img, file_list_txt=None): super().__init__(config, in_folder, file_list_txt=file_list_txt) self._out_data_folder = DataFolder.get_data_folder_obj( config, out_folder, data_list_txt=file_list_txt) mkdir_p(out_folder) self._average_img = ScanWrapper(average_img)
def main(): parser = argparse.ArgumentParser(description='Load a saved pca object') parser.add_argument('--load-pca-bin-path', type=str, help='Location of the pca bin') parser.add_argument('--in-ori-folder', type=str) parser.add_argument('--in-jac-folder', type=str) parser.add_argument('--file-list-txt', type=str) parser.add_argument('--save-bin-path', type=str) parser.add_argument('--num-process', type=int, default=10) args = parser.parse_args() in_ori_folder_obj = DataFolder(args.in_ori_folder, args.file_list_txt) in_jac_folder_obj = DataFolder(args.in_jac_folder, args.file_list_txt) paral_obj = ParalDimensionReductionConcat( args.load_pca_bin_path, in_ori_folder_obj, in_jac_folder_obj, args.num_process) paral_obj.run_dimension_reduction(args.save_bin_path)
def __init__(self, config, in_scan_folder, in_region_mask_folder, file_list_txt, out_average_img, ambient=-5000): super().__init__(config, in_scan_folder, file_list_txt) self._in_region_mask_folder = DataFolder.get_data_folder_obj( config, in_region_mask_folder, data_list_txt=file_list_txt) self._out_img_path = out_average_img self._ref_img = ScanWrapper(self._in_data_folder.get_first_path()) self._ambient = ambient
def __init__(self, config, in_folder, out_folder, roi_img, file_list_txt=None): super().__init__(config, in_folder, file_list_txt=file_list_txt) # self._out_data_folder = DataFolder.get_data_folder_obj(config, out_folder, data_list_txt=file_list_txt) self._out_data_folder = DataFolder.get_data_folder_obj_with_list( out_folder, self._in_data_folder.get_data_file_list()) mkdir_p(out_folder) self._roi_img_path = roi_img self._reg_resample_path = config['niftyreg_resample']
def main(): # yaml_config = 'simg_bmi_regression_3.6.3_nfs.yaml' yaml_config = 'simg_bmi_regression_3.6.4_nfs.yaml' # yaml_config = 'simg_bmi_regression_3.6.6.1_nfs.yaml' SRC_ROOT = os.path.dirname(os.path.realpath(__file__)) + '/..' yaml_config = os.path.join(SRC_ROOT, f'src/yaml/{yaml_config}') logger.info(f'Read yaml file {yaml_config}') f = open(yaml_config, 'r').read() config = yaml.safe_load(f) num_fold = config['fold_num'] exp_dir = config['exp_dir'] layer_flag = config['gcam_target_layer'] gcam_overlay_out_folder = os.path.join(exp_dir, 'gcam_overlay') gcam_link_out_folder = os.path.join(exp_dir, 'gcam_link') mkdir_p(gcam_overlay_out_folder) mkdir_p(gcam_link_out_folder) pred_result_csv = os.path.join(exp_dir, f'pred_total.csv') file_list = pd.read_csv(pred_result_csv, index_col=False)['file_name'] in_int_folder_obj = DataFolder(atlas_intensity_folder, file_list) in_jac_folder_obj = DataFolder(atlas_jacobian_folder, file_list) in_att_folder_obj = DataFolder(gcam_link_out_folder, file_list) paral_plot_obj = ParaPlotClip( in_int_folder_obj, in_jac_folder_obj, in_att_folder_obj, gcam_overlay_out_folder, 10 ) paral_plot_obj.run_parallel()
def main(): parser = argparse.ArgumentParser(description='Load a saved pca object') parser.add_argument('--pc-folder', type=str, help='Location of the principle feature images') parser.add_argument('--file-list-txt', type=str) parser.add_argument('--out-png', type=str) parser.add_argument('--step-axial', type=int, default=50) parser.add_argument('--step-sagittal', type=int, default=75) parser.add_argument('--step-coronal', type=int, default=30) parser.add_argument('--num-show-pc', type=int, default=10) args = parser.parse_args() pc_folder_obj = DataFolder(args.pc_folder, args.file_list_txt) plt_obj = PlotPCGrid(pc_folder_obj, args.step_axial, args.step_sagittal, args.step_coronal, args.num_show_pc) plt_obj.plot_pc(args.out_png)
class GetLossBetweenFolder(AbstractParallelRoutine): def __init__(self, config, in_folder_1, in_folder_2, file_list_txt): super().__init__(config, in_folder_1, file_list_txt) self._in_data_folder_2 = DataFolder(in_folder_2, file_list_txt) self._nrmse_diff = [] def get_nrmse(self): return self._nrmse_diff def print_file_list(self): file_list = self._in_data_folder.get_data_file_list() for idx in range(len(file_list)): print(f'The {idx}th file is {file_list[idx]}') def _run_single_scan(self, idx): in_file_1_path = self._in_data_folder.get_file_path(idx) in_file_2_path = self._in_data_folder_2.get_file_path(idx) in_img_1 = ScanWrapper(in_file_1_path).get_data() in_img_2 = ScanWrapper(in_file_2_path).get_data() nrmse = compare_nrmse(np.abs(in_img_1), np.abs(in_img_2)) self._nrmse_diff.append(nrmse)
class AbstractParallelRoutine: def __init__(self, config, in_folder, file_list=None): self._config = config self._in_data_folder = DataFolder(in_folder, file_list) # self._num_processes = config['num_processes'] self._num_processes = 1 def run_parallel(self): pool = Pool(processes=self._num_processes) chunk_list = self._in_data_folder.get_chunks_list(self._num_processes) result_obj_list = [ pool.apply_async(self._run_chunk, (file_idx_chunk, )) for file_idx_chunk in chunk_list ] result_list = [] for thread_idx in range(len(result_obj_list)): result_obj = result_obj_list[thread_idx] result_obj.wait() logger.info( f'Thread with idx {thread_idx} / {len(result_obj_list)} is completed' ) result_list = result_list + result_obj.get() return result_list def run_non_parallel(self, idx_list=None): run_idx_list = idx_list if idx_list else range(self.num_files()) for idx in run_idx_list: self._in_data_folder.print_idx(idx) self._run_single_scan(idx) # print('Done', flush=True) def _run_single_scan(self, idx): raise NotImplementedError def _run_chunk(self, chunk_list): result_list = [] for idx in chunk_list: self._in_data_folder.print_idx(idx) result = self._run_single_scan(idx) result_list.append(result) return result_list def num_files(self): return self._in_data_folder.num_files()
def get_data_dict(config, file_list_txt): task = config['task'] in_folder = config['input_img_dir'] label_csv = config['label_csv'] in_folder_obj = DataFolder(in_folder, read_file_contents_list(file_list_txt)) file_list = in_folder_obj.get_data_file_list() clinical_reader = ClinicalDataReaderSPORE.create_spore_data_reader_csv( label_csv) label_array = None file_list_with_valid_label = None if task == 'BMI': label_array, file_list_with_valid_label = clinical_reader.get_gt_value_BMI( file_list) subject_list = [ ClinicalDataReaderSPORE._get_subject_id_from_file_name(file_name) for file_name in file_list_with_valid_label ] in_folder_obj.set_file_list(file_list_with_valid_label) file_path_list = in_folder_obj.get_file_path_list() data_dict = { 'img_names': file_list_with_valid_label, 'img_subs': subject_list, 'img_files': file_path_list, 'gt_val': label_array } if config['add_jacobian_map']: in_jacobian_folder = config['input_jac_dir'] in_jacobian_folder_obj = DataFolder(in_jacobian_folder, file_list_with_valid_label) jacobian_map_path_list = in_jacobian_folder_obj.get_file_path_list() data_dict['jacobian_maps'] = jacobian_map_path_list if config['add_valid_mask_map'] | config['apply_random_valid_mask']: in_valid_mask_folder = config['input_valid_mask_dir'] in_valid_mask_folder_obj = DataFolder(in_valid_mask_folder, file_list_with_valid_label) valid_mask_path_list = in_valid_mask_folder_obj.get_file_path_list() data_dict['valid_masks'] = valid_mask_path_list if config['add_d_index_map']: in_d_index_map_folder = config['input_d_index_dir'] in_d_index_map_folder_obj = DataFolder(in_d_index_map_folder, file_list_with_valid_label) d_index_map_path_list = in_d_index_map_folder_obj.get_file_path_list() data_dict['d_index_maps'] = d_index_map_path_list if config['add_jac_elem_maps']: in_jac_elem_folder = config['input_jac_elem_dir'] in_jac_elem_folder_obj = DataFolder(in_jac_elem_folder, file_list_with_valid_label) for idx_elem in range(9): in_jac_elem_path_list = [ map_path.replace('.nii.gz', f'_{idx_elem}.nii.gz') for map_path in in_jac_elem_folder_obj.get_file_path_list() ] data_dict[f'jac_elem_{idx_elem}_map'] = in_jac_elem_path_list return data_dict
class ScanFolderConcatBatchReader(AbstractParallelRoutine): def __init__(self, config, in_ori_folder, in_jac_folder, batch_size, file_list_txt=None): super().__init__(config, in_ori_folder, file_list_txt) self._in_jac_folder = DataFolder(in_jac_folder, file_list_txt) self._ref_ori = ScanWrapper(self._in_data_folder.get_file_path(0)) self._ref_jac = ScanWrapper(self._in_jac_folder.get_file_path(0)) self._chunk_list = self._in_data_folder.get_chunks_list_batch_size( batch_size) self._data_matrix = [] self._cur_idx = 0 def read_data(self, idx_batch): self._reset_cur_idx() print(f'Reading scans from folder {self._in_data_folder.get_folder()}', flush=True) tic = time.perf_counter() cur_batch = self._chunk_list[idx_batch] self._init_data_matrix(len(cur_batch)) self.run_non_parallel(cur_batch) toc = time.perf_counter() print(f'Done. {toc - tic:0.4f} (s)', flush=True) def num_batch(self): return len(self._chunk_list) def get_data_matrix(self): return self._data_matrix def save_flat_data(self, data_array, idx, out_folder): out_path_ori = os.path.join(out_folder, f'pc_ori_{idx}.nii.gz') out_path_jac = os.path.join(out_folder, f'pc_jac_{idx}.nii.gz') ori_data_flat = data_array[:self._ref_ori.get_number_voxel()] jac_data_flat = data_array[self._ref_ori.get_number_voxel():] self._ref_ori.save_scan_flat_img(ori_data_flat, out_path_ori) self._ref_jac.save_scan_flat_img(jac_data_flat, out_path_jac) def get_ref(self): return self._ref_ori def _run_single_scan(self, idx): in_ori_data = ScanWrapper( self._in_data_folder.get_file_path(idx)).get_data() in_jac_data = ScanWrapper( self._in_jac_folder.get_file_path(idx)).get_data() self._data_matrix[self._cur_idx, :self._ref_ori.get_number_voxel( )] = convert_3d_2_flat(in_ori_data) self._data_matrix[ self._cur_idx, self._ref_ori.get_number_voxel():] = convert_3d_2_flat(in_jac_data) self._cur_idx += 1 def _init_data_matrix(self, num_sample): num_features = self._get_number_of_voxel() del self._data_matrix self._data_matrix = np.zeros((num_sample, num_features)) def _get_number_of_voxel(self): return self._get_number_of_voxel_ori() + self._get_number_of_voxel_jac( ) def _get_number_of_voxel_ori(self): return self._ref_ori.get_number_voxel() def _get_number_of_voxel_jac(self): return self._ref_jac.get_number_voxel() def _reset_cur_idx(self): self._cur_idx = 0
def __init__(self, config, in_folder, file_list=None): self._config = config self._in_data_folder = DataFolder(in_folder, file_list) # self._num_processes = config['num_processes'] self._num_processes = 1
class AverageScans: def __init__(self, config, in_folder=None, data_file_txt=None, in_data_folder_obj=None): self._data_folder = None if in_data_folder_obj is None: self._data_folder = DataFolder(in_folder, data_file_txt) else: self._data_folder = in_data_folder_obj self._standard_ref = ScanWrapper(self._data_folder.get_first_path()) self._num_processes = config['num_processes'] def get_average_image_union(self, save_path): im_shape = self._get_std_shape() average_union = np.zeros(im_shape) average_union.fill(np.nan) non_null_mask_count_image = np.zeros(im_shape) chunk_list = self._data_folder.get_chunks_list(self._num_processes) pool = Pool(processes=self._num_processes) print('Average in union') print('Step.1 Summation') image_average_union_result_list = [ pool.apply_async(self._sum_images_union, (file_idx_chunk, )) for file_idx_chunk in chunk_list ] for thread_idx in range(len(image_average_union_result_list)): result = image_average_union_result_list[thread_idx] result.wait() print( f'Thread with idx {thread_idx} / {len(image_average_union_result_list)} is completed' ) print('Adding to averaged_image...') averaged_image_chunk = result.get() average_union = self._add_image_union(average_union, averaged_image_chunk) print('Done.') print('Step.2 Non-nan counter') non_null_mask_count_result = [ pool.apply_async(self._sum_non_null_count, (file_idx_chunk, )) for file_idx_chunk in chunk_list ] for thread_idx in range(len(non_null_mask_count_result)): result = non_null_mask_count_result[thread_idx] result.wait() print( f'Thread with idx {thread_idx} / {len(non_null_mask_count_result)} is completed' ) print('Adding to averaged_image...') averaged_image_chunk = result.get() non_null_mask_count_image = np.add(non_null_mask_count_image, averaged_image_chunk) print('Done.') average_union = np.divide(average_union, non_null_mask_count_image, out=average_union, where=non_null_mask_count_image > 0) self._standard_ref.save_scan_same_space(save_path, average_union) print('Done.') def _sum_images_union(self, chunk_list): print('Sum images, union non-null region. Loading images...') im_shape = self._get_std_shape() sum_image = np.zeros(im_shape) sum_image.fill(np.nan) for id_file in chunk_list: file_path = self._data_folder.get_file_path(id_file) self._data_folder.print_idx(id_file) im = nib.load(file_path) im_data = im.get_data() sum_image = self._add_image_union(sum_image, im_data) return sum_image def _sum_non_null_count(self, chunk_list): print('Count non-null per voxel. Loading images...') im_shape = self._get_std_shape() sum_image = np.zeros(im_shape) for id_file in chunk_list: file_path = self._data_folder.get_file_path(id_file) self._data_folder.print_idx(id_file) im = nib.load(file_path) im_data = im.get_data() sum_image = np.add(sum_image, 1, out=sum_image, where=np.logical_not(np.isnan(im_data))) return sum_image def _get_std_shape(self): return self._standard_ref.get_data().shape @staticmethod def _add_image_inter(image1, image2): return np.add(image1, image2, out=np.full_like(image1, np.nan), where=np.logical_not( np.logical_or(np.isnan(image1), np.isnan(image2)))) @staticmethod def _add_image_union(image1, image2): add_image = np.full_like(image1, np.nan) add_image[np.logical_not( np.logical_and(np.isnan(image1), np.isnan(image2)))] = 0 add_image = np.add(add_image, image1, out=add_image, where=np.logical_not(np.isnan(image1))) add_image = np.add(add_image, image2, out=add_image, where=np.logical_not(np.isnan(image2))) return add_image @staticmethod def sum_non_null_count(file_list, in_folder): print('Count non-null per voxel. Loading images...') im_temp = nib.load(os.path.join(in_folder, file_list[0])) im_temp_data = im_temp.get_data() sum_image = np.zeros_like(im_temp_data) for id_file in range(len(file_list)): file_name = file_list[id_file] print('%s (%d/%d)' % (file_name, id_file, len(file_list))) file_path = os.path.join(in_folder, file_name) im = nib.load(file_path) im_data = im.get_data() sum_image = np.add(sum_image, 1, out=sum_image, where=np.logical_not(np.isnan(im_data))) return sum_image
def __init__(self, config, in_folder, out_folder, file_list_txt): super().__init__(config, in_folder, file_list_txt) self._out_folder = DataFolder.get_data_folder_obj( config, out_folder, data_list_txt=file_list_txt)
def __init__(self, config, in_folder_1, in_folder_2, file_list_txt): super().__init__(config, in_folder_1, file_list_txt) self._in_data_folder_2 = DataFolder(in_folder_2, file_list_txt) self._nrmse_diff = []