def main():
parser = argparse.ArgumentParser(description='Load a saved pca object')
parser.add_argument('--load-pca-bin-path', type=str,
help='Location of the pca bin')
parser.add_argument('--in-folder', type=str)
parser.add_argument('--out-folder', type=str)
parser.add_argument('--mask-img', type=str)
parser.add_argument('--num-mode', type=int)
parser.add_argument('--file-list-txt', type=str)
parser.add_argument('--num-process', type=int, default=10)
args = parser.parse_args()
in_folder_obj = DataFolder(args.in_folder, args.file_list_txt)
out_folder_obj = DataFolder(args.out_folder, args.file_list_txt)
mask_img_obj = ScanWrapper(args.mask_img)
paral_obj = ParalGetResidual(
args.load_pca_bin_path,
in_folder_obj,
out_folder_obj,
mask_img_obj,
args.num_mode,
args.num_process
)
paral_obj.run_parallel()
def __init__(self, config, in_jac_folder, in_affine_mat_folder, out_folder,
ref_img, file_list_txt):
super().__init__(config, in_jac_folder, file_list_txt)
self._ref_img = ScanWrapper(ref_img)
self._in_affine_mat_folder = DataFolder.get_data_folder_obj(
config, in_affine_mat_folder, data_list_txt=file_list_txt)
self._out_data_folder = DataFolder.get_data_folder_obj(
config, out_folder, data_list_txt=file_list_txt)
def __init__(self,
config,
in_folder=None,
data_file_txt=None,
in_data_folder_obj=None):
self._data_folder = None
if in_data_folder_obj is None:
self._data_folder = DataFolder(in_folder, data_file_txt)
else:
self._data_folder = in_data_folder_obj
self._standard_ref = ScanWrapper(self._data_folder.get_first_path())
self._num_processes = config['num_processes']
def __init__(self,
config,
in_ori_folder,
in_jac_folder,
batch_size,
file_list_txt=None):
super().__init__(config, in_ori_folder, file_list_txt)
self._in_jac_folder = DataFolder(in_jac_folder, file_list_txt)
self._ref_ori = ScanWrapper(self._in_data_folder.get_file_path(0))
self._ref_jac = ScanWrapper(self._in_jac_folder.get_file_path(0))
self._chunk_list = self._in_data_folder.get_chunks_list_batch_size(
batch_size)
self._data_matrix = []
self._cur_idx = 0
def get_csv(args):
file_list = read_file_contents_list(args.file_list_txt)
in_ori_folder_obj = DataFolder(args.in_ori_folder, file_list)
in_mask_folder_obj = DataFolder(args.in_mask_folder, file_list)
exe_obj = MeanIntensityMask(in_ori_folder_obj, in_mask_folder_obj, [2, 4],
20)
result_dict_list = exe_obj.run_parallel()
result_df = pd.DataFrame(result_dict_list)
result_df = result_df.set_index('file_name')
print(f'Output csv to {args.out_csv}')
result_df.to_csv(args.out_csv)
def _get_slices_dim_list(pc_folder, file_list_txt):
center_slices_x = []
center_slices_y = []
center_slices_z = []
data_folder = DataFolder(pc_folder, file_list_txt)
for file_idx in range(show_pc_number):
data_folder.print_idx(file_idx)
file_path = data_folder.get_file_path(file_idx)
scan = ScanWrapper(file_path)
slice_x, slice_y, slice_z = scan.get_center_slices()
center_slices_x.append(slice_x)
center_slices_y.append(slice_y)
center_slices_z.append(slice_z)
return center_slices_x, center_slices_y, center_slices_z
def __init__(self, config, in_dat_folder, out_folder, ref_img,
file_list_txt):
super().__init__(config, in_dat_folder, file_list_txt)
self._ref_img = ScanWrapper(ref_img)
self._out_data_folder = DataFolder.get_data_folder_obj(
config, out_folder, data_list_txt=file_list_txt)
self._jacobian_tool = config['deedsBCV_jacobian']
def main():
parser = argparse.ArgumentParser(description='ThoraxPCA')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--save-path',
type=str,
help='Output location for union average',
required=True)
parser.add_argument('--data-file-list', type=str, help='Data file list')
args = parser.parse_args()
config = load_config(args.config)
average_tool = None
if args.data_file_list:
data_folder_obj = DataFolder(args.in_folder, args.data_file_list)
average_tool = AverageScans(config, in_data_folder_obj=data_folder_obj)
else:
average_tool = AverageScans(config, args.in_folder)
average_tool.get_average_image_union(args.save_path)
def __init__(self, config, in_folder, mask_img, out_png_folder, file_list_txt):
super().__init__(config, in_folder, file_list_txt)
self._mask_img = ScanWrapper(mask_img)
self._out_png_folder = DataFolder.get_data_folder_obj(config, out_png_folder, data_list_txt=file_list_txt)
self._out_png_folder.change_suffix('.png')
self._vmax = 500
self._vmin = -1000
def __init__(self, config, in_folder, out_folder, ref_img, order=3):
super().__init__(config, in_folder)
self._out_data_folder = DataFolder.get_data_folder_obj(
config, out_folder)
self._ref_img = ScanWrapper(ref_img)
self._order = order
self._reg_resample = config['niftyreg_resample']
self._reg_resmaple_ref = config['reg_resample_ref_img']
def __init__(self,
config,
in_folder,
out_folder,
average_img,
file_list_txt=None):
super().__init__(config, in_folder, file_list_txt=file_list_txt)
self._out_data_folder = DataFolder.get_data_folder_obj(
config, out_folder, data_list_txt=file_list_txt)
mkdir_p(out_folder)
self._average_img = ScanWrapper(average_img)
def main():
parser = argparse.ArgumentParser(description='Load a saved pca object')
parser.add_argument('--load-pca-bin-path', type=str,
help='Location of the pca bin')
parser.add_argument('--in-ori-folder', type=str)
parser.add_argument('--in-jac-folder', type=str)
parser.add_argument('--file-list-txt', type=str)
parser.add_argument('--save-bin-path', type=str)
parser.add_argument('--num-process', type=int, default=10)
args = parser.parse_args()
in_ori_folder_obj = DataFolder(args.in_ori_folder, args.file_list_txt)
in_jac_folder_obj = DataFolder(args.in_jac_folder, args.file_list_txt)
paral_obj = ParalDimensionReductionConcat(
args.load_pca_bin_path,
in_ori_folder_obj,
in_jac_folder_obj,
args.num_process)
paral_obj.run_dimension_reduction(args.save_bin_path)
def __init__(self,
config,
in_scan_folder,
in_region_mask_folder,
file_list_txt,
out_average_img,
ambient=-5000):
super().__init__(config, in_scan_folder, file_list_txt)
self._in_region_mask_folder = DataFolder.get_data_folder_obj(
config, in_region_mask_folder, data_list_txt=file_list_txt)
self._out_img_path = out_average_img
self._ref_img = ScanWrapper(self._in_data_folder.get_first_path())
self._ambient = ambient
def __init__(self,
config,
in_folder,
out_folder,
roi_img,
file_list_txt=None):
super().__init__(config, in_folder, file_list_txt=file_list_txt)
# self._out_data_folder = DataFolder.get_data_folder_obj(config, out_folder, data_list_txt=file_list_txt)
self._out_data_folder = DataFolder.get_data_folder_obj_with_list(
out_folder, self._in_data_folder.get_data_file_list())
mkdir_p(out_folder)
self._roi_img_path = roi_img
self._reg_resample_path = config['niftyreg_resample']
def main():
# yaml_config = 'simg_bmi_regression_3.6.3_nfs.yaml'
yaml_config = 'simg_bmi_regression_3.6.4_nfs.yaml'
# yaml_config = 'simg_bmi_regression_3.6.6.1_nfs.yaml'
SRC_ROOT = os.path.dirname(os.path.realpath(__file__)) + '/..'
yaml_config = os.path.join(SRC_ROOT, f'src/yaml/{yaml_config}')
logger.info(f'Read yaml file {yaml_config}')
f = open(yaml_config, 'r').read()
config = yaml.safe_load(f)
num_fold = config['fold_num']
exp_dir = config['exp_dir']
layer_flag = config['gcam_target_layer']
gcam_overlay_out_folder = os.path.join(exp_dir, 'gcam_overlay')
gcam_link_out_folder = os.path.join(exp_dir, 'gcam_link')
mkdir_p(gcam_overlay_out_folder)
mkdir_p(gcam_link_out_folder)
pred_result_csv = os.path.join(exp_dir, f'pred_total.csv')
file_list = pd.read_csv(pred_result_csv, index_col=False)['file_name']
in_int_folder_obj = DataFolder(atlas_intensity_folder, file_list)
in_jac_folder_obj = DataFolder(atlas_jacobian_folder, file_list)
in_att_folder_obj = DataFolder(gcam_link_out_folder, file_list)
paral_plot_obj = ParaPlotClip(
in_int_folder_obj,
in_jac_folder_obj,
in_att_folder_obj,
gcam_overlay_out_folder,
10
)
paral_plot_obj.run_parallel()
def main():
parser = argparse.ArgumentParser(description='Load a saved pca object')
parser.add_argument('--pc-folder',
type=str,
help='Location of the principle feature images')
parser.add_argument('--file-list-txt', type=str)
parser.add_argument('--out-png', type=str)
parser.add_argument('--step-axial', type=int, default=50)
parser.add_argument('--step-sagittal', type=int, default=75)
parser.add_argument('--step-coronal', type=int, default=30)
parser.add_argument('--num-show-pc', type=int, default=10)
args = parser.parse_args()
pc_folder_obj = DataFolder(args.pc_folder, args.file_list_txt)
plt_obj = PlotPCGrid(pc_folder_obj, args.step_axial, args.step_sagittal,
args.step_coronal, args.num_show_pc)
plt_obj.plot_pc(args.out_png)
class GetLossBetweenFolder(AbstractParallelRoutine):
def __init__(self, config, in_folder_1, in_folder_2, file_list_txt):
super().__init__(config, in_folder_1, file_list_txt)
self._in_data_folder_2 = DataFolder(in_folder_2, file_list_txt)
self._nrmse_diff = []
def get_nrmse(self):
return self._nrmse_diff
def print_file_list(self):
file_list = self._in_data_folder.get_data_file_list()
for idx in range(len(file_list)):
print(f'The {idx}th file is {file_list[idx]}')
def _run_single_scan(self, idx):
in_file_1_path = self._in_data_folder.get_file_path(idx)
in_file_2_path = self._in_data_folder_2.get_file_path(idx)
in_img_1 = ScanWrapper(in_file_1_path).get_data()
in_img_2 = ScanWrapper(in_file_2_path).get_data()
nrmse = compare_nrmse(np.abs(in_img_1), np.abs(in_img_2))
self._nrmse_diff.append(nrmse)
class AbstractParallelRoutine:
def __init__(self, config, in_folder, file_list=None):
self._config = config
self._in_data_folder = DataFolder(in_folder, file_list)
# self._num_processes = config['num_processes']
self._num_processes = 1
def run_parallel(self):
pool = Pool(processes=self._num_processes)
chunk_list = self._in_data_folder.get_chunks_list(self._num_processes)
result_obj_list = [
pool.apply_async(self._run_chunk, (file_idx_chunk, ))
for file_idx_chunk in chunk_list
]
result_list = []
for thread_idx in range(len(result_obj_list)):
result_obj = result_obj_list[thread_idx]
result_obj.wait()
logger.info(
f'Thread with idx {thread_idx} / {len(result_obj_list)} is completed'
)
result_list = result_list + result_obj.get()
return result_list
def run_non_parallel(self, idx_list=None):
run_idx_list = idx_list if idx_list else range(self.num_files())
for idx in run_idx_list:
self._in_data_folder.print_idx(idx)
self._run_single_scan(idx)
# print('Done', flush=True)
def _run_single_scan(self, idx):
raise NotImplementedError
def _run_chunk(self, chunk_list):
result_list = []
for idx in chunk_list:
self._in_data_folder.print_idx(idx)
result = self._run_single_scan(idx)
result_list.append(result)
return result_list
def num_files(self):
return self._in_data_folder.num_files()
def get_data_dict(config, file_list_txt):
task = config['task']
in_folder = config['input_img_dir']
label_csv = config['label_csv']
in_folder_obj = DataFolder(in_folder,
read_file_contents_list(file_list_txt))
file_list = in_folder_obj.get_data_file_list()
clinical_reader = ClinicalDataReaderSPORE.create_spore_data_reader_csv(
label_csv)
label_array = None
file_list_with_valid_label = None
if task == 'BMI':
label_array, file_list_with_valid_label = clinical_reader.get_gt_value_BMI(
file_list)
subject_list = [
ClinicalDataReaderSPORE._get_subject_id_from_file_name(file_name)
for file_name in file_list_with_valid_label
]
in_folder_obj.set_file_list(file_list_with_valid_label)
file_path_list = in_folder_obj.get_file_path_list()
data_dict = {
'img_names': file_list_with_valid_label,
'img_subs': subject_list,
'img_files': file_path_list,
'gt_val': label_array
}
if config['add_jacobian_map']:
in_jacobian_folder = config['input_jac_dir']
in_jacobian_folder_obj = DataFolder(in_jacobian_folder,
file_list_with_valid_label)
jacobian_map_path_list = in_jacobian_folder_obj.get_file_path_list()
data_dict['jacobian_maps'] = jacobian_map_path_list
if config['add_valid_mask_map'] | config['apply_random_valid_mask']:
in_valid_mask_folder = config['input_valid_mask_dir']
in_valid_mask_folder_obj = DataFolder(in_valid_mask_folder,
file_list_with_valid_label)
valid_mask_path_list = in_valid_mask_folder_obj.get_file_path_list()
data_dict['valid_masks'] = valid_mask_path_list
if config['add_d_index_map']:
in_d_index_map_folder = config['input_d_index_dir']
in_d_index_map_folder_obj = DataFolder(in_d_index_map_folder,
file_list_with_valid_label)
d_index_map_path_list = in_d_index_map_folder_obj.get_file_path_list()
data_dict['d_index_maps'] = d_index_map_path_list
if config['add_jac_elem_maps']:
in_jac_elem_folder = config['input_jac_elem_dir']
in_jac_elem_folder_obj = DataFolder(in_jac_elem_folder,
file_list_with_valid_label)
for idx_elem in range(9):
in_jac_elem_path_list = [
map_path.replace('.nii.gz', f'_{idx_elem}.nii.gz')
for map_path in in_jac_elem_folder_obj.get_file_path_list()
]
data_dict[f'jac_elem_{idx_elem}_map'] = in_jac_elem_path_list
return data_dict
class ScanFolderConcatBatchReader(AbstractParallelRoutine):
def __init__(self,
config,
in_ori_folder,
in_jac_folder,
batch_size,
file_list_txt=None):
super().__init__(config, in_ori_folder, file_list_txt)
self._in_jac_folder = DataFolder(in_jac_folder, file_list_txt)
self._ref_ori = ScanWrapper(self._in_data_folder.get_file_path(0))
self._ref_jac = ScanWrapper(self._in_jac_folder.get_file_path(0))
self._chunk_list = self._in_data_folder.get_chunks_list_batch_size(
batch_size)
self._data_matrix = []
self._cur_idx = 0
def read_data(self, idx_batch):
self._reset_cur_idx()
print(f'Reading scans from folder {self._in_data_folder.get_folder()}',
flush=True)
tic = time.perf_counter()
cur_batch = self._chunk_list[idx_batch]
self._init_data_matrix(len(cur_batch))
self.run_non_parallel(cur_batch)
toc = time.perf_counter()
print(f'Done. {toc - tic:0.4f} (s)', flush=True)
def num_batch(self):
return len(self._chunk_list)
def get_data_matrix(self):
return self._data_matrix
def save_flat_data(self, data_array, idx, out_folder):
out_path_ori = os.path.join(out_folder, f'pc_ori_{idx}.nii.gz')
out_path_jac = os.path.join(out_folder, f'pc_jac_{idx}.nii.gz')
ori_data_flat = data_array[:self._ref_ori.get_number_voxel()]
jac_data_flat = data_array[self._ref_ori.get_number_voxel():]
self._ref_ori.save_scan_flat_img(ori_data_flat, out_path_ori)
self._ref_jac.save_scan_flat_img(jac_data_flat, out_path_jac)
def get_ref(self):
return self._ref_ori
def _run_single_scan(self, idx):
in_ori_data = ScanWrapper(
self._in_data_folder.get_file_path(idx)).get_data()
in_jac_data = ScanWrapper(
self._in_jac_folder.get_file_path(idx)).get_data()
self._data_matrix[self._cur_idx, :self._ref_ori.get_number_voxel(
)] = convert_3d_2_flat(in_ori_data)
self._data_matrix[
self._cur_idx,
self._ref_ori.get_number_voxel():] = convert_3d_2_flat(in_jac_data)
self._cur_idx += 1
def _init_data_matrix(self, num_sample):
num_features = self._get_number_of_voxel()
del self._data_matrix
self._data_matrix = np.zeros((num_sample, num_features))
def _get_number_of_voxel(self):
return self._get_number_of_voxel_ori() + self._get_number_of_voxel_jac(
)
def _get_number_of_voxel_ori(self):
return self._ref_ori.get_number_voxel()
def _get_number_of_voxel_jac(self):
return self._ref_jac.get_number_voxel()
def _reset_cur_idx(self):
self._cur_idx = 0
def __init__(self, config, in_folder, file_list=None):
self._config = config
self._in_data_folder = DataFolder(in_folder, file_list)
# self._num_processes = config['num_processes']
self._num_processes = 1
class AverageScans:
def __init__(self,
config,
in_folder=None,
data_file_txt=None,
in_data_folder_obj=None):
self._data_folder = None
if in_data_folder_obj is None:
self._data_folder = DataFolder(in_folder, data_file_txt)
else:
self._data_folder = in_data_folder_obj
self._standard_ref = ScanWrapper(self._data_folder.get_first_path())
self._num_processes = config['num_processes']
def get_average_image_union(self, save_path):
im_shape = self._get_std_shape()
average_union = np.zeros(im_shape)
average_union.fill(np.nan)
non_null_mask_count_image = np.zeros(im_shape)
chunk_list = self._data_folder.get_chunks_list(self._num_processes)
pool = Pool(processes=self._num_processes)
print('Average in union')
print('Step.1 Summation')
image_average_union_result_list = [
pool.apply_async(self._sum_images_union, (file_idx_chunk, ))
for file_idx_chunk in chunk_list
]
for thread_idx in range(len(image_average_union_result_list)):
result = image_average_union_result_list[thread_idx]
result.wait()
print(
f'Thread with idx {thread_idx} / {len(image_average_union_result_list)} is completed'
)
print('Adding to averaged_image...')
averaged_image_chunk = result.get()
average_union = self._add_image_union(average_union,
averaged_image_chunk)
print('Done.')
print('Step.2 Non-nan counter')
non_null_mask_count_result = [
pool.apply_async(self._sum_non_null_count, (file_idx_chunk, ))
for file_idx_chunk in chunk_list
]
for thread_idx in range(len(non_null_mask_count_result)):
result = non_null_mask_count_result[thread_idx]
result.wait()
print(
f'Thread with idx {thread_idx} / {len(non_null_mask_count_result)} is completed'
)
print('Adding to averaged_image...')
averaged_image_chunk = result.get()
non_null_mask_count_image = np.add(non_null_mask_count_image,
averaged_image_chunk)
print('Done.')
average_union = np.divide(average_union,
non_null_mask_count_image,
out=average_union,
where=non_null_mask_count_image > 0)
self._standard_ref.save_scan_same_space(save_path, average_union)
print('Done.')
def _sum_images_union(self, chunk_list):
print('Sum images, union non-null region. Loading images...')
im_shape = self._get_std_shape()
sum_image = np.zeros(im_shape)
sum_image.fill(np.nan)
for id_file in chunk_list:
file_path = self._data_folder.get_file_path(id_file)
self._data_folder.print_idx(id_file)
im = nib.load(file_path)
im_data = im.get_data()
sum_image = self._add_image_union(sum_image, im_data)
return sum_image
def _sum_non_null_count(self, chunk_list):
print('Count non-null per voxel. Loading images...')
im_shape = self._get_std_shape()
sum_image = np.zeros(im_shape)
for id_file in chunk_list:
file_path = self._data_folder.get_file_path(id_file)
self._data_folder.print_idx(id_file)
im = nib.load(file_path)
im_data = im.get_data()
sum_image = np.add(sum_image,
1,
out=sum_image,
where=np.logical_not(np.isnan(im_data)))
return sum_image
def _get_std_shape(self):
return self._standard_ref.get_data().shape
@staticmethod
def _add_image_inter(image1, image2):
return np.add(image1,
image2,
out=np.full_like(image1, np.nan),
where=np.logical_not(
np.logical_or(np.isnan(image1), np.isnan(image2))))
@staticmethod
def _add_image_union(image1, image2):
add_image = np.full_like(image1, np.nan)
add_image[np.logical_not(
np.logical_and(np.isnan(image1), np.isnan(image2)))] = 0
add_image = np.add(add_image,
image1,
out=add_image,
where=np.logical_not(np.isnan(image1)))
add_image = np.add(add_image,
image2,
out=add_image,
where=np.logical_not(np.isnan(image2)))
return add_image
@staticmethod
def sum_non_null_count(file_list, in_folder):
print('Count non-null per voxel. Loading images...')
im_temp = nib.load(os.path.join(in_folder, file_list[0]))
im_temp_data = im_temp.get_data()
sum_image = np.zeros_like(im_temp_data)
for id_file in range(len(file_list)):
file_name = file_list[id_file]
print('%s (%d/%d)' % (file_name, id_file, len(file_list)))
file_path = os.path.join(in_folder, file_name)
im = nib.load(file_path)
im_data = im.get_data()
sum_image = np.add(sum_image,
1,
out=sum_image,
where=np.logical_not(np.isnan(im_data)))
return sum_image
def __init__(self, config, in_folder, out_folder, file_list_txt):
super().__init__(config, in_folder, file_list_txt)
self._out_folder = DataFolder.get_data_folder_obj(
config, out_folder, data_list_txt=file_list_txt)
def __init__(self, config, in_folder_1, in_folder_2, file_list_txt):
super().__init__(config, in_folder_1, file_list_txt)
self._in_data_folder_2 = DataFolder(in_folder_2, file_list_txt)
self._nrmse_diff = []