def test_parse_idx_different_order(): im_name = 'im_t50000_z010_p020_c005.png' meta = aux_utils.parse_idx_from_name(im_name, order="tzpc") nose.tools.assert_equal(meta["channel_idx"], 5) nose.tools.assert_equal(meta["slice_idx"], 10) nose.tools.assert_equal(meta["time_idx"], 50000) nose.tools.assert_equal(meta["pos_idx"], 20)
def setUp(self): """ Set up a directory with some images to resample """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.output_dir = os.path.join(self.temp_path, 'out_dir') # Start frames meta file self.meta_name = 'frames_meta.csv' self.frames_meta = aux_utils.make_dataframe() # Write images self.time_idx = 5 self.slice_idx = 6 self.pos_idx = 7 self.im = 1500 * np.ones((30, 20), dtype=np.uint16) for c in range(4): for p in range(self.pos_idx, self.pos_idx + 2): im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=self.slice_idx, time_idx=self.time_idx, pos_idx=p, ) cv2.imwrite(os.path.join(self.temp_path, im_name), self.im + c * 100) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) # Write metadata self.frames_meta.to_csv( os.path.join(self.temp_path, self.meta_name), sep=',', )
def test_parse_idx_from_name(): im_name = 'im_c005_z010_t50000_p020.png' meta = aux_utils.parse_idx_from_name(im_name, order="cztp") nose.tools.assert_equal(meta["channel_idx"], 5) nose.tools.assert_equal(meta["slice_idx"], 10) nose.tools.assert_equal(meta["time_idx"], 50000) nose.tools.assert_equal(meta["pos_idx"], 20)
def setUp(self): """Set up a dictionary with images""" self.tempdir = TempDirectory() self.temp_path = self.tempdir.path meta_fname = 'frames_meta.csv' self.df_columns = ['channel_idx', 'slice_idx', 'time_idx', 'channel_name', 'file_name', 'pos_idx'] self.frames_meta = pd.DataFrame(columns=self.df_columns) x = np.linspace(-4, 4, 32) y = x.copy() z = np.linspace(-3, 3, 8) xx, yy, zz = np.meshgrid(x, y, z) sph = (xx ** 2 + yy ** 2 + zz ** 2) sph = (sph <= 8) * (8 - sph) sph = (sph / sph.max()) * 255 sph = sph.astype('uint8') self.sph = sph self.channel_idx = 1 self.time_idx = 0 self.pos_idx = 1 self.int2str_len = 3 for z in range(sph.shape[2]): im_name = aux_utils.get_im_name( channel_idx=1, slice_idx=z, time_idx=self.time_idx, pos_idx=self.pos_idx, ) cv2.imwrite(os.path.join(self.temp_path, im_name), sph[:, :, z]) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, self.df_columns), ignore_index=True, ) # Write metadata self.frames_meta.to_csv(os.path.join(self.temp_path, meta_fname), sep=',') # Write 3D sphere data self.sph_fname = os.path.join( self.temp_path, 'im_c001_z000_t000_p001_3d.npy', ) np.save(self.sph_fname, self.sph, allow_pickle=True, fix_imports=True) meta_3d = pd.DataFrame.from_dict([{ 'channel_idx': 1, 'slice_idx': 0, 'time_idx': 0, 'channel_name': '3d_test', 'file_name': 'im_c001_z000_t000_p001_3d.npy', 'pos_idx': 1, }]) self.meta_3d = meta_3d
def setUp(self): """ Set up a directory with some images to resample """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.tempdir.makedir('mask_dir') self.input_dir = os.path.join(self.temp_path, 'input_dir') self.tempdir.makedir('input_dir') self.mask_channel = 1 self.slice_idx = 7 self.time_idx = 8 # Mask meta file self.csv_name = 'mask_image_matchup.csv' input_meta = aux_utils.make_dataframe() # Make input meta for c in range(4): for p in range(10): im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=self.slice_idx, time_idx=self.time_idx, pos_idx=p, ) input_meta = input_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) input_meta.to_csv( os.path.join(self.input_dir, 'frames_meta.csv'), sep=',', ) # Make mask meta mask_meta = pd.DataFrame() for p in range(10): im_name = aux_utils.get_im_name( channel_idx=self.mask_channel, slice_idx=self.slice_idx, time_idx=self.time_idx, pos_idx=p, ) # Indexing can be different mask_name = 'mask_{}.png'.format(p + 1) mask_meta = mask_meta.append( {'mask_name': mask_name, 'file_name': im_name}, ignore_index=True, ) mask_meta.to_csv( os.path.join(self.mask_dir, self.csv_name), sep=',', )
def write_data_in_meta_csv(self, array, frames_meta, ch_idx): for z in range(array.shape[2]): im_name = self.get_name(ch_idx, z, self.time_ids, self.pos_ids) with warnings.catch_warnings(): warnings.simplefilter("ignore") sk_im_io.imsave( os.path.join(self.temp_path, im_name), array[:, :, z].astype('uint8'), ) frames_meta = frames_meta.append(aux_utils.parse_idx_from_name( im_name, self.df_columns), ignore_index=True) return frames_meta
def setUp(self): """ Set up a dataframe for training table """ # Start frames meta file self.meta_name = 'frames_meta.csv' self.frames_meta = aux_utils.make_dataframe() self.time_ids = [3, 4, 5] self.pos_ids = [7, 8, 10, 12, 15] self.channel_ids = [0, 1, 2, 3] self.slice_ids = [0, 1, 2, 3, 4, 5] # Tiles will typically be split into image subsections # but it doesn't matter for testing for c in self.channel_ids: for p in self.pos_ids: for z in self.slice_ids: for t in self.time_ids: im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=z, time_idx=t, pos_idx=p, ) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) self.tiles_meta = aux_utils.sort_meta_by_channel(self.frames_meta) self.input_channels = [0, 2] self.target_channels = [3] self.mask_channels = [1] self.split_ratio = { 'train': 0.6, 'val': 0.2, 'test': 0.2, } # Instantiate class self.table_inst = training_table.BaseTrainingTable( df_metadata=self.tiles_meta, input_channels=self.input_channels, target_channels=self.target_channels, split_by_column='pos_idx', split_ratio=self.split_ratio, mask_channels=[1], random_seed=42, )
def test_adjust_slice_indices(self): # First create new frames meta with more slices temp_meta = aux_utils.make_dataframe() for s in range(10): im_name = aux_utils.get_im_name( time_idx=2, channel_idx=4, slice_idx=s, pos_idx=6, ) temp_meta = temp_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) self.data_inst.iteration_meta = temp_meta self.data_inst.depth = 5 # This should remove first and last two slices self.data_inst.adjust_slice_indices() # Original slice ids are 0-9 so after removing margins should be 2-7 self.assertListEqual( self.data_inst.iteration_meta.slice_idx.unique().tolist(), [2, 3, 4, 5, 6, 7])
def test_resize_volumes(self): """Test resizing volumes""" # set up a volume with 5 slices, 2 channels slice_ids = [0, 1, 2, 3, 4] channel_ids = [2, 3] frames_meta = aux_utils.make_dataframe() exp_meta_dict = [] for c in channel_ids: for s in slice_ids: im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=s, time_idx=self.time_idx, pos_idx=self.pos_idx, ) cv2.imwrite(os.path.join(self.temp_path, im_name), self.im + c * 100) frames_meta = frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) op_fname = 'im_c00{}_z000_t005_p007_3.3-0.8-1.0.npy'.format(c) exp_meta_dict.append({'time_idx': self.time_idx, 'pos_idx': self.pos_idx, 'channel_idx': c, 'slice_idx': 0, 'file_name': op_fname}) # Write metadata frames_meta.to_csv( os.path.join(self.temp_path, self.meta_name), sep=',', ) scale_factor = [3.3, 0.8, 1.0] resize_inst = resize_images.ImageResizer( input_dir=self.temp_path, output_dir=self.output_dir, scale_factor=scale_factor, ) # save all slices in one volume resize_inst.resize_volumes() saved_meta = pd.read_csv(os.path.join(self.output_dir, 'resized_images', 'frames_meta.csv')) del saved_meta['Unnamed: 0'] exp_meta_df = pd.DataFrame.from_dict(exp_meta_dict) pd.testing.assert_frame_equal(saved_meta, exp_meta_df) # num_slices_subvolume = 3, save vol chunks exp_meta_dict = [] for c in channel_ids: for s in [0, 2]: op_fname = 'im_c00{}_z00{}_t005_p007_3.3-0.8-1.0.npy'.format(c, s) exp_meta_dict.append({'time_idx': self.time_idx, 'pos_idx': self.pos_idx, 'channel_idx': c, 'slice_idx': s, 'file_name': op_fname}) resize_inst.resize_volumes(num_slices_subvolume=3) saved_meta = pd.read_csv(os.path.join(self.output_dir, 'resized_images', 'frames_meta.csv')) del saved_meta['Unnamed: 0'] exp_meta_df = pd.DataFrame.from_dict(exp_meta_dict) pd.testing.assert_frame_equal(saved_meta, exp_meta_df)
def setUp(self): """Set up a dir for tiling with flatfield""" self.tempdir = TempDirectory() self.temp_path = self.tempdir.path # Start frames meta file self.meta_name = 'frames_meta.csv' frames_meta = aux_utils.make_dataframe() self.im = 127 * np.ones((15, 11), dtype=np.uint8) self.im2 = 234 * np.ones((15, 11), dtype=np.uint8) self.int2str_len = 3 self.channel_idx = [1, 2] self.pos_idx1 = 7 self.pos_idx2 = 8 # write pos1 with 3 time points and 5 slices for z in range(5): for t in range(3): for c in self.channel_idx: im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=z, time_idx=t, pos_idx=self.pos_idx1, ) with warnings.catch_warnings(): warnings.simplefilter("ignore") sk_im_io.imsave( os.path.join(self.temp_path, im_name), self.im, ) frames_meta = frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) # write pos2 with 2 time points and 3 slices for z in range(3): for t in range(2): for c in self.channel_idx: im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=z, time_idx=t, pos_idx=self.pos_idx2, ) with warnings.catch_warnings(): warnings.simplefilter("ignore") sk_im_io.imsave( os.path.join(self.temp_path, im_name), self.im, ) frames_meta = frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) # Write metadata frames_meta.to_csv(os.path.join(self.temp_path, self.meta_name), sep=',',) # Instantiate tiler class self.output_dir = os.path.join(self.temp_path, 'tile_dir') self.tile_inst = tile_images.ImageTilerNonUniform( input_dir=self.temp_path, output_dir=self.output_dir, tile_size=[5, 5], step_size=[4, 4], depths=3, channel_ids=[1, 2], normalize_channels=[False, True] )
def test_generate_masks_nonuni(self): """Test generate_masks with non-uniform structure""" rec = self.rec_object[:, :, 3:6] channel_ids = 0 time_ids = 0 pos_ids = [1, 2] frames_meta = aux_utils.make_dataframe() for z in range(self.sph_object.shape[2]): im_name = aux_utils.get_im_name( time_idx=time_ids, channel_idx=channel_ids, slice_idx=z, pos_idx=pos_ids[0], ) sk_im_io.imsave(os.path.join(self.temp_path, im_name), self.sph_object[:, :, z].astype('uint8')) frames_meta = frames_meta.append(aux_utils.parse_idx_from_name( im_name, aux_utils.DF_NAMES), ignore_index=True) for z in range(rec.shape[2]): im_name = aux_utils.get_im_name( time_idx=time_ids, channel_idx=channel_ids, slice_idx=z, pos_idx=pos_ids[1], ) sk_im_io.imsave(os.path.join(self.temp_path, im_name), rec[:, :, z].astype('uint8')) frames_meta = frames_meta.append(aux_utils.parse_idx_from_name( im_name, aux_utils.DF_NAMES), ignore_index=True) # Write metadata frames_meta.to_csv(os.path.join(self.temp_path, self.meta_fname), sep=',') self.output_dir = os.path.join(self.temp_path, 'mask_dir') mask_gen_inst = MaskProcessor(input_dir=self.temp_path, output_dir=self.output_dir, channel_ids=channel_ids, uniform_struct=False) exp_nested_id_dict = { 0: { 0: { 1: [0, 1, 2, 3, 4, 5, 6, 7], 2: [0, 1, 2] } } } numpy.testing.assert_array_equal(mask_gen_inst.nested_id_dict[0][0][1], exp_nested_id_dict[0][0][1]) numpy.testing.assert_array_equal(mask_gen_inst.nested_id_dict[0][0][2], exp_nested_id_dict[0][0][2]) mask_gen_inst.generate_masks(str_elem_radius=1) frames_meta = pd.read_csv( os.path.join(mask_gen_inst.get_mask_dir(), 'frames_meta.csv'), index_col=0, ) # pos1: 8 slices, pos2: 3 slices exp_len = 8 + 3 nose.tools.assert_equal(len(frames_meta), exp_len) mask_fnames = frames_meta['file_name'].tolist() exp_mask_fnames = [ 'im_c001_z000_t000_p001.npy', 'im_c001_z000_t000_p002.npy', 'im_c001_z001_t000_p001.npy', 'im_c001_z001_t000_p002.npy', 'im_c001_z002_t000_p001.npy', 'im_c001_z002_t000_p002.npy', 'im_c001_z003_t000_p001.npy', 'im_c001_z004_t000_p001.npy', 'im_c001_z005_t000_p001.npy', 'im_c001_z006_t000_p001.npy', 'im_c001_z007_t000_p001.npy' ] nose.tools.assert_list_equal(mask_fnames, exp_mask_fnames)
def setUp(self): """ Set up a directory with some images to generate frames_meta.csv for """ self.tempdir = TempDirectory() self.temp_dir = self.tempdir.path self.model_dir = os.path.join(self.temp_dir, 'model_dir') self.pred_dir = os.path.join(self.model_dir, 'predictions') self.image_dir = os.path.join(self.temp_dir, 'image_dir') self.tempdir.makedir(self.model_dir) self.tempdir.makedir(self.pred_dir) self.tempdir.makedir(self.image_dir) # Write images self.time_idx = 5 self.pos_idx = 7 self.im = 1500 * np.ones((30, 20), dtype=np.uint16) im_add = np.zeros((30, 20), dtype=np.uint16) im_add[15:, :] = 10 self.ext = '.tif' # Start frames meta file self.meta_name = 'frames_meta.csv' self.frames_meta = aux_utils.make_dataframe() for c in range(3): for z in range(5, 10): im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=z, time_idx=self.time_idx, pos_idx=self.pos_idx, ext=self.ext, ) cv2.imwrite(os.path.join(self.image_dir, im_name), self.im) if c == 2: norm_im = normalize.zscore(self.im + im_add).astype(np.float32) cv2.imwrite( os.path.join(self.pred_dir, im_name), norm_im, ) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) # Write metadata self.frames_meta.to_csv( os.path.join(self.image_dir, self.meta_name), sep=',', ) # Write as test metadata in model dir too self.frames_meta.to_csv( os.path.join(self.model_dir, 'test_metadata.csv'), sep=',', ) # Write split samples split_idx_fname = os.path.join(self.model_dir, 'split_samples.json') split_samples = {'test': [5, 6, 7, 8, 9]} aux_utils.write_json(split_samples, split_idx_fname) # Write config in model dir config = { 'dataset': { 'input_channels': [0, 1], 'target_channels': [2], 'split_by_column': 'slice_idx' }, 'network': {} } config_name = os.path.join(self.model_dir, 'config.yml') with open(config_name, 'w') as outfile: yaml.dump(config, outfile, default_flow_style=False)
def setUp(self): """ Set up a directory with some images to resample """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.image_dir = self.temp_path self.output_dir = os.path.join(self.temp_path, 'out_dir') self.tempdir.makedir(self.output_dir) # Start frames meta file self.meta_name = 'frames_meta.csv' self.frames_meta = aux_utils.make_dataframe() # Write images self.time_idx = 0 self.pos_ids = [7, 8, 10] self.channel_ids = [0, 1, 2, 3] self.slice_ids = [0, 1, 2, 3, 4, 5] self.im = 1500 * np.ones((30, 20), dtype=np.uint16) self.im[10:20, 5:15] = 3000 for c in self.channel_ids: for p in self.pos_ids: for z in self.slice_ids: im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=z, time_idx=self.time_idx, pos_idx=p, ) cv2.imwrite( os.path.join(self.image_dir, im_name), self.im + c * 100, ) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) # Write metadata self.frames_meta.to_csv( os.path.join(self.image_dir, self.meta_name), sep=',', ) # Make input masks self.input_mask_channel = 111 self.input_mask_dir = os.path.join(self.temp_path, 'input_mask_dir') self.tempdir.makedir(self.input_mask_dir) # Must have at least two foreground classes in mask for weight map to work mask = np.zeros((30, 20), dtype=np.uint16) mask[5:10, 5:15] = 1 mask[20:25, 5:10] = 2 mask_meta = aux_utils.make_dataframe() for p in self.pos_ids: for z in self.slice_ids: im_name = aux_utils.get_im_name( channel_idx=self.input_mask_channel, slice_idx=z, time_idx=self.time_idx, pos_idx=p, ) cv2.imwrite( os.path.join(self.input_mask_dir, im_name), mask, ) mask_meta = mask_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) mask_meta.to_csv( os.path.join(self.input_mask_dir, self.meta_name), sep=',', ) # Create preprocessing config self.pp_config = { 'output_dir': self.output_dir, 'input_dir': self.image_dir, 'channel_ids': [0, 1, 3], 'num_workers': 4, 'flat_field': {'estimate': True, 'block_size': 2, 'correct': True}, 'masks': {'channels': [3], 'str_elem_radius': 3, 'normalize_im': False}, 'tile': {'tile_size': [10, 10], 'step_size': [10, 10], 'depths': [1, 1, 1], 'mask_depth': 1, 'image_format': 'zyx', 'normalize_channels': [True, True, True] }, } # Create base config, generated party from pp_config in script self.base_config = { 'input_dir': self.image_dir, 'output_dir': self.output_dir, 'slice_ids': -1, 'time_ids': -1, 'pos_ids': -1, 'channel_ids': self.pp_config['channel_ids'], 'uniform_struct': True, 'int2strlen': 3, 'num_workers': 4, 'normalize_channels': [True, True, True] }
def setUp(self): """Set up a dir for tiling with flatfield""" self.tempdir = TempDirectory() self.temp_path = self.tempdir.path # Start frames meta file self.meta_name = 'frames_meta.csv' frames_meta = aux_utils.make_dataframe() # Write images self.im = 127 * np.ones((15, 11), dtype=np.uint8) self.im2 = 234 * np.ones((15, 11), dtype=np.uint8) self.channel_idx = 1 self.time_idx = 5 self.pos_idx1 = 7 self.pos_idx2 = 8 self.int2str_len = 3 # Write test images with 4 z and 2 pos idx for z in range(15, 20): im_name = aux_utils.get_im_name( channel_idx=self.channel_idx, slice_idx=z, time_idx=self.time_idx, pos_idx=self.pos_idx1, ) cv2.imwrite( os.path.join(self.temp_path, im_name), self.im, ) frames_meta = frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) for z in range(15, 20): im_name = aux_utils.get_im_name( channel_idx=self.channel_idx, slice_idx=z, time_idx=self.time_idx, pos_idx=self.pos_idx2, ) cv2.imwrite( os.path.join(self.temp_path, im_name), self.im2, ) frames_meta = frames_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) # Write metadata frames_meta.to_csv( os.path.join(self.temp_path, self.meta_name), sep=',', ) # Add flatfield self.flat_field_dir = os.path.join(self.temp_path, 'ff_dir') self.tempdir.makedir('ff_dir') self.ff_im = 4. * np.ones((15, 11)) self.ff_name = os.path.join( self.flat_field_dir, 'flat-field_channel-1.npy', ) np.save(self.ff_name, self.ff_im, allow_pickle=True, fix_imports=True) # Instantiate tiler class self.output_dir = os.path.join(self.temp_path, 'tile_dir') self.tile_inst = tile_images.ImageTilerUniform( input_dir=self.temp_path, output_dir=self.output_dir, tile_size=[5, 5], step_size=[4, 4], depths=3, channel_ids=[1], normalize_channels=[True], flat_field_dir=self.flat_field_dir, ) exp_fnames = [ 'im_c001_z015_t005_p007.png', 'im_c001_z016_t005_p007.png', 'im_c001_z017_t005_p007.png' ] self.exp_fnames = [ os.path.join(self.temp_path, fname) for fname in exp_fnames ] self.exp_tile_indices = [ [0, 5, 0, 5], [0, 5, 4, 9], [0, 5, 6, 11], [10, 15, 0, 5], [10, 15, 4, 9], [10, 15, 6, 11], [4, 9, 0, 5], [4, 9, 4, 9], [4, 9, 6, 11], [8, 13, 0, 5], [8, 13, 4, 9], [8, 13, 6, 11], ] # create a mask mask_dir = os.path.join(self.temp_path, 'mask_dir') os.makedirs(mask_dir, exist_ok=True) mask_images = np.zeros((15, 11, 5), dtype='bool') mask_images[4:12, 4:9, 2:4] = 1 # write mask images and add meta to frames_meta self.mask_channel = 3 mask_meta = [] for z in range(5): cur_im = mask_images[:, :, z] im_name = aux_utils.get_im_name( channel_idx=3, slice_idx=z + 15, time_idx=self.time_idx, pos_idx=self.pos_idx1, ext='.npy', ) np.save(os.path.join(mask_dir, im_name), cur_im) cur_meta = { 'channel_idx': 3, 'slice_idx': z + 15, 'time_idx': self.time_idx, 'pos_idx': self.pos_idx1, 'file_name': im_name } mask_meta.append(cur_meta) mask_meta_df = pd.DataFrame.from_dict(mask_meta) mask_meta_df.to_csv(os.path.join(mask_dir, 'frames_meta.csv'), sep=',') self.mask_dir = mask_dir exp_tile_indices = [[0, 5, 0, 5], [0, 5, 4, 9], [0, 5, 6, 11], [10, 15, 0, 5], [10, 15, 4, 9], [10, 15, 6, 11], [4, 9, 0, 5], [4, 9, 4, 9], [4, 9, 6, 11], [8, 13, 0, 5], [8, 13, 4, 9], [8, 13, 6, 11]] self.exp_tile_indices = exp_tile_indices
def test_parse_idx_from_name_no_channel(): file_name = 'img_phase_t500_p400_z300.tif' aux_utils.parse_idx_from_name(file_name)
def setUp(self): """Set up a directory for mask generation, no flatfield""" self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.meta_fname = 'frames_meta.csv' frames_meta = aux_utils.make_dataframe() # create an image with bimodal hist x = np.linspace(-4, 4, 32) y = x.copy() z = np.linspace(-3, 3, 8) xx, yy, zz = np.meshgrid(x, y, z) sph = (xx**2 + yy**2 + zz**2) fg = (sph <= 8) * (8 - sph) fg[fg > 1e-8] = (fg[fg > 1e-8] / np.max(fg)) * 127 + 128 fg = np.around(fg).astype('uint8') bg = np.around((sph > 8) * sph).astype('uint8') object1 = fg + bg # create an image with a rect rec = np.zeros(sph.shape) rec[3:30, 14:18, 3:6] = 120 rec[14:18, 3:30, 3:6] = 120 self.sph_object = object1 self.rec_object = rec self.channel_ids = [1, 2] self.time_ids = 0 self.pos_ids = 1 self.int2str_len = 3 for z in range(sph.shape[2]): im_name = aux_utils.get_im_name( time_idx=self.time_ids, channel_idx=1, slice_idx=z, pos_idx=self.pos_ids, ) with warnings.catch_warnings(): warnings.simplefilter("ignore") sk_im_io.imsave( os.path.join(self.temp_path, im_name), object1[:, :, z].astype('uint8'), ) frames_meta = frames_meta.append(aux_utils.parse_idx_from_name( im_name, aux_utils.DF_NAMES), ignore_index=True) for z in range(rec.shape[2]): im_name = aux_utils.get_im_name( time_idx=self.time_ids, channel_idx=2, slice_idx=z, pos_idx=self.pos_ids, ) with warnings.catch_warnings(): warnings.simplefilter("ignore") sk_im_io.imsave( os.path.join(self.temp_path, im_name), rec[:, :, z].astype('uint8'), ) frames_meta = frames_meta.append(aux_utils.parse_idx_from_name( im_name, aux_utils.DF_NAMES), ignore_index=True) # Write metadata frames_meta.to_csv(os.path.join(self.temp_path, self.meta_fname), sep=',') self.output_dir = os.path.join(self.temp_path, 'mask_dir') self.mask_gen_inst = MaskProcessor(input_dir=self.temp_path, output_dir=self.output_dir, channel_ids=self.channel_ids)
def setUp(self, mock_model): """ Set up a directory with images """ mock_model.return_value = 'dummy_model' self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('mask_dir') self.tempdir.makedir('model_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') # Create a temp image dir self.im = np.zeros((10, 16), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 for p in range(5): for c in range(3): for z in range(6): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.image_dir, im_name), self.im + c * 10) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 for p in range(5): for z in range(6): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=self.mask_channel, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.mask_dir, im_name), self.im + 1) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to mask dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Setup model dir split_samples = { "train": [0, 1], "val": [2], "test": [3, 4], } aux_utils.write_json( split_samples, os.path.join(self.model_dir, 'split_samples.json'), ) # Make configs with fields necessary for 2.5D segmentation inference self.train_config = { 'network': { 'class': 'UNetStackTo2D', 'data_format': 'channels_first', 'depth': 5, 'width': 10, 'height': 10 }, 'dataset': { 'split_by_column': 'pos_idx', 'input_channels': [1], 'target_channels': [self.mask_channel], 'model_task': 'segmentation', }, } self.inference_config = { 'model_dir': self.model_dir, 'model_fname': 'dummy_weights.hdf5', 'image_dir': self.image_dir, 'data_split': 'test', 'images': { 'image_format': 'zyx', 'image_ext': '.png', }, 'metrics': { 'metrics': ['dice'], 'metrics_orientations': ['xy'], }, 'masks': { 'mask_dir': self.mask_dir, 'mask_type': 'target', 'mask_channel': 50, } } # Instantiate class self.infer_inst = image_inference.ImagePredictor( train_config=self.train_config, inference_config=self.inference_config, )
def compute_metrics(model_dir, image_dir, metrics_list, orientations_list, test_data=True): """ Compute specified metrics for given orientations for predictions, which are assumed to be stored in model_dir/predictions. Targets are stored in image_dir. Writes metrics csv files for each orientation in model_dir/predictions. :param str model_dir: Assumed to contain config, split_samples.json and subdirectory predictions/ :param str image_dir: Directory containing target images with frames_meta.csv :param list metrics_list: See inference/evaluation_metrics.py for options :param list orientations_list: Any subset of {xy, xz, yz, xyz} (see evaluation_metrics) :param bool test_data: Uses test indices in split_samples.json, otherwise all indices """ # Load config file config_name = os.path.join(model_dir, 'config.yml') with open(config_name, 'r') as f: config = yaml.safe_load(f) # Load frames metadata and determine indices frames_meta = pd.read_csv(os.path.join(image_dir, 'frames_meta.csv')) if isinstance(metrics_list, str): metrics_list = [metrics_list] metrics_inst = metrics.MetricsEstimator(metrics_list=metrics_list) split_idx_name = config['dataset']['split_by_column'] if test_data: idx_fname = os.path.join(model_dir, 'split_samples.json') try: split_samples = aux_utils.read_json(idx_fname) test_ids = split_samples['test'] except FileNotFoundError as e: print("No split_samples file. Will predict all images in dir.") else: test_ids = np.unique(frames_meta[split_idx_name]) # Find other indices to iterate over than split index name # E.g. if split is position, we also need to iterate over time and slice test_meta = pd.read_csv(os.path.join(model_dir, 'test_metadata.csv')) metadata_ids = {split_idx_name: test_ids} iter_ids = ['slice_idx', 'pos_idx', 'time_idx'] for id in iter_ids: if id != split_idx_name: metadata_ids[id] = np.unique(test_meta[id]) # Create image subdirectory to write predicted images pred_dir = os.path.join(model_dir, 'predictions') target_channel = config['dataset']['target_channels'][0] # If network depth is > 3 determine depth margins for +-z depth = 1 if 'depth' in config['network']: depth = config['network']['depth'] # Get channel name and extension for predictions pred_fnames = [f for f in os.listdir(pred_dir) if f.startswith('im_')] meta_row = aux_utils.parse_idx_from_name(pred_fnames[0]) pred_channel = meta_row['channel_idx'] _, ext = os.path.splitext(pred_fnames[0]) if isinstance(orientations_list, str): orientations_list = [orientations_list] available_orientations = {'xy', 'xz', 'yz', 'xyz'} assert set(orientations_list).issubset(available_orientations), \ "Orientations must be subset of {}".format(available_orientations) fn_mapping = { 'xy': metrics_inst.estimate_xy_metrics, 'xz': metrics_inst.estimate_xz_metrics, 'yz': metrics_inst.estimate_yz_metrics, 'xyz': metrics_inst.estimate_xyz_metrics, } metrics_mapping = { 'xy': metrics_inst.get_metrics_xy, 'xz': metrics_inst.get_metrics_xz, 'yz': metrics_inst.get_metrics_yz, 'xyz': metrics_inst.get_metrics_xyz, } df_mapping = { 'xy': pd.DataFrame(), 'xz': pd.DataFrame(), 'yz': pd.DataFrame(), 'xyz': pd.DataFrame(), } # Iterate over all indices for test data for time_idx in metadata_ids['time_idx']: for pos_idx in metadata_ids['pos_idx']: target_fnames = [] pred_fnames = [] for slice_idx in metadata_ids['slice_idx']: im_idx = aux_utils.get_meta_idx( frames_metadata=frames_meta, time_idx=time_idx, channel_idx=target_channel, slice_idx=slice_idx, pos_idx=pos_idx, ) target_fname = os.path.join( image_dir, frames_meta.loc[im_idx, 'file_name'], ) target_fnames.append(target_fname) pred_fname = aux_utils.get_im_name( time_idx=time_idx, channel_idx=pred_channel, slice_idx=slice_idx, pos_idx=pos_idx, ext=ext, ) pred_fname = os.path.join(pred_dir, pred_fname) pred_fnames.append(pred_fname) target_stack = image_utils.read_imstack( input_fnames=tuple(target_fnames), ) pred_stack = image_utils.read_imstack( input_fnames=tuple(pred_fnames), normalize_im=False, ) if depth == 1: # Remove singular z dimension for 2D image target_stack = np.squeeze(target_stack) pred_stack = np.squeeze(pred_stack) if target_stack.dtype == np.float64: target_stack = target_stack.astype(np.float32) pred_name = "t{}_p{}".format(time_idx, pos_idx) for orientation in orientations_list: metric_fn = fn_mapping[orientation] metric_fn( target=target_stack, prediction=pred_stack, pred_name=pred_name, ) df_mapping[orientation] = df_mapping[orientation].append( metrics_mapping[orientation](), ignore_index=True, ) # Save non-empty dataframes for orientation in orientations_list: metrics_df = df_mapping[orientation] df_name = 'metrics_{}.csv'.format(orientation) metrics_name = os.path.join(pred_dir, df_name) metrics_df.to_csv(metrics_name, sep=",", index=False)
def setUp(self, mock_model): """ Set up a directory with 3D images """ mock_model.return_value = 'dummy_model' self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('mask_dir') self.tempdir.makedir('model_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') # Create a temp image dir self.im = np.zeros((10, 10, 8), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 self.slice_idx = 0 for p in range(5): for c in range(3): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=self.slice_idx, pos_idx=p, ext='.npy', ) np.save(os.path.join(self.image_dir, im_name), self.im + c * 10, allow_pickle=True, fix_imports=True) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 # Mask half the image mask = np.zeros_like(self.im) mask[:5, ...] = 1 for p in range(5): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=self.mask_channel, slice_idx=self.slice_idx, pos_idx=p, ext='.npy', ) np.save(os.path.join(self.mask_dir, im_name), mask) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to mask dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Setup model dir split_samples = { "train": [0, 1], "val": [2], "test": [3, 4], } aux_utils.write_json( split_samples, os.path.join(self.model_dir, 'split_samples.json'), ) # Make configs with fields necessary for 2.5D segmentation inference self.train_config = { 'network': { 'class': 'UNet3D', 'data_format': 'channels_first', 'num_filters_per_block': [8, 16], 'depth': 5, 'width': 5, 'height': 5 }, 'dataset': { 'split_by_column': 'pos_idx', 'input_channels': [1], 'target_channels': [2], 'model_task': 'regression', }, } self.inference_config = { 'model_dir': self.model_dir, 'model_fname': 'dummy_weights.hdf5', 'image_dir': self.image_dir, 'data_split': 'test', 'images': { 'image_format': 'zyx', 'image_ext': '.png', }, 'metrics': { 'metrics': ['mse'], 'metrics_orientations': ['xyz'], }, 'masks': { 'mask_dir': self.mask_dir, 'mask_type': 'metrics', 'mask_channel': 50, }, 'inference_3d': { 'tile_shape': [5, 5, 5], 'num_overlap': [1, 1, 1], 'overlap_operation': 'mean', }, } # Instantiate class self.infer_inst = image_inference.ImagePredictor( train_config=self.train_config, inference_config=self.inference_config, )
def setUp(self): """ Set up a directory with images """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('model_dir') self.tempdir.makedir('mask_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') # Create a temp image dir im = np.zeros((10, 16), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 for p in range(5): for z in range(4): for c in range(3): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.image_dir, im_name), im + c * 10) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 for p in range(5): for z in range(4): im_name = aux_utils.get_im_name( time_idx=2, channel_idx=self.mask_channel, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.mask_dir, im_name), im + 1) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Select inference split of dataset self.split_col_ids = ('pos_idx', [1, 3]) # Make configs with fields necessary for inference dataset dataset_config = { 'input_channels': [2], 'target_channels': [self.mask_channel], 'model_task': 'segmentation', } self.network_config = { 'class': 'UNetStackTo2D', 'depth': 3, 'data_format': 'channels_first', } # Instantiate class self.data_inst = inference_dataset.InferenceDataSet( image_dir=self.image_dir, dataset_config=dataset_config, network_config=self.network_config, split_col_ids=self.split_col_ids, mask_dir=self.mask_dir, )
import micro_dl.utils.aux_utils as aux_utils # Create test metadata table meta_df = aux_utils.make_dataframe() channel_idx = 5 time_idx = 6 for s in range(3): for p in range(4): im_temp = aux_utils.get_im_name( channel_idx=channel_idx, slice_idx=s, time_idx=time_idx, pos_idx=p, ) meta_df = meta_df.append( aux_utils.parse_idx_from_name(im_temp), ignore_index=True, ) def test_import_object(): module_name = 'networks' class_name = 'InterpUpSampling2D' class_inst = aux_utils.import_object(module_name, class_name) nose.tools.assert_true(inspect.isclass(class_inst)) nose.tools.assert_equal(class_inst.__name__, class_name) def test_read_config(): with TempDirectory() as tempdir: # The function doesn't care about file format, names just have to start with im_