def test_tile_stack(self):
"""Test tile_stack"""
self.tile_inst.tile_stack()
# Read and validate the saved metadata
tile_dir = self.tile_inst.get_tile_dir()
frames_meta = pd.read_csv(os.path.join(tile_dir, 'frames_meta.csv'))
self.assertSetEqual(set(frames_meta.channel_idx.tolist()), {1})
self.assertSetEqual(set(frames_meta.slice_idx.tolist()), {16, 17, 18})
self.assertSetEqual(set(frames_meta.time_idx.tolist()), {5})
self.assertSetEqual(set(frames_meta.pos_idx.tolist()), {7, 8})
# 15 rows and step size 4, so it can take 3 full steps and 1 short step
self.assertSetEqual(set(frames_meta.row_start.tolist()), {0, 4, 8, 10})
# 11 cols and step size 4, so it can take 2 full steps and 1 short step
self.assertSetEqual(set(frames_meta.col_start.tolist()), {0, 4, 6})
# Read and validate tiles
im_val = np.mean(norm_util.zscore(self.im / self.ff_im))
im_norm = im_val * np.ones((3, 5, 5))
im_val = np.mean(norm_util.zscore(self.im2 / self.ff_im))
im2_norm = im_val * np.ones((3, 5, 5))
for i, row in frames_meta.iterrows():
tile = np.load(os.path.join(tile_dir, row.file_name))
if row.pos_idx == 7:
np.testing.assert_array_equal(tile, im_norm)
else:
np.testing.assert_array_equal(tile, im2_norm)
def test_preprocess_imstack(self):
"""Test preprocess_imstack"""
im_stack = image_utils.preprocess_imstack(
self.frames_meta,
self.temp_path,
depth=3,
time_idx=self.time_idx,
channel_idx=self.channel_idx,
slice_idx=2,
pos_idx=self.pos_idx,
normalize_im=True,
)
np.testing.assert_equal(im_stack.shape, (32, 32, 3))
exp_stack = normalize.zscore(self.sph[:, :, 1:4])
np.testing.assert_array_equal(im_stack, exp_stack)
# preprocess a 3D image
im_stack = image_utils.preprocess_imstack(
self.meta_3d,
self.temp_path,
depth=1,
time_idx=0,
channel_idx=1,
slice_idx=0,
pos_idx=1,
normalize_im=True,
)
np.testing.assert_equal(im_stack.shape, (32, 32, 8))
def _read_one(tp_dir, channel_ids, fname, flat_field_dir=None):
"""Read one image set
:param str tp_dir: timepoint dir
:param list channel_ids: list of channels to read from
:param str fname: fname of the image. Expects the fname to be the same
in all channels
:param str flat_field_dir: dir where flat field images are stored
:returns: np.array of shape nb_channels, im_size (with or without
flat field correction)
"""
cur_images = []
for ch in channel_ids:
cur_fname = os.path.join(tp_dir, 'channel_{}'.format(ch), fname)
cur_image = np.load(cur_fname)
if flat_field_dir is not None:
ff_fname = os.path.join(flat_field_dir,
'flat-field_channel-{}.npy'.format(ch))
ff_image = np.load(ff_fname)
cur_image = image_utils.apply_flat_field_correction(
cur_image, flat_field_image=ff_image)
cur_image = zscore(cur_image)
cur_images.append(cur_image)
cur_images = np.stack(cur_images)
return cur_images
def read_imstack(input_fnames,
flat_field_fname=None,
hist_clip_limits=None,
is_mask=False,
normalize_im=True):
"""
Read the images in the fnames and assembles a stack.
If images are masks, make sure they're boolean by setting >0 to True
:param tuple input_fnames: tuple of input fnames with full path
:param str flat_field_fname: fname of flat field image
:param tuple hist_clip_limits: limits for histogram clipping
:param bool is_mask: Indicator for if files contain masks
:param bool normalize_im: Whether to zscore normalize im stack
:return np.array: input stack flat_field correct and z-scored if regular
images, booleans if they're masks
"""
im_stack = []
for idx, fname in enumerate(input_fnames):
im = read_image(fname)
if flat_field_fname is not None:
# multiple flat field images are passed in case of mask generation
if isinstance(flat_field_fname, (list, tuple)):
flat_field_image = np.load(flat_field_fname[idx])
else:
flat_field_image = np.load(flat_field_fname)
if not is_mask and not normalize_im:
im = apply_flat_field_correction(
im,
flat_field_image=flat_field_image,
)
im_stack.append(im)
input_image = np.stack(im_stack, axis=-1)
# remove singular dimension for 3D images
if len(input_image.shape) > 3:
input_image = np.squeeze(input_image)
if not is_mask:
if hist_clip_limits is not None:
input_image = normalize.hist_clipping(
input_image,
hist_clip_limits[0],
hist_clip_limits[1]
)
if normalize_im:
input_image = normalize.zscore(input_image)
else:
if input_image.dtype != bool:
input_image = input_image > 0
return input_image
def test_read_imstack(self):
"""Test read_imstack"""
fnames = self.frames_meta['file_name'][:3]
fnames = [os.path.join(self.temp_path, fname) for fname in fnames]
# non-boolean
im_stack = image_utils.read_imstack(fnames)
exp_stack = normalize.zscore(self.sph[:, :, :3])
np.testing.assert_equal(im_stack.shape, (32, 32, 3))
np.testing.assert_array_equal(exp_stack[:, :, :3],
im_stack)
# read a 3D image
im_stack = image_utils.read_imstack([self.sph_fname])
np.testing.assert_equal(im_stack.shape, (32, 32, 8))
# read multiple 3D images
im_stack = image_utils.read_imstack((self.sph_fname, self.sph_fname))
np.testing.assert_equal(im_stack.shape, (32, 32, 8, 2))
def _get_volume(self, fname_list, normalize=True, aug_idx=0):
"""
Read tiles from fname_list and stack them into an image volume.
:param list fname_list: list of file names of input/target images
:param bool normalize: Whether to zscore normalize tiles
:param int aug_idx: type of augmentation to be applied (if any)
:return: np.ndarray of stacked images
"""
image_volume = []
for fname in fname_list:
cur_tile = np.load(os.path.join(self.tile_dir, fname))
if self.augmentations:
cur_tile = self._augment_image(cur_tile, aug_idx)
if self.squeeze:
cur_tile = np.squeeze(cur_tile)
image_volume.append(cur_tile)
# Stack images channels first
image_volume = np.stack(image_volume)
if image_volume.dtype == bool:
image_volume = image_volume.astype(np.float32)
elif normalize:
image_volume = norm.zscore(image_volume)
return image_volume
def preprocess_imstack(frames_metadata,
input_dir,
depth,
time_idx,
channel_idx,
slice_idx,
pos_idx,
flat_field_im=None,
hist_clip_limits=None,
normalize_im=False):
"""
Preprocess image given by indices: flatfield correction, histogram
clipping and z-score normalization is performed.
:param pd.DataFrame frames_metadata: DF with meta info for all images
:param str input_dir: dir containing input images
:param int depth: num of slices in stack if 2.5D or depth for 3D
:param int time_idx: Time index
:param int channel_idx: Channel index
:param int slice_idx: Slice (z) index
:param int pos_idx: Position (FOV) index
:param np.array flat_field_im: Flat field image for channel
:param list hist_clip_limits: Limits for histogram clipping (size 2)
:param bool normalize_im: indicator to normalize image based on z-score or not
:return np.array im: 3D preprocessed image
"""
margin = 0 if depth == 1 else depth // 2
im_stack = []
for z in range(slice_idx - margin, slice_idx + margin + 1):
meta_idx = aux_utils.get_meta_idx(
frames_metadata,
time_idx,
channel_idx,
z,
pos_idx,
)
file_path = os.path.join(
input_dir,
frames_metadata.loc[meta_idx, "file_name"],
)
im = read_image(file_path)
# Only flatfield correct images that will be normalized
if flat_field_im is not None and not normalize_im:
im = apply_flat_field_correction(
im,
flat_field_image=flat_field_im,
)
im_stack.append(im)
if len(im.shape) == 3:
# each channel is tiled independently and stacked later in dataset cls
im_stack = im
assert depth == 1, 'more than one 3D volume gets read'
else:
# Stack images in same channel
im_stack = np.stack(im_stack, axis=2)
# normalize
if hist_clip_limits is not None:
im_stack = normalize.hist_clipping(
im_stack,
hist_clip_limits[0],
hist_clip_limits[1],
)
if normalize_im:
im_stack = normalize.zscore(im_stack)
return im_stack
def setUp(self):
"""
Set up a directory with some images to generate frames_meta.csv for
"""
self.tempdir = TempDirectory()
self.temp_dir = self.tempdir.path
self.model_dir = os.path.join(self.temp_dir, 'model_dir')
self.pred_dir = os.path.join(self.model_dir, 'predictions')
self.image_dir = os.path.join(self.temp_dir, 'image_dir')
self.tempdir.makedir(self.model_dir)
self.tempdir.makedir(self.pred_dir)
self.tempdir.makedir(self.image_dir)
# Write images
self.time_idx = 5
self.pos_idx = 7
self.im = 1500 * np.ones((30, 20), dtype=np.uint16)
im_add = np.zeros((30, 20), dtype=np.uint16)
im_add[15:, :] = 10
self.ext = '.tif'
# Start frames meta file
self.meta_name = 'frames_meta.csv'
self.frames_meta = aux_utils.make_dataframe()
for c in range(3):
for z in range(5, 10):
im_name = aux_utils.get_im_name(
channel_idx=c,
slice_idx=z,
time_idx=self.time_idx,
pos_idx=self.pos_idx,
ext=self.ext,
)
cv2.imwrite(os.path.join(self.image_dir, im_name), self.im)
if c == 2:
norm_im = normalize.zscore(self.im + im_add).astype(np.float32)
cv2.imwrite(
os.path.join(self.pred_dir, im_name),
norm_im,
)
self.frames_meta = self.frames_meta.append(
aux_utils.parse_idx_from_name(im_name),
ignore_index=True,
)
# Write metadata
self.frames_meta.to_csv(
os.path.join(self.image_dir, self.meta_name),
sep=',',
)
# Write as test metadata in model dir too
self.frames_meta.to_csv(
os.path.join(self.model_dir, 'test_metadata.csv'),
sep=',',
)
# Write split samples
split_idx_fname = os.path.join(self.model_dir, 'split_samples.json')
split_samples = {'test': [5, 6, 7, 8, 9]}
aux_utils.write_json(split_samples, split_idx_fname)
# Write config in model dir
config = {
'dataset': {
'input_channels': [0, 1],
'target_channels': [2],
'split_by_column': 'slice_idx'
},
'network': {}
}
config_name = os.path.join(self.model_dir, 'config.yml')
with open(config_name, 'w') as outfile:
yaml.dump(config, outfile, default_flow_style=False)