def get_config(config_path):
"""Load experiment configuration
Args:
config_path: Either a path to a configuration file or a directory containing a
configuration file with the default name (controlled by CYTOKIT_CONFIG_DEFAULT_FILENAME)
Returns:
Configuration object
"""
# Load experiment configuration and "register" the environment meaning that any variables not
# explicitly defined by env variables should set based on what is present in the configuration
# (it is crucial that this happen first)
config = cytokit_config.load(config_path)
config.register_environment()
return config
def _get_example_conf(self):
conf_dir = osp.join(cytokit.conf_dir, 'v0.1', 'examples', 'ex1')
return cytokit_config.load(conf_dir)
def _get_example_conf(self, file_type='yaml'):
cytokit.set_config_default_filename('experiment.' + file_type)
conf_dir = osp.join(cytokit.conf_dir, 'v0.1', 'examples', 'ex1')
return cytokit_config.load(conf_dir)
def get_example_config(example_name='ex1'):
path = osp.join(cytokit.test_data_dir, 'configs', 'v0.1', 'examples', example_name)
return cytokit_config.load(path)
def test_pipeline_01(self):
out_dir = tempfile.mkdtemp(prefix='cytokit_test_pipeline_01_')
print('Initialized output dir {} for pipeline test 01'.format(out_dir))
raw_dir = osp.join(cytokit.test_data_dir, 'experiment',
'cellular-marker-small', 'raw')
val_dir = osp.join(cytokit.test_data_dir, 'experiment',
'cellular-marker-small', 'validation')
config_dir = osp.join(cytokit.test_data_dir, 'experiment',
'cellular-marker-small', 'config')
config = ck_config.load(config_dir)
# Run processor and extractions/aggregations
processor.Processor(data_dir=raw_dir,
config_path=config_dir).run_all(output_dir=out_dir)
operator.Operator(data_dir=out_dir, config_path=config_dir).run_all()
analysis.Analysis(data_dir=out_dir, config_path=config_dir).run_all()
# ##################### #
# Processor Data Checks #
# ##################### #
df = ck_fn.get_processor_data(out_dir)['drift_compensator']
# Expect one drift comp record since there are two cycles and one is the reference
self.assertEqual(len(df), 1)
# Expecting 12 row and -3 col translation introduced in synthetic data
self.assertEqual(df.iloc[0]['translation'], [12, -3])
df = ck_fn.get_processor_data(out_dir)['focal_plane_selector']
# Expect one focal selection record (there is only 1 tile in experiment and these
# records are per-tile)
self.assertEqual(len(df), 1)
# Expecting second of 3 z planes to have the best focus (data was generated this way)
self.assertEqual(df.iloc[0]['best_z'], 1)
# ##################### #
# Cytometry Stats Check #
# ##################### #
df = ck_fn.get_cytometry_data(out_dir, config, mode='best_z_plane')
# Verify that the overall cell count and size found are in the expected ranges
self.assertTrue(
20 <= len(df) <= 25,
'Expecting between 20 and 25 cells, found {} instead'.format(
len(df)))
nuc_diam, cell_diam = df['nucleus_diameter'].mean(
), df['cell_diameter'].mean()
self.assertTrue(
4 < nuc_diam < 6,
'Expecting mean nucleus diameter in [4, 6] um, found {} instead'.
format(nuc_diam))
self.assertTrue(
8 < cell_diam < 10,
'Expecting mean cell diameter in [8, 10] um, found {} instead'.
format(cell_diam))
# The drift align dapi channels should be nearly identical across cycles, but in this case there are border
# cells that end up with dapi=0 for cval=0 in drift compensation translation function so make the check
# on a threshold (the ratio is < .5 with no drift compensation)
dapi_ratio = df['ni:DAPI2'].mean() / df['ni:DAPI1'].mean()
self.assertTrue(
.8 < dapi_ratio <= 1,
'Expecting cycle 2 DAPI averages to be similar to cycle 1 DAPI after drift compensation, '
'found ratio {} (not in (.8, 1])'.format(dapi_ratio))
# Check that all records are for single z plane (with known best focus)
self.assertEqual(df['z'].nunique(), 1)
self.assertEqual(int(df['z'].unique()[0]), 1)
# Verify that single cell image generation works
df = ck_fn.get_single_cell_image_data(out_dir,
df,
'best_z_segm',
image_size=(64, 64))
self.assertEqual(df['image'].iloc[0].shape, (64, 64, 3))
self.assertTrue(df['image'].notnull().all())
# ################## #
# Segmentation Check #
# ################## #
# Load extract with object masks
img, meta = ck_io.read_tile(osp.join(
out_dir,
ck_io.get_extract_image_path(ireg=0,
tx=0,
ty=0,
name='best_z_segm')),
return_metadata=True)
# Ensure that the 8 channels set for extraction showed up in the resulting hyperstack
self.assertEqual(len(meta['labels']), 8)
# Verify that IoU for both nuclei and cell masks vs ground-truth is > 80%
img_seg_cell = img[0, 0, meta['labels'].index('cyto_cell_mask')]
img_seg_nucl = img[0, 0, meta['labels'].index('cyto_nucleus_mask')]
img_val_cell = sk_io.imread(osp.join(val_dir, 'cells.tif'))
img_val_nucl = sk_io.imread(osp.join(val_dir, 'nuclei.tif'))
def iou(im1, im2):
return ((im1 > 0) & (im2 > 0)).sum() / ((im1 > 0) |
(im2 > 0)).sum()
self.assertGreater(iou(img_seg_cell, img_val_cell), .8)
self.assertGreater(iou(img_seg_nucl, img_val_nucl), .8)
# ############# #
# Montage Check #
# ############# #
# Load montage and check that it has the same dimensions as the extract image above,
# since there is only one tile in this case
img_mntg = ck_io.read_tile(
osp.join(out_dir,
ck_io.get_montage_image_path(ireg=0, name='best_z_segm')))
self.assertEqual(img.shape, img_mntg.shape)
self.assertEqual(img.dtype, img_mntg.dtype)
def test_quantification(self):
"""Validate quantification of objects in the "Random Shapes" test dataset"""
exp_dir = osp.join(cytokit.test_data_dir, 'experiment',
'random-shapes')
config_path = osp.join(exp_dir, 'config', 'experiment.yaml')
config = ck_config.load(config_path)
config.register_environment()
# Pull shape of grid (i.e. region)
region_shape = config.region_height, config.region_width
#########################
# Load tiles and original
#########################
# Load each tile for the experiment
tiles = [
tile_generator.CytokitTileGenerator(config,
osp.join(exp_dir, 'raw'),
region_index=0,
tile_index=i).run()
for i in range(config.n_tiles_per_region)
]
self.assertEqual(
tiles[0].ndim, 5,
'Expecting 5D tiles, got shape {}'.format(tiles[0].shape))
# Load original image used to create individual tile images (i.e. at region scale) and compare
# to a montage generated from the tiles just loaded
img_mtv = ck_io.read_image(
osp.join(exp_dir, 'validation', 'original_shapes_image.tif'))
# Create montage from first channel (which contains object ids for reference)
img_mtg = ck_core.montage(tiles, config)[0, 0, 0]
assert_array_equal(img_mtg, img_mtv)
# Classify objects as either free or on border
# * First create a stacked image containing cleared tiles
img_clr = np.stack(
[segmentation.clear_border(t[0, 0, 0]) for t in tiles])
# Split ids into 2 groups based on the cleared stack image
ids = np.unique(img_mtg[img_mtg > 0])
ids_free = np.setdiff1d(np.unique(img_clr), [0])
ids_brdr = np.setdiff1d(ids, ids_free)
# Check that the background id is not included in any of the above
self.assertTrue(
np.all(ids_free > 0) and np.all(ids_brdr > 0) and np.all(ids > 0))
####################
# Run quantification
####################
def create_segments(im):
# Create segmentation images as (z, ch, h, w)
imb = segmentation.find_boundaries(im, mode='inner')
segments = np.stack([im, im, imb, imb])[np.newaxis]
assert segments.ndim == 4
return segments
# Quantify each tile image and concatenate results
df = pd.concat([
cytometer.CytometerBase.quantify(
tiles[i],
create_segments(tiles[i][0, 0, 0]),
channel_names=config.channel_names,
cell_intensity=['mean', 'median', 'sum', 'var'],
nucleus_intensity=False,
cell_graph=True,
border_features=True,
morphology_features=True).assign(tile_x=c,
tile_y=r,
tile_index=i)
for i, (r, c) in enumerate(np.ndindex(region_shape))
])
#########################
# Validate quantification
#########################
# Ensure that all objects in original image are also present in cytometry data
self.assertTrue(
len(np.intersect1d(ids, df['id'].unique())) == len(ids),
'Object ids expected do not match those found\nIds found: {}\nIds expected: {}'
.format(sorted(df['id'].unique()), sorted(ids)))
# Check that objects classified as on border or not are correct
assert_array_equal(sorted(df[df['cb:on_border'] > 0]['id'].unique()),
sorted(ids_brdr))
assert_array_equal(sorted(df[df['cb:on_border'] < 1]['id'].unique()),
sorted(ids_free))
# Loop through objects identified and validate each one
for i, r in df.iterrows():
# Fetch tile image from tile list and make sure that size of cell returned matches that in image
area = (tiles[r['tile_index']][0, 0, 0] == r['id']).sum()
self.assertEquals(r['cm:size'], area)
# For each channel validate that:
# - mean and median equal the id times channel index (1-based)
# - sum equals area times id times channel index
# - variance is 0
for j, c in enumerate(config.channel_names):
for f in ['mean', 'median']:
self.assertEquals(r['id'] * (j + 1),
r['ci:{}:{}'.format(c, f)])
self.assertEquals(r['id'] * (j + 1) * area,
r['ci:{}:sum'.format(c)])
self.assertEquals(0, r['ci:{}:var'.format(c)])
def __init__(self):
self._exp_config = cytokit_config.load(self.exp_config_path)
self._exp_config.register_environment()
def exp_config(self):
if not self._exp_config:
self._exp_config = cytokit_config.load(self.exp_config_path)
return self._exp_config