def save(self, tile_indices, output_dir, data, compress=6):
region_index, tile_index, tx, ty = tile_indices
img_label, stats = data
# Save label volumes if present (use compression as these are often highly redundant)
label_tile_path = None
if img_label is not None:
label_tile_path = cytokit_io.get_cytometry_image_path(
region_index, tx, ty)
cytokit_io.save_tile(osp.join(output_dir, label_tile_path),
img_label,
config=self.config,
compress=compress)
# Save statistics if present
stats_path = None
if stats is not None:
# Append useful metadata to cytometry stats (align these names to those used in config.TileDims)
# and export as csv
stats.insert(0, 'tile_y', ty)
stats.insert(0, 'tile_x', tx)
stats.insert(0, 'tile_index', tile_index)
stats.insert(0, 'region_index', region_index)
stats_path = cytokit_io.get_cytometry_stats_path(
region_index, tx, ty)
cytokit_io.save_csv(osp.join(output_dir, stats_path),
stats,
index=False)
return label_tile_path, stats_path
def save(self, tile_indices, output_dir, data):
region_index, tile_index, tx, ty = tile_indices
best_focus_tile, best_z, scores = data
path = cytokit_io.get_best_focus_img_path(region_index, tx, ty, best_z)
if self.save_tile:
cytokit_io.save_tile(osp.join(output_dir, path),
best_focus_tile,
config=self.config)
return [path]
def save(self, tile_indices, output_dir, tile):
# Overwrite the original preprocessed tile with corrected version
path = cytokit_io.get_processor_img_path(tile_indices.region_index,
tile_indices.tile_x,
tile_indices.tile_y)
cytokit_io.save_tile(osp.join(output_dir, path),
tile,
config=self.config)
return path
def postprocess_tile(tile, tile_indices, ops, log_fn, task_config):
output_dir = task_config.output_dir
# Illumination Correction
if ops.illumination_op:
# Prepare and save illumination images, if not already done
ops.illumination_op.prepare_region_data(output_dir)
path = ops.illumination_op.save_region_data(output_dir)
if path is not None:
log_fn('Illumination data saved to "{}"'.format(path))
# Run correction for tile
tile = ops.illumination_op.run(tile, tile_indices)
log_fn('Illumination correction complete', tile)
else:
log_fn('Skipping illumination correction', debug=True)
# Spectral Unmixing
if ops.unmixing_op:
# Prepare unmixing models for each region
ops.unmixing_op.prepare_region_data(output_dir)
# Run correction for tile
tile = ops.unmixing_op.run(tile, tile_indices)
log_fn('Spectral unmixing complete', tile)
else:
log_fn('Skipping spectral unmixing', debug=True)
# Get best focus data
# TODO Prevent needing to re-read the processor data file each time
best_focus_data = function_data.get_best_focus_coord_map(output_dir)
best_focus_z_plane = best_focus_data[(tile_indices.region_index,
tile_indices.tile_x,
tile_indices.tile_y)]
# Rerun cytometry based on corrected tile
tile, cyto_data = ops.cytometry_op.run(
tile, best_focus_z_plane=best_focus_z_plane)
paths = ops.cytometry_op.save(tile_indices, output_dir, cyto_data)
log_fn(
'Postprocessing cytometry complete; Statistics saved to "{}"'.format(
paths[-1]), cyto_data[0])
# Save resulting tile
path = cytokit_io.get_processor_img_path(tile_indices.region_index,
tile_indices.tile_x,
tile_indices.tile_y)
cytokit_io.save_tile(osp.join(output_dir, path),
tile,
config=task_config.exp_config)
log_fn('Saved postprocessed tile to "{}"'.format(path), tile)
def create_montage(output_dir,
config,
extract,
name,
region_indexes,
prep_fn=None,
compress=6):
from cytokit.utils import ij_utils
# Loop through regions and generate a montage for each, skipping any (with a warning) that
# do not have focal plane selection information
if region_indexes is None:
region_indexes = config.region_indexes
path = None
for ireg in region_indexes:
logger.info('Generating montage for region %d of %d', ireg + 1,
len(region_indexes))
tiles = []
labels = None
for itile in range(config.n_tiles_per_region):
tx, ty = config.get_tile_coordinates(itile)
path = cytokit_io.get_extract_image_path(ireg, tx, ty, extract)
tile, meta = cytokit_io.read_tile(osp.join(output_dir, path),
return_metadata=True)
if labels is None:
labels = meta['labels']
tiles.append(tile)
reg_img_montage = montage(tiles, config)
if prep_fn is not None:
reg_img_montage = prep_fn(reg_img_montage)
path = osp.join(output_dir,
cytokit_io.get_montage_image_path(ireg, name))
logger.info('Saving montage to file "%s"', path)
tags = [] if labels is None else ij_utils.get_slice_label_tags(labels)
cytokit_io.save_tile(path,
reg_img_montage,
config=config,
infer_labels=False,
extratags=tags,
compress=compress)
logger.info('Montage generation complete; results saved to "%s"',
None if path is None else osp.dirname(path))
def extract(self,
name,
channels,
z='best',
region_indexes=None,
tile_indexes=None,
raw_dir=None):
"""Create a new data extraction include either raw, processed, or cytometric imaging data
Args:
name: Name of extraction to be created; This will be used to construct result path like
EXP_DIR/output/extract/`name`
channels: List of strings indicating channel names (case-insensitive) prefixed by source for that
channel (e.g. proc_DAPI, raw_CD4, cyto_nucleus_boundary); Available sources are:
- "raw": Raw data images
- "proc": Data generated as a results of preprocessing
- "cyto": Cytometric object data (nuclei and cell boundaries)
z: String or 1-based index selector for z indexes constructed as any of the following:
- "best": Indicates that z slices should be inferred based on focal quality (default option)
- "all": Indicates that a slice for all z-planes should be used
- str or int: A single value will be interpreted as a single index
- tuple: A 2-item or 3-item tuple forming the slice (start, stop[, step]); stop is inclusive
- list: A list of integers will be used as is
region_indexes: 1-based sequence of region indexes to process; can be specified as:
- None: Region indexes will be inferred from experiment configuration
- str or int: A single value will be interpreted as a single index
- tuple: A 2-item or 3-item tuple forming the slice (start, stop[, step]); stop is inclusive
- list: A list of integers will be used as is
tile_indexes: 1-based sequence of tile indexes to process; has same semantics as `region_indexes`
raw_dir: If using any channels sourced from raw data, this directory must be specified and should
be equivalent to the same raw directory used during processing (i.e. nearly all operations like
this are run relative to an `output_dir` -- the result of processing -- but in this case
the original raw data path is needed as well)
"""
channel_map = _map_channels(self.config, channels).groupby('source')
channel_sources = sorted(list(channel_map.groups.keys()))
z_slice_fn = _get_z_slice_fn(z, self.data_dir)
region_indexes = cli.resolve_index_list_arg(region_indexes,
zero_based=True)
tile_indexes = cli.resolve_index_list_arg(tile_indexes,
zero_based=True)
logging.info('Creating extraction "%s"', name)
tile_locations = _get_tile_locations(self.config, region_indexes,
tile_indexes)
extract_path = None
for i, loc in enumerate(tile_locations):
logging.info('Extracting tile {} of {}'.format(
i + 1, len(tile_locations)))
extract_tile = []
# Create function used to crop out z-slices from extracted volumes
z_slice = z_slice_fn(loc.region_index, loc.tile_x, loc.tile_y)
slice_labels = []
for src in channel_sources:
# Initialize tile generator for this data source (which are all the same except
# for when using raw data, which does not have pre-assembled tiles available)
tile_gen_dir = self.data_dir
tile_gen_mode = 'stack'
if src == CH_SRC_RAW:
if not raw_dir:
raise ValueError(
'When extracting raw data channels, the `raw_dir` argument must be provided'
)
tile_gen_dir = raw_dir
tile_gen_mode = 'raw'
generator = tile_generator.CytokitTileGenerator(
self.config,
tile_gen_dir,
loc.region_index,
loc.tile_index,
mode=tile_gen_mode,
path_fmt_name=PATH_FMT_MAP[src])
tile = generator.run(None)
# Crop raw images if necessary
if src == CH_SRC_RAW:
tile = tile_crop.CytokitTileCrop(self.config).run(tile)
# Sort channels by name to make extract channel order deterministic
for _, r in channel_map.get_group(src).sort_values(
'channel_name').iterrows():
# Extract (z, h, w) subtile
sub_tile = tile[r['cycle_index'], z_slice,
r['channel_index']]
logging.debug(
'Extraction for cycle %s, channel %s (%s), z slice %s, source "%s" complete (tile shape = %s)',
r['cycle_index'], r['channel_index'],
r['channel_name'], z_slice, src, sub_tile.shape)
assert sub_tile.ndim == 3, \
'Expecting sub_tile to have 3 dimensions but got shape {}'.format(sub_tile.shape)
slice_labels.append('{}_{}'.format(src, r['channel_name']))
extract_tile.append(sub_tile)
# Stack the subtiles to give array with shape (z, channels, h, w) and then reshape to 5D
# format like (cycles, z, channels, h, w)
extract_tile = np.stack(extract_tile, axis=1)[np.newaxis]
assert extract_tile.ndim == 5, \
'Expecting extract tile to have 5 dimensions but got shape {}'.format(extract_tile.shape)
extract_path = cytokit_io.get_extract_image_path(
loc.region_index, loc.tile_x, loc.tile_y, name)
extract_path = osp.join(self.data_dir, extract_path)
logging.debug('Saving tile with shape %s (dtype = %s) to "%s"',
extract_tile.shape, extract_tile.dtype, extract_path)
# Construct slice labels as repeats across z-dimension (there is only one time/cycle dimension)
slice_label_tags = ij_utils.get_channel_label_tags(
slice_labels, z=extract_tile.shape[1], t=1)
cytokit_io.save_tile(extract_path,
extract_tile,
config=self.config,
infer_labels=False,
extratags=slice_label_tags)
logging.info('Extraction complete (results saved to %s)',
osp.dirname(extract_path) if extract_path else None)
def preprocess_tile(tile, tile_indices, ops, log_fn, task_config):
output_dir = task_config.output_dir
# Drift Compensation
if ops.align_op:
tile = ops.align_op.run(tile)
log_fn('Drift compensation complete', tile)
else:
log_fn('Skipping drift compensation', debug=True)
# Crop off overlap in imaging process
if ops.crop_op:
tile = ops.crop_op.run(tile)
log_fn('Tile overlap crop complete', tile)
else:
log_fn('Skipping tile crop', debug=True)
# Resample images for improved downstream speed
if ops.resize_op:
tile = ops.resize_op.run(tile)
log_fn('Tile resize complete', tile)
else:
log_fn('Skipping tile resize', debug=True)
# Best Focal Plane Selection
best_focus_data = None
if ops.focus_op:
# Used the cropped, but un-deconvolved tile for focal plane selection
best_focus_data = ops.focus_op.run(tile)
ops.focus_op.save(tile_indices, output_dir, best_focus_data)
log_fn('Focal plane selection complete', best_focus_data[0])
else:
log_fn('Skipping focal plane selection', debug=True)
# Deconvolution
if ops.decon_op:
tile = ops.decon_op.run(tile)
log_fn('Deconvolution complete', tile)
else:
log_fn('Skipping deconvolution', debug=True)
# Cytometry (segmentation + quantification)
if ops.cytometry_op:
best_focus_z_plane = best_focus_data[1] if best_focus_data else None
tile, cyto_data = ops.cytometry_op.run(
tile,
best_focus_z_plane=best_focus_z_plane,
tile_indices=tile_indices)
paths = ops.cytometry_op.save(tile_indices, output_dir, cyto_data)
log_fn(
'Tile cytometry complete; Statistics saved to "{}"'.format(
paths[-1]), cyto_data[0])
else:
log_fn('Skipping tile cytometry', debug=True)
# Tile summary statistic operations
if ops.summary_op:
ops.summary_op.run(tile)
log_fn('Tile statistic summary complete')
else:
log_fn('Skipping tile statistic summary', debug=True)
# Save the output tile if tile generation/assembly was enabled
if task_config.op_flags.run_tile_generator:
path = cytokit_io.get_processor_img_path(tile_indices.region_index,
tile_indices.tile_x,
tile_indices.tile_y)
cytokit_io.save_tile(osp.join(output_dir, path),
tile,
config=task_config.exp_config)
log_fn('Saved preprocessed tile to path "{}"'.format(path), tile)
