def save(self, tile_indices, output_dir, data, compress=6): region_index, tile_index, tx, ty = tile_indices img_label, stats = data # Save label volumes if present (use compression as these are often highly redundant) label_tile_path = None if img_label is not None: label_tile_path = cytokit_io.get_cytometry_image_path( region_index, tx, ty) cytokit_io.save_tile(osp.join(output_dir, label_tile_path), img_label, config=self.config, compress=compress) # Save statistics if present stats_path = None if stats is not None: # Append useful metadata to cytometry stats (align these names to those used in config.TileDims) # and export as csv stats.insert(0, 'tile_y', ty) stats.insert(0, 'tile_x', tx) stats.insert(0, 'tile_index', tile_index) stats.insert(0, 'region_index', region_index) stats_path = cytokit_io.get_cytometry_stats_path( region_index, tx, ty) cytokit_io.save_csv(osp.join(output_dir, stats_path), stats, index=False) return label_tile_path, stats_path
def save(self, tile_indices, output_dir, data): region_index, tile_index, tx, ty = tile_indices best_focus_tile, best_z, scores = data path = cytokit_io.get_best_focus_img_path(region_index, tx, ty, best_z) if self.save_tile: cytokit_io.save_tile(osp.join(output_dir, path), best_focus_tile, config=self.config) return [path]
def save(self, tile_indices, output_dir, tile): # Overwrite the original preprocessed tile with corrected version path = cytokit_io.get_processor_img_path(tile_indices.region_index, tile_indices.tile_x, tile_indices.tile_y) cytokit_io.save_tile(osp.join(output_dir, path), tile, config=self.config) return path
def postprocess_tile(tile, tile_indices, ops, log_fn, task_config): output_dir = task_config.output_dir # Illumination Correction if ops.illumination_op: # Prepare and save illumination images, if not already done ops.illumination_op.prepare_region_data(output_dir) path = ops.illumination_op.save_region_data(output_dir) if path is not None: log_fn('Illumination data saved to "{}"'.format(path)) # Run correction for tile tile = ops.illumination_op.run(tile, tile_indices) log_fn('Illumination correction complete', tile) else: log_fn('Skipping illumination correction', debug=True) # Spectral Unmixing if ops.unmixing_op: # Prepare unmixing models for each region ops.unmixing_op.prepare_region_data(output_dir) # Run correction for tile tile = ops.unmixing_op.run(tile, tile_indices) log_fn('Spectral unmixing complete', tile) else: log_fn('Skipping spectral unmixing', debug=True) # Get best focus data # TODO Prevent needing to re-read the processor data file each time best_focus_data = function_data.get_best_focus_coord_map(output_dir) best_focus_z_plane = best_focus_data[(tile_indices.region_index, tile_indices.tile_x, tile_indices.tile_y)] # Rerun cytometry based on corrected tile tile, cyto_data = ops.cytometry_op.run( tile, best_focus_z_plane=best_focus_z_plane) paths = ops.cytometry_op.save(tile_indices, output_dir, cyto_data) log_fn( 'Postprocessing cytometry complete; Statistics saved to "{}"'.format( paths[-1]), cyto_data[0]) # Save resulting tile path = cytokit_io.get_processor_img_path(tile_indices.region_index, tile_indices.tile_x, tile_indices.tile_y) cytokit_io.save_tile(osp.join(output_dir, path), tile, config=task_config.exp_config) log_fn('Saved postprocessed tile to "{}"'.format(path), tile)
def create_montage(output_dir, config, extract, name, region_indexes, prep_fn=None, compress=6): from cytokit.utils import ij_utils # Loop through regions and generate a montage for each, skipping any (with a warning) that # do not have focal plane selection information if region_indexes is None: region_indexes = config.region_indexes path = None for ireg in region_indexes: logger.info('Generating montage for region %d of %d', ireg + 1, len(region_indexes)) tiles = [] labels = None for itile in range(config.n_tiles_per_region): tx, ty = config.get_tile_coordinates(itile) path = cytokit_io.get_extract_image_path(ireg, tx, ty, extract) tile, meta = cytokit_io.read_tile(osp.join(output_dir, path), return_metadata=True) if labels is None: labels = meta['labels'] tiles.append(tile) reg_img_montage = montage(tiles, config) if prep_fn is not None: reg_img_montage = prep_fn(reg_img_montage) path = osp.join(output_dir, cytokit_io.get_montage_image_path(ireg, name)) logger.info('Saving montage to file "%s"', path) tags = [] if labels is None else ij_utils.get_slice_label_tags(labels) cytokit_io.save_tile(path, reg_img_montage, config=config, infer_labels=False, extratags=tags, compress=compress) logger.info('Montage generation complete; results saved to "%s"', None if path is None else osp.dirname(path))
def extract(self, name, channels, z='best', region_indexes=None, tile_indexes=None, raw_dir=None): """Create a new data extraction include either raw, processed, or cytometric imaging data Args: name: Name of extraction to be created; This will be used to construct result path like EXP_DIR/output/extract/`name` channels: List of strings indicating channel names (case-insensitive) prefixed by source for that channel (e.g. proc_DAPI, raw_CD4, cyto_nucleus_boundary); Available sources are: - "raw": Raw data images - "proc": Data generated as a results of preprocessing - "cyto": Cytometric object data (nuclei and cell boundaries) z: String or 1-based index selector for z indexes constructed as any of the following: - "best": Indicates that z slices should be inferred based on focal quality (default option) - "all": Indicates that a slice for all z-planes should be used - str or int: A single value will be interpreted as a single index - tuple: A 2-item or 3-item tuple forming the slice (start, stop[, step]); stop is inclusive - list: A list of integers will be used as is region_indexes: 1-based sequence of region indexes to process; can be specified as: - None: Region indexes will be inferred from experiment configuration - str or int: A single value will be interpreted as a single index - tuple: A 2-item or 3-item tuple forming the slice (start, stop[, step]); stop is inclusive - list: A list of integers will be used as is tile_indexes: 1-based sequence of tile indexes to process; has same semantics as `region_indexes` raw_dir: If using any channels sourced from raw data, this directory must be specified and should be equivalent to the same raw directory used during processing (i.e. nearly all operations like this are run relative to an `output_dir` -- the result of processing -- but in this case the original raw data path is needed as well) """ channel_map = _map_channels(self.config, channels).groupby('source') channel_sources = sorted(list(channel_map.groups.keys())) z_slice_fn = _get_z_slice_fn(z, self.data_dir) region_indexes = cli.resolve_index_list_arg(region_indexes, zero_based=True) tile_indexes = cli.resolve_index_list_arg(tile_indexes, zero_based=True) logging.info('Creating extraction "%s"', name) tile_locations = _get_tile_locations(self.config, region_indexes, tile_indexes) extract_path = None for i, loc in enumerate(tile_locations): logging.info('Extracting tile {} of {}'.format( i + 1, len(tile_locations))) extract_tile = [] # Create function used to crop out z-slices from extracted volumes z_slice = z_slice_fn(loc.region_index, loc.tile_x, loc.tile_y) slice_labels = [] for src in channel_sources: # Initialize tile generator for this data source (which are all the same except # for when using raw data, which does not have pre-assembled tiles available) tile_gen_dir = self.data_dir tile_gen_mode = 'stack' if src == CH_SRC_RAW: if not raw_dir: raise ValueError( 'When extracting raw data channels, the `raw_dir` argument must be provided' ) tile_gen_dir = raw_dir tile_gen_mode = 'raw' generator = tile_generator.CytokitTileGenerator( self.config, tile_gen_dir, loc.region_index, loc.tile_index, mode=tile_gen_mode, path_fmt_name=PATH_FMT_MAP[src]) tile = generator.run(None) # Crop raw images if necessary if src == CH_SRC_RAW: tile = tile_crop.CytokitTileCrop(self.config).run(tile) # Sort channels by name to make extract channel order deterministic for _, r in channel_map.get_group(src).sort_values( 'channel_name').iterrows(): # Extract (z, h, w) subtile sub_tile = tile[r['cycle_index'], z_slice, r['channel_index']] logging.debug( 'Extraction for cycle %s, channel %s (%s), z slice %s, source "%s" complete (tile shape = %s)', r['cycle_index'], r['channel_index'], r['channel_name'], z_slice, src, sub_tile.shape) assert sub_tile.ndim == 3, \ 'Expecting sub_tile to have 3 dimensions but got shape {}'.format(sub_tile.shape) slice_labels.append('{}_{}'.format(src, r['channel_name'])) extract_tile.append(sub_tile) # Stack the subtiles to give array with shape (z, channels, h, w) and then reshape to 5D # format like (cycles, z, channels, h, w) extract_tile = np.stack(extract_tile, axis=1)[np.newaxis] assert extract_tile.ndim == 5, \ 'Expecting extract tile to have 5 dimensions but got shape {}'.format(extract_tile.shape) extract_path = cytokit_io.get_extract_image_path( loc.region_index, loc.tile_x, loc.tile_y, name) extract_path = osp.join(self.data_dir, extract_path) logging.debug('Saving tile with shape %s (dtype = %s) to "%s"', extract_tile.shape, extract_tile.dtype, extract_path) # Construct slice labels as repeats across z-dimension (there is only one time/cycle dimension) slice_label_tags = ij_utils.get_channel_label_tags( slice_labels, z=extract_tile.shape[1], t=1) cytokit_io.save_tile(extract_path, extract_tile, config=self.config, infer_labels=False, extratags=slice_label_tags) logging.info('Extraction complete (results saved to %s)', osp.dirname(extract_path) if extract_path else None)
def preprocess_tile(tile, tile_indices, ops, log_fn, task_config): output_dir = task_config.output_dir # Drift Compensation if ops.align_op: tile = ops.align_op.run(tile) log_fn('Drift compensation complete', tile) else: log_fn('Skipping drift compensation', debug=True) # Crop off overlap in imaging process if ops.crop_op: tile = ops.crop_op.run(tile) log_fn('Tile overlap crop complete', tile) else: log_fn('Skipping tile crop', debug=True) # Resample images for improved downstream speed if ops.resize_op: tile = ops.resize_op.run(tile) log_fn('Tile resize complete', tile) else: log_fn('Skipping tile resize', debug=True) # Best Focal Plane Selection best_focus_data = None if ops.focus_op: # Used the cropped, but un-deconvolved tile for focal plane selection best_focus_data = ops.focus_op.run(tile) ops.focus_op.save(tile_indices, output_dir, best_focus_data) log_fn('Focal plane selection complete', best_focus_data[0]) else: log_fn('Skipping focal plane selection', debug=True) # Deconvolution if ops.decon_op: tile = ops.decon_op.run(tile) log_fn('Deconvolution complete', tile) else: log_fn('Skipping deconvolution', debug=True) # Cytometry (segmentation + quantification) if ops.cytometry_op: best_focus_z_plane = best_focus_data[1] if best_focus_data else None tile, cyto_data = ops.cytometry_op.run( tile, best_focus_z_plane=best_focus_z_plane, tile_indices=tile_indices) paths = ops.cytometry_op.save(tile_indices, output_dir, cyto_data) log_fn( 'Tile cytometry complete; Statistics saved to "{}"'.format( paths[-1]), cyto_data[0]) else: log_fn('Skipping tile cytometry', debug=True) # Tile summary statistic operations if ops.summary_op: ops.summary_op.run(tile) log_fn('Tile statistic summary complete') else: log_fn('Skipping tile statistic summary', debug=True) # Save the output tile if tile generation/assembly was enabled if task_config.op_flags.run_tile_generator: path = cytokit_io.get_processor_img_path(tile_indices.region_index, tile_indices.tile_x, tile_indices.tile_y) cytokit_io.save_tile(osp.join(output_dir, path), tile, config=task_config.exp_config) log_fn('Saved preprocessed tile to path "{}"'.format(path), tile)