def setup_match_blobs_roi(blobs, tol): """Set up tolerances for matching blobs in an ROI. Args: blobs (:obj:`np.ndarray`): Sequence of blobs to resize if the first ROI profile (:attr:`config.roi_profiles`) ``resize_blobs`` value is given. tol (List[int, float]): Sequence of tolerances. Returns: float, List[float], List[float], List[float], :obj:`np.ndarray`: Distance map threshold, scaling normalized by ``tol``, ROI padding shape, resize sequence retrieved from ROI profile, and ``blobs`` after any resizing. """ # convert tolerance seq to scaling and single number distance # threshold for point distance map, which assumes isotropy; use # custom tol rather than calculating isotropy since may need to give # greater latitude along a given axis, such as poorer res in z thresh = np.amax(tol) # similar to longest radius from the tol bounding box scaling = thresh / tol # casting to int causes improper offset import into db inner_padding = np.floor(tol[::-1]) libmag.printv( "verifying blobs with tol {} leading to thresh {}, scaling {}, " "inner_padding {}".format(tol, thresh, scaling, inner_padding)) # resize blobs based only on first profile resize = config.get_roi_profile(0)["resize_blobs"] if resize: blobs = detector.multiply_blob_rel_coords(blobs, resize) libmag.printv("resized blobs by {}:\n{}".format(resize, blobs)) return thresh, scaling, inner_padding, resize, blobs
def prepare_subimg(image5d, offset, size, ndim_base=5): """Extracts a subimage from a larger image. Args: image5d (:obj:`np.ndarray`): 5D image array in the order, ``t,z,y,x[,c]``, where the final dimension is optional as with many one channel images. offset (List[int]): Tuple of offset given as ``z,y,x`` for the region of interest. Defaults to ``(0, 0, 0)``. size (List[int]): Size of the region of interest as ``z,y,x``. ndim_base (int): Number of dimensions on which ``image5d`` is based. Typically 3 or 5, defaulting to 5 as ``t,z,y,x[,c]``. If 3, the ``t`` dimension is removed. Returns: :obj:`np.ndarray`: The sub-imge without separate time dimension as a 3D (or 4-D array if channel dimension exists) array. """ cube_slices = [slice(o, o + s) for o, s in zip(offset, size)] libmag.printv( "preparing sub-image at offset: {}, size: {}, slices: {}".format( offset, size, cube_slices)) # cube with corner at offset and shape given by size img = image5d if ndim_base >= 5: # remove time axis img = image5d[0] return img[cube_slices[0], cube_slices[1], cube_slices[2]]
def scale_coords(coord, scaling=None, clip_shape=None): """Get the atlas label IDs for the given coordinates. Args: coord (:class:`numpy.ndarray`): Coordinates of experiment image in ``z,y,x`` order. Can be an ``[n, 3]`` array of coordinates. scaling (Sequence[int]): Scaling factor for the labels image size compared with the experiment image as ``z,y,x``; defaults to None. clip_shape (Sequence[int]): Max image shape as ``z,y,x``, used to round coordinates for extra precision. For simplicity, scaled values are simply floored. Repeated scaling such as upsampling after downsampling can lead to errors. If this parameter is given, values will instead by rounded to minimize errors while giving ints. Rounded values will be clipped to this shape minus 1 to stay within bounds. Returns: :class:`numpy.ndarray`: An scaled array of the same shape as ``coord``. """ libmag.printv("getting label IDs from coordinates using scaling", scaling) coord_scaled = coord if scaling is not None: # scale coordinates to atlas image size coord_scaled = np.multiply(coord, scaling) if clip_shape is not None: # round when extra precision is necessary, such as during reverse # scaling, which requires clipping so coordinates don't exceed labels # image shape coord_scaled = np.around(coord_scaled).astype(np.int) coord_scaled = np.clip(coord_scaled, None, np.subtract(clip_shape, 1)) else: # typically don't round to stay within bounds coord_scaled = coord_scaled.astype(np.int) return coord_scaled
def saturate_roi(roi, clip_vmin=-1, clip_vmax=-1, max_thresh_factor=-1, channel=None): """Saturates an image, clipping extreme values and stretching remaining values to fit the full range. Args: roi (:obj:`np.ndarray`): Region of interest. clip_vmin (float): Percent for lower clipping. Defaults to -1 to use the profile setting. clip_vmax (float): Percent for upper clipping. Defaults to -1 to use the profile setting. max_thresh_factor (float): Multiplier of :attr:`config.near_max` for ROI's scaled maximum value. If the max data range value adjusted through``clip_vmax``is below this product, this max value will be set to this product. Defaults to -1 to use the profile setting. channel (List[int]): Sequence of channel indices in ``roi`` to saturate. Defaults to None to use all channels. Returns: Saturated region of interest. """ multichannel, channels = setup_channels(roi, channel, 3) roi_out = None for chl in channels: roi_show = roi[..., chl] if multichannel else roi settings = config.get_roi_profile(chl) if clip_vmin == -1: clip_vmin = settings["clip_vmin"] if clip_vmax == -1: clip_vmax = settings["clip_vmax"] if max_thresh_factor == -1: max_thresh_factor = settings["max_thresh_factor"] # enhance contrast and normalize to 0-1 scale vmin, vmax = np.percentile(roi_show, (clip_vmin, clip_vmax)) libmag.printv("vmin:", vmin, "vmax:", vmax, "near max:", config.near_max[chl]) # adjust the near max value derived globally from image5d for the chl max_thresh = config.near_max[chl] * max_thresh_factor if vmax < max_thresh: vmax = max_thresh libmag.printv("adjusted vmax to {}".format(vmax)) saturated = np.clip(roi_show, vmin, vmax) saturated = (saturated - vmin) / (vmax - vmin) if multichannel: if roi_out is None: roi_out = np.zeros(roi.shape, dtype=saturated.dtype) roi_out[..., chl] = saturated else: roi_out = saturated return roi_out
def get_label_ids_from_position(coord, labels_img, scaling=None, rounding=False, return_coord_scaled=False): """Get the atlas label IDs for the given coordinates. Args: coord: Coordinates of experiment image in (z, y, x) order. Can be an [n, 3] array of coordinates. labels_img: The registered image whose intensity values correspond to label IDs. scaling: Scaling factor for the labels image size compared with the experiment image; defaults to None. rounding: True to round coordinates after scaling, which should be used rounding to reverse coordinates that were previously scaled inversely to avoid size degredation with repeated scaling. Typically rounding is False (default) so that coordinates fall evenly to their lowest integer, without exceeding max bounds. return_coord_scaled: True to return the array of scaled coordinates; defaults to False. Returns: An array of label IDs corresponding to ``coord``, or a scalar of one ID if only one coordinate is given. If ``return_coord_scaled`` is True, also returns a Numpy array of the same shape as ``coord`` scaled based on ``scaling``. """ libmag.printv("getting label IDs from coordinates using scaling", scaling) coord_scaled = coord if scaling is not None: # scale coordinates to atlas image size coord_scaled = np.multiply(coord, scaling) if rounding: # round when extra precision is necessary, such as during reverse # scaling, which requires clipping so coordinates don't exceed labels # image shape coord_scaled = np.around(coord_scaled).astype(np.int) coord_scaled = np.clip(coord_scaled, None, np.subtract(labels_img.shape, 1)) else: # typically don't round to stay within bounds coord_scaled = coord_scaled.astype(np.int) # index blob coordinates into labels image by int array indexing to # get the corresponding label IDs coordsi = libmag.coords_for_indexing(coord_scaled) label_ids = labels_img[tuple(coordsi)][0] if return_coord_scaled: return label_ids, coord_scaled return label_ids
def make_isotropic(roi, scale): resize_factor = calc_isotropic_factor(scale) isotropic_shape = np.array(roi.shape) isotropic_shape[:3] = (isotropic_shape[:3] * resize_factor).astype(np.int) libmag.printv("original ROI shape: {}, isotropic: {}".format( roi.shape, isotropic_shape)) mode = "reflect" if np.any(np.array(roi.shape) == 1): # may crash with floating point exception if 1px thick (see # https://github.com/scikit-image/scikit-image/issues/3001, which # causes multiprocessing Pool to hang since the exception isn't # raised), so need to change mode in this case mode = "edge" return transform.resize(roi, isotropic_shape, preserve_range=True, mode=mode, anti_aliasing=True)
def get_split_stack_total_shape(sub_rois, overlap=None): """Get the shape of a chunked stack. Useful for determining the final shape of a stack that has been chunked and potentially scaled before merging the stack into an output array of this shape. Attributes: sub_rois: Array of sub regions, in (z, y, x, ...) dimensions. overlap: Overlap size between sub-ROIs. Defaults to None for no overlap. Returns: The shape of the chunked stack after it would be merged. """ size = sub_rois.shape shape_sub_roi = sub_rois[0, 0, 0].shape # for number of dimensions merged_shape = np.zeros(len(shape_sub_roi)).astype(np.int) final_shape = np.zeros(len(shape_sub_roi)).astype(np.int) edges = None for z in range(size[0]): for y in range(size[1]): for x in range(size[2]): coord = (z, y, x) sub_roi = sub_rois[coord] edges = list(sub_roi.shape[0:3]) if overlap is not None: # remove overlap if not at last sub_roi or row or column for n in range(len(edges)): if coord[n] != size[n] - 1: edges[n] -= overlap[n] #print("edges: {}".format(edges)) merged_shape[2] += edges[2] if final_shape[2] <= 0: final_shape[2] = merged_shape[2] merged_shape[1] += edges[1] if final_shape[1] <= 0: final_shape[1] = merged_shape[1] final_shape[0] += edges[0] channel_dim = 3 if len(shape_sub_roi) > channel_dim: final_shape[channel_dim] = shape_sub_roi[channel_dim] libmag.printv("final_shape: {}".format(final_shape)) return final_shape
def prepare_roi(image5d, roi_size, roi_offset): """Extracts a region of interest (ROI). Calls :meth:`prepare_subimage` but expects size and offset variables to be in x,y,z order following this software's legacy convention. Args: image5d: Image array as a 5D array (t, z, y, x, c), or 4D if no separate channel dimension exists as with most one channel images. roi_size: Size of the region of interest as (x, y, z). roi_offset: Tuple of offset given as (x, y, z) for the region of interest. Defaults to (0, 0, 0). Returns: The region of interest without separate time dimension as a 3D if ``image5d`` is 4D, without a separate channel dimension, or 4-D array if channel dimension exists. """ libmag.printv("preparing ROI at x,y,z:") return prepare_subimg(image5d, roi_size[::-1], roi_offset[::-1])
def get_label(coord, labels_img, labels_ref, scaling, level=None, rounding=False): """Get the atlas label for the given coordinates. Args: coord: Coordinates of experiment image in (z, y, x) order. labels_img: The registered image whose intensity values correspond to label IDs. labels_ref: The labels reference lookup, assumed to be generated by :func:`ontology.create_reverse_lookup` to look up by ID. scaling: Scaling factor for the labels image size compared with the experiment image. level: The ontology level as an integer to target; defaults to None. If None, level will be ignored, and the exact matching label to the given coordinates will be returned. If a level is given, the label at the highest (numerically lowest) level encompassing this region will be returned. rounding: True to round coordinates after scaling (see :func:``get_label_ids_from_position``); defaults to False. Returns: The label dictionary at those coordinates, or None if no label is found. """ label_id = get_label_ids_from_position(coord, labels_img, scaling, rounding) libmag.printv("found label_id: {}".format(label_id)) mirrored = label_id < 0 if mirrored: label_id = -1 * label_id label = None try: label = labels_ref[label_id] if level is not None and label[NODE][ config.ABAKeys.LEVEL.value] > level: # search for parent at "higher" (numerically lower) level # that matches the target level parents = label[PARENT_IDS] label = None if label_id < 0: parents = np.multiply(parents, -1) for parent in parents: parent_label = labels_ref[parent] if parent_label[NODE][config.ABAKeys.LEVEL.value] == level: label = parent_label break if label is not None: label[MIRRORED] = mirrored libmag.printv("label ID at level {}: {}".format(level, label_id)) except KeyError as e: libmag.printv( "could not find label id {} or its parent (error {})".format( label_id, e)) return label
def prepare_subimg(image5d, size, offset): """Extracts a subimage from a larger image. Args: image5d: Image array as a 5D array (t, z, y, x, c), or 4D if no separate channel dimension exists as with most one channel images. size: Size of the region of interest as (z, y, x). offset: Tuple of offset given as (z, y, x) for the region of interest. Defaults to (0, 0, 0). Returns: The sub-imge without separate time dimension as a 3D (or 4-D array if channel dimension exists) array. """ cube_slices = [slice(o, o + s) for o, s in zip(offset, size)] libmag.printv( "preparing sub-image at offset: {}, size: {}, slices: {}".format( offset, size, cube_slices)) # cube with corner at offset, side of cube_len return image5d[0, cube_slices[0], cube_slices[1], cube_slices[2]]
def merge_blobs(blob_rois): """Combine all blobs into a master list so that each overlapping region will contain all blobs from all sub-ROIs that had blobs in those regions, obviating the need to pair sub-ROIs with one another. Args: blob_rois (:obj:`np.ndarray`): Blob from each sub-region defined by :meth:`stack_splitter`. Blobs are assumed to be a 2D array in the format ``[[z, y, x, ...], ...]``. Returns: :obj:`np.ndarray`: Merged blobs in 2D format of the format, ``[[z, y, x, ..., sub_roi_z, sub_roi_y, sub_roi_x], ...]``, where sub-ROI coordinates have been added as the final columns. """ blobs_all = [] for z in range(blob_rois.shape[0]): for y in range(blob_rois.shape[1]): for x in range(blob_rois.shape[2]): coord = (z, y, x) blobs = blob_rois[coord] #print("checking blobs in {}:\n{}".format(coord, blobs)) if blobs is None: libmag.printv("no blobs to add, skipping") else: # add temporary tag with sub-ROI coordinate extras = np.zeros((blobs.shape[0], 3), dtype=int) extras[:] = coord blobs = np.concatenate((blobs, extras), axis=1) blobs_all.append(blobs) if len(blobs_all) > 0: blobs_all = np.vstack(blobs_all) else: blobs_all = None return blobs_all
def prepare_roi(image5d, roi_offset, roi_size, ndim_base=5): """Extracts a region of interest (ROI). Calls :meth:`prepare_subimage` but expects size and offset variables to be in x,y,z order following this software's legacy convention. Args: image5d (:obj:`np.ndarray`): 5D image array in the order, ``t,z,y,x[,c]``, where the final dimension is optional as with many one channel images. roi_offset (List[int]): Tuple of offset given as ``x,y,z`` for the region of interest. Defaults to ``(0, 0, 0)``. roi_size (List[int]): Size of the region of interest as ``x,y,z``. ndim_base (int): Number of dimensions on which ``image5d`` is based. Typically 3 or 5, defaulting to 5 as ``t,z,y,x[,c]``. If 3, the ``t`` dimension is removed. Returns: :obj:`np.ndarray`: The region of interest without separate time dimension as a 3D (or 4-D array if channel dimension exists) array. """ libmag.printv("preparing ROI at x,y,z:") return prepare_subimg(image5d, roi_offset[::-1], roi_size[::-1], ndim_base)
def merge_blobs(blob_rois): # combine all blobs into a master list so that each overlapping region # will contain all blobs from all sub-ROIs that had blobs in those regions, # obviating the need to pair sub-ROIs with one another blobs_all = [] for z in range(blob_rois.shape[0]): for y in range(blob_rois.shape[1]): for x in range(blob_rois.shape[2]): coord = (z, y, x) blobs = blob_rois[coord] #print("checking blobs in {}:\n{}".format(coord, blobs)) if blobs is None: libmag.printv("no blobs to add, skipping") else: # add temporary tag with sub-ROI coordinate extras = np.zeros((blobs.shape[0], 3), dtype=int) extras[:] = coord blobs = np.concatenate((blobs, extras), axis=1) blobs_all.append(blobs) if len(blobs_all) > 0: blobs_all = np.vstack(blobs_all) else: blobs_all = None return blobs_all
def setup_dbs(filename_base, db_path=None, truth_db_config=None): """Set up databases for the given image file if the given database has not been set up already. Args: filename_base (str): Image base path. db_path (str): Main database path; defaults to None to use a default path. truth_db_config (List[str]): Sequence of truth database configuration settings; defaults to None to not load truth-related databases. """ if db_path: config.db_name = db_path print("Set database name to {}".format(config.db_name)) # load "truth blobs" from separate database for viewing if truth_db_config is not None: # set the truth database mode config.truth_db_params = args_to_dict(truth_db_config, config.TruthDB, config.truth_db_params, sep_vals="|") mode = config.truth_db_params[config.TruthDB.MODE] config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes) libmag.printv(config.truth_db_params) print("Mapped \"{}\" truth_db mode to {}".format( mode, config.truth_db_mode)) truth_db_path = config.truth_db_params[config.TruthDB.PATH] truth_db_name_base = filename_base if filename_base else sqlite.DB_NAME_BASE if config.truth_db_mode is config.TruthDBModes.VIEW: # loads truth DB as a separate database in parallel with the given # editable database, with name based on filename by default unless # truth DB name explicitly given path = truth_db_path if truth_db_path else truth_db_name_base try: sqlite.load_truth_db(path) except FileNotFoundError as e: print(e) print("Could not load truth DB from current image path") elif config.truth_db_mode is config.TruthDBModes.VERIFY: if not config.verified_db: # creates a new verified DB to store all ROC results config.verified_db = sqlite.ClrDB() config.verified_db.load_db(sqlite.DB_NAME_VERIFIED, True) if truth_db_path: # load truth DB path to verify against if explicitly given try: sqlite.load_truth_db(truth_db_path) except FileNotFoundError as e: print(e) print("Could not load truth DB from {}".format(truth_db_path)) elif config.truth_db_mode is config.TruthDBModes.VERIFIED: # loads verified DB as the main DB, which includes copies of truth # values with flags for whether they were detected path = sqlite.DB_NAME_VERIFIED if truth_db_path: path = truth_db_path try: config.db = sqlite.ClrDB() config.db.load_db(path) config.verified_db = config.db except FileNotFoundError as e: print(e) print("Could not load verified DB from {}".format( sqlite.DB_NAME_VERIFIED)) elif config.truth_db_mode is config.TruthDBModes.EDIT: # loads truth DB as the main database for editing rather than # loading as a truth database config.db_name = truth_db_path if not config.db_name: config.db_name = "{}{}".format( os.path.basename(truth_db_name_base), sqlite.DB_SUFFIX_TRUTH) print("Editing truth database at {}".format(config.db_name)) if config.db is None: config.db = sqlite.ClrDB() config.db.load_db(None, False)
def prune_blobs_mp(cls, img, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels, overlap_padding=None): """Prune close blobs within overlapping regions by checking within entire planes across the ROI in parallel with multiprocessing. Args: img (:obj:`np.ndarray`): Array in which to detect blobs. seg_rois (:obj:`np.ndarray`): Blobs from each sub-region. overlap: 1D array of size 3 with the number of overlapping pixels for each image axis. tol: Tolerance as (z, y, x), within which a segment will be considered a duplicate of a segment in the master array and removed. sub_roi_slices (:obj:`np.ndarray`): Object array of ub-regions, used to check size. sub_rois_offsets: Offsets of each sub-region. channels (Sequence[int]): Sequence of channels; defaults to None to detect in all channels. overlap_padding: Sequence of z,y,x for additional padding beyond ``overlap``. Defaults to None to use ``tol`` as padding. Returns: :obj:`np.ndarray`, :obj:`pd.DataFrame`: All blobs as a Numpy array and a data frame of pruning stats, or None for both if no blobs are in the ``seg_rois``. """ # collect all blobs in master array to group all overlapping regions, # with sub-ROI coordinates as last 3 columns blobs_merged = chunking.merge_blobs(seg_rois) if blobs_merged is None: return None, None print("total blobs before pruning:", len(blobs_merged)) print("pruning with overlap: {}, overlap tol: {}, pruning tol: {}" .format(overlap, overlap_padding, tol)) blobs_all = [] blob_ratios = {} cols = ("blobs", "ratio_pruning", "ratio_adjacent") if overlap_padding is None: overlap_padding = tol for i in channels: # prune blobs from each channel separately to avoid pruning based on # co-localized channel signals blobs = detector.blobs_in_channel(blobs_merged, i) for axis in range(3): # prune planes with all the overlapping regions within a given axis, # skipping if this axis has no overlapping sub-regions num_sections = sub_rois_offsets.shape[axis] if num_sections <= 1: continue # multiprocess pruning by overlapping planes blobs_all_non_ol = None # all blobs from non-overlapping regions blobs_to_prune = [] coord_last = tuple(np.subtract(sub_roi_slices.shape, 1)) for j in range(num_sections): # build overlapping region dimensions based on size of # sub-region in the given axis coord = np.zeros(3, dtype=np.int) coord[axis] = j print("** setting up blob pruning in axis {}, section {} " "of {}".format(axis, j, num_sections - 1)) offset = sub_rois_offsets[tuple(coord)] sub_roi = img[sub_roi_slices[tuple(coord)]] size = sub_roi.shape _logger.debug(f"offset: {offset}, size: {size}") # overlapping region: each region but the last extends # into the next region, with the overlapping volume from # the end of the region, minus the overlap and a tolerance # space, to the region's end plus this tolerance space; # non-overlapping region: the region before the overlap, # after any overlap with the prior region (n > 1) # to the start of the overlap (n < last region) blobs_ol = None blobs_ol_next = None blobs_in_non_ol = [] shift = overlap[axis] + overlap_padding[axis] offset_axis = offset[axis] if j < num_sections - 1: bounds = [offset_axis + size[axis] - shift, offset_axis + size[axis] + overlap_padding[axis]] libmag.printv( "axis {}, boundaries: {}".format(axis, bounds)) blobs_ol = blobs[np.all([ blobs[:, axis] >= bounds[0], blobs[:, axis] < bounds[1]], axis=0)] # get blobs from immediatley adjacent region of the same # size as that of the overlapping region; keep same # starting point with or without overlap_tol start = offset_axis + size[axis] + tol[axis] bounds_next = [ start, start + overlap[axis] + 2 * overlap_padding[axis]] shape = np.add( sub_rois_offsets[coord_last], sub_roi.shape[:3]) libmag.printv( "axis {}, boundaries (next): {}, max bounds: {}" .format(axis, bounds_next, shape[axis])) if np.all(np.less(bounds_next, shape[axis])): # ensure that next overlapping region is within ROI blobs_ol_next = blobs[np.all([ blobs[:, axis] >= bounds_next[0], blobs[:, axis] < bounds_next[1]], axis=0)] # non-overlapping region extends up this overlap blobs_in_non_ol.append(blobs[:, axis] < bounds[0]) else: # last non-overlapping region extends to end of region blobs_in_non_ol.append( blobs[:, axis] < offset_axis + size[axis]) # get non-overlapping area start = offset_axis if j > 0: # shift past overlapping part at start of region start += shift blobs_in_non_ol.append(blobs[:, axis] >= start) blobs_non_ol = blobs[np.all(blobs_in_non_ol, axis=0)] # collect all non-overlapping region blobs if blobs_all_non_ol is None: blobs_all_non_ol = blobs_non_ol elif blobs_non_ol is not None: blobs_all_non_ol = np.concatenate( (blobs_all_non_ol, blobs_non_ol)) blobs_to_prune.append((blobs_ol, axis, tol, blobs_ol_next)) is_fork = chunking.is_fork() if is_fork: # set data as class variables to share across forks cls.blobs_to_prune = blobs_to_prune pool = chunking.get_mp_pool() pool_results = [] for j in range(len(blobs_to_prune)): if is_fork: # prune blobs in overlapping regions by multiprocessing, # using a separate class to avoid pickling input blobs pool_results.append(pool.apply_async( StackPruner.prune_overlap_by_index, args=(j, ))) else: # for spawned methods, need to pickle the blobs pool_results.append(pool.apply_async( StackPruner.prune_overlap, args=( j, blobs_to_prune[j]))) # collect all the pruned blob lists blobs_all_ol = None for result in pool_results: blobs_ol_pruned, ratios = result.get() if blobs_all_ol is None: blobs_all_ol = blobs_ol_pruned elif blobs_ol_pruned is not None: blobs_all_ol = np.concatenate( (blobs_all_ol, blobs_ol_pruned)) if ratios: for col, val in zip(cols, ratios): blob_ratios.setdefault(col, []).append(val) # recombine blobs from the non-overlapping with the pruned # overlapping regions from the entire stack to re-prune along # any remaining axes pool.close() pool.join() if blobs_all_ol is None: blobs = blobs_all_non_ol elif blobs_all_non_ol is None: blobs = blobs_all_ol else: blobs = np.concatenate((blobs_all_non_ol, blobs_all_ol)) # build up list from each channel blobs_all.append(blobs) # merge blobs into Numpy array and remove sub-ROI coordinate columns blobs_all = np.vstack(blobs_all)[:, :-3] print("total blobs after pruning:", len(blobs_all)) # export blob ratios as data frame df = pd.DataFrame(blob_ratios) return blobs_all, df
def match_blobs_roi(blobs, blobs_base, offset, size, thresh, scaling, inner_padding, resize=None): """Match blobs from two sets of blobs in an ROI, prioritizing the inner portion of ROIs to avoid missing detections because of edge effects while also adding matches between a blob in the inner ROI and another blob in the remaining portion of the ROI. Args: blobs (:obj:`np.ndarray`): The blobs to be matched against ``blobs_base``, given as 2D array of ``[[z, row, column, radius, ...], ...]``. blobs_base (:obj:`np.ndarray`): The blobs to which ``blobs`` will be matched, in the same format as ``blobs``. offset (List[int]): ROI offset from which to select blobs in x,y,z. size (List[int]): ROI size in x,y,z. thresh (float): Distance map threshold scaling (List[float]): Scaling normalized by ``tol``. inner_padding (List[float]): ROI padding shape. resize (List[float]): Resize sequence retrieved from ROI profile; defaults to None. Returns: :class:`numpy.ndarray`, :class:`numpy.ndarray`, list[int], list[int], list[list[:class:`numpy.ndarray`, :class:`numpy.ndarray`, float]]: Array of blobs from ``blobs``; corresponding array from ``blobs_base`` matching blobs in ``blobs``; offset of the inner portion of the ROI in absolute coordinates of x,y,z; shape of this inner portion of the ROI; and list of blob matches, each given as a list of ``blob_master, blob, distance``. """ # get all blobs in inner and total ROI offset_inner = np.add(offset, inner_padding) size_inner = np.subtract(size, inner_padding * 2) libmag.printv( "offset: {}, offset_inner: {}, size: {}, size_inner: {}" .format(offset, offset_inner, size, size_inner)) blobs_roi, _ = detector.get_blobs_in_roi(blobs, offset, size) if resize is not None: # TODO: doesn't align with exported ROIs padding = config.plot_labels[config.PlotLabels.PADDING] libmag.printv("shifting blobs in ROI by offset {}, border {}" .format(offset, padding)) blobs_roi = detector.shift_blob_rel_coords(blobs_roi, offset) if padding: blobs_roi = detector.shift_blob_rel_coords(blobs_roi, padding) blobs_inner, blobs_inner_mask = detector.get_blobs_in_roi( blobs_roi, offset_inner, size_inner) blobs_base_roi, _ = detector.get_blobs_in_roi(blobs_base, offset, size) blobs_base_inner, blobs_base_inner_mask = detector.get_blobs_in_roi( blobs_base_roi, offset_inner, size_inner) # compare blobs from inner region of ROI with all base blobs, # prioritizing the closest matches found, found_base, dists = find_closest_blobs_cdist( blobs_inner, blobs_base_roi, thresh, scaling) blobs_inner[:, 4] = 0 blobs_inner[found, 4] = 1 blobs_base_roi[blobs_base_inner_mask, 5] = 0 blobs_base_roi[found_base, 5] = 1 # add any base blobs missed in the inner ROI by comparing with # test blobs from outer ROI blobs_base_inner_missed = blobs_base_roi[blobs_base_roi[:, 5] == 0] blobs_outer = blobs_roi[np.invert(blobs_inner_mask)] found_out, found_base_out, dists_out = find_closest_blobs_cdist( blobs_outer, blobs_base_inner_missed, thresh, scaling) blobs_base_inner_missed[found_base_out, 5] = 1 # combine inner and outer groups blobs_truth_inner_plus = np.concatenate( (blobs_base_roi[blobs_base_roi[:, 5] == 1], blobs_base_inner_missed)) blobs_outer[found_out, 4] = 1 blobs_inner_plus = np.concatenate((blobs_inner, blobs_outer[found_out])) matches_inner = _match_blobs( blobs_inner, blobs_base_roi, found, found_base, dists) matches_outer = _match_blobs( blobs_outer, blobs_base_inner_missed, found_out, found_base_out, dists_out) matches = colocalizer.BlobMatch([*matches_inner, *matches_outer]) if config.verbose: ''' print("blobs_roi:\n{}".format(blobs_roi)) print("blobs_inner:\n{}".format(blobs_inner)) print("blobs_base_inner:\n{}".format(blobs_base_inner)) print("blobs_base_roi:\n{}".format(blobs_base_roi)) print("found inner:\n{}" .format(blobs_inner[found])) print("truth found:\n{}" .format(blobs_base_roi[found_base])) print("blobs_outer:\n{}".format(blobs_outer)) print("blobs_base_inner_missed:\n{}" .format(blobs_base_inner_missed)) print("truth blobs detected by an outside blob:\n{}" .format(blobs_base_inner_missed[found_base_out])) print("all those outside detection blobs:\n{}" .format(blobs_roi_extra)) print("blobs_inner_plus:\n{}".format(blobs_inner_plus)) print("blobs_truth_inner_plus:\n{}".format(blobs_truth_inner_plus)) ''' print("Closest matches found (truth, detected, distance):") msgs = ("\n- Inner ROI:", "\n- Outer ROI:") for msg, matches_sub in zip(msgs, (matches_inner, matches_outer)): print(msg) for match in matches_sub: print( "Blob1:", match[0][:3], "chl", detector.get_blob_channel(match[0]), "Blob2:", match[1][:3], "chl", detector.get_blob_channel(match[1]), "dist:", match[2]) print() return blobs_inner_plus, blobs_truth_inner_plus, offset_inner, size_inner, \ matches
def load_truth_blobs(self): self.blobs_truth = select_blobs_confirmed(self.cur, 1) libmag.printv("truth blobs:\n{}".format(self.blobs_truth))
def export_rois(db, image5d, channel, path, padding=None, unit_factor=None, truth_mode=None, exp_name=None): """Export all ROIs from database. If the current processing profile includes isotropic interpolation, the ROIs will be resized to make isotropic according to this factor. Args: db: Database from which to export. image5d: The image with the ROIs. channel (List[int]): Channels to export; currently only the first channel is used. path: Path with filename base from which to save the exported files. padding (List[int]): Padding in x,y,z to exclude from the ROI; defaults to None. unit_factor (float): Linear conversion factor for units (eg 1000.0 to convert um to mm). truth_mode (:obj:`config.TruthDBModes`): Truth mode enum; defaults to None. exp_name (str): Name of experiment to export; defaults to None to export all experiments in ``db``. Returns: :obj:`pd.DataFrame`: ROI metrics in a data frame. """ if padding is not None: padding = np.array(padding) # TODO: consider iterating through all channels channel = channel[0] if channel else 0 # convert volume base on scaling and unit factor phys_mult = np.prod(detector.calc_scaling_factor()) if unit_factor: phys_mult /= unit_factor**3 metrics_all = {} exps = sqlite.select_experiment(db.cur, None) for exp in exps: if exp_name and exp["name"] != exp_name: # DBs may contain many experiments, which may not correspond to # image5d, eg verified DBs from many truth sets continue rois = sqlite.select_rois(db.cur, exp["id"]) for roi in rois: # get ROI as a small image size = sqlite.get_roi_size(roi) offset = sqlite.get_roi_offset(roi) img3d = plot_3d.prepare_roi(image5d, size, offset) # get blobs and change confirmation flag to avoid confirmation # color in 2D plots roi_id = roi["id"] blobs = sqlite.select_blobs(db.cur, roi_id) blobs_detected = None if truth_mode is config.TruthDBModes.VERIFIED: # verified DBs use a truth value of -1 to indicate "detected", # non-truth blobs, including both correct and incorrect # detections, while the rest of blobs are "truth" blobs truth_vals = detector.get_blob_truth(blobs) blobs_detected = blobs[truth_vals == -1] blobs = blobs[truth_vals != -1] else: # default to include only confirmed blobs; truth sets # ironically do not use the truth flag but instead # assume all confirmed blobs are "truth" blobs = blobs[detector.get_blob_confirmed(blobs) == 1] blobs[:, 4] = -1 # adjust ROI size and offset if border set if padding is not None: size = np.subtract(img3d.shape[::-1], 2 * padding) img3d = plot_3d.prepare_roi(img3d, size, padding) blobs[:, 0:3] = np.subtract(blobs[:, 0:3], np.add(offset, padding)[::-1]) print("exporting ROI of shape {}".format(img3d.shape)) isotropic = config.roi_profile["isotropic"] blobs_orig = blobs if isotropic is not None: # interpolation for isotropy if set in first processing profile img3d = cv_nd.make_isotropic(img3d, isotropic) isotropic_factor = cv_nd.calc_isotropic_factor(isotropic) blobs_orig = np.copy(blobs) blobs = detector.multiply_blob_rel_coords( blobs, isotropic_factor) # export ROI and 2D plots path_base, path_dir_nifti, path_img, path_img_nifti, path_blobs, \ path_img_annot, path_img_annot_nifti = make_roi_paths( path, roi_id, channel, make_dirs=True) np.save(path_img, img3d) print("saved 3D image to {}".format(path_img)) # WORKAROUND: for some reason SimpleITK gives a conversion error # when converting from uint16 (>u2) Numpy array img3d = img3d.astype(np.float64) img3d_sitk = sitk.GetImageFromArray(img3d) ''' print(img3d_sitk) print("orig img:\n{}".format(img3d[0])) img3d_back = sitk.GetArrayFromImage(img3d_sitk) print(img3d.shape, img3d.dtype, img3d_back.shape, img3d_back.dtype) print("sitk img:\n{}".format(img3d_back[0])) ''' sitk.WriteImage(img3d_sitk, path_img_nifti, False) roi_ed = roi_editor.ROIEditor(img3d) roi_ed.plot_roi(blobs, channel, show=False, title=os.path.splitext(path_img)[0]) libmag.show_full_arrays() # export image and blobs, stripping blob flags and adjusting # user-added segments' radii; use original rather than blobs with # any interpolation since the ground truth will itself be # interpolated blobs = blobs_orig blobs = blobs[:, 0:4] # prior to v.0.5.0, user-added segments had a radius of 0.0 blobs[np.isclose(blobs[:, 3], 0), 3] = 5.0 # as of v.0.5.0, user-added segments have neg radii whose abs # value corresponds to the displayed radius blobs[:, 3] = np.abs(blobs[:, 3]) # make more rounded since near-integer values appear to give # edges of 5 straight pixels # https://github.com/scikit-image/scikit-image/issues/2112 #blobs[:, 3] += 1E-1 blobs[:, 3] -= 0.5 libmag.printv("blobs:\n{}".format(blobs)) np.save(path_blobs, blobs) # convert blobs to ground truth img3d_truth = plot_3d.build_ground_truth( np.zeros(size[::-1], dtype=np.uint8), blobs) if isotropic is not None: img3d_truth = cv_nd.make_isotropic(img3d_truth, isotropic) # remove fancy blending since truth set must be binary img3d_truth[img3d_truth >= 0.5] = 1 img3d_truth[img3d_truth < 0.5] = 0 print("exporting truth ROI of shape {}".format(img3d_truth.shape)) np.save(path_img_annot, img3d_truth) #print(img3d_truth) sitk.WriteImage(sitk.GetImageFromArray(img3d_truth), path_img_annot_nifti, False) # avoid smoothing interpolation, using "nearest" instead with plt.style.context(config.rc_params_mpl2_img_interp): roi_ed.plot_roi(img3d_truth, None, channel, show=False, title=os.path.splitext(path_img_annot)[0]) # measure ROI metrics and export to data frame; use AtlasMetrics # enum vals since will need LabelMetrics names instead metrics = { config.AtlasMetrics.SAMPLE.value: exp["name"], config.AtlasMetrics.CONDITION.value: "truth", config.AtlasMetrics.CHANNEL.value: channel, config.AtlasMetrics.OFFSET.value: offset, config.AtlasMetrics.SIZE.value: size, } # get basic counts for ROI and update volume for physical units vols.MeasureLabel.set_data(img3d, np.ones_like(img3d, dtype=np.int8)) _, metrics_counts = vols.MeasureLabel.measure_counts(1) metrics_counts[vols.LabelMetrics.Volume] *= phys_mult for key, val in metrics_counts.items(): # convert LabelMetrics to their name metrics[key.name] = val metrics[vols.LabelMetrics.Nuclei.name] = len(blobs) metrics_dicts = [metrics] if blobs_detected is not None: # add another row for detected blobs metrics_detected = dict(metrics) metrics_detected[ config.AtlasMetrics.CONDITION.value] = "detected" metrics_detected[vols.LabelMetrics.Nuclei.name] = len( blobs_detected) metrics_dicts.append(metrics_detected) for m in metrics_dicts: for key, val in m.items(): metrics_all.setdefault(key, []).append(val) print("exported {}".format(path_base)) #_test_loading_rois(db, channel, path) # convert to data frame and compute densities for nuclei and intensity df = df_io.dict_to_data_frame(metrics_all) vol = df[vols.LabelMetrics.Volume.name] df.loc[:, vols.LabelMetrics.DensityIntens.name] = ( df[vols.LabelMetrics.Intensity.name] / vol) df.loc[:, vols.LabelMetrics.Density.name] = ( df[vols.LabelMetrics.Nuclei.name] / vol) df = df_io.data_frames_to_csv(df, "{}_rois.csv".format(path)) return df
def colocalize_stack(cls, shape, blobs): """Entry point to colocalizing blobs within a stack. Args: shape (List[int]): Image shape in z,y,x. blobs (:obj:`np.ndarray`): 2D Numpy array of blobs. Returns: dict[tuple[int, int], :class:`BlobMatch`]: The dictionary of matches, where keys are tuples of the channel pairs, and values are blob match objects. """ print("Colocalizing blobs based on matching blobs in each pair of " "channels") # set up ROI blocks from which to select blobs in each block sub_roi_slices, sub_rois_offsets, _, _, _, overlap_base, _, _ \ = stack_detect.setup_blocks(config.roi_profile, shape) match_tol = np.multiply( overlap_base, config.roi_profile["verify_tol_factor"]) is_fork = chunking.is_fork() if is_fork: # set shared data in forked multiprocessing cls.blobs = blobs cls.match_tol = match_tol pool = mp.Pool(processes=config.cpus) pool_results = [] for z in range(sub_roi_slices.shape[0]): for y in range(sub_roi_slices.shape[1]): for x in range(sub_roi_slices.shape[2]): coord = (z, y, x) offset = sub_rois_offsets[coord] slices = sub_roi_slices[coord] shape = [s.stop - s.start for s in slices] if is_fork: # use variables stored as class attributes pool_results.append(pool.apply_async( StackColocalizer.colocalize_block, args=(coord, offset, shape))) else: # pickle full set of variables pool_results.append(pool.apply_async( StackColocalizer.colocalize_block, args=(coord, offset, shape, detector.get_blobs_in_roi( blobs, offset, shape)[0], match_tol, True))) # dict of channel combos to blob matches data frame matches_all = {} for result in pool_results: coord, matches = result.get() count = 0 for key, val in matches.items(): matches_all.setdefault(key, []).append(val.df) count += len(val.df) print("adding {} matches from block at {} of {}" .format(count, coord, np.add(sub_roi_slices.shape, -1))) pool.close() pool.join() # prune duplicates by taking matches with shortest distance for key in matches_all.keys(): matches_all[key] = pd.concat(matches_all[key]) for blobi in (BlobMatch.Cols.BLOB1, BlobMatch.Cols.BLOB2): # convert blob column to ndarray to extract coords by column matches = matches_all[key] matches_uniq, matches_i, matches_inv, matches_cts = np.unique( np.vstack(matches[blobi.value])[:, :3], axis=0, return_index=True, return_inverse=True, return_counts=True) if np.sum(matches_cts > 1) > 0: # prune if at least one blob has been matched to multiple # other blobs singles = matches.iloc[matches_i[matches_cts == 1]] dups = [] for i, ct in enumerate(matches_cts): # include non-duplicates to retain index if ct <= 1: continue # get indices in orig matches at given unique array # index and take match with lowest dist matches_mult = matches.loc[matches_inv == i] dists = matches_mult[BlobMatch.Cols.DIST.value] min_dist = np.amin(dists) num_matches = len(matches_mult) if config.verbose and num_matches > 1: print("pruning from", num_matches, "matches of dist:", dists) matches_mult = matches_mult.loc[dists == min_dist] # take first in case of any ties dups.append(matches_mult.iloc[[0]]) matches_all[key] = pd.concat((singles, pd.concat(dups))) print("Colocalization matches for channels {}: {}" .format(key, len(matches_all[key]))) libmag.printv(print(matches_all[key])) # store data frame in BlobMatch object matches_all[key] = BlobMatch(df=matches_all[key]) return matches_all
def detect_blobs(roi, channel, exclude_border=None): """Detects objects using 3D blob detection technique. Args: roi: Region of interest to segment. channel (Sequence[int]): Sequence of channels to select, which can be None to indicate all channels. exclude_border: Sequence of border pixels in x,y,z to exclude; defaults to None. Returns: Array of detected blobs, each given as (z, row, column, radius, confirmation). """ time_start = time() shape = roi.shape multichannel, channels = plot_3d.setup_channels(roi, channel, 3) isotropic = config.get_roi_profile(channels[0])["isotropic"] if isotropic is not None: # interpolate for (near) isotropy during detection, using only the # first process settings since applies to entire ROI roi = cv_nd.make_isotropic(roi, isotropic) blobs_all = [] for chl in channels: roi_detect = roi[..., chl] if multichannel else roi settings = config.get_roi_profile(chl) # scaling as a factor in pixel/um, where scaling of 1um/pixel # corresponds to factor of 1, and 0.25um/pixel corresponds to # 1 / 0.25 = 4 pixels/um; currently simplified to be based on # x scaling alone scale = calc_scaling_factor() scaling_factor = scale[2] # find blobs; sigma factors can be sequences by axes for anisotropic # detection in skimage >= 0.15, or images can be interpolated to # isotropy using the "isotropic" MagellanMapper setting min_sigma = settings["min_sigma_factor"] * scaling_factor max_sigma = settings["max_sigma_factor"] * scaling_factor num_sigma = settings["num_sigma"] threshold = settings["detection_threshold"] overlap = settings["overlap"] blobs_log = blob_log( roi_detect, min_sigma=min_sigma, max_sigma=max_sigma, num_sigma=num_sigma, threshold=threshold, overlap=overlap) if config.verbose: print("detecting blobs with min size {}, max {}, num std {}, " "threshold {}, overlap {}" .format(min_sigma, max_sigma, num_sigma, threshold, overlap)) print("time for 3D blob detection: {}".format(time() - time_start)) if blobs_log.size < 1: libmag.printv("no blobs detected") continue blobs_log[:, 3] = blobs_log[:, 3] * math.sqrt(3) blobs = format_blobs(blobs_log, chl) #print(blobs) blobs_all.append(blobs) if not blobs_all: return None blobs_all = np.vstack(blobs_all) if isotropic is not None: # if detected on isotropic ROI, need to reposition blob coordinates # for original, non-isotropic ROI isotropic_factor = cv_nd.calc_isotropic_factor(isotropic) blobs_all = multiply_blob_rel_coords(blobs_all, 1 / isotropic_factor) blobs_all = multiply_blob_abs_coords(blobs_all, 1 / isotropic_factor) if exclude_border is not None: # exclude blobs from the border in x,y,z blobs_all = get_blobs_interior(blobs_all, shape, *exclude_border) return blobs_all
def process_cli_args(): """Parse command-line arguments. Typically stores values as :mod:`magmap.settings.config` attributes. """ parser = argparse.ArgumentParser( description="Setup environment for MagellanMapper") parser.add_argument("--version", action="store_true", help="Show version information and exit") # image specification arguments # image path(s) specified as an optional argument; takes precedence # over positional argument parser.add_argument( "--img", nargs="*", default=None, help="Main image path(s); after import, the filename is often " "given as the original name without its extension") # alternatively specified as the first and only positional parameter # with as many arguments as desired parser.add_argument( "img_paths", nargs="*", default=None, help="Main image path(s); can also be given as --img, which takes " "precedence over this argument") parser.add_argument( "--meta", nargs="*", help="Metadata path(s), which can be given as multiple files " "corresponding to each image") parser.add_argument( "--prefix", nargs="*", type=str, help="Path prefix(es), typically used as the base path for file output" ) parser.add_argument( "--prefix_out", nargs="*", type=str, help="Path prefix(es), typically used as the base path for file output " "when --prefix modifies the input path") parser.add_argument( "--suffix", nargs="*", type=str, help="Path suffix(es), typically inserted just before the extension") parser.add_argument("--channel", nargs="*", type=int, help="Channel index") parser.add_argument("--series", help="Series index") parser.add_argument("--subimg_offset", nargs="*", help="Sub-image offset in x,y,z") parser.add_argument("--subimg_size", nargs="*", help="Sub-image size in x,y,z") parser.add_argument("--offset", nargs="*", help="ROI offset in x,y,z") parser.add_argument("--size", nargs="*", help="ROI size in x,y,z") parser.add_argument("--db", help="Database path") parser.add_argument( "--cpus", help="Maximum number of CPUs/processes to use for multiprocessing " "tasks. Use \"none\" or 0 to auto-detect this number (default).") parser.add_argument( "--load", nargs="*", help="Load associated data files; see config.LoadData for settings") # task arguments parser.add_argument( "--proc", nargs="*", help=_get_args_dict_help( "Image processing mode; see config.ProcessTypes for keys " "and config.PreProcessKeys for PREPROCESS values", config.ProcessTypes)) parser.add_argument("--register", type=str.lower, choices=libmag.enum_names_aslist(config.RegisterTypes), help="Image registration task") parser.add_argument("--df", type=str.lower, choices=libmag.enum_names_aslist(config.DFTasks), help="Data frame task") parser.add_argument("--plot_2d", type=str.lower, choices=libmag.enum_names_aslist(config.Plot2DTypes), help="2D plot task; see config.Plot2DTypes") parser.add_argument("--ec2_start", nargs="*", help="AWS EC2 instance start") parser.add_argument("--ec2_list", nargs="*", help="AWS EC2 instance list") parser.add_argument("--ec2_terminate", nargs="*", help="AWS EC2 instance termination") parser.add_argument( "--notify", nargs="*", help="Notification message URL, message, and attachment strings") # profile arguments parser.add_argument( "--roi_profile", nargs="*", help="ROI profile, which can be separated by underscores " "for multiple profiles and given as paths to custom profiles " "in YAML format. Multiple profile groups can be given, which " "will each be applied to the corresponding channel. See " "docs/settings.md for more details.") parser.add_argument( "--atlas_profile", help="Atlas profile, which can be separated by underscores " "for multiple profiles and given as paths to custom profiles " "in YAML format. See docs/settings.md for more details.") parser.add_argument( "--grid_search", help="Grid search hyperparameter tuning profile(s), which can be " "separated by underscores for multiple profiles and given as " "paths to custom profiles in YAML format. See docs/settings.md " "for more details.") parser.add_argument( "--theme", nargs="*", type=str.lower, choices=libmag.enum_names_aslist(config.Themes), help="UI theme, which can be given as multiple themes to apply " "on top of one another") # grouped arguments parser.add_argument( "--truth_db", nargs="*", help="Truth database; see config.TruthDB for settings and " "config.TruthDBModes for modes") parser.add_argument("--labels", nargs="*", help=_get_args_dict_help( "Atlas labels; see config.AtlasLabels.", config.AtlasLabels)) parser.add_argument("--transform", nargs="*", help=_get_args_dict_help( "Image transformations; see config.Transforms.", config.Transforms)) parser.add_argument( "--reg_suffixes", nargs="*", help=_get_args_dict_help( "Registered image suffixes; see config.RegSuffixes for keys " "and config.RegNames for values", config.RegSuffixes)) parser.add_argument( "--plot_labels", nargs="*", help=_get_args_dict_help( "Plot label customizations; see config.PlotLabels ", config.PlotLabels)) parser.add_argument( "--set_meta", nargs="*", help="Set metadata values; see config.MetaKeys for settings") # image and figure display arguments parser.add_argument("--plane", type=str.lower, choices=config.PLANE, help="Planar orientation") parser.add_argument( "--show", nargs="?", const="1", help="If applicable, show images after completing the given task") parser.add_argument( "--alphas", help="Alpha opacity levels, which can be comma-delimited for " "multichannel images") parser.add_argument( "--vmin", help="Minimum intensity levels, which can be comma-delimited " "for multichannel images") parser.add_argument( "--vmax", help="Maximum intensity levels, which can be comma-delimited " "for multichannel images") parser.add_argument("--seed", help="Random number generator seed") # export arguments parser.add_argument("--save_subimg", action="store_true", help="Save sub-image as separate file") parser.add_argument("--slice", help="Slice given as start,stop,step") parser.add_argument("--delay", help="Animation delay in ms") parser.add_argument("--savefig", help="Extension for saved figures") parser.add_argument("--groups", nargs="*", help="Group values corresponding to each image") parser.add_argument( "-v", "--verbose", nargs="*", help=_get_args_dict_help( "Verbose output to assist with debugging; see config.Verbosity.", config.Verbosity)) # only parse recognized arguments to avoid error for unrecognized ones args, args_unknown = parser.parse_known_args() # set up application directories user_dir = config.user_app_dirs.user_data_dir if not os.path.isdir(user_dir): # make application data directory if os.path.exists(user_dir): # backup any non-directory file libmag.backup_file(user_dir) os.makedirs(user_dir) if args.verbose is not None: # verbose mode and logging setup config.verbose = True config.verbosity = args_to_dict(args.verbose, config.Verbosity, config.verbosity) if config.verbosity[config.Verbosity.LEVEL] is None: # default to debug mode if any verbose flag is set without level config.verbosity[config.Verbosity.LEVEL] = logging.DEBUG logs.update_log_level(config.logger, config.verbosity[config.Verbosity.LEVEL]) # print longer Numpy arrays for debugging np.set_printoptions(linewidth=200, threshold=10000) _logger.info("Set verbose to %s", config.verbosity) # set up logging to given file unless explicitly given an empty string log_path = config.verbosity[config.Verbosity.LOG_PATH] if log_path != "": if log_path is None: log_path = os.path.join(config.user_app_dirs.user_data_dir, "out.log") # log to file config.log_path = logs.add_file_handler(config.logger, log_path) # redirect standard out/error to logging sys.stdout = logs.LogWriter(config.logger.info) sys.stderr = logs.LogWriter(config.logger.error) # load preferences file config.prefs = prefs_prof.PrefsProfile() config.prefs.add_profiles(str(config.PREFS_PATH)) if args.version: # print version info and exit _logger.info(f"{config.APP_NAME}-{libmag.get_version(True)}") shutdown() # log the app launch path path_launch = (sys._MEIPASS if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS") else sys.argv[0]) _logger.info(f"Launched MagellanMapper from {path_launch}") if args.img is not None or args.img_paths: # set image file path and convert to basis for additional paths config.filenames = args.img if args.img else args.img_paths config.filename = config.filenames[0] print("Set filenames to {}, current filename {}".format( config.filenames, config.filename)) if args.meta is not None: # set metadata paths config.metadata_paths = args.meta print("Set metadata paths to", config.metadata_paths) config.metadatas = [] for path in config.metadata_paths: # load metadata to dictionary md, _ = importer.load_metadata(path, assign=False) config.metadatas.append(md) if args.channel is not None: # set the channels config.channel = args.channel print("Set channel to {}".format(config.channel)) config.series_list = [config.series] # list of series if args.series is not None: series_split = args.series.split(",") config.series_list = [] for ser in series_split: ser_split = ser.split("-") if len(ser_split) > 1: ser_range = np.arange(int(ser_split[0]), int(ser_split[1]) + 1) config.series_list.extend(ser_range.tolist()) else: config.series_list.append(int(ser_split[0])) config.series = config.series_list[0] print("Set to series_list to {}, current series {}".format( config.series_list, config.series)) if args.savefig is not None: # save figure with file type of this extension; remove leading period config.savefig = _parse_none(args.savefig.lstrip(".")) print("Set savefig extension to {}".format(config.savefig)) # parse sub-image offsets and sizes; # expects x,y,z input but stores as z,y,x by convention if args.subimg_offset is not None: config.subimg_offsets = _parse_coords(args.subimg_offset, True) print("Set sub-image offsets to {} (z,y,x)".format( config.subimg_offsets)) if args.subimg_size is not None: config.subimg_sizes = _parse_coords(args.subimg_size, True) print("Set sub-image sizes to {} (z,y,x)".format(config.subimg_sizes)) # parse ROI offsets and sizes, which are relative to any sub-image; # expects x,y,z input and output if args.offset is not None: config.roi_offsets = _parse_coords(args.offset) if config.roi_offsets: config.roi_offset = config.roi_offsets[0] print("Set ROI offsets to {}, current offset {} (x,y,z)".format( config.roi_offsets, config.roi_offset)) if args.size is not None: config.roi_sizes = _parse_coords(args.size) if config.roi_sizes: config.roi_size = config.roi_sizes[0] print("Set ROI sizes to {}, current size {} (x,y,z)".format( config.roi_sizes, config.roi_size)) if args.cpus is not None: # set maximum number of CPUs config.cpus = _parse_none(args.cpus.lower(), int) print("Set maximum number of CPUs for multiprocessing tasks to", config.cpus) if args.load is not None: # flag loading data sources with default sub-arg indicating that the # data should be loaded from a default path; otherwise, load from # path given by the sub-arg; change delimiter to allow paths with "," config.load_data = args_to_dict(args.load, config.LoadData, config.load_data, sep_vals="|", default=True) print("Set to load the data types: {}".format(config.load_data)) # set up main processing mode if args.proc is not None: config.proc_type = args_to_dict(args.proc, config.ProcessTypes, config.proc_type, default=True) print("Set main processing tasks to:", config.proc_type) if args.set_meta is not None: # set individual metadata values, currently used for image import # TODO: take precedence over loaded metadata archives config.meta_dict = args_to_dict(args.set_meta, config.MetaKeys, config.meta_dict, sep_vals="|") print("Set metadata values to {}".format(config.meta_dict)) res = config.meta_dict[config.MetaKeys.RESOLUTIONS] if res: # set image resolutions, taken as a single set of x,y,z and # converting to a nested list of z,y,x res_split = res.split(",") if len(res_split) >= 3: res_float = tuple(float(i) for i in res_split)[::-1] config.resolutions = [res_float] print("Set resolutions to {}".format(config.resolutions)) else: res_float = None print("Resolution ({}) should be given as 3 values (x,y,z)". format(res)) # store single set of resolutions, similar to input config.meta_dict[config.MetaKeys.RESOLUTIONS] = res_float mag = config.meta_dict[config.MetaKeys.MAGNIFICATION] if mag: # set objective magnification config.magnification = mag print("Set magnification to {}".format(config.magnification)) zoom = config.meta_dict[config.MetaKeys.ZOOM] if zoom: # set objective zoom config.zoom = zoom print("Set zoom to {}".format(config.zoom)) shape = config.meta_dict[config.MetaKeys.SHAPE] if shape: # parse shape, storing only in dict config.meta_dict[config.MetaKeys.SHAPE] = [ int(n) for n in shape.split(",")[::-1] ] # set up ROI and register profiles setup_roi_profiles(args.roi_profile) setup_atlas_profiles(args.atlas_profile) setup_grid_search_profiles(args.grid_search) if args.plane is not None: config.plane = args.plane print("Set plane to {}".format(config.plane)) if args.save_subimg: config.save_subimg = args.save_subimg print("Set to save the sub-image") if args.labels: # set up atlas labels setup_labels(args.labels) if args.transform is not None: # image transformations such as flipping, rotation config.transform = args_to_dict(args.transform, config.Transforms, config.transform) print("Set transformations to {}".format(config.transform)) if args.register: # register type to process in register module config.register_type = args.register print("Set register type to {}".format(config.register_type)) if args.df: # data frame processing task config.df_task = args.df print("Set data frame processing task to {}".format(config.df_task)) if args.plot_2d: # 2D plot type to process in plot_2d module config.plot_2d_type = args.plot_2d print("Set plot_2d type to {}".format(config.plot_2d_type)) if args.slice: # specify a generic slice by command-line, assuming same order # of arguments as for slice built-in function and interpreting # "none" string as None config.slice_vals = args.slice.split(",") config.slice_vals = [ _parse_none(val.lower(), int) for val in config.slice_vals ] print("Set slice values to {}".format(config.slice_vals)) if args.delay: config.delay = int(args.delay) print("Set delay to {}".format(config.delay)) if args.show: # show images after task is performed, if supported config.show = _is_arg_true(args.show) print("Set show to {}".format(config.show)) if args.groups: config.groups = args.groups print("Set groups to {}".format(config.groups)) if args.ec2_start is not None: # start EC2 instances config.ec2_start = args_with_dict(args.ec2_start) print("Set ec2 start to {}".format(config.ec2_start)) if args.ec2_list: # list EC2 instances config.ec2_list = args_with_dict(args.ec2_list) print("Set ec2 list to {}".format(config.ec2_list)) if args.ec2_terminate: config.ec2_terminate = args.ec2_terminate print("Set ec2 terminate to {}".format(config.ec2_terminate)) if args.notify: notify_len = len(args.notify) if notify_len > 0: config.notify_url = args.notify[0] print("Set notification URL to {}".format(config.notify_url)) if notify_len > 1: config.notify_msg = args.notify[1] print("Set notification message to {}".format(config.notify_msg)) if notify_len > 2: config.notify_attach = args.notify[2] print("Set notification attachment path to {}".format( config.notify_attach)) if args.prefix is not None: # path input/output prefixes config.prefixes = args.prefix config.prefix = config.prefixes[0] print("Set path prefixes to {}".format(config.prefixes)) if args.prefix_out is not None: # path output prefixes config.prefixes_out = args.prefix_out config.prefix_out = config.prefixes_out[0] print("Set path prefixes to {}".format(config.prefixes_out)) if args.suffix is not None: # path suffixes config.suffixes = args.suffix config.suffix = config.suffixes[0] print("Set path suffixes to {}".format(config.suffixes)) if args.alphas: # specify alpha levels config.alphas = [float(val) for val in args.alphas.split(",")] print("Set alphas to", config.alphas) if args.vmin: # specify vmin levels config.vmins = [libmag.get_int(val) for val in args.vmin.split(",")] print("Set vmins to", config.vmins) if args.vmax: # specify vmax levels and copy to vmax overview used for plotting # and updated for normalization config.vmaxs = [libmag.get_int(val) for val in args.vmax.split(",")] config.vmax_overview = list(config.vmaxs) print("Set vmaxs to", config.vmaxs) if args.reg_suffixes is not None: # specify suffixes of registered images to load config.reg_suffixes = args_to_dict(args.reg_suffixes, config.RegSuffixes, config.reg_suffixes) print("Set registered image suffixes to {}".format( config.reg_suffixes)) if args.seed: # specify random number generator seed config.seed = int(args.seed) print("Set random number generator seed to", config.seed) if args.plot_labels is not None: # specify general plot labels config.plot_labels = args_to_dict(args.plot_labels, config.PlotLabels, config.plot_labels) print("Set plot labels to {}".format(config.plot_labels)) if args.theme is not None: # specify themes, currently applied to Matplotlib elements theme_names = [] for theme in args.theme: # add theme enum if found theme_enum = libmag.get_enum(theme, config.Themes) if theme_enum: config.rc_params.append(theme_enum) theme_names.append(theme_enum.name) print("Set to use themes to {}".format(theme_names)) # set up Matplotlib styles/themes plot_2d.setup_style() if args.db: # set main database path to user arg config.db_path = args.db print("Set database name to {}".format(config.db_path)) else: # set default path config.db_path = os.path.join(user_dir, config.db_path) if args.truth_db: # set settings for separate database of "truth blobs" config.truth_db_params = args_to_dict(args.truth_db, config.TruthDB, config.truth_db_params, sep_vals="|") mode = config.truth_db_params[config.TruthDB.MODE] config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes) libmag.printv(config.truth_db_params) print("Mapped \"{}\" truth_db mode to {}".format( mode, config.truth_db_mode)) # notify user of full args list, including unrecognized args _logger.debug(f"All command-line arguments: {sys.argv}") if args_unknown: _logger.info( f"The following command-line arguments were unrecognized and " f"ignored: {args_unknown}")