def setup_match_blobs_roi(blobs, tol):
"""Set up tolerances for matching blobs in an ROI.
Args:
blobs (:obj:`np.ndarray`): Sequence of blobs to resize if the
first ROI profile (:attr:`config.roi_profiles`) ``resize_blobs``
value is given.
tol (List[int, float]): Sequence of tolerances.
Returns:
float, List[float], List[float], List[float], :obj:`np.ndarray`:
Distance map threshold, scaling normalized by ``tol``, ROI padding
shape, resize sequence retrieved from ROI profile, and ``blobs``
after any resizing.
"""
# convert tolerance seq to scaling and single number distance
# threshold for point distance map, which assumes isotropy; use
# custom tol rather than calculating isotropy since may need to give
# greater latitude along a given axis, such as poorer res in z
thresh = np.amax(tol) # similar to longest radius from the tol bounding box
scaling = thresh / tol
# casting to int causes improper offset import into db
inner_padding = np.floor(tol[::-1])
libmag.printv(
"verifying blobs with tol {} leading to thresh {}, scaling {}, "
"inner_padding {}".format(tol, thresh, scaling, inner_padding))
# resize blobs based only on first profile
resize = config.get_roi_profile(0)["resize_blobs"]
if resize:
blobs = detector.multiply_blob_rel_coords(blobs, resize)
libmag.printv("resized blobs by {}:\n{}".format(resize, blobs))
return thresh, scaling, inner_padding, resize, blobs
def prepare_subimg(image5d, offset, size, ndim_base=5):
"""Extracts a subimage from a larger image.
Args:
image5d (:obj:`np.ndarray`): 5D image array in the order,
``t,z,y,x[,c]``, where the final dimension is optional as with
many one channel images.
offset (List[int]): Tuple of offset given as ``z,y,x`` for the region
of interest. Defaults to ``(0, 0, 0)``.
size (List[int]): Size of the region of interest as ``z,y,x``.
ndim_base (int): Number of dimensions on which ``image5d`` is based.
Typically 3 or 5, defaulting to 5 as ``t,z,y,x[,c]``.
If 3, the ``t`` dimension is removed.
Returns:
:obj:`np.ndarray`: The sub-imge without separate time dimension as
a 3D (or 4-D array if channel dimension exists) array.
"""
cube_slices = [slice(o, o + s) for o, s in zip(offset, size)]
libmag.printv(
"preparing sub-image at offset: {}, size: {}, slices: {}".format(
offset, size, cube_slices))
# cube with corner at offset and shape given by size
img = image5d
if ndim_base >= 5:
# remove time axis
img = image5d[0]
return img[cube_slices[0], cube_slices[1], cube_slices[2]]
def scale_coords(coord, scaling=None, clip_shape=None):
"""Get the atlas label IDs for the given coordinates.
Args:
coord (:class:`numpy.ndarray`): Coordinates of experiment image in
``z,y,x`` order. Can be an ``[n, 3]`` array of coordinates.
scaling (Sequence[int]): Scaling factor for the labels image size
compared with the experiment image as ``z,y,x``; defaults to None.
clip_shape (Sequence[int]): Max image shape as ``z,y,x``, used to
round coordinates for extra precision. For simplicity, scaled
values are simply floored. Repeated scaling such as upsampling
after downsampling can lead to errors. If this parameter is given,
values will instead by rounded to minimize errors while giving
ints. Rounded values will be clipped to this shape minus 1 to
stay within bounds.
Returns:
:class:`numpy.ndarray`: An scaled array of the same shape as ``coord``.
"""
libmag.printv("getting label IDs from coordinates using scaling", scaling)
coord_scaled = coord
if scaling is not None:
# scale coordinates to atlas image size
coord_scaled = np.multiply(coord, scaling)
if clip_shape is not None:
# round when extra precision is necessary, such as during reverse
# scaling, which requires clipping so coordinates don't exceed labels
# image shape
coord_scaled = np.around(coord_scaled).astype(np.int)
coord_scaled = np.clip(coord_scaled, None, np.subtract(clip_shape, 1))
else:
# typically don't round to stay within bounds
coord_scaled = coord_scaled.astype(np.int)
return coord_scaled
def saturate_roi(roi,
clip_vmin=-1,
clip_vmax=-1,
max_thresh_factor=-1,
channel=None):
"""Saturates an image, clipping extreme values and stretching remaining
values to fit the full range.
Args:
roi (:obj:`np.ndarray`): Region of interest.
clip_vmin (float): Percent for lower clipping. Defaults to -1
to use the profile setting.
clip_vmax (float): Percent for upper clipping. Defaults to -1
to use the profile setting.
max_thresh_factor (float): Multiplier of :attr:`config.near_max`
for ROI's scaled maximum value. If the max data range value
adjusted through``clip_vmax``is below this product, this max
value will be set to this product. Defaults to -1 to use the
profile setting.
channel (List[int]): Sequence of channel indices in ``roi`` to
saturate. Defaults to None to use all channels.
Returns:
Saturated region of interest.
"""
multichannel, channels = setup_channels(roi, channel, 3)
roi_out = None
for chl in channels:
roi_show = roi[..., chl] if multichannel else roi
settings = config.get_roi_profile(chl)
if clip_vmin == -1:
clip_vmin = settings["clip_vmin"]
if clip_vmax == -1:
clip_vmax = settings["clip_vmax"]
if max_thresh_factor == -1:
max_thresh_factor = settings["max_thresh_factor"]
# enhance contrast and normalize to 0-1 scale
vmin, vmax = np.percentile(roi_show, (clip_vmin, clip_vmax))
libmag.printv("vmin:", vmin, "vmax:", vmax, "near max:",
config.near_max[chl])
# adjust the near max value derived globally from image5d for the chl
max_thresh = config.near_max[chl] * max_thresh_factor
if vmax < max_thresh:
vmax = max_thresh
libmag.printv("adjusted vmax to {}".format(vmax))
saturated = np.clip(roi_show, vmin, vmax)
saturated = (saturated - vmin) / (vmax - vmin)
if multichannel:
if roi_out is None:
roi_out = np.zeros(roi.shape, dtype=saturated.dtype)
roi_out[..., chl] = saturated
else:
roi_out = saturated
return roi_out
def get_label_ids_from_position(coord,
labels_img,
scaling=None,
rounding=False,
return_coord_scaled=False):
"""Get the atlas label IDs for the given coordinates.
Args:
coord: Coordinates of experiment image in (z, y, x) order. Can be an
[n, 3] array of coordinates.
labels_img: The registered image whose intensity values correspond to
label IDs.
scaling: Scaling factor for the labels image size compared with the
experiment image; defaults to None.
rounding: True to round coordinates after scaling, which should be
used rounding to reverse coordinates that were previously scaled
inversely to avoid size degredation with repeated scaling.
Typically rounding is False (default) so that coordinates fall
evenly to their lowest integer, without exceeding max bounds.
return_coord_scaled: True to return the array of scaled coordinates;
defaults to False.
Returns:
An array of label IDs corresponding to ``coord``, or a scalar of
one ID if only one coordinate is given. If ``return_coord_scaled`` is
True, also returns a Numpy array of the same shape as ``coord``
scaled based on ``scaling``.
"""
libmag.printv("getting label IDs from coordinates using scaling", scaling)
coord_scaled = coord
if scaling is not None:
# scale coordinates to atlas image size
coord_scaled = np.multiply(coord, scaling)
if rounding:
# round when extra precision is necessary, such as during reverse
# scaling, which requires clipping so coordinates don't exceed labels
# image shape
coord_scaled = np.around(coord_scaled).astype(np.int)
coord_scaled = np.clip(coord_scaled, None,
np.subtract(labels_img.shape, 1))
else:
# typically don't round to stay within bounds
coord_scaled = coord_scaled.astype(np.int)
# index blob coordinates into labels image by int array indexing to
# get the corresponding label IDs
coordsi = libmag.coords_for_indexing(coord_scaled)
label_ids = labels_img[tuple(coordsi)][0]
if return_coord_scaled:
return label_ids, coord_scaled
return label_ids
def make_isotropic(roi, scale):
resize_factor = calc_isotropic_factor(scale)
isotropic_shape = np.array(roi.shape)
isotropic_shape[:3] = (isotropic_shape[:3] * resize_factor).astype(np.int)
libmag.printv("original ROI shape: {}, isotropic: {}".format(
roi.shape, isotropic_shape))
mode = "reflect"
if np.any(np.array(roi.shape) == 1):
# may crash with floating point exception if 1px thick (see
# https://github.com/scikit-image/scikit-image/issues/3001, which
# causes multiprocessing Pool to hang since the exception isn't
# raised), so need to change mode in this case
mode = "edge"
return transform.resize(roi,
isotropic_shape,
preserve_range=True,
mode=mode,
anti_aliasing=True)
def get_split_stack_total_shape(sub_rois, overlap=None):
"""Get the shape of a chunked stack.
Useful for determining the final shape of a stack that has been
chunked and potentially scaled before merging the stack into
an output array of this shape.
Attributes:
sub_rois: Array of sub regions, in (z, y, x, ...) dimensions.
overlap: Overlap size between sub-ROIs. Defaults to None for no overlap.
Returns:
The shape of the chunked stack after it would be merged.
"""
size = sub_rois.shape
shape_sub_roi = sub_rois[0, 0, 0].shape # for number of dimensions
merged_shape = np.zeros(len(shape_sub_roi)).astype(np.int)
final_shape = np.zeros(len(shape_sub_roi)).astype(np.int)
edges = None
for z in range(size[0]):
for y in range(size[1]):
for x in range(size[2]):
coord = (z, y, x)
sub_roi = sub_rois[coord]
edges = list(sub_roi.shape[0:3])
if overlap is not None:
# remove overlap if not at last sub_roi or row or column
for n in range(len(edges)):
if coord[n] != size[n] - 1:
edges[n] -= overlap[n]
#print("edges: {}".format(edges))
merged_shape[2] += edges[2]
if final_shape[2] <= 0:
final_shape[2] = merged_shape[2]
merged_shape[1] += edges[1]
if final_shape[1] <= 0:
final_shape[1] = merged_shape[1]
final_shape[0] += edges[0]
channel_dim = 3
if len(shape_sub_roi) > channel_dim:
final_shape[channel_dim] = shape_sub_roi[channel_dim]
libmag.printv("final_shape: {}".format(final_shape))
return final_shape
def prepare_roi(image5d, roi_size, roi_offset):
"""Extracts a region of interest (ROI).
Calls :meth:`prepare_subimage` but expects size and offset variables to
be in x,y,z order following this software's legacy convention.
Args:
image5d: Image array as a 5D array (t, z, y, x, c), or 4D if
no separate channel dimension exists as with most one channel
images.
roi_size: Size of the region of interest as (x, y, z).
roi_offset: Tuple of offset given as (x, y, z) for the region
of interest. Defaults to (0, 0, 0).
Returns:
The region of interest without separate time dimension as a 3D
if ``image5d`` is 4D, without a separate channel dimension, or 4-D
array if channel dimension exists.
"""
libmag.printv("preparing ROI at x,y,z:")
return prepare_subimg(image5d, roi_size[::-1], roi_offset[::-1])
def get_label(coord,
labels_img,
labels_ref,
scaling,
level=None,
rounding=False):
"""Get the atlas label for the given coordinates.
Args:
coord: Coordinates of experiment image in (z, y, x) order.
labels_img: The registered image whose intensity values correspond to
label IDs.
labels_ref: The labels reference lookup, assumed to be generated by
:func:`ontology.create_reverse_lookup` to look up by ID.
scaling: Scaling factor for the labels image size compared with the
experiment image.
level: The ontology level as an integer to target; defaults to None.
If None, level will be ignored, and the exact matching label
to the given coordinates will be returned. If a level is given,
the label at the highest (numerically lowest) level encompassing
this region will be returned.
rounding: True to round coordinates after scaling (see
:func:``get_label_ids_from_position``); defaults to False.
Returns:
The label dictionary at those coordinates, or None if no label is
found.
"""
label_id = get_label_ids_from_position(coord, labels_img, scaling,
rounding)
libmag.printv("found label_id: {}".format(label_id))
mirrored = label_id < 0
if mirrored:
label_id = -1 * label_id
label = None
try:
label = labels_ref[label_id]
if level is not None and label[NODE][
config.ABAKeys.LEVEL.value] > level:
# search for parent at "higher" (numerically lower) level
# that matches the target level
parents = label[PARENT_IDS]
label = None
if label_id < 0:
parents = np.multiply(parents, -1)
for parent in parents:
parent_label = labels_ref[parent]
if parent_label[NODE][config.ABAKeys.LEVEL.value] == level:
label = parent_label
break
if label is not None:
label[MIRRORED] = mirrored
libmag.printv("label ID at level {}: {}".format(level, label_id))
except KeyError as e:
libmag.printv(
"could not find label id {} or its parent (error {})".format(
label_id, e))
return label
def prepare_subimg(image5d, size, offset):
"""Extracts a subimage from a larger image.
Args:
image5d: Image array as a 5D array (t, z, y, x, c), or 4D if
no separate channel dimension exists as with most one channel
images.
size: Size of the region of interest as (z, y, x).
offset: Tuple of offset given as (z, y, x) for the region
of interest. Defaults to (0, 0, 0).
Returns:
The sub-imge without separate time dimension as a 3D (or 4-D
array if channel dimension exists) array.
"""
cube_slices = [slice(o, o + s) for o, s in zip(offset, size)]
libmag.printv(
"preparing sub-image at offset: {}, size: {}, slices: {}".format(
offset, size, cube_slices))
# cube with corner at offset, side of cube_len
return image5d[0, cube_slices[0], cube_slices[1], cube_slices[2]]
def merge_blobs(blob_rois):
"""Combine all blobs into a master list so that each overlapping region
will contain all blobs from all sub-ROIs that had blobs in those regions,
obviating the need to pair sub-ROIs with one another.
Args:
blob_rois (:obj:`np.ndarray`): Blob from each sub-region defined by
:meth:`stack_splitter`. Blobs are assumed to be a 2D array
in the format ``[[z, y, x, ...], ...]``.
Returns:
:obj:`np.ndarray`: Merged blobs in 2D format of the format,
``[[z, y, x, ..., sub_roi_z, sub_roi_y, sub_roi_x], ...]``, where
sub-ROI coordinates have been added as the final columns.
"""
blobs_all = []
for z in range(blob_rois.shape[0]):
for y in range(blob_rois.shape[1]):
for x in range(blob_rois.shape[2]):
coord = (z, y, x)
blobs = blob_rois[coord]
#print("checking blobs in {}:\n{}".format(coord, blobs))
if blobs is None:
libmag.printv("no blobs to add, skipping")
else:
# add temporary tag with sub-ROI coordinate
extras = np.zeros((blobs.shape[0], 3), dtype=int)
extras[:] = coord
blobs = np.concatenate((blobs, extras), axis=1)
blobs_all.append(blobs)
if len(blobs_all) > 0:
blobs_all = np.vstack(blobs_all)
else:
blobs_all = None
return blobs_all
def prepare_roi(image5d, roi_offset, roi_size, ndim_base=5):
"""Extracts a region of interest (ROI).
Calls :meth:`prepare_subimage` but expects size and offset variables to
be in x,y,z order following this software's legacy convention.
Args:
image5d (:obj:`np.ndarray`): 5D image array in the order,
``t,z,y,x[,c]``, where the final dimension is optional as with
many one channel images.
roi_offset (List[int]): Tuple of offset given as ``x,y,z`` for the region
of interest. Defaults to ``(0, 0, 0)``.
roi_size (List[int]): Size of the region of interest as ``x,y,z``.
ndim_base (int): Number of dimensions on which ``image5d`` is based.
Typically 3 or 5, defaulting to 5 as ``t,z,y,x[,c]``.
If 3, the ``t`` dimension is removed.
Returns:
:obj:`np.ndarray`: The region of interest without separate time
dimension as a 3D (or 4-D array if channel dimension exists) array.
"""
libmag.printv("preparing ROI at x,y,z:")
return prepare_subimg(image5d, roi_offset[::-1], roi_size[::-1], ndim_base)
def merge_blobs(blob_rois):
# combine all blobs into a master list so that each overlapping region
# will contain all blobs from all sub-ROIs that had blobs in those regions,
# obviating the need to pair sub-ROIs with one another
blobs_all = []
for z in range(blob_rois.shape[0]):
for y in range(blob_rois.shape[1]):
for x in range(blob_rois.shape[2]):
coord = (z, y, x)
blobs = blob_rois[coord]
#print("checking blobs in {}:\n{}".format(coord, blobs))
if blobs is None:
libmag.printv("no blobs to add, skipping")
else:
# add temporary tag with sub-ROI coordinate
extras = np.zeros((blobs.shape[0], 3), dtype=int)
extras[:] = coord
blobs = np.concatenate((blobs, extras), axis=1)
blobs_all.append(blobs)
if len(blobs_all) > 0:
blobs_all = np.vstack(blobs_all)
else:
blobs_all = None
return blobs_all
def setup_dbs(filename_base, db_path=None, truth_db_config=None):
"""Set up databases for the given image file if the given database has
not been set up already.
Args:
filename_base (str): Image base path.
db_path (str): Main database path; defaults to None to use a default
path.
truth_db_config (List[str]): Sequence of truth database configuration
settings; defaults to None to not load truth-related databases.
"""
if db_path:
config.db_name = db_path
print("Set database name to {}".format(config.db_name))
# load "truth blobs" from separate database for viewing
if truth_db_config is not None:
# set the truth database mode
config.truth_db_params = args_to_dict(truth_db_config,
config.TruthDB,
config.truth_db_params,
sep_vals="|")
mode = config.truth_db_params[config.TruthDB.MODE]
config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes)
libmag.printv(config.truth_db_params)
print("Mapped \"{}\" truth_db mode to {}".format(
mode, config.truth_db_mode))
truth_db_path = config.truth_db_params[config.TruthDB.PATH]
truth_db_name_base = filename_base if filename_base else sqlite.DB_NAME_BASE
if config.truth_db_mode is config.TruthDBModes.VIEW:
# loads truth DB as a separate database in parallel with the given
# editable database, with name based on filename by default unless
# truth DB name explicitly given
path = truth_db_path if truth_db_path else truth_db_name_base
try:
sqlite.load_truth_db(path)
except FileNotFoundError as e:
print(e)
print("Could not load truth DB from current image path")
elif config.truth_db_mode is config.TruthDBModes.VERIFY:
if not config.verified_db:
# creates a new verified DB to store all ROC results
config.verified_db = sqlite.ClrDB()
config.verified_db.load_db(sqlite.DB_NAME_VERIFIED, True)
if truth_db_path:
# load truth DB path to verify against if explicitly given
try:
sqlite.load_truth_db(truth_db_path)
except FileNotFoundError as e:
print(e)
print("Could not load truth DB from {}".format(truth_db_path))
elif config.truth_db_mode is config.TruthDBModes.VERIFIED:
# loads verified DB as the main DB, which includes copies of truth
# values with flags for whether they were detected
path = sqlite.DB_NAME_VERIFIED
if truth_db_path: path = truth_db_path
try:
config.db = sqlite.ClrDB()
config.db.load_db(path)
config.verified_db = config.db
except FileNotFoundError as e:
print(e)
print("Could not load verified DB from {}".format(
sqlite.DB_NAME_VERIFIED))
elif config.truth_db_mode is config.TruthDBModes.EDIT:
# loads truth DB as the main database for editing rather than
# loading as a truth database
config.db_name = truth_db_path
if not config.db_name:
config.db_name = "{}{}".format(
os.path.basename(truth_db_name_base), sqlite.DB_SUFFIX_TRUTH)
print("Editing truth database at {}".format(config.db_name))
if config.db is None:
config.db = sqlite.ClrDB()
config.db.load_db(None, False)
def prune_blobs_mp(cls, img, seg_rois, overlap, tol, sub_roi_slices,
sub_rois_offsets, channels, overlap_padding=None):
"""Prune close blobs within overlapping regions by checking within
entire planes across the ROI in parallel with multiprocessing.
Args:
img (:obj:`np.ndarray`): Array in which to detect blobs.
seg_rois (:obj:`np.ndarray`): Blobs from each sub-region.
overlap: 1D array of size 3 with the number of overlapping pixels
for each image axis.
tol: Tolerance as (z, y, x), within which a segment will be
considered a duplicate of a segment in the master array and
removed.
sub_roi_slices (:obj:`np.ndarray`): Object array of ub-regions, used
to check size.
sub_rois_offsets: Offsets of each sub-region.
channels (Sequence[int]): Sequence of channels; defaults to None
to detect in all channels.
overlap_padding: Sequence of z,y,x for additional padding beyond
``overlap``. Defaults to None to use ``tol`` as padding.
Returns:
:obj:`np.ndarray`, :obj:`pd.DataFrame`: All blobs as a Numpy array
and a data frame of pruning stats, or None for both if no blobs are
in the ``seg_rois``.
"""
# collect all blobs in master array to group all overlapping regions,
# with sub-ROI coordinates as last 3 columns
blobs_merged = chunking.merge_blobs(seg_rois)
if blobs_merged is None:
return None, None
print("total blobs before pruning:", len(blobs_merged))
print("pruning with overlap: {}, overlap tol: {}, pruning tol: {}"
.format(overlap, overlap_padding, tol))
blobs_all = []
blob_ratios = {}
cols = ("blobs", "ratio_pruning", "ratio_adjacent")
if overlap_padding is None: overlap_padding = tol
for i in channels:
# prune blobs from each channel separately to avoid pruning based on
# co-localized channel signals
blobs = detector.blobs_in_channel(blobs_merged, i)
for axis in range(3):
# prune planes with all the overlapping regions within a given axis,
# skipping if this axis has no overlapping sub-regions
num_sections = sub_rois_offsets.shape[axis]
if num_sections <= 1:
continue
# multiprocess pruning by overlapping planes
blobs_all_non_ol = None # all blobs from non-overlapping regions
blobs_to_prune = []
coord_last = tuple(np.subtract(sub_roi_slices.shape, 1))
for j in range(num_sections):
# build overlapping region dimensions based on size of
# sub-region in the given axis
coord = np.zeros(3, dtype=np.int)
coord[axis] = j
print("** setting up blob pruning in axis {}, section {} "
"of {}".format(axis, j, num_sections - 1))
offset = sub_rois_offsets[tuple(coord)]
sub_roi = img[sub_roi_slices[tuple(coord)]]
size = sub_roi.shape
_logger.debug(f"offset: {offset}, size: {size}")
# overlapping region: each region but the last extends
# into the next region, with the overlapping volume from
# the end of the region, minus the overlap and a tolerance
# space, to the region's end plus this tolerance space;
# non-overlapping region: the region before the overlap,
# after any overlap with the prior region (n > 1)
# to the start of the overlap (n < last region)
blobs_ol = None
blobs_ol_next = None
blobs_in_non_ol = []
shift = overlap[axis] + overlap_padding[axis]
offset_axis = offset[axis]
if j < num_sections - 1:
bounds = [offset_axis + size[axis] - shift,
offset_axis + size[axis]
+ overlap_padding[axis]]
libmag.printv(
"axis {}, boundaries: {}".format(axis, bounds))
blobs_ol = blobs[np.all([
blobs[:, axis] >= bounds[0],
blobs[:, axis] < bounds[1]], axis=0)]
# get blobs from immediatley adjacent region of the same
# size as that of the overlapping region; keep same
# starting point with or without overlap_tol
start = offset_axis + size[axis] + tol[axis]
bounds_next = [
start,
start + overlap[axis] + 2 * overlap_padding[axis]]
shape = np.add(
sub_rois_offsets[coord_last], sub_roi.shape[:3])
libmag.printv(
"axis {}, boundaries (next): {}, max bounds: {}"
.format(axis, bounds_next, shape[axis]))
if np.all(np.less(bounds_next, shape[axis])):
# ensure that next overlapping region is within ROI
blobs_ol_next = blobs[np.all([
blobs[:, axis] >= bounds_next[0],
blobs[:, axis] < bounds_next[1]], axis=0)]
# non-overlapping region extends up this overlap
blobs_in_non_ol.append(blobs[:, axis] < bounds[0])
else:
# last non-overlapping region extends to end of region
blobs_in_non_ol.append(
blobs[:, axis] < offset_axis + size[axis])
# get non-overlapping area
start = offset_axis
if j > 0:
# shift past overlapping part at start of region
start += shift
blobs_in_non_ol.append(blobs[:, axis] >= start)
blobs_non_ol = blobs[np.all(blobs_in_non_ol, axis=0)]
# collect all non-overlapping region blobs
if blobs_all_non_ol is None:
blobs_all_non_ol = blobs_non_ol
elif blobs_non_ol is not None:
blobs_all_non_ol = np.concatenate(
(blobs_all_non_ol, blobs_non_ol))
blobs_to_prune.append((blobs_ol, axis, tol, blobs_ol_next))
is_fork = chunking.is_fork()
if is_fork:
# set data as class variables to share across forks
cls.blobs_to_prune = blobs_to_prune
pool = chunking.get_mp_pool()
pool_results = []
for j in range(len(blobs_to_prune)):
if is_fork:
# prune blobs in overlapping regions by multiprocessing,
# using a separate class to avoid pickling input blobs
pool_results.append(pool.apply_async(
StackPruner.prune_overlap_by_index, args=(j, )))
else:
# for spawned methods, need to pickle the blobs
pool_results.append(pool.apply_async(
StackPruner.prune_overlap, args=(
j, blobs_to_prune[j])))
# collect all the pruned blob lists
blobs_all_ol = None
for result in pool_results:
blobs_ol_pruned, ratios = result.get()
if blobs_all_ol is None:
blobs_all_ol = blobs_ol_pruned
elif blobs_ol_pruned is not None:
blobs_all_ol = np.concatenate(
(blobs_all_ol, blobs_ol_pruned))
if ratios:
for col, val in zip(cols, ratios):
blob_ratios.setdefault(col, []).append(val)
# recombine blobs from the non-overlapping with the pruned
# overlapping regions from the entire stack to re-prune along
# any remaining axes
pool.close()
pool.join()
if blobs_all_ol is None:
blobs = blobs_all_non_ol
elif blobs_all_non_ol is None:
blobs = blobs_all_ol
else:
blobs = np.concatenate((blobs_all_non_ol, blobs_all_ol))
# build up list from each channel
blobs_all.append(blobs)
# merge blobs into Numpy array and remove sub-ROI coordinate columns
blobs_all = np.vstack(blobs_all)[:, :-3]
print("total blobs after pruning:", len(blobs_all))
# export blob ratios as data frame
df = pd.DataFrame(blob_ratios)
return blobs_all, df
def match_blobs_roi(blobs, blobs_base, offset, size, thresh, scaling,
inner_padding, resize=None):
"""Match blobs from two sets of blobs in an ROI, prioritizing the inner
portion of ROIs to avoid missing detections because of edge effects
while also adding matches between a blob in the inner ROI and another
blob in the remaining portion of the ROI.
Args:
blobs (:obj:`np.ndarray`): The blobs to be matched against
``blobs_base``, given as 2D array of
``[[z, row, column, radius, ...], ...]``.
blobs_base (:obj:`np.ndarray`): The blobs to which ``blobs`` will
be matched, in the same format as ``blobs``.
offset (List[int]): ROI offset from which to select blobs in x,y,z.
size (List[int]): ROI size in x,y,z.
thresh (float): Distance map threshold
scaling (List[float]): Scaling normalized by ``tol``.
inner_padding (List[float]): ROI padding shape.
resize (List[float]): Resize sequence retrieved from ROI profile;
defaults to None.
Returns:
:class:`numpy.ndarray`, :class:`numpy.ndarray`, list[int], list[int],
list[list[:class:`numpy.ndarray`, :class:`numpy.ndarray`, float]]:
Array of blobs from ``blobs``; corresponding array from ``blobs_base``
matching blobs in ``blobs``; offset of the inner portion of the ROI
in absolute coordinates of x,y,z; shape of this inner portion of the
ROI; and list of blob matches, each given as a list of
``blob_master, blob, distance``.
"""
# get all blobs in inner and total ROI
offset_inner = np.add(offset, inner_padding)
size_inner = np.subtract(size, inner_padding * 2)
libmag.printv(
"offset: {}, offset_inner: {}, size: {}, size_inner: {}"
.format(offset, offset_inner, size, size_inner))
blobs_roi, _ = detector.get_blobs_in_roi(blobs, offset, size)
if resize is not None:
# TODO: doesn't align with exported ROIs
padding = config.plot_labels[config.PlotLabels.PADDING]
libmag.printv("shifting blobs in ROI by offset {}, border {}"
.format(offset, padding))
blobs_roi = detector.shift_blob_rel_coords(blobs_roi, offset)
if padding:
blobs_roi = detector.shift_blob_rel_coords(blobs_roi, padding)
blobs_inner, blobs_inner_mask = detector.get_blobs_in_roi(
blobs_roi, offset_inner, size_inner)
blobs_base_roi, _ = detector.get_blobs_in_roi(blobs_base, offset, size)
blobs_base_inner, blobs_base_inner_mask = detector.get_blobs_in_roi(
blobs_base_roi, offset_inner, size_inner)
# compare blobs from inner region of ROI with all base blobs,
# prioritizing the closest matches
found, found_base, dists = find_closest_blobs_cdist(
blobs_inner, blobs_base_roi, thresh, scaling)
blobs_inner[:, 4] = 0
blobs_inner[found, 4] = 1
blobs_base_roi[blobs_base_inner_mask, 5] = 0
blobs_base_roi[found_base, 5] = 1
# add any base blobs missed in the inner ROI by comparing with
# test blobs from outer ROI
blobs_base_inner_missed = blobs_base_roi[blobs_base_roi[:, 5] == 0]
blobs_outer = blobs_roi[np.invert(blobs_inner_mask)]
found_out, found_base_out, dists_out = find_closest_blobs_cdist(
blobs_outer, blobs_base_inner_missed, thresh, scaling)
blobs_base_inner_missed[found_base_out, 5] = 1
# combine inner and outer groups
blobs_truth_inner_plus = np.concatenate(
(blobs_base_roi[blobs_base_roi[:, 5] == 1],
blobs_base_inner_missed))
blobs_outer[found_out, 4] = 1
blobs_inner_plus = np.concatenate((blobs_inner, blobs_outer[found_out]))
matches_inner = _match_blobs(
blobs_inner, blobs_base_roi, found, found_base, dists)
matches_outer = _match_blobs(
blobs_outer, blobs_base_inner_missed, found_out,
found_base_out, dists_out)
matches = colocalizer.BlobMatch([*matches_inner, *matches_outer])
if config.verbose:
'''
print("blobs_roi:\n{}".format(blobs_roi))
print("blobs_inner:\n{}".format(blobs_inner))
print("blobs_base_inner:\n{}".format(blobs_base_inner))
print("blobs_base_roi:\n{}".format(blobs_base_roi))
print("found inner:\n{}"
.format(blobs_inner[found]))
print("truth found:\n{}"
.format(blobs_base_roi[found_base]))
print("blobs_outer:\n{}".format(blobs_outer))
print("blobs_base_inner_missed:\n{}"
.format(blobs_base_inner_missed))
print("truth blobs detected by an outside blob:\n{}"
.format(blobs_base_inner_missed[found_base_out]))
print("all those outside detection blobs:\n{}"
.format(blobs_roi_extra))
print("blobs_inner_plus:\n{}".format(blobs_inner_plus))
print("blobs_truth_inner_plus:\n{}".format(blobs_truth_inner_plus))
'''
print("Closest matches found (truth, detected, distance):")
msgs = ("\n- Inner ROI:", "\n- Outer ROI:")
for msg, matches_sub in zip(msgs, (matches_inner, matches_outer)):
print(msg)
for match in matches_sub:
print(
"Blob1:", match[0][:3], "chl",
detector.get_blob_channel(match[0]), "Blob2:", match[1][:3],
"chl", detector.get_blob_channel(match[1]),
"dist:", match[2])
print()
return blobs_inner_plus, blobs_truth_inner_plus, offset_inner, size_inner, \
matches
def load_truth_blobs(self):
self.blobs_truth = select_blobs_confirmed(self.cur, 1)
libmag.printv("truth blobs:\n{}".format(self.blobs_truth))
def export_rois(db,
image5d,
channel,
path,
padding=None,
unit_factor=None,
truth_mode=None,
exp_name=None):
"""Export all ROIs from database.
If the current processing profile includes isotropic interpolation, the
ROIs will be resized to make isotropic according to this factor.
Args:
db: Database from which to export.
image5d: The image with the ROIs.
channel (List[int]): Channels to export; currently only the first
channel is used.
path: Path with filename base from which to save the exported files.
padding (List[int]): Padding in x,y,z to exclude from the ROI;
defaults to None.
unit_factor (float): Linear conversion factor for units (eg 1000.0
to convert um to mm).
truth_mode (:obj:`config.TruthDBModes`): Truth mode enum; defaults
to None.
exp_name (str): Name of experiment to export; defaults to None to
export all experiments in ``db``.
Returns:
:obj:`pd.DataFrame`: ROI metrics in a data frame.
"""
if padding is not None:
padding = np.array(padding)
# TODO: consider iterating through all channels
channel = channel[0] if channel else 0
# convert volume base on scaling and unit factor
phys_mult = np.prod(detector.calc_scaling_factor())
if unit_factor: phys_mult /= unit_factor**3
metrics_all = {}
exps = sqlite.select_experiment(db.cur, None)
for exp in exps:
if exp_name and exp["name"] != exp_name:
# DBs may contain many experiments, which may not correspond to
# image5d, eg verified DBs from many truth sets
continue
rois = sqlite.select_rois(db.cur, exp["id"])
for roi in rois:
# get ROI as a small image
size = sqlite.get_roi_size(roi)
offset = sqlite.get_roi_offset(roi)
img3d = plot_3d.prepare_roi(image5d, size, offset)
# get blobs and change confirmation flag to avoid confirmation
# color in 2D plots
roi_id = roi["id"]
blobs = sqlite.select_blobs(db.cur, roi_id)
blobs_detected = None
if truth_mode is config.TruthDBModes.VERIFIED:
# verified DBs use a truth value of -1 to indicate "detected",
# non-truth blobs, including both correct and incorrect
# detections, while the rest of blobs are "truth" blobs
truth_vals = detector.get_blob_truth(blobs)
blobs_detected = blobs[truth_vals == -1]
blobs = blobs[truth_vals != -1]
else:
# default to include only confirmed blobs; truth sets
# ironically do not use the truth flag but instead
# assume all confirmed blobs are "truth"
blobs = blobs[detector.get_blob_confirmed(blobs) == 1]
blobs[:, 4] = -1
# adjust ROI size and offset if border set
if padding is not None:
size = np.subtract(img3d.shape[::-1], 2 * padding)
img3d = plot_3d.prepare_roi(img3d, size, padding)
blobs[:, 0:3] = np.subtract(blobs[:, 0:3],
np.add(offset, padding)[::-1])
print("exporting ROI of shape {}".format(img3d.shape))
isotropic = config.roi_profile["isotropic"]
blobs_orig = blobs
if isotropic is not None:
# interpolation for isotropy if set in first processing profile
img3d = cv_nd.make_isotropic(img3d, isotropic)
isotropic_factor = cv_nd.calc_isotropic_factor(isotropic)
blobs_orig = np.copy(blobs)
blobs = detector.multiply_blob_rel_coords(
blobs, isotropic_factor)
# export ROI and 2D plots
path_base, path_dir_nifti, path_img, path_img_nifti, path_blobs, \
path_img_annot, path_img_annot_nifti = make_roi_paths(
path, roi_id, channel, make_dirs=True)
np.save(path_img, img3d)
print("saved 3D image to {}".format(path_img))
# WORKAROUND: for some reason SimpleITK gives a conversion error
# when converting from uint16 (>u2) Numpy array
img3d = img3d.astype(np.float64)
img3d_sitk = sitk.GetImageFromArray(img3d)
'''
print(img3d_sitk)
print("orig img:\n{}".format(img3d[0]))
img3d_back = sitk.GetArrayFromImage(img3d_sitk)
print(img3d.shape, img3d.dtype, img3d_back.shape, img3d_back.dtype)
print("sitk img:\n{}".format(img3d_back[0]))
'''
sitk.WriteImage(img3d_sitk, path_img_nifti, False)
roi_ed = roi_editor.ROIEditor(img3d)
roi_ed.plot_roi(blobs,
channel,
show=False,
title=os.path.splitext(path_img)[0])
libmag.show_full_arrays()
# export image and blobs, stripping blob flags and adjusting
# user-added segments' radii; use original rather than blobs with
# any interpolation since the ground truth will itself be
# interpolated
blobs = blobs_orig
blobs = blobs[:, 0:4]
# prior to v.0.5.0, user-added segments had a radius of 0.0
blobs[np.isclose(blobs[:, 3], 0), 3] = 5.0
# as of v.0.5.0, user-added segments have neg radii whose abs
# value corresponds to the displayed radius
blobs[:, 3] = np.abs(blobs[:, 3])
# make more rounded since near-integer values appear to give
# edges of 5 straight pixels
# https://github.com/scikit-image/scikit-image/issues/2112
#blobs[:, 3] += 1E-1
blobs[:, 3] -= 0.5
libmag.printv("blobs:\n{}".format(blobs))
np.save(path_blobs, blobs)
# convert blobs to ground truth
img3d_truth = plot_3d.build_ground_truth(
np.zeros(size[::-1], dtype=np.uint8), blobs)
if isotropic is not None:
img3d_truth = cv_nd.make_isotropic(img3d_truth, isotropic)
# remove fancy blending since truth set must be binary
img3d_truth[img3d_truth >= 0.5] = 1
img3d_truth[img3d_truth < 0.5] = 0
print("exporting truth ROI of shape {}".format(img3d_truth.shape))
np.save(path_img_annot, img3d_truth)
#print(img3d_truth)
sitk.WriteImage(sitk.GetImageFromArray(img3d_truth),
path_img_annot_nifti, False)
# avoid smoothing interpolation, using "nearest" instead
with plt.style.context(config.rc_params_mpl2_img_interp):
roi_ed.plot_roi(img3d_truth,
None,
channel,
show=False,
title=os.path.splitext(path_img_annot)[0])
# measure ROI metrics and export to data frame; use AtlasMetrics
# enum vals since will need LabelMetrics names instead
metrics = {
config.AtlasMetrics.SAMPLE.value: exp["name"],
config.AtlasMetrics.CONDITION.value: "truth",
config.AtlasMetrics.CHANNEL.value: channel,
config.AtlasMetrics.OFFSET.value: offset,
config.AtlasMetrics.SIZE.value: size,
}
# get basic counts for ROI and update volume for physical units
vols.MeasureLabel.set_data(img3d, np.ones_like(img3d,
dtype=np.int8))
_, metrics_counts = vols.MeasureLabel.measure_counts(1)
metrics_counts[vols.LabelMetrics.Volume] *= phys_mult
for key, val in metrics_counts.items():
# convert LabelMetrics to their name
metrics[key.name] = val
metrics[vols.LabelMetrics.Nuclei.name] = len(blobs)
metrics_dicts = [metrics]
if blobs_detected is not None:
# add another row for detected blobs
metrics_detected = dict(metrics)
metrics_detected[
config.AtlasMetrics.CONDITION.value] = "detected"
metrics_detected[vols.LabelMetrics.Nuclei.name] = len(
blobs_detected)
metrics_dicts.append(metrics_detected)
for m in metrics_dicts:
for key, val in m.items():
metrics_all.setdefault(key, []).append(val)
print("exported {}".format(path_base))
#_test_loading_rois(db, channel, path)
# convert to data frame and compute densities for nuclei and intensity
df = df_io.dict_to_data_frame(metrics_all)
vol = df[vols.LabelMetrics.Volume.name]
df.loc[:, vols.LabelMetrics.DensityIntens.name] = (
df[vols.LabelMetrics.Intensity.name] / vol)
df.loc[:, vols.LabelMetrics.Density.name] = (
df[vols.LabelMetrics.Nuclei.name] / vol)
df = df_io.data_frames_to_csv(df, "{}_rois.csv".format(path))
return df
def colocalize_stack(cls, shape, blobs):
"""Entry point to colocalizing blobs within a stack.
Args:
shape (List[int]): Image shape in z,y,x.
blobs (:obj:`np.ndarray`): 2D Numpy array of blobs.
Returns:
dict[tuple[int, int], :class:`BlobMatch`]: The
dictionary of matches, where keys are tuples of the channel pairs,
and values are blob match objects.
"""
print("Colocalizing blobs based on matching blobs in each pair of "
"channels")
# set up ROI blocks from which to select blobs in each block
sub_roi_slices, sub_rois_offsets, _, _, _, overlap_base, _, _ \
= stack_detect.setup_blocks(config.roi_profile, shape)
match_tol = np.multiply(
overlap_base, config.roi_profile["verify_tol_factor"])
is_fork = chunking.is_fork()
if is_fork:
# set shared data in forked multiprocessing
cls.blobs = blobs
cls.match_tol = match_tol
pool = mp.Pool(processes=config.cpus)
pool_results = []
for z in range(sub_roi_slices.shape[0]):
for y in range(sub_roi_slices.shape[1]):
for x in range(sub_roi_slices.shape[2]):
coord = (z, y, x)
offset = sub_rois_offsets[coord]
slices = sub_roi_slices[coord]
shape = [s.stop - s.start for s in slices]
if is_fork:
# use variables stored as class attributes
pool_results.append(pool.apply_async(
StackColocalizer.colocalize_block,
args=(coord, offset, shape)))
else:
# pickle full set of variables
pool_results.append(pool.apply_async(
StackColocalizer.colocalize_block,
args=(coord, offset, shape,
detector.get_blobs_in_roi(
blobs, offset, shape)[0], match_tol,
True)))
# dict of channel combos to blob matches data frame
matches_all = {}
for result in pool_results:
coord, matches = result.get()
count = 0
for key, val in matches.items():
matches_all.setdefault(key, []).append(val.df)
count += len(val.df)
print("adding {} matches from block at {} of {}"
.format(count, coord, np.add(sub_roi_slices.shape, -1)))
pool.close()
pool.join()
# prune duplicates by taking matches with shortest distance
for key in matches_all.keys():
matches_all[key] = pd.concat(matches_all[key])
for blobi in (BlobMatch.Cols.BLOB1, BlobMatch.Cols.BLOB2):
# convert blob column to ndarray to extract coords by column
matches = matches_all[key]
matches_uniq, matches_i, matches_inv, matches_cts = np.unique(
np.vstack(matches[blobi.value])[:, :3], axis=0,
return_index=True, return_inverse=True, return_counts=True)
if np.sum(matches_cts > 1) > 0:
# prune if at least one blob has been matched to multiple
# other blobs
singles = matches.iloc[matches_i[matches_cts == 1]]
dups = []
for i, ct in enumerate(matches_cts):
# include non-duplicates to retain index
if ct <= 1: continue
# get indices in orig matches at given unique array
# index and take match with lowest dist
matches_mult = matches.loc[matches_inv == i]
dists = matches_mult[BlobMatch.Cols.DIST.value]
min_dist = np.amin(dists)
num_matches = len(matches_mult)
if config.verbose and num_matches > 1:
print("pruning from", num_matches,
"matches of dist:", dists)
matches_mult = matches_mult.loc[dists == min_dist]
# take first in case of any ties
dups.append(matches_mult.iloc[[0]])
matches_all[key] = pd.concat((singles, pd.concat(dups)))
print("Colocalization matches for channels {}: {}"
.format(key, len(matches_all[key])))
libmag.printv(print(matches_all[key]))
# store data frame in BlobMatch object
matches_all[key] = BlobMatch(df=matches_all[key])
return matches_all
def detect_blobs(roi, channel, exclude_border=None):
"""Detects objects using 3D blob detection technique.
Args:
roi: Region of interest to segment.
channel (Sequence[int]): Sequence of channels to select, which can
be None to indicate all channels.
exclude_border: Sequence of border pixels in x,y,z to exclude;
defaults to None.
Returns:
Array of detected blobs, each given as
(z, row, column, radius, confirmation).
"""
time_start = time()
shape = roi.shape
multichannel, channels = plot_3d.setup_channels(roi, channel, 3)
isotropic = config.get_roi_profile(channels[0])["isotropic"]
if isotropic is not None:
# interpolate for (near) isotropy during detection, using only the
# first process settings since applies to entire ROI
roi = cv_nd.make_isotropic(roi, isotropic)
blobs_all = []
for chl in channels:
roi_detect = roi[..., chl] if multichannel else roi
settings = config.get_roi_profile(chl)
# scaling as a factor in pixel/um, where scaling of 1um/pixel
# corresponds to factor of 1, and 0.25um/pixel corresponds to
# 1 / 0.25 = 4 pixels/um; currently simplified to be based on
# x scaling alone
scale = calc_scaling_factor()
scaling_factor = scale[2]
# find blobs; sigma factors can be sequences by axes for anisotropic
# detection in skimage >= 0.15, or images can be interpolated to
# isotropy using the "isotropic" MagellanMapper setting
min_sigma = settings["min_sigma_factor"] * scaling_factor
max_sigma = settings["max_sigma_factor"] * scaling_factor
num_sigma = settings["num_sigma"]
threshold = settings["detection_threshold"]
overlap = settings["overlap"]
blobs_log = blob_log(
roi_detect, min_sigma=min_sigma, max_sigma=max_sigma,
num_sigma=num_sigma, threshold=threshold, overlap=overlap)
if config.verbose:
print("detecting blobs with min size {}, max {}, num std {}, "
"threshold {}, overlap {}"
.format(min_sigma, max_sigma, num_sigma, threshold, overlap))
print("time for 3D blob detection: {}".format(time() - time_start))
if blobs_log.size < 1:
libmag.printv("no blobs detected")
continue
blobs_log[:, 3] = blobs_log[:, 3] * math.sqrt(3)
blobs = format_blobs(blobs_log, chl)
#print(blobs)
blobs_all.append(blobs)
if not blobs_all:
return None
blobs_all = np.vstack(blobs_all)
if isotropic is not None:
# if detected on isotropic ROI, need to reposition blob coordinates
# for original, non-isotropic ROI
isotropic_factor = cv_nd.calc_isotropic_factor(isotropic)
blobs_all = multiply_blob_rel_coords(blobs_all, 1 / isotropic_factor)
blobs_all = multiply_blob_abs_coords(blobs_all, 1 / isotropic_factor)
if exclude_border is not None:
# exclude blobs from the border in x,y,z
blobs_all = get_blobs_interior(blobs_all, shape, *exclude_border)
return blobs_all
def process_cli_args():
"""Parse command-line arguments.
Typically stores values as :mod:`magmap.settings.config` attributes.
"""
parser = argparse.ArgumentParser(
description="Setup environment for MagellanMapper")
parser.add_argument("--version",
action="store_true",
help="Show version information and exit")
# image specification arguments
# image path(s) specified as an optional argument; takes precedence
# over positional argument
parser.add_argument(
"--img",
nargs="*",
default=None,
help="Main image path(s); after import, the filename is often "
"given as the original name without its extension")
# alternatively specified as the first and only positional parameter
# with as many arguments as desired
parser.add_argument(
"img_paths",
nargs="*",
default=None,
help="Main image path(s); can also be given as --img, which takes "
"precedence over this argument")
parser.add_argument(
"--meta",
nargs="*",
help="Metadata path(s), which can be given as multiple files "
"corresponding to each image")
parser.add_argument(
"--prefix",
nargs="*",
type=str,
help="Path prefix(es), typically used as the base path for file output"
)
parser.add_argument(
"--prefix_out",
nargs="*",
type=str,
help="Path prefix(es), typically used as the base path for file output "
"when --prefix modifies the input path")
parser.add_argument(
"--suffix",
nargs="*",
type=str,
help="Path suffix(es), typically inserted just before the extension")
parser.add_argument("--channel", nargs="*", type=int, help="Channel index")
parser.add_argument("--series", help="Series index")
parser.add_argument("--subimg_offset",
nargs="*",
help="Sub-image offset in x,y,z")
parser.add_argument("--subimg_size",
nargs="*",
help="Sub-image size in x,y,z")
parser.add_argument("--offset", nargs="*", help="ROI offset in x,y,z")
parser.add_argument("--size", nargs="*", help="ROI size in x,y,z")
parser.add_argument("--db", help="Database path")
parser.add_argument(
"--cpus",
help="Maximum number of CPUs/processes to use for multiprocessing "
"tasks. Use \"none\" or 0 to auto-detect this number (default).")
parser.add_argument(
"--load",
nargs="*",
help="Load associated data files; see config.LoadData for settings")
# task arguments
parser.add_argument(
"--proc",
nargs="*",
help=_get_args_dict_help(
"Image processing mode; see config.ProcessTypes for keys "
"and config.PreProcessKeys for PREPROCESS values",
config.ProcessTypes))
parser.add_argument("--register",
type=str.lower,
choices=libmag.enum_names_aslist(config.RegisterTypes),
help="Image registration task")
parser.add_argument("--df",
type=str.lower,
choices=libmag.enum_names_aslist(config.DFTasks),
help="Data frame task")
parser.add_argument("--plot_2d",
type=str.lower,
choices=libmag.enum_names_aslist(config.Plot2DTypes),
help="2D plot task; see config.Plot2DTypes")
parser.add_argument("--ec2_start",
nargs="*",
help="AWS EC2 instance start")
parser.add_argument("--ec2_list", nargs="*", help="AWS EC2 instance list")
parser.add_argument("--ec2_terminate",
nargs="*",
help="AWS EC2 instance termination")
parser.add_argument(
"--notify",
nargs="*",
help="Notification message URL, message, and attachment strings")
# profile arguments
parser.add_argument(
"--roi_profile",
nargs="*",
help="ROI profile, which can be separated by underscores "
"for multiple profiles and given as paths to custom profiles "
"in YAML format. Multiple profile groups can be given, which "
"will each be applied to the corresponding channel. See "
"docs/settings.md for more details.")
parser.add_argument(
"--atlas_profile",
help="Atlas profile, which can be separated by underscores "
"for multiple profiles and given as paths to custom profiles "
"in YAML format. See docs/settings.md for more details.")
parser.add_argument(
"--grid_search",
help="Grid search hyperparameter tuning profile(s), which can be "
"separated by underscores for multiple profiles and given as "
"paths to custom profiles in YAML format. See docs/settings.md "
"for more details.")
parser.add_argument(
"--theme",
nargs="*",
type=str.lower,
choices=libmag.enum_names_aslist(config.Themes),
help="UI theme, which can be given as multiple themes to apply "
"on top of one another")
# grouped arguments
parser.add_argument(
"--truth_db",
nargs="*",
help="Truth database; see config.TruthDB for settings and "
"config.TruthDBModes for modes")
parser.add_argument("--labels",
nargs="*",
help=_get_args_dict_help(
"Atlas labels; see config.AtlasLabels.",
config.AtlasLabels))
parser.add_argument("--transform",
nargs="*",
help=_get_args_dict_help(
"Image transformations; see config.Transforms.",
config.Transforms))
parser.add_argument(
"--reg_suffixes",
nargs="*",
help=_get_args_dict_help(
"Registered image suffixes; see config.RegSuffixes for keys "
"and config.RegNames for values", config.RegSuffixes))
parser.add_argument(
"--plot_labels",
nargs="*",
help=_get_args_dict_help(
"Plot label customizations; see config.PlotLabels ",
config.PlotLabels))
parser.add_argument(
"--set_meta",
nargs="*",
help="Set metadata values; see config.MetaKeys for settings")
# image and figure display arguments
parser.add_argument("--plane",
type=str.lower,
choices=config.PLANE,
help="Planar orientation")
parser.add_argument(
"--show",
nargs="?",
const="1",
help="If applicable, show images after completing the given task")
parser.add_argument(
"--alphas",
help="Alpha opacity levels, which can be comma-delimited for "
"multichannel images")
parser.add_argument(
"--vmin",
help="Minimum intensity levels, which can be comma-delimited "
"for multichannel images")
parser.add_argument(
"--vmax",
help="Maximum intensity levels, which can be comma-delimited "
"for multichannel images")
parser.add_argument("--seed", help="Random number generator seed")
# export arguments
parser.add_argument("--save_subimg",
action="store_true",
help="Save sub-image as separate file")
parser.add_argument("--slice", help="Slice given as start,stop,step")
parser.add_argument("--delay", help="Animation delay in ms")
parser.add_argument("--savefig", help="Extension for saved figures")
parser.add_argument("--groups",
nargs="*",
help="Group values corresponding to each image")
parser.add_argument(
"-v",
"--verbose",
nargs="*",
help=_get_args_dict_help(
"Verbose output to assist with debugging; see config.Verbosity.",
config.Verbosity))
# only parse recognized arguments to avoid error for unrecognized ones
args, args_unknown = parser.parse_known_args()
# set up application directories
user_dir = config.user_app_dirs.user_data_dir
if not os.path.isdir(user_dir):
# make application data directory
if os.path.exists(user_dir):
# backup any non-directory file
libmag.backup_file(user_dir)
os.makedirs(user_dir)
if args.verbose is not None:
# verbose mode and logging setup
config.verbose = True
config.verbosity = args_to_dict(args.verbose, config.Verbosity,
config.verbosity)
if config.verbosity[config.Verbosity.LEVEL] is None:
# default to debug mode if any verbose flag is set without level
config.verbosity[config.Verbosity.LEVEL] = logging.DEBUG
logs.update_log_level(config.logger,
config.verbosity[config.Verbosity.LEVEL])
# print longer Numpy arrays for debugging
np.set_printoptions(linewidth=200, threshold=10000)
_logger.info("Set verbose to %s", config.verbosity)
# set up logging to given file unless explicitly given an empty string
log_path = config.verbosity[config.Verbosity.LOG_PATH]
if log_path != "":
if log_path is None:
log_path = os.path.join(config.user_app_dirs.user_data_dir,
"out.log")
# log to file
config.log_path = logs.add_file_handler(config.logger, log_path)
# redirect standard out/error to logging
sys.stdout = logs.LogWriter(config.logger.info)
sys.stderr = logs.LogWriter(config.logger.error)
# load preferences file
config.prefs = prefs_prof.PrefsProfile()
config.prefs.add_profiles(str(config.PREFS_PATH))
if args.version:
# print version info and exit
_logger.info(f"{config.APP_NAME}-{libmag.get_version(True)}")
shutdown()
# log the app launch path
path_launch = (sys._MEIPASS if getattr(sys, "frozen", False)
and hasattr(sys, "_MEIPASS") else sys.argv[0])
_logger.info(f"Launched MagellanMapper from {path_launch}")
if args.img is not None or args.img_paths:
# set image file path and convert to basis for additional paths
config.filenames = args.img if args.img else args.img_paths
config.filename = config.filenames[0]
print("Set filenames to {}, current filename {}".format(
config.filenames, config.filename))
if args.meta is not None:
# set metadata paths
config.metadata_paths = args.meta
print("Set metadata paths to", config.metadata_paths)
config.metadatas = []
for path in config.metadata_paths:
# load metadata to dictionary
md, _ = importer.load_metadata(path, assign=False)
config.metadatas.append(md)
if args.channel is not None:
# set the channels
config.channel = args.channel
print("Set channel to {}".format(config.channel))
config.series_list = [config.series] # list of series
if args.series is not None:
series_split = args.series.split(",")
config.series_list = []
for ser in series_split:
ser_split = ser.split("-")
if len(ser_split) > 1:
ser_range = np.arange(int(ser_split[0]), int(ser_split[1]) + 1)
config.series_list.extend(ser_range.tolist())
else:
config.series_list.append(int(ser_split[0]))
config.series = config.series_list[0]
print("Set to series_list to {}, current series {}".format(
config.series_list, config.series))
if args.savefig is not None:
# save figure with file type of this extension; remove leading period
config.savefig = _parse_none(args.savefig.lstrip("."))
print("Set savefig extension to {}".format(config.savefig))
# parse sub-image offsets and sizes;
# expects x,y,z input but stores as z,y,x by convention
if args.subimg_offset is not None:
config.subimg_offsets = _parse_coords(args.subimg_offset, True)
print("Set sub-image offsets to {} (z,y,x)".format(
config.subimg_offsets))
if args.subimg_size is not None:
config.subimg_sizes = _parse_coords(args.subimg_size, True)
print("Set sub-image sizes to {} (z,y,x)".format(config.subimg_sizes))
# parse ROI offsets and sizes, which are relative to any sub-image;
# expects x,y,z input and output
if args.offset is not None:
config.roi_offsets = _parse_coords(args.offset)
if config.roi_offsets:
config.roi_offset = config.roi_offsets[0]
print("Set ROI offsets to {}, current offset {} (x,y,z)".format(
config.roi_offsets, config.roi_offset))
if args.size is not None:
config.roi_sizes = _parse_coords(args.size)
if config.roi_sizes:
config.roi_size = config.roi_sizes[0]
print("Set ROI sizes to {}, current size {} (x,y,z)".format(
config.roi_sizes, config.roi_size))
if args.cpus is not None:
# set maximum number of CPUs
config.cpus = _parse_none(args.cpus.lower(), int)
print("Set maximum number of CPUs for multiprocessing tasks to",
config.cpus)
if args.load is not None:
# flag loading data sources with default sub-arg indicating that the
# data should be loaded from a default path; otherwise, load from
# path given by the sub-arg; change delimiter to allow paths with ","
config.load_data = args_to_dict(args.load,
config.LoadData,
config.load_data,
sep_vals="|",
default=True)
print("Set to load the data types: {}".format(config.load_data))
# set up main processing mode
if args.proc is not None:
config.proc_type = args_to_dict(args.proc,
config.ProcessTypes,
config.proc_type,
default=True)
print("Set main processing tasks to:", config.proc_type)
if args.set_meta is not None:
# set individual metadata values, currently used for image import
# TODO: take precedence over loaded metadata archives
config.meta_dict = args_to_dict(args.set_meta,
config.MetaKeys,
config.meta_dict,
sep_vals="|")
print("Set metadata values to {}".format(config.meta_dict))
res = config.meta_dict[config.MetaKeys.RESOLUTIONS]
if res:
# set image resolutions, taken as a single set of x,y,z and
# converting to a nested list of z,y,x
res_split = res.split(",")
if len(res_split) >= 3:
res_float = tuple(float(i) for i in res_split)[::-1]
config.resolutions = [res_float]
print("Set resolutions to {}".format(config.resolutions))
else:
res_float = None
print("Resolution ({}) should be given as 3 values (x,y,z)".
format(res))
# store single set of resolutions, similar to input
config.meta_dict[config.MetaKeys.RESOLUTIONS] = res_float
mag = config.meta_dict[config.MetaKeys.MAGNIFICATION]
if mag:
# set objective magnification
config.magnification = mag
print("Set magnification to {}".format(config.magnification))
zoom = config.meta_dict[config.MetaKeys.ZOOM]
if zoom:
# set objective zoom
config.zoom = zoom
print("Set zoom to {}".format(config.zoom))
shape = config.meta_dict[config.MetaKeys.SHAPE]
if shape:
# parse shape, storing only in dict
config.meta_dict[config.MetaKeys.SHAPE] = [
int(n) for n in shape.split(",")[::-1]
]
# set up ROI and register profiles
setup_roi_profiles(args.roi_profile)
setup_atlas_profiles(args.atlas_profile)
setup_grid_search_profiles(args.grid_search)
if args.plane is not None:
config.plane = args.plane
print("Set plane to {}".format(config.plane))
if args.save_subimg:
config.save_subimg = args.save_subimg
print("Set to save the sub-image")
if args.labels:
# set up atlas labels
setup_labels(args.labels)
if args.transform is not None:
# image transformations such as flipping, rotation
config.transform = args_to_dict(args.transform, config.Transforms,
config.transform)
print("Set transformations to {}".format(config.transform))
if args.register:
# register type to process in register module
config.register_type = args.register
print("Set register type to {}".format(config.register_type))
if args.df:
# data frame processing task
config.df_task = args.df
print("Set data frame processing task to {}".format(config.df_task))
if args.plot_2d:
# 2D plot type to process in plot_2d module
config.plot_2d_type = args.plot_2d
print("Set plot_2d type to {}".format(config.plot_2d_type))
if args.slice:
# specify a generic slice by command-line, assuming same order
# of arguments as for slice built-in function and interpreting
# "none" string as None
config.slice_vals = args.slice.split(",")
config.slice_vals = [
_parse_none(val.lower(), int) for val in config.slice_vals
]
print("Set slice values to {}".format(config.slice_vals))
if args.delay:
config.delay = int(args.delay)
print("Set delay to {}".format(config.delay))
if args.show:
# show images after task is performed, if supported
config.show = _is_arg_true(args.show)
print("Set show to {}".format(config.show))
if args.groups:
config.groups = args.groups
print("Set groups to {}".format(config.groups))
if args.ec2_start is not None:
# start EC2 instances
config.ec2_start = args_with_dict(args.ec2_start)
print("Set ec2 start to {}".format(config.ec2_start))
if args.ec2_list:
# list EC2 instances
config.ec2_list = args_with_dict(args.ec2_list)
print("Set ec2 list to {}".format(config.ec2_list))
if args.ec2_terminate:
config.ec2_terminate = args.ec2_terminate
print("Set ec2 terminate to {}".format(config.ec2_terminate))
if args.notify:
notify_len = len(args.notify)
if notify_len > 0:
config.notify_url = args.notify[0]
print("Set notification URL to {}".format(config.notify_url))
if notify_len > 1:
config.notify_msg = args.notify[1]
print("Set notification message to {}".format(config.notify_msg))
if notify_len > 2:
config.notify_attach = args.notify[2]
print("Set notification attachment path to {}".format(
config.notify_attach))
if args.prefix is not None:
# path input/output prefixes
config.prefixes = args.prefix
config.prefix = config.prefixes[0]
print("Set path prefixes to {}".format(config.prefixes))
if args.prefix_out is not None:
# path output prefixes
config.prefixes_out = args.prefix_out
config.prefix_out = config.prefixes_out[0]
print("Set path prefixes to {}".format(config.prefixes_out))
if args.suffix is not None:
# path suffixes
config.suffixes = args.suffix
config.suffix = config.suffixes[0]
print("Set path suffixes to {}".format(config.suffixes))
if args.alphas:
# specify alpha levels
config.alphas = [float(val) for val in args.alphas.split(",")]
print("Set alphas to", config.alphas)
if args.vmin:
# specify vmin levels
config.vmins = [libmag.get_int(val) for val in args.vmin.split(",")]
print("Set vmins to", config.vmins)
if args.vmax:
# specify vmax levels and copy to vmax overview used for plotting
# and updated for normalization
config.vmaxs = [libmag.get_int(val) for val in args.vmax.split(",")]
config.vmax_overview = list(config.vmaxs)
print("Set vmaxs to", config.vmaxs)
if args.reg_suffixes is not None:
# specify suffixes of registered images to load
config.reg_suffixes = args_to_dict(args.reg_suffixes,
config.RegSuffixes,
config.reg_suffixes)
print("Set registered image suffixes to {}".format(
config.reg_suffixes))
if args.seed:
# specify random number generator seed
config.seed = int(args.seed)
print("Set random number generator seed to", config.seed)
if args.plot_labels is not None:
# specify general plot labels
config.plot_labels = args_to_dict(args.plot_labels, config.PlotLabels,
config.plot_labels)
print("Set plot labels to {}".format(config.plot_labels))
if args.theme is not None:
# specify themes, currently applied to Matplotlib elements
theme_names = []
for theme in args.theme:
# add theme enum if found
theme_enum = libmag.get_enum(theme, config.Themes)
if theme_enum:
config.rc_params.append(theme_enum)
theme_names.append(theme_enum.name)
print("Set to use themes to {}".format(theme_names))
# set up Matplotlib styles/themes
plot_2d.setup_style()
if args.db:
# set main database path to user arg
config.db_path = args.db
print("Set database name to {}".format(config.db_path))
else:
# set default path
config.db_path = os.path.join(user_dir, config.db_path)
if args.truth_db:
# set settings for separate database of "truth blobs"
config.truth_db_params = args_to_dict(args.truth_db,
config.TruthDB,
config.truth_db_params,
sep_vals="|")
mode = config.truth_db_params[config.TruthDB.MODE]
config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes)
libmag.printv(config.truth_db_params)
print("Mapped \"{}\" truth_db mode to {}".format(
mode, config.truth_db_mode))
# notify user of full args list, including unrecognized args
_logger.debug(f"All command-line arguments: {sys.argv}")
if args_unknown:
_logger.info(
f"The following command-line arguments were unrecognized and "
f"ignored: {args_unknown}")
