def stack_split_remerge(roi: np.ndarray, max_pixels: Sequence[int],
overlap: Sequence[int]) -> np.ndarray:
"""Split and remerge a stack.
Args:
roi: Region of interest.
max_pixels: Maximum pixels along each dimension.
overlap: Number of overlapping pixels along each dimension, to
be added to ``max_pixels`` if possible.
Returns:
The remerged stack
"""
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
roi.shape, max_pixels, overlap)
print("sub_rois shape: {}".format(sub_roi_slices.shape))
print("sub_roi_slices:\n{}".format(sub_roi_slices))
print("overlap: {}".format(overlap))
print("sub_rois_offsets:\n{}".format(sub_rois_offsets))
for z in range(sub_roi_slices.shape[0]):
for y in range(sub_roi_slices.shape[1]):
for x in range(sub_roi_slices.shape[2]):
coord = (z, y, x)
sub_roi_slices[coord] = roi[sub_roi_slices[coord]]
print(coord, "shape:", sub_roi_slices[coord].shape,
sub_roi_slices[coord])
# print("sub_rois:\n{}".format(sub_roi_slices))
merged = chunking.merge_split_stack(sub_roi_slices, max_pixels,
overlap)
print("merged:\n{}".format(merged))
return merged
def setup_blocks(settings: "profiles.SettingsDict",
shape: Sequence[int]) -> Blocks:
"""Set up blocks for block processing
Each block is a chunk of a larger image processed sequentially or in
parallel to optimize resource usage.
Args:
settings: Settings dictionary that defines that the blocks.
shape: Shape of full image in ``z, y, x``.
Returns:
A named tuple of the block parameters.
"""
scaling_factor = detector.calc_scaling_factor()
print("microsope scaling factor based on resolutions: {}".format(
scaling_factor))
denoise_size = settings["denoise_size"]
denoise_max_shape = None
if denoise_size:
# further subdivide each sub-ROI for local preprocessing
denoise_max_shape = np.ceil(np.multiply(scaling_factor,
denoise_size)).astype(int)
# overlap sub-ROIs to minimize edge effects
overlap_base = detector.calc_overlap()
tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int)
overlap_padding = np.copy(tol)
overlap = np.copy(overlap_base)
exclude_border = settings["exclude_border"]
if exclude_border is not None:
# exclude border to avoid blob detector edge effects, where blobs
# often collect at the faces of the sub-ROI;
# ensure that overlap is greater than twice the border exclusion per
# axis so that no plane will be excluded from both overlapping sub-ROIs
exclude_border_thresh = np.multiply(2, exclude_border)
overlap_less = np.less(overlap, exclude_border_thresh)
overlap[overlap_less] = exclude_border_thresh[overlap_less]
excluded = np.greater(exclude_border, 0)
overlap[excluded] += 1 # additional padding
overlap_padding[excluded] = 0 # no need to prune past excluded border
print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond "
"overlap for pruning: {}, exclude borders: {}".format(
overlap, tol, overlap_padding, exclude_border))
max_pixels = np.ceil(np.multiply(scaling_factor,
settings["segment_size"])).astype(int)
print("preprocessing max shape: {}, detection max pixels: {}".format(
denoise_max_shape, max_pixels))
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
shape, max_pixels, overlap)
return Blocks(sub_roi_slices, sub_rois_offsets, denoise_max_shape,
exclude_border, tol, overlap_base, overlap, overlap_padding,
max_pixels)
def detect_sub_roi(cls, coord, offset, last_coord, denoise_max_shape,
exclude_border, sub_roi, channel, img_path=None,
coloc=False):
"""Perform 3D blob detection within a sub-ROI without accessing
class attributes, such as for spawned multiprocessing.
Args:
coord (Tuple[int]): Coordinate of the sub-ROI in the order z,y,x.
offset (Tuple[int]): Offset of the sub-ROI within the full ROI,
in z,y,x.
last_coord (:obj:`np.ndarray`): See attributes.
denoise_max_shape (Tuple[int]): See attributes.
exclude_border (bool): See attributes.
sub_roi (:obj:`np.ndarray`): Array in which to perform detections.
img_path (str): Path from which to load metadatat; defaults to None.
If given, the command line arguments will be reloaded to
set up the image and processing parameters.
coloc (bool): True to perform blob co-localizations; defaults
to False.
channel (Sequence[int]): Sequence of channels, where None detects
in all channels.
Returns:
Tuple[int], :obj:`np.ndarray`: The coordinate given back again to
identify the sub-ROI position and an array of detected blobs.
"""
if img_path:
# reload command-line parameters and image metadata, which is
# required if run from a spawned (not forked) process
cli.process_cli_args()
_, orig_info = importer.make_filenames(img_path)
importer.load_metadata(orig_info)
print("detecting blobs in sub-ROI at {} of {}, offset {}, shape {}..."
.format(coord, last_coord, tuple(offset.astype(int)),
sub_roi.shape))
if denoise_max_shape is not None:
# further split sub-ROI for preprocessing locally
denoise_roi_slices, _ = chunking.stack_splitter(
sub_roi.shape, denoise_max_shape)
for z in range(denoise_roi_slices.shape[0]):
for y in range(denoise_roi_slices.shape[1]):
for x in range(denoise_roi_slices.shape[2]):
denoise_coord = (z, y, x)
denoise_roi = sub_roi[denoise_roi_slices[denoise_coord]]
_logger.debug(
f"preprocessing sub-sub-ROI {denoise_coord} of "
f"{np.subtract(denoise_roi_slices.shape, 1)} "
f"(shape {denoise_roi.shape} within sub-ROI shape "
f"{sub_roi.shape})")
denoise_roi = plot_3d.saturate_roi(
denoise_roi, channel=channel)
denoise_roi = plot_3d.denoise_roi(
denoise_roi, channel=channel)
# replace slices with denoised ROI
denoise_roi_slices[denoise_coord] = denoise_roi
# re-merge back into the sub-ROI
merged_shape = chunking.get_split_stack_total_shape(
denoise_roi_slices)
merged = np.zeros(
tuple(merged_shape), dtype=denoise_roi_slices[0, 0, 0].dtype)
chunking.merge_split_stack2(denoise_roi_slices, None, 0, merged)
sub_roi = merged
if exclude_border is None:
exclude = None
else:
exclude = np.array([exclude_border, exclude_border])
exclude[0, np.equal(coord, 0)] = 0
exclude[1, np.equal(coord, last_coord)] = 0
segments = detector.detect_blobs(sub_roi, channel, exclude)
if coloc and segments is not None:
# co-localize blobs and append to blobs array
colocs = colocalizer.colocalize_blobs(sub_roi, segments)
segments = np.hstack((segments, colocs))
#print("segs before (offset: {}):\n{}".format(offset, segments))
if segments is not None:
# shift both coordinate sets (at beginning and end of array) to
# absolute positioning, using the latter set to store shifted
# coordinates based on duplicates and the former for initial
# positions to check for multiple duplicates
detector.shift_blob_rel_coords(segments, offset)
detector.shift_blob_abs_coords(segments, offset)
#print("segs after:\n{}".format(segments))
return coord, segments
def setup_blocks(settings, shape):
"""Set up blocks for block processing, where each block is a chunk of
a larger image processed sequentially or in parallel to optimize
resource usage.
Args:
settings (:obj:`magmap.settings.profiles.SettingsDict`): Settings
dictionary that defines that the blocks.
shape (List[int]): Shape of full image in z,y,x.
Returns:
:obj:`np.ndarray`, :obj:`np.ndarray`, :obj:`np.ndarray`, List[int],
:obj:`np.ndarray`, :obj:`np.ndarray`, :obj:`np.ndarray`,
:obj:`np.ndarray`: Numpy object array of tuples containing slices
of each block; similar Numpy array but with tuples of offsets;
Numpy int array of max shape for each sub-block used for denoising;
List of ints for border pixels to exclude in z,y,x;
match tolerance as a Numpy float array in z,y,x;
Numpy float array of overlapping pixels in z,y,x;
similar overlap array but modified by the border exclusion; and
similar overlap array but for padding beyond the overlap.
"""
scaling_factor = detector.calc_scaling_factor()
print("microsope scaling factor based on resolutions: {}"
.format(scaling_factor))
denoise_size = settings["denoise_size"]
denoise_max_shape = None
if denoise_size:
# further subdivide each sub-ROI for local preprocessing
denoise_max_shape = np.ceil(
np.multiply(scaling_factor, denoise_size)).astype(int)
# overlap sub-ROIs to minimize edge effects
overlap_base = chunking.calc_overlap()
tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int)
overlap_padding = np.copy(tol)
overlap = np.copy(overlap_base)
exclude_border = settings["exclude_border"]
if exclude_border is not None:
# exclude border to avoid blob detector edge effects, where blobs
# often collect at the faces of the sub-ROI;
# ensure that overlap is greater than twice the border exclusion per
# axis so that no plane will be excluded from both overlapping sub-ROIs
exclude_border_thresh = np.multiply(2, exclude_border)
overlap_less = np.less(overlap, exclude_border_thresh)
overlap[overlap_less] = exclude_border_thresh[overlap_less]
excluded = np.greater(exclude_border, 0)
overlap[excluded] += 1 # additional padding
overlap_padding[excluded] = 0 # no need to prune past excluded border
print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond "
"overlap for pruning: {}, exclude borders: {}"
.format(overlap, tol, overlap_padding, exclude_border))
max_pixels = np.ceil(np.multiply(
scaling_factor, settings["segment_size"])).astype(int)
print("preprocessing max shape: {}, detection max pixels: {}"
.format(denoise_max_shape, max_pixels))
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
shape, max_pixels, overlap)
return sub_roi_slices, sub_rois_offsets, denoise_max_shape, \
exclude_border, tol, overlap_base, overlap, overlap_padding
def colocalize_stack(cls, shape, blobs):
"""Entry point to colocalizing blobs within a stack.
Args:
shape (List[int]): Image shape in z,y,x.
blobs (:obj:`np.ndarray`): 2D Numpy array of blobs.
Returns:
dict[tuple[int, int], :class:`BlobMatch`]: The
dictionary of matches, where keys are tuples of the channel pairs,
and values are blob match objects.
"""
_logger.info(
"Colocalizing blobs based on matching blobs in each pair of "
"channels")
# scale match tolerance based on block processing ROI size
blocks = stack_detect.setup_blocks(config.roi_profile, shape)
match_tol = np.multiply(blocks.overlap_base,
config.roi_profile["verify_tol_factor"])
# re-split stack with ROI size adjusted to the inner padding plus
# the raw overlap size
inner_pad = verifier.setup_match_blobs_roi(match_tol)[2]
inner_pad = np.add(inner_pad, blocks.overlap_base)
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
shape, blocks.max_pixels, inner_pad[::-1])
is_fork = chunking.is_fork()
if is_fork:
# set shared data in forked multiprocessing
cls.blobs = blobs
cls.match_tol = match_tol
pool = mp.Pool(processes=config.cpus)
pool_results = []
for z in range(sub_roi_slices.shape[0]):
for y in range(sub_roi_slices.shape[1]):
for x in range(sub_roi_slices.shape[2]):
coord = (z, y, x)
offset = sub_rois_offsets[coord]
slices = sub_roi_slices[coord]
shape = [s.stop - s.start for s in slices]
if is_fork:
# use variables stored as class attributes
pool_results.append(
pool.apply_async(StackColocalizer.colocalize_block,
args=(coord, offset, shape)))
else:
# pickle full set of variables
pool_results.append(
pool.apply_async(StackColocalizer.colocalize_block,
args=(coord, offset, shape,
detector.get_blobs_in_roi(
blobs, offset,
shape)[0], match_tol,
True)))
# dict of channel combos to blob matches data frame
matches_all = {}
for result in pool_results:
coord, matches = result.get()
count = 0
for key, val in matches.items():
matches_all.setdefault(key, []).append(val.df)
count += len(val.df)
_logger.info("Adding %s matches from block at %s of %s", count,
coord, np.add(sub_roi_slices.shape, -1))
pool.close()
pool.join()
# prune duplicates by taking matches with shortest distance
for key in matches_all.keys():
matches_all[key] = pd.concat(matches_all[key])
if matches_all[key].size > 0:
for blobi in (BlobMatch.Cols.BLOB1, BlobMatch.Cols.BLOB2):
# convert blob column to ndarray to extract coords by column
matches = matches_all[key]
matches_uniq, matches_i, matches_inv, matches_cts = np.unique(
np.vstack(matches[blobi.value])[:, :3],
axis=0,
return_index=True,
return_inverse=True,
return_counts=True)
if np.sum(matches_cts > 1) > 0:
# prune if at least one blob has been matched to
# multiple other blobs
singles = matches.iloc[matches_i[matches_cts == 1]]
dups = []
for i, ct in enumerate(matches_cts):
# include non-duplicates to retain index
if ct <= 1: continue
# get indices in orig matches at given unique array
# index and take match with lowest dist
matches_mult = matches.loc[matches_inv == i]
dists = matches_mult[BlobMatch.Cols.DIST.value]
min_dist = np.amin(dists)
num_matches = len(matches_mult)
if num_matches > 1:
_logger.debug(
"Pruning from %s matches of dist: %s",
num_matches, dists)
matches_mult = matches_mult.loc[dists == min_dist]
# take first in case of any ties
dups.append(matches_mult.iloc[[0]])
matches_all[key] = pd.concat(
(singles, pd.concat(dups)))
_logger.info("Colocalization matches for channels %s: %s", key,
len(matches_all[key]))
_logger.debug("Blob matches for %s after pruning:\n%s", key,
matches_all[key])
# store data frame in BlobMatch object
matches_all[key] = BlobMatch(df=matches_all[key])
return matches_all
def transpose_img(filename,
series,
plane=None,
rescale=None,
target_size=None):
"""Transpose Numpy NPY saved arrays into new planar orientations and
rescaling or resizing.
Rescaling/resizing take place in multiprocessing. Files are saved
through memmap-based arrays to minimize RAM usage. Output filenames
are based on the ``make_modifer_[task]`` functions. Currently transposes
all channels, ignoring :attr:``config.channel`` parameter.
Args:
filename: Full file path in :attribute:cli:`filename` format.
series: Series within multi-series file.
plane: Planar orientation (see :attribute:plot_2d:`PLANES`). Defaults
to None, in which case no planar transformation will occur.
rescale: Rescaling factor; defaults to None. Takes precedence over
``target_size``.
target_size (List[int]): Target shape in x,y,z; defaults to None,
in which case the target size will be extracted from the register
profile if available if available.
"""
if target_size is None:
target_size = config.atlas_profile["target_size"]
if plane is None and rescale is None and target_size is None:
print("No transposition to perform, skipping")
return
time_start = time()
# even if loaded already, reread to get image metadata
# TODO: consider saving metadata in config and retrieving from there
img5d = importer.read_file(filename, series)
info = img5d.meta
image5d = img5d.img
sizes = info["sizes"]
# make filenames based on transpositions
modifier = ""
if plane is not None:
modifier = make_modifier_plane(plane)
# either rescaling or resizing
if rescale is not None:
modifier += make_modifier_scale(rescale)
elif target_size:
# target size may differ from final output size but allows a known
# size to be used for finding the file later
modifier += make_modifier_resized(target_size)
filename_image5d_npz, filename_info_npz = importer.make_filenames(
filename, series, modifier=modifier)
# TODO: image5d should assume 4/5 dimensions
offset = 0 if image5d.ndim <= 3 else 1
multichannel = image5d.ndim >= 5
image5d_swapped = image5d
if plane is not None and plane != config.PLANE[0]:
# swap z-y to get (y, z, x) order for xz orientation
image5d_swapped = np.swapaxes(image5d_swapped, offset, offset + 1)
config.resolutions[0] = libmag.swap_elements(config.resolutions[0], 0,
1)
if plane == config.PLANE[2]:
# swap new y-x to get (x, z, y) order for yz orientation
image5d_swapped = np.swapaxes(image5d_swapped, offset, offset + 2)
config.resolutions[0] = libmag.swap_elements(
config.resolutions[0], 0, 2)
scaling = None
if rescale is not None or target_size is not None:
# rescale based on scaling factor or target specific size
rescaled = image5d_swapped
# TODO: generalize for more than 1 preceding dimension?
if offset > 0:
rescaled = rescaled[0]
max_pixels = [100, 500, 500]
sub_roi_size = None
if target_size:
# to avoid artifacts from thin chunks, fit image into even
# number of pixels per chunk by rounding up number of chunks
# and resizing each chunk by ratio of total size to chunk num
target_size = target_size[::-1] # change to z,y,x
shape = rescaled.shape[:3]
num_chunks = np.ceil(np.divide(shape, max_pixels))
max_pixels = np.ceil(np.divide(shape, num_chunks)).astype(np.int)
sub_roi_size = np.floor(np.divide(target_size,
num_chunks)).astype(np.int)
print("Resizing image of shape {} to target_size: {}, using "
"num_chunks: {}, max_pixels: {}, sub_roi_size: {}".format(
rescaled.shape, target_size, num_chunks, max_pixels,
sub_roi_size))
else:
print("Rescaling image of shape {} by factor of {}".format(
rescaled.shape, rescale))
# rescale in chunks with multiprocessing
sub_roi_slices, _ = chunking.stack_splitter(rescaled.shape, max_pixels)
is_fork = chunking.is_fork()
if is_fork:
Downsampler.set_data(rescaled)
sub_rois = np.zeros_like(sub_roi_slices)
pool = chunking.get_mp_pool()
pool_results = []
for z in range(sub_roi_slices.shape[0]):
for y in range(sub_roi_slices.shape[1]):
for x in range(sub_roi_slices.shape[2]):
coord = (z, y, x)
slices = sub_roi_slices[coord]
args = [coord, slices, rescale, sub_roi_size, multichannel]
if not is_fork:
# pickle chunk if img not directly available
args.append(rescaled[slices])
pool_results.append(
pool.apply_async(Downsampler.rescale_sub_roi,
args=args))
for result in pool_results:
coord, sub_roi = result.get()
print("replacing sub_roi at {} of {}".format(
coord, np.add(sub_roi_slices.shape, -1)))
sub_rois[coord] = sub_roi
pool.close()
pool.join()
rescaled_shape = chunking.get_split_stack_total_shape(sub_rois)
if offset > 0:
rescaled_shape = np.concatenate(([1], rescaled_shape))
print("rescaled_shape: {}".format(rescaled_shape))
# rescale chunks directly into memmap-backed array to minimize RAM usage
image5d_transposed = np.lib.format.open_memmap(
filename_image5d_npz,
mode="w+",
dtype=sub_rois[0, 0, 0].dtype,
shape=tuple(rescaled_shape))
chunking.merge_split_stack2(sub_rois, None, offset, image5d_transposed)
if rescale is not None:
# scale resolutions based on single rescaling factor
config.resolutions = np.multiply(config.resolutions, 1 / rescale)
else:
# scale resolutions based on size ratio for each dimension
config.resolutions = np.multiply(config.resolutions,
(image5d_swapped.shape /
rescaled_shape)[1:4])
sizes[0] = rescaled_shape
scaling = importer.calc_scaling(image5d_swapped, image5d_transposed)
else:
# transfer directly to memmap-backed array
image5d_transposed = np.lib.format.open_memmap(
filename_image5d_npz,
mode="w+",
dtype=image5d_swapped.dtype,
shape=image5d_swapped.shape)
if plane == config.PLANE[1] or plane == config.PLANE[2]:
# flip upside-down if re-orienting planes
if offset:
image5d_transposed[0, :] = np.fliplr(image5d_swapped[0, :])
else:
image5d_transposed[:] = np.fliplr(image5d_swapped[:])
else:
image5d_transposed[:] = image5d_swapped[:]
sizes[0] = image5d_swapped.shape
# save image metadata
print("detector.resolutions: {}".format(config.resolutions))
print("sizes: {}".format(sizes))
image5d.flush()
importer.save_image_info(
filename_info_npz, info["names"], sizes, config.resolutions,
info["magnification"], info["zoom"],
*importer.calc_intensity_bounds(image5d_transposed), scaling, plane)
print("saved transposed file to {} with shape {}".format(
filename_image5d_npz, image5d_transposed.shape))
print("time elapsed (s): {}".format(time() - time_start))
def detect_blobs_large_image(filename_base, image5d, offset, size,
verify=False, save_dfs=True, full_roi=False):
"""Detect blobs within a large image through parallel processing of
smaller chunks.
Args:
filename_base: Base path to use file output.
image5d: Large image to process as a Numpy array of t,z,y,x,[c]
offset: Sub-image offset given as coordinates in z,y,x.
size: Sub-image shape given in z,y,x.
verify: True to verify detections against truth database; defaults
to False.
save_dfs: True to save data frames to file; defaults to True.
full_roi (bool): True to treat ``image5d`` as the full ROI; defaults
to False.
"""
time_start = time()
if size is None or offset is None:
# uses the entire stack if no size or offset specified
size = image5d.shape[1:4]
offset = (0, 0, 0)
else:
# change base filename for ROI-based partial stack
filename_base = make_subimage_name(filename_base, offset, size)
filename_subimg = libmag.combine_paths(filename_base, config.SUFFIX_SUBIMG)
filename_blobs = libmag.combine_paths(filename_base, config.SUFFIX_BLOBS)
# get ROI for given region, including all channels
if full_roi:
# treat the full image as the ROI
roi = image5d[0]
else:
roi = plot_3d.prepare_subimg(image5d, size, offset)
_, channels = plot_3d.setup_channels(roi, config.channel, 3)
# prep chunking ROI into sub-ROIs with size based on segment_size, scaling
# by physical units to make more independent of resolution
time_detection_start = time()
settings = config.roi_profile # use default settings
scaling_factor = detector.calc_scaling_factor()
print("microsope scaling factor based on resolutions: {}"
.format(scaling_factor))
denoise_size = config.roi_profile["denoise_size"]
denoise_max_shape = None
if denoise_size:
# further subdivide each sub-ROI for local preprocessing
denoise_max_shape = np.ceil(
np.multiply(scaling_factor, denoise_size)).astype(int)
# overlap sub-ROIs to minimize edge effects
overlap_base = chunking.calc_overlap()
tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int)
overlap_padding = np.copy(tol)
overlap = np.copy(overlap_base)
exclude_border = config.roi_profile["exclude_border"]
if exclude_border is not None:
# exclude border to avoid blob detector edge effects, where blobs
# often collect at the faces of the sub-ROI;
# ensure that overlap is greater than twice the border exclusion per
# axis so that no plane will be excluded from both overlapping sub-ROIs
exclude_border_thresh = np.multiply(2, exclude_border)
overlap_less = np.less(overlap, exclude_border_thresh)
overlap[overlap_less] = exclude_border_thresh[overlap_less]
excluded = np.greater(exclude_border, 0)
overlap[excluded] += 1 # additional padding
overlap_padding[excluded] = 0 # no need to prune past excluded border
print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond "
"overlap for pruning: {}, exclude borders: {}"
.format(overlap, tol, overlap_padding, exclude_border))
max_pixels = np.ceil(np.multiply(
scaling_factor,
config.roi_profile["segment_size"])).astype(int)
print("preprocessing max shape: {}, detection max pixels: {}"
.format(denoise_max_shape, max_pixels))
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
roi.shape, max_pixels, overlap)
# TODO: option to distribute groups of sub-ROIs to different servers
# for blob detection
seg_rois = detect_blobs_sub_rois(
roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border)
detection_time = time() - time_detection_start
print("blob detection time (s):", detection_time)
# prune blobs in overlapping portions of sub-ROIs
time_pruning_start = time()
segments_all, df_pruning = _prune_blobs_mp(
roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels,
overlap_padding)
pruning_time = time() - time_pruning_start
print("blob pruning time (s):", pruning_time)
#print("maxes:", np.amax(segments_all, axis=0))
# get weighted mean of ratios
if df_pruning is not None:
print("\nBlob pruning ratios:")
path_pruning = "blob_ratios.csv" if save_dfs else None
df_pruning_all = df_io.data_frames_to_csv(
df_pruning, path_pruning, show=" ")
cols = df_pruning_all.columns.tolist()
blob_pruning_means = {}
if "blobs" in cols:
blobs_unpruned = df_pruning_all["blobs"]
num_blobs_unpruned = np.sum(blobs_unpruned)
for col in cols[1:]:
blob_pruning_means["mean_{}".format(col)] = [
np.sum(np.multiply(df_pruning_all[col], blobs_unpruned))
/ num_blobs_unpruned]
path_pruning_means = "blob_ratios_means.csv" if save_dfs else None
df_pruning_means = df_io.dict_to_data_frame(
blob_pruning_means, path_pruning_means, show=" ")
else:
print("no blob ratios found")
'''# report any remaining duplicates
np.set_printoptions(linewidth=500, threshold=10000000)
print("all blobs (len {}):".format(len(segments_all)))
sort = np.lexsort(
(segments_all[:, 2], segments_all[:, 1], segments_all[:, 0]))
blobs = segments_all[sort]
print(blobs)
print("checking for duplicates in all:")
print(detector.remove_duplicate_blobs(blobs, slice(0, 3)))
'''
stats_detection = None
fdbk = None
if segments_all is not None:
# remove the duplicated elements that were used for pruning
detector.replace_rel_with_abs_blob_coords(segments_all)
segments_all = detector.remove_abs_blob_coords(segments_all)
# compare detected blobs with truth blobs
# TODO: assumes ground truth is relative to any ROI offset,
# but should make customizable
if verify:
db_path_base = None
exp_name = os.path.splitext(os.path.basename(config.filename))[0]
try:
if config.truth_db is None:
# find and load truth DB based on filename and subimage
db_path_base = os.path.basename(filename_base)
print("about to verify with truth db from {}"
.format(db_path_base))
sqlite.load_truth_db(db_path_base)
if config.truth_db is not None:
# truth DB may contain multiple experiments for different
# subimages; series not included in exp name since in ROI
rois = config.truth_db.get_rois(exp_name)
if rois is None:
# exp may have been named by ROI
print("{} experiment name not found, will try with"
"ROI offset/size".format(exp_name))
exp_name = make_subimage_name(exp_name, offset, size)
rois = config.truth_db.get_rois(exp_name)
if rois is None:
raise LookupError(
"No truth set ROIs found for experiment {}, will "
"skip detection verification".format(exp_name))
print("load ROIs from exp: {}".format(exp_name))
exp_id = sqlite.insert_experiment(
config.verified_db.conn, config.verified_db.cur,
exp_name, None)
verify_tol = np.multiply(
overlap_base, settings["verify_tol_factor"])
stats_detection, fdbk = detector.verify_rois(
rois, segments_all, config.truth_db.blobs_truth,
verify_tol, config.verified_db, exp_id, config.channel)
except FileNotFoundError:
libmag.warn("Could not load truth DB from {}; "
"will not verify ROIs".format(db_path_base))
except LookupError as e:
libmag.warn(str(e))
file_time_start = time()
if config.save_subimg:
if (isinstance(config.image5d, np.memmap) and
config.image5d.filename == os.path.abspath(filename_subimg)):
# file at sub-image save path may have been opened as a memmap
# file, in which case saving would fail
libmag.warn("{} is currently open, cannot save sub-image"
.format(filename_subimg))
else:
# write sub-image, which is in ROI (3D) format
with open(filename_subimg, "wb") as f:
np.save(f, roi)
# save blobs
# TODO: only segments used; consider removing the rest except ver
outfile_blobs = open(filename_blobs, "wb")
np.savez(outfile_blobs, ver=BLOBS_NP_VER, segments=segments_all,
resolutions=config.resolutions,
basename=os.path.basename(config.filename), # only save name
offset=offset, roi_size=size) # None unless explicitly set
outfile_blobs.close()
file_save_time = time() - file_time_start
# whole image benchmarking time
times = (
[detection_time],
[pruning_time],
time() - time_start)
times_dict = {}
for key, val in zip(StackTimes, times):
times_dict[key] = val
if segments_all is None:
print("\nNo blobs detected")
else:
print("\nTotal blobs found:", len(segments_all))
detector.show_blobs_per_channel(segments_all)
print("file save time:", file_save_time)
print("\nTotal detection processing times (s):")
path_times = "stack_detection_times.csv" if save_dfs else None
df_io.dict_to_data_frame(times_dict, path_times, show=" ")
return stats_detection, fdbk, segments_all