def cluster_blobs(img_path, suffix=None):
"""Cluster blobs and save to Numpy archive.
Args:
img_path (str): Base path from which registered labels and blobs files
will be found and output blobs file save location will be
constructed.
suffix (str): Suffix for ``path``; defaults to None.
Returns:
"""
mod_path = img_path
if suffix is not None:
mod_path = libmag.insert_before_ext(img_path, suffix)
labels_img_np = sitk_io.load_registered_img(
mod_path, config.RegNames.IMG_LABELS.value)
blobs = detector.Blobs().load_blobs(np_io.img_to_blobs_path(img_path))
scaling, res = np_io.find_scaling(img_path, labels_img_np.shape)
if blobs is None:
libmag.warn("unable to load nuclei coordinates")
return
# append label IDs to blobs and scale to make isotropic
blobs_clus = ClusterByLabel.cluster_by_label(blobs.blobs[:, :3],
labels_img_np, scaling, res)
print(blobs_clus)
out_path = libmag.combine_paths(mod_path, config.SUFFIX_BLOB_CLUSTERS)
np.save(out_path, blobs_clus)
def animate_imgs(base_path, plotted_imgs, delay, ext=None, suffix=None):
"""Export to an animated image.
Defaults to an animated GIF unless ``ext`` specifies otherwise.
Requires ``FFMpeg`` for MP4 file format exports and ``ImageMagick`` for
all other types of exports.
Args:
base_path (str): String from which an output path will be constructed.
plotted_imgs (List[:obj:`matplotlib.image.AxesImage]): Sequence of
images to include in the animation.
delay (int): Delay between image display in ms. If None, the delay will
defaul to 100ms.
ext (str): Extension to use when saving, without the period. Defaults
to None, in which case "gif" will be used.
suffix (str): String to append to output path before extension;
defaults to None to ignore.
"""
# set up animation output path and time interval
if ext is None: ext = "gif"
out_path = libmag.combine_paths(base_path, "animated", ext=ext)
if suffix: out_path = libmag.insert_before_ext(out_path, suffix, "_")
libmag.backup_file(out_path)
if delay is None:
delay = 100
if plotted_imgs and len(plotted_imgs[0]) > 0:
fig = plotted_imgs[0][0].figure
else:
libmag.warn("No images available to animate")
return
# WORKAROUND: FFMpeg may give a "height not divisible by 2" error, fixed
# by padding with a pixel
# TODO: check if needed for width
# TODO: account for difference in FFMpeg height and fig height
for fn, size in {
# fig.set_figwidth: fig.get_figwidth(),
fig.set_figheight:
fig.get_figheight()
}.items():
if size * fig.dpi % 2 != 0:
fn(size + 1. / fig.dpi)
print("Padded size with", fn, fig.get_figwidth(), "to new size of",
fig.get_figheight())
# generate and save animation
anim = animation.ArtistAnimation(fig,
plotted_imgs,
interval=delay,
repeat_delay=0,
blit=False)
try:
writer = "ffmpeg" if ext == "mp4" else "imagemagick"
anim.save(out_path, writer=writer)
print("saved animation file to {}".format(out_path))
except ValueError as e:
print(e)
libmag.warn("No animation writer available for Matplotlib")
def read_sitk_files(filename_sitk, reg_names=None):
"""Read files through SimpleITK and export to Numpy array format,
loading associated metadata.
Args:
filename_sitk: Path to file in a format that can be read by SimpleITK.
reg_names: Path or sequence of paths of registered names. Can
be a registered suffix or a full path. Defaults to None.
Returns:
Image array in MagellanMapper image5d format. Associated metadata will
have been loaded into module-level variables.
Raises:
``FileNotFoundError`` if ``filename_sitk`` cannot be found, after
attempting to load metadata from ``filename_np``.
"""
# load image via SimpleITK
img_sitk = None
loaded_path = filename_sitk
if reg_names:
img_nps = []
if not libmag.is_seq(reg_names):
reg_names = [reg_names]
for reg_name in reg_names:
# load each registered suffix into list of images with same shape,
# keeping first image in sitk format
img, path = _load_reg_img_to_combine(filename_sitk, reg_name,
img_nps)
if img_sitk is None:
img_sitk = img
loaded_path = path
if len(img_nps) > 1:
# merge images into separate channels
img_np = np.stack(img_nps, axis=img_nps[0].ndim)
else:
img_np = img_nps[0]
else:
# load filename_sitk directly
if not os.path.exists(filename_sitk):
raise FileNotFoundError(
"could not find file {}".format(filename_sitk))
img_sitk, _ = read_sitk(filename_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if config.resolutions is None:
# fallback to determining metadata directly from sitk file
libmag.warn(
"MagellanMapper image metadata file not loaded; will fallback to "
"{} for metadata".format(loaded_path))
config.resolutions = np.array([img_sitk.GetSpacing()[::-1]])
print("set resolutions to {}".format(config.resolutions))
return img_np
def verify_stack(filename_base, subimg_path_base, settings, segments_all,
channels, overlap_base):
db_path_base = os.path.basename(subimg_path_base)
stats_detection = None
fdbk = None
try:
# Truth databases are any database stored with manually
# verified blobs and loaded at command-line with the
# `--truth_db` flag or loaded here. While all experiments
# can be stored in a single database, this verification also
# supports experiments saved to separate databases in the
# software root directory and named as a sub-image but with
# the `sqlite.DB_SUFFIX_TRUTH` suffix. Experiments in the
# database are also assumed to be named based on the full
# image or the sub-image filename, without any directories.
# load ROIs from previously loaded truth database or one loaded
# based on sub-image filename
exp_name, rois = _get_truth_db_rois(
subimg_path_base, filename_base,
db_path_base if config.truth_db is None else None)
if rois is None:
# load alternate truth database based on sub-image filename
print("Loading truth ROIs from experiment:", exp_name)
exp_name, rois = _get_truth_db_rois(
subimg_path_base, filename_base, db_path_base)
if config.truth_db is None:
raise LookupError(
"No truth database found for experiment {}, will "
"skip detection verification".format(exp_name))
if rois is None:
raise LookupError(
"No truth set ROIs found for experiment {}, will "
"skip detection verification".format(exp_name))
# verify each ROI and store results in a separate database
exp_id = sqlite.insert_experiment(
config.verified_db.conn, config.verified_db.cur,
exp_name, None)
verify_tol = np.multiply(
overlap_base, settings["verify_tol_factor"])
stats_detection, fdbk, df_verify = verify_rois(
rois, segments_all, config.truth_db.blobs_truth,
verify_tol, config.verified_db, exp_id, exp_name,
channels)
df_io.data_frames_to_csv(df_verify, libmag.combine_paths(
exp_name, "verify.csv"))
except FileNotFoundError:
libmag.warn("Could not load truth DB from {}; "
"will not verify ROIs".format(db_path_base))
except LookupError as e:
libmag.warn(str(e))
return stats_detection, fdbk
def animate_imgs(base_path, plotted_imgs, delay, ext=None, suffix=None):
"""Export to an animated image.
Defaults to an animated GIF unless ``ext`` specifies otherwise.
Requires ``FFMpeg`` for MP4 file format exports and ``ImageMagick`` for
all other types of exports.
Args:
base_path (str): String from which an output path will be constructed.
plotted_imgs (List[:obj:`matplotlib.image.AxesImage]): Sequence of
images to include in the animation.
delay (int): Delay between image display in ms. If None, the delay will
defaul to 100ms.
ext (str): Extension to use when saving, without the period. Defaults
to None, in which case "gif" will be used.
suffix (str): String to append to output path before extension;
defaults to None to ignore.
"""
if ext is None: ext = "gif"
out_path = libmag.combine_paths(base_path, "animated", ext=ext)
if suffix: out_path = libmag.insert_before_ext(out_path, suffix, "_")
libmag.backup_file(out_path)
if delay is None:
delay = 100
if plotted_imgs and len(plotted_imgs[0]) > 0:
fig = plotted_imgs[0][0].figure
else:
libmag.warn("No images available to animate")
return
anim = animation.ArtistAnimation(fig,
plotted_imgs,
interval=delay,
repeat_delay=0,
blit=False)
try:
writer = "ffmpeg" if ext == "mp4" else "imagemagick"
anim.save(out_path, writer=writer)
print("saved animation file to {}".format(out_path))
except ValueError as e:
print(e)
libmag.warn("No animation writer available for Matplotlib")
def setup_images(path=None,
series=None,
offset=None,
size=None,
proc_mode=None,
allow_import=True):
"""Sets up an image and all associated images and metadata.
Paths for related files such as registered images will generally be
constructed from ``path``. If :attr:`config.prefix` is set, it will
be used in place of ``path`` for registered labels.
Args:
path (str): Path to image from which MagellanMapper-style paths will
be generated.
series (int): Image series number; defaults to None.
offset (List[int]): Sub-image offset given in z,y,x; defaults to None.
size (List[int]): Sub-image shape given in z,y,x; defaults to None.
proc_mode (str): Processing mode, which should be a key in
:class:`config.ProcessTypes`, case-insensitive; defaults to None.
allow_import (bool): True to allow importing the image if it
cannot be loaded; defaults to True.
"""
def add_metadata():
# override metadata set from command-line metadata args if available
md = {
config.MetaKeys.RESOLUTIONS:
config.meta_dict[config.MetaKeys.RESOLUTIONS],
config.MetaKeys.MAGNIFICATION:
config.meta_dict[config.MetaKeys.MAGNIFICATION],
config.MetaKeys.ZOOM:
config.meta_dict[config.MetaKeys.ZOOM],
config.MetaKeys.SHAPE:
config.meta_dict[config.MetaKeys.SHAPE],
config.MetaKeys.DTYPE:
config.meta_dict[config.MetaKeys.DTYPE],
}
for key, val in md.items():
if val is not None:
# explicitly set metadata takes precedence over extracted vals
import_md[key] = val
# LOAD MAIN IMAGE
# reset image5d
config.image5d = None
config.image5d_is_roi = False
load_subimage = offset is not None and size is not None
config.resolutions = None
# reset label images
config.labels_img = None
config.borders_img = None
filename_base = importer.filename_to_base(path, series)
subimg_base = None
if load_subimage and not config.save_subimg:
# load a saved sub-image file if available and not set to save one
subimg_base = stack_detect.make_subimage_name(filename_base, offset,
size)
filename_subimg = libmag.combine_paths(subimg_base,
config.SUFFIX_SUBIMG)
try:
# load sub-image if available
config.image5d = np.load(filename_subimg, mmap_mode="r")
config.image5d = importer.roi_to_image5d(config.image5d)
config.image5d_is_roi = True
config.image5d_io = config.LoadIO.NP
print("Loaded sub-image from {} with shape {}".format(
filename_subimg, config.image5d.shape))
# after loading sub-image, load original image's metadata
# for essential data such as vmin/vmax; will only warn if
# fails to load since metadata could be specified elsewhere
_, orig_info = importer.make_filenames(path, series)
print("load original image metadata from:", orig_info)
importer.load_metadata(orig_info)
except IOError:
print("Ignored sub-image file from {} as unable to load".format(
filename_subimg))
proc_type = libmag.get_enum(proc_mode, config.ProcessTypes)
if proc_type in (config.ProcessTypes.LOAD, config.ProcessTypes.EXPORT_ROIS,
config.ProcessTypes.EXPORT_BLOBS,
config.ProcessTypes.DETECT):
# load a blobs archive
try:
if subimg_base:
try:
# load blobs generated from sub-image
config.blobs = load_blobs(subimg_base)
except (FileNotFoundError, KeyError):
# fallback to loading from full image blobs and getting
# a subset, shifting them relative to sub-image offset
print("Unable to load blobs file based on {}, will try "
"from {}".format(subimg_base, filename_base))
config.blobs = load_blobs(filename_base)
config.blobs, _ = detector.get_blobs_in_roi(config.blobs,
offset,
size,
reverse=False)
detector.shift_blob_rel_coords(config.blobs,
np.multiply(offset, -1))
else:
# load full image blobs
config.blobs = load_blobs(filename_base)
except (FileNotFoundError, KeyError) as e2:
print("Unable to load blobs file")
if proc_type in (config.ProcessTypes.LOAD,
config.ProcessTypes.EXPORT_BLOBS):
# blobs expected but not found
raise e2
if path and config.image5d is None:
# load or import the main image stack
print("Loading main image")
try:
if path.endswith(sitk_io.EXTS_3D):
# attempt to format supported by SimpleITK and prepend time axis
config.image5d = sitk_io.read_sitk_files(path)[None]
config.image5d_io = config.LoadIO.SITK
else:
# load or import from MagellanMapper Numpy format
import_only = proc_type is config.ProcessTypes.IMPORT_ONLY
if not import_only:
# load previously imported image
config.image5d = importer.read_file(path, series)
if allow_import:
# re-import over existing image or import new image
if os.path.isdir(path) and all(
[r is None for r in config.reg_suffixes.values()]):
# import directory of single plane images to single
# stack if no register suffixes are set
chls, import_md = importer.setup_import_dir(path)
add_metadata()
prefix = config.prefix
if not prefix:
prefix = os.path.join(
os.path.dirname(path),
importer.DEFAULT_IMG_STACK_NAME)
config.image5d = importer.import_planes_to_stack(
chls, prefix, import_md)
elif import_only or config.image5d is None:
# import multi-plane image
chls, import_path = importer.setup_import_multipage(
path)
prefix = config.prefix if config.prefix else import_path
import_md = importer.setup_import_metadata(
chls, config.channel, series)
add_metadata()
config.image5d = importer.import_multiplane_images(
chls,
prefix,
import_md,
series,
channel=config.channel)
config.image5d_io = config.LoadIO.NP
except FileNotFoundError as e:
print(e)
print("Could not load {}, will fall back to any associated "
"registered image".format(path))
if config.metadatas and config.metadatas[0]:
# assign metadata from alternate file if given to supersede settings
# for any loaded image5d
# TODO: access metadata directly from given image5d's dict to allow
# loading multiple image5d images simultaneously
importer.assign_metadata(config.metadatas[0])
# main image is currently required since many parameters depend on it
atlas_suffix = config.reg_suffixes[config.RegSuffixes.ATLAS]
if atlas_suffix is None and config.image5d is None:
# fallback to atlas if main image not already loaded
atlas_suffix = config.RegNames.IMG_ATLAS.value
print(
"main image is not set, falling back to registered "
"image with suffix", atlas_suffix)
# use prefix to get images registered to a different image, eg a
# downsampled version, or a different version of registered images
path = config.prefix if config.prefix else path
if path and atlas_suffix is not None:
try:
# will take the place of any previously loaded image5d
config.image5d = sitk_io.read_sitk_files(
path, reg_names=atlas_suffix)[None]
config.image5d_io = config.LoadIO.SITK
except FileNotFoundError as e:
print(e)
annotation_suffix = config.reg_suffixes[config.RegSuffixes.ANNOTATION]
if annotation_suffix is not None:
# load labels image, set up scaling, and load labels file
try:
# TODO: need to support multichannel labels images
config.labels_img = sitk_io.read_sitk_files(
path, reg_names=annotation_suffix)
if config.image5d is not None:
config.labels_scaling = importer.calc_scaling(
config.image5d, config.labels_img)
if config.load_labels is not None:
labels_ref = ontology.load_labels_ref(config.load_labels)
if isinstance(labels_ref, pd.DataFrame):
# parse CSV files loaded into data frame
config.labels_ref_lookup = ontology.create_lookup_pd(
labels_ref)
else:
# parse dict from ABA JSON file
config.labels_ref_lookup = (
ontology.create_aba_reverse_lookup(labels_ref))
except FileNotFoundError as e:
print(e)
borders_suffix = config.reg_suffixes[config.RegSuffixes.BORDERS]
if borders_suffix is not None:
# load borders image, which can also be another labels image
try:
config.borders_img = sitk_io.read_sitk_files(
path, reg_names=borders_suffix)
except FileNotFoundError as e:
print(e)
if (config.atlas_labels[config.AtlasLabels.ORIG_COLORS]
and config.load_labels is not None):
# load original labels image from same directory as ontology
# file for consistent ID-color mapping, even if labels are missing
try:
config.labels_img_orig = sitk_io.load_registered_img(
config.load_labels, config.RegNames.IMG_LABELS.value)
except FileNotFoundError as e:
print(e)
libmag.warn(
"could not load original labels image; colors may differ"
"differ from it")
load_rot90 = config.roi_profile["load_rot90"]
if load_rot90 and config.image5d is not None:
# rotate main image specified num of times x90deg after loading since
# need to rotate images output by deep learning toolkit
config.image5d = np.rot90(config.image5d, load_rot90, (2, 3))
if (config.image5d is not None and load_subimage
and not config.image5d_is_roi):
# crop full image to bounds of sub-image
config.image5d = plot_3d.prepare_subimg(config.image5d, size,
offset)[None]
config.image5d_is_roi = True
# add any additional image5d thresholds for multichannel images, such
# as those loaded without metadata for these settings
colormaps.setup_cmaps()
num_channels = get_num_channels(config.image5d)
config.near_max = libmag.pad_seq(config.near_max, num_channels, -1)
config.near_min = libmag.pad_seq(config.near_min, num_channels, 0)
config.vmax_overview = libmag.pad_seq(config.vmax_overview, num_channels)
colormaps.setup_colormaps(num_channels)
def plot_knns(img_paths, suffix=None, show=False, names=None):
"""Plot k-nearest-neighbor distances for multiple sets of blobs,
overlaying on a single plot.
Args:
img_paths (List[str]): Base paths from which registered labels and
blobs files will be found and output blobs file save location
will be constructed.
suffix (str): Suffix for ``path``; defaults to None.
show (bool): True to plot the distances; defaults to False.
names (List[str]): Sequence of names corresponding to ``img_paths``
for the plot legend.
"""
cluster_settings = config.atlas_profile[profiles.RegKeys.METRICS_CLUSTER]
knn_n = cluster_settings[profiles.RegKeys.KNN_N]
if not knn_n:
knn_n = cluster_settings[profiles.RegKeys.DBSCAN_MINPTS] - 1
print("Calculating k-nearest-neighbor distances and plotting distances "
"for neighbor {}".format(knn_n))
# set up combined data frames for all samples at each zoom level
df_keys = ("ov", "zoom")
dfs_comb = {key: [] for key in df_keys}
names_disp = names if names else []
for i, img_path in enumerate(img_paths):
# load blobs associated with image
mod_path = img_path
if suffix is not None:
mod_path = libmag.insert_before_ext(img_path, suffix)
labels_img_np = sitk_io.load_registered_img(
mod_path, config.RegNames.IMG_LABELS.value)
blobs = detector.Blobs().load_blobs(np_io.img_to_blobs_path(img_path))
scaling, res = np_io.find_scaling(img_path, labels_img_np.shape)
if blobs is None:
libmag.warn("unable to load nuclei coordinates for", img_path)
continue
# convert to physical units and display k-nearest-neighbors for nuclei
blobs_phys = np.multiply(blobs.blobs[:, :3], res)
# TESTING: given the same blobs, simply shift
#blobs = np.multiply(blobs[i*10000000:, :3], res)
_, _, dfs = knn_dist(blobs_phys, knn_n, 2, 1000000, False)
if names is None:
# default to naming from filename
names_disp.append(os.path.basename(mod_path))
for j, df in enumerate(dfs):
dfs_comb[df_keys[j]].append(df)
for key in dfs_comb:
# combine data frames at each zoom level, save, and plot with
# different colors for each image
df = df_io.join_dfs(dfs_comb[key], "point")
dist_cols = [col for col in df.columns if col.startswith("dist")]
rename_cols = {col: name for col, name in zip(dist_cols, names_disp)}
df = df.rename(rename_cols, axis=1)
out_path = "knn_dist_combine_{}".format(key)
df_io.data_frames_to_csv(df, out_path)
plot_2d.plot_lines(out_path,
"point",
rename_cols.values(),
df=df,
show=show,
title=config.plot_labels[config.PlotLabels.TITLE])
def detect_blobs_blocks(filename_base, image5d, offset, size, channels,
verify=False, save_dfs=True, full_roi=False,
coloc=False):
"""Detect blobs by block processing of a large image.
All channels are processed in the same blocks.
Args:
filename_base: Base path to use file output.
image5d: Large image to process as a Numpy array of t,z,y,x,[c]
offset: Sub-image offset given as coordinates in z,y,x.
size: Sub-image shape given in z,y,x.
channels (Sequence[int]): Sequence of channels, where None detects
in all channels.
verify: True to verify detections against truth database; defaults
to False.
save_dfs: True to save data frames to file; defaults to True.
full_roi (bool): True to treat ``image5d`` as the full ROI; defaults
to False.
coloc (bool): True to perform blob co-localizations; defaults to False.
Returns:
tuple[int, int, int], str, :class:`magmap.cv.detector.Blobs`:
Accuracy metrics from :class:`magmap.cv.detector.verify_rois`,
feedback message from this same function, and detected blobs.
"""
time_start = time()
subimg_path_base = filename_base
if size is None or offset is None:
# uses the entire stack if no size or offset specified
size = image5d.shape[1:4]
offset = (0, 0, 0)
else:
# get base path for sub-image
subimg_path_base = naming.make_subimage_name(
filename_base, offset, size)
filename_blobs = libmag.combine_paths(subimg_path_base, config.SUFFIX_BLOBS)
# get ROI for given region, including all channels
if full_roi:
# treat the full image as the ROI
roi = image5d[0]
else:
roi = plot_3d.prepare_subimg(image5d, offset, size)
num_chls_roi = 1 if len(roi.shape) < 4 else roi.shape[3]
if num_chls_roi < 2:
coloc = False
print("Unable to co-localize as image has only 1 channel")
# prep chunking ROI into sub-ROIs with size based on segment_size, scaling
# by physical units to make more independent of resolution; use profile
# from first channel to be processed for block settings
time_detection_start = time()
settings = config.get_roi_profile(channels[0])
print("Profile for block settings:", settings[settings.NAME_KEY])
sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border, \
tol, overlap_base, overlap, overlap_padding = setup_blocks(
settings, roi.shape)
# TODO: option to distribute groups of sub-ROIs to different servers
# for blob detection
seg_rois = StackDetector.detect_blobs_sub_rois(
roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape,
exclude_border, coloc, channels)
detection_time = time() - time_detection_start
print("blob detection time (s):", detection_time)
# prune blobs in overlapping portions of sub-ROIs
time_pruning_start = time()
segments_all, df_pruning = StackPruner.prune_blobs_mp(
roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels,
overlap_padding)
pruning_time = time() - time_pruning_start
print("blob pruning time (s):", pruning_time)
#print("maxes:", np.amax(segments_all, axis=0))
# get weighted mean of ratios
if df_pruning is not None:
print("\nBlob pruning ratios:")
path_pruning = "blob_ratios.csv" if save_dfs else None
df_pruning_all = df_io.data_frames_to_csv(
df_pruning, path_pruning, show=" ")
cols = df_pruning_all.columns.tolist()
blob_pruning_means = {}
if "blobs" in cols:
blobs_unpruned = df_pruning_all["blobs"]
num_blobs_unpruned = np.sum(blobs_unpruned)
for col in cols[1:]:
blob_pruning_means["mean_{}".format(col)] = [
np.sum(np.multiply(df_pruning_all[col], blobs_unpruned))
/ num_blobs_unpruned]
path_pruning_means = "blob_ratios_means.csv" if save_dfs else None
df_pruning_means = df_io.dict_to_data_frame(
blob_pruning_means, path_pruning_means, show=" ")
else:
print("no blob ratios found")
'''# report any remaining duplicates
np.set_printoptions(linewidth=500, threshold=10000000)
print("all blobs (len {}):".format(len(segments_all)))
sort = np.lexsort(
(segments_all[:, 2], segments_all[:, 1], segments_all[:, 0]))
blobs = segments_all[sort]
print(blobs)
print("checking for duplicates in all:")
print(detector.remove_duplicate_blobs(blobs, slice(0, 3)))
'''
stats_detection = None
fdbk = None
colocs = None
if segments_all is not None:
# remove the duplicated elements that were used for pruning
detector.replace_rel_with_abs_blob_coords(segments_all)
if coloc:
colocs = segments_all[:, 10:10+num_chls_roi].astype(np.uint8)
# remove absolute coordinate and any co-localization columns
segments_all = detector.remove_abs_blob_coords(segments_all)
# compare detected blobs with truth blobs
# TODO: assumes ground truth is relative to any ROI offset,
# but should make customizable
if verify:
stats_detection, fdbk = verifier.verify_stack(
filename_base, subimg_path_base, settings, segments_all,
channels, overlap_base)
if config.save_subimg:
subimg_base_path = libmag.combine_paths(
subimg_path_base, config.SUFFIX_SUBIMG)
if (isinstance(config.image5d, np.memmap) and
config.image5d.filename == os.path.abspath(subimg_base_path)):
# file at sub-image save path may have been opened as a memmap
# file, in which case saving would fail
libmag.warn("{} is currently open, cannot save sub-image"
.format(subimg_base_path))
else:
# write sub-image, which is in ROI (3D) format
with open(subimg_base_path, "wb") as f:
np.save(f, roi)
# store blobs in Blobs instance
# TODO: consider separating into blobs and blobs metadata archives
blobs = detector.Blobs(
segments_all, colocalizations=colocs, path=filename_blobs)
blobs.resolutions = config.resolutions
blobs.basename = os.path.basename(config.filename)
blobs.roi_offset = offset
blobs.roi_size = size
# whole image benchmarking time
times = (
[detection_time],
[pruning_time],
time() - time_start)
times_dict = {}
for key, val in zip(StackTimes, times):
times_dict[key] = val
if segments_all is None:
print("\nNo blobs detected")
else:
print("\nTotal blobs found:", len(segments_all))
detector.show_blobs_per_channel(segments_all)
print("\nTotal detection processing times (s):")
path_times = "stack_detection_times.csv" if save_dfs else None
df_io.dict_to_data_frame(times_dict, path_times, show=" ")
return stats_detection, fdbk, blobs
def process_cli_args():
"""Parse command-line arguments.
Typically stores values as :mod:`magmap.settings.config` attributes.
"""
parser = argparse.ArgumentParser(
description="Setup environment for MagellanMapper")
# image specification arguments
# image path(s) specified as an optional argument; takes precedence
# over positional argument
parser.add_argument(
"--img",
nargs="*",
default=None,
help="Main image path(s); after import, the filename is often "
"given as the original name without its extension")
# alternatively specified as the first and only positional parameter
# with as many arguments as desired
parser.add_argument(
"img_paths",
nargs="*",
default=None,
help="Main image path(s); can also be given as --img, which takes "
"precedence over this argument")
parser.add_argument(
"--meta",
nargs="*",
help="Metadata path(s), which can be given as multiple files "
"corresponding to each image")
parser.add_argument("--prefix", help="Path prefix")
parser.add_argument("--suffix", help="Filename suffix")
parser.add_argument("--channel", nargs="*", type=int, help="Channel index")
parser.add_argument("--series", help="Series index")
parser.add_argument("--subimg_offset",
nargs="*",
help="Sub-image offset in x,y,z")
parser.add_argument("--subimg_size",
nargs="*",
help="Sub-image size in x,y,z")
parser.add_argument("--offset", nargs="*", help="ROI offset in x,y,z")
parser.add_argument("--size", nargs="*", help="ROI size in x,y,z")
parser.add_argument("--db", help="Database path")
parser.add_argument(
"--cpus",
help="Maximum number of CPUs/processes to use for multiprocessing "
"tasks. Use \"none\" or 0 to auto-detect this number (default).")
parser.add_argument(
"--load",
nargs="*",
help="Load associated data files; see config.LoadData for settings")
# task arguments
parser.add_argument("--proc",
type=str.lower,
choices=libmag.enum_names_aslist(config.ProcessTypes),
help="Image processing mode")
parser.add_argument("--register",
type=str.lower,
choices=libmag.enum_names_aslist(config.RegisterTypes),
help="Image registration task")
parser.add_argument("--df",
type=str.lower,
choices=libmag.enum_names_aslist(config.DFTasks),
help="Data frame task")
parser.add_argument("--plot_2d",
type=str.lower,
choices=libmag.enum_names_aslist(config.Plot2DTypes),
help="2D plot task; see config.Plot2DTypes")
parser.add_argument("--ec2_start",
nargs="*",
help="AWS EC2 instance start")
parser.add_argument("--ec2_list", nargs="*", help="AWS EC2 instance list")
parser.add_argument("--ec2_terminate",
nargs="*",
help="AWS EC2 instance termination")
parser.add_argument(
"--notify",
nargs="*",
help="Notification message URL, message, and attachment strings")
parser.add_argument("--grid_search",
help="Grid search hyperparameter tuning profile(s)")
# profile arguments
parser.add_argument(
"--roi_profile",
nargs="*",
help="ROI profile, which can be separated by underscores "
"for multiple profiles and given as paths to custom profiles "
"in YAML format. Multiple profile groups can be given, which "
"will each be applied to the corresponding channel. See "
"docs/settings.md for more details.")
parser.add_argument(
"--atlas_profile",
help="Atlas profile, which can be separated by underscores "
"for multiple profiles and given as paths to custom profiles "
"in YAML format. See docs/settings.md for more details.")
parser.add_argument(
"--theme",
nargs="*",
type=str.lower,
choices=libmag.enum_names_aslist(config.Themes),
help="UI theme, which can be given as multiple themes to apply "
"on top of one another")
# grouped arguments
parser.add_argument(
"--truth_db",
nargs="*",
help="Truth database; see config.TruthDB for settings and "
"config.TruthDBModes for modes")
parser.add_argument("--labels",
nargs="*",
help=_get_args_dict_help(
"Atlas labels; see config.AtlasLabels.",
config.AtlasLabels))
parser.add_argument("--transform",
nargs="*",
help=_get_args_dict_help(
"Image transformations; see config.Transforms.",
config.Transforms))
parser.add_argument(
"--reg_suffixes",
nargs="*",
help=_get_args_dict_help(
"Registered image suffixes; see config.RegSuffixes for keys "
"and config.RegNames for values", config.RegSuffixes))
parser.add_argument(
"--plot_labels",
nargs="*",
help=_get_args_dict_help(
"Plot label customizations; see config.PlotLabels ",
config.PlotLabels))
parser.add_argument(
"--set_meta",
nargs="*",
help="Set metadata values; see config.MetaKeys for settings")
# image and figure display arguments
parser.add_argument("--plane",
type=str.lower,
choices=config.PLANE,
help="Planar orientation")
parser.add_argument(
"--show",
nargs="?",
const="1",
help="If applicable, show images after completing the given task")
parser.add_argument(
"--alphas",
help="Alpha opacity levels, which can be comma-delimited for "
"multichannel images")
parser.add_argument(
"--vmin",
help="Minimum intensity levels, which can be comma-delimited "
"for multichannel images")
parser.add_argument(
"--vmax",
help="Maximum intensity levels, which can be comma-delimited "
"for multichannel images")
parser.add_argument("--seed", help="Random number generator seed")
# export arguments
parser.add_argument("--save_subimg",
action="store_true",
help="Save sub-image as separate file")
parser.add_argument("--slice", help="Slice given as start,stop,step")
parser.add_argument("--delay", help="Animation delay in ms")
parser.add_argument("--savefig", help="Extension for saved figures")
parser.add_argument("--groups",
nargs="*",
help="Group values corresponding to each image")
parser.add_argument(
"-v",
"--verbose",
nargs="*",
help=_get_args_dict_help(
"Verbose output to assist with debugging; see config.Verbosity.",
config.Verbosity))
# only parse recognized arguments to avoid error for unrecognized ones
args, args_unknown = parser.parse_known_args()
if args.verbose:
# verbose mode, including printing longer Numpy arrays for debugging
config.verbose = True
config.verbosity = args_to_dict(args.verbose, config.Verbosity,
config.verbosity)
logs.update_log_level(config.logger,
config.verbosity[config.Verbosity.LEVEL])
np.set_printoptions(linewidth=200, threshold=10000)
_logger.info("Set verbose to {}".format(config.verbosity))
# set up logging to given file unless explicitly given an empty string
log_path = config.verbosity[config.Verbosity.LOG_PATH]
if log_path != "":
if log_path is None:
log_path = os.path.join(config.user_app_dirs.user_data_dir,
"out.log")
# log to file
logs.add_file_handler(config.logger, log_path)
# redirect standard out/error to logging
sys.stdout = logs.LogWriter(config.logger.info)
sys.stderr = logs.LogWriter(config.logger.error)
if args.img is not None or args.img_paths:
# set image file path and convert to basis for additional paths
config.filenames = args.img if args.img else args.img_paths
config.filename = config.filenames[0]
print("Set filenames to {}, current filename {}".format(
config.filenames, config.filename))
if args.meta is not None:
# set metadata paths
config.metadata_paths = args.meta
print("Set metadata paths to", config.metadata_paths)
config.metadatas = []
for path in config.metadata_paths:
# load metadata to dictionary
md, _ = importer.load_metadata(path, assign=False)
config.metadatas.append(md)
if args.channel is not None:
# set the channels
config.channel = args.channel
print("Set channel to {}".format(config.channel))
config.series_list = [config.series] # list of series
if args.series is not None:
series_split = args.series.split(",")
config.series_list = []
for ser in series_split:
ser_split = ser.split("-")
if len(ser_split) > 1:
ser_range = np.arange(int(ser_split[0]), int(ser_split[1]) + 1)
config.series_list.extend(ser_range.tolist())
else:
config.series_list.append(int(ser_split[0]))
config.series = config.series_list[0]
print("Set to series_list to {}, current series {}".format(
config.series_list, config.series))
if args.savefig is not None:
# save figure with file type of this extension; remove leading period
config.savefig = _parse_none(args.savefig.lstrip("."))
print("Set savefig extension to {}".format(config.savefig))
# parse sub-image offsets and sizes;
# expects x,y,z input but stores as z,y,x by convention
if args.subimg_offset is not None:
config.subimg_offsets = _parse_coords(args.subimg_offset, True)
print("Set sub-image offsets to {} (z,y,x)".format(
config.subimg_offsets))
if args.subimg_size is not None:
config.subimg_sizes = _parse_coords(args.subimg_size, True)
print("Set sub-image sizes to {} (z,y,x)".format(config.subimg_sizes))
# parse ROI offsets and sizes, which are relative to any sub-image;
# expects x,y,z input and output
if args.offset is not None:
config.roi_offsets = _parse_coords(args.offset)
if config.roi_offsets:
config.roi_offset = config.roi_offsets[0]
print("Set ROI offsets to {}, current offset {} (x,y,z)".format(
config.roi_offsets, config.roi_offset))
if args.size is not None:
config.roi_sizes = _parse_coords(args.size)
if config.roi_sizes:
config.roi_size = config.roi_sizes[0]
print("Set ROI sizes to {}, current size {} (x,y,z)".format(
config.roi_sizes, config.roi_size))
if args.cpus is not None:
# set maximum number of CPUs
config.cpus = _parse_none(args.cpus.lower(), int)
print("Set maximum number of CPUs for multiprocessing tasks to",
config.cpus)
if args.load is not None:
# flag loading data sources with default sub-arg indicating that the
# data should be loaded from a default path; otherwise, load from
# path given by the sub-arg; change delimiter to allow paths with ","
config.load_data = args_to_dict(args.load,
config.LoadData,
config.load_data,
sep_vals="|",
default=True)
print("Set to load the data types: {}".format(config.load_data))
# set up main processing mode
if args.proc is not None:
config.proc_type = args.proc
print("processing type set to {}".format(config.proc_type))
proc_type = libmag.get_enum(config.proc_type, config.ProcessTypes)
if config.proc_type and proc_type not in config.ProcessTypes:
libmag.warn("\"{}\" processing type not found".format(
config.proc_type))
if args.set_meta is not None:
# set individual metadata values, currently used for image import
# TODO: take precedence over loaded metadata archives
config.meta_dict = args_to_dict(args.set_meta,
config.MetaKeys,
config.meta_dict,
sep_vals="|")
print("Set metadata values to {}".format(config.meta_dict))
res = config.meta_dict[config.MetaKeys.RESOLUTIONS]
if res:
# set image resolutions, taken as a single set of x,y,z and
# converting to a nested list of z,y,x
res_split = res.split(",")
if len(res_split) >= 3:
res_float = tuple(float(i) for i in res_split)[::-1]
config.resolutions = [res_float]
print("Set resolutions to {}".format(config.resolutions))
else:
res_float = None
print("Resolution ({}) should be given as 3 values (x,y,z)".
format(res))
# store single set of resolutions, similar to input
config.meta_dict[config.MetaKeys.RESOLUTIONS] = res_float
mag = config.meta_dict[config.MetaKeys.MAGNIFICATION]
if mag:
# set objective magnification
config.magnification = mag
print("Set magnification to {}".format(config.magnification))
zoom = config.meta_dict[config.MetaKeys.ZOOM]
if zoom:
# set objective zoom
config.zoom = zoom
print("Set zoom to {}".format(config.zoom))
shape = config.meta_dict[config.MetaKeys.SHAPE]
if shape:
# parse shape, storing only in dict
config.meta_dict[config.MetaKeys.SHAPE] = [
int(n) for n in shape.split(",")[::-1]
]
# set up ROI and register profiles
setup_roi_profiles(args.roi_profile)
setup_atlas_profiles(args.atlas_profile)
setup_grid_search_profiles(args.grid_search)
if args.plane is not None:
config.plane = args.plane
print("Set plane to {}".format(config.plane))
if args.save_subimg:
config.save_subimg = args.save_subimg
print("Set to save the sub-image")
if args.labels:
# set up atlas labels
setup_labels(args.labels)
if args.transform is not None:
# image transformations such as flipping, rotation
config.transform = args_to_dict(args.transform, config.Transforms,
config.transform)
print("Set transformations to {}".format(config.transform))
if args.register:
# register type to process in register module
config.register_type = args.register
print("Set register type to {}".format(config.register_type))
if args.df:
# data frame processing task
config.df_task = args.df
print("Set data frame processing task to {}".format(config.df_task))
if args.plot_2d:
# 2D plot type to process in plot_2d module
config.plot_2d_type = args.plot_2d
print("Set plot_2d type to {}".format(config.plot_2d_type))
if args.slice:
# specify a generic slice by command-line, assuming same order
# of arguments as for slice built-in function and interpreting
# "none" string as None
config.slice_vals = args.slice.split(",")
config.slice_vals = [
_parse_none(val.lower(), int) for val in config.slice_vals
]
print("Set slice values to {}".format(config.slice_vals))
if args.delay:
config.delay = int(args.delay)
print("Set delay to {}".format(config.delay))
if args.show:
# show images after task is performed, if supported
config.show = _is_arg_true(args.show)
print("Set show to {}".format(config.show))
if args.groups:
config.groups = args.groups
print("Set groups to {}".format(config.groups))
if args.ec2_start is not None:
# start EC2 instances
config.ec2_start = args_with_dict(args.ec2_start)
print("Set ec2 start to {}".format(config.ec2_start))
if args.ec2_list:
# list EC2 instances
config.ec2_list = args_with_dict(args.ec2_list)
print("Set ec2 list to {}".format(config.ec2_list))
if args.ec2_terminate:
config.ec2_terminate = args.ec2_terminate
print("Set ec2 terminate to {}".format(config.ec2_terminate))
if args.notify:
notify_len = len(args.notify)
if notify_len > 0:
config.notify_url = args.notify[0]
print("Set notification URL to {}".format(config.notify_url))
if notify_len > 1:
config.notify_msg = args.notify[1]
print("Set notification message to {}".format(config.notify_msg))
if notify_len > 2:
config.notify_attach = args.notify[2]
print("Set notification attachment path to {}".format(
config.notify_attach))
if args.prefix:
config.prefix = args.prefix
print("Set path prefix to {}".format(config.prefix))
if args.suffix:
config.suffix = args.suffix
print("Set path suffix to {}".format(config.suffix))
if args.alphas:
# specify alpha levels
config.alphas = [float(val) for val in args.alphas.split(",")]
print("Set alphas to", config.alphas)
if args.vmin:
# specify vmin levels
config.vmins = [libmag.get_int(val) for val in args.vmin.split(",")]
print("Set vmins to", config.vmins)
if args.vmax:
# specify vmax levels and copy to vmax overview used for plotting
# and updated for normalization
config.vmaxs = [libmag.get_int(val) for val in args.vmax.split(",")]
config.vmax_overview = list(config.vmaxs)
print("Set vmaxs to", config.vmaxs)
if args.reg_suffixes is not None:
# specify suffixes of registered images to load
config.reg_suffixes = args_to_dict(args.reg_suffixes,
config.RegSuffixes,
config.reg_suffixes)
print("Set registered image suffixes to {}".format(
config.reg_suffixes))
if args.seed:
# specify random number generator seed
config.seed = int(args.seed)
print("Set random number generator seed to", config.seed)
if args.plot_labels is not None:
# specify general plot labels
config.plot_labels = args_to_dict(args.plot_labels, config.PlotLabels,
config.plot_labels)
print("Set plot labels to {}".format(config.plot_labels))
if args.theme is not None:
# specify themes, currently applied to Matplotlib elements
theme_names = []
for theme in args.theme:
# add theme enum if found
theme_enum = libmag.get_enum(theme, config.Themes)
if theme_enum:
config.rc_params.append(theme_enum)
theme_names.append(theme_enum.name)
print("Set to use themes to {}".format(theme_names))
# set up Matplotlib styles/themes
plot_2d.setup_style()
# set up application directories
user_dir = config.user_app_dirs.user_data_dir
if not os.path.isdir(user_dir):
# make application data directory
if os.path.exists(user_dir):
# backup any non-directory file
libmag.backup_file(user_dir)
os.makedirs(user_dir)
if args.db:
# set main database path to user arg
config.db_path = args.db
print("Set database name to {}".format(config.db_path))
else:
# set default path
config.db_path = os.path.join(user_dir, config.db_path)
if args.truth_db:
# set settings for separate database of "truth blobs"
config.truth_db_params = args_to_dict(args.truth_db,
config.TruthDB,
config.truth_db_params,
sep_vals="|")
mode = config.truth_db_params[config.TruthDB.MODE]
config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes)
libmag.printv(config.truth_db_params)
print("Mapped \"{}\" truth_db mode to {}".format(
mode, config.truth_db_mode))
# notify user of full args list, including unrecognized args
_logger.debug(f"All command-line arguments: {sys.argv}")
if args_unknown:
_logger.info(
f"The following command-line arguments were unrecognized and "
f"ignored: {args_unknown}")
def detect_blobs_large_image(filename_base, image5d, offset, size,
verify=False, save_dfs=True, full_roi=False):
"""Detect blobs within a large image through parallel processing of
smaller chunks.
Args:
filename_base: Base path to use file output.
image5d: Large image to process as a Numpy array of t,z,y,x,[c]
offset: Sub-image offset given as coordinates in z,y,x.
size: Sub-image shape given in z,y,x.
verify: True to verify detections against truth database; defaults
to False.
save_dfs: True to save data frames to file; defaults to True.
full_roi (bool): True to treat ``image5d`` as the full ROI; defaults
to False.
"""
time_start = time()
if size is None or offset is None:
# uses the entire stack if no size or offset specified
size = image5d.shape[1:4]
offset = (0, 0, 0)
else:
# change base filename for ROI-based partial stack
filename_base = make_subimage_name(filename_base, offset, size)
filename_subimg = libmag.combine_paths(filename_base, config.SUFFIX_SUBIMG)
filename_blobs = libmag.combine_paths(filename_base, config.SUFFIX_BLOBS)
# get ROI for given region, including all channels
if full_roi:
# treat the full image as the ROI
roi = image5d[0]
else:
roi = plot_3d.prepare_subimg(image5d, size, offset)
_, channels = plot_3d.setup_channels(roi, config.channel, 3)
# prep chunking ROI into sub-ROIs with size based on segment_size, scaling
# by physical units to make more independent of resolution
time_detection_start = time()
settings = config.roi_profile # use default settings
scaling_factor = detector.calc_scaling_factor()
print("microsope scaling factor based on resolutions: {}"
.format(scaling_factor))
denoise_size = config.roi_profile["denoise_size"]
denoise_max_shape = None
if denoise_size:
# further subdivide each sub-ROI for local preprocessing
denoise_max_shape = np.ceil(
np.multiply(scaling_factor, denoise_size)).astype(int)
# overlap sub-ROIs to minimize edge effects
overlap_base = chunking.calc_overlap()
tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int)
overlap_padding = np.copy(tol)
overlap = np.copy(overlap_base)
exclude_border = config.roi_profile["exclude_border"]
if exclude_border is not None:
# exclude border to avoid blob detector edge effects, where blobs
# often collect at the faces of the sub-ROI;
# ensure that overlap is greater than twice the border exclusion per
# axis so that no plane will be excluded from both overlapping sub-ROIs
exclude_border_thresh = np.multiply(2, exclude_border)
overlap_less = np.less(overlap, exclude_border_thresh)
overlap[overlap_less] = exclude_border_thresh[overlap_less]
excluded = np.greater(exclude_border, 0)
overlap[excluded] += 1 # additional padding
overlap_padding[excluded] = 0 # no need to prune past excluded border
print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond "
"overlap for pruning: {}, exclude borders: {}"
.format(overlap, tol, overlap_padding, exclude_border))
max_pixels = np.ceil(np.multiply(
scaling_factor,
config.roi_profile["segment_size"])).astype(int)
print("preprocessing max shape: {}, detection max pixels: {}"
.format(denoise_max_shape, max_pixels))
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
roi.shape, max_pixels, overlap)
# TODO: option to distribute groups of sub-ROIs to different servers
# for blob detection
seg_rois = detect_blobs_sub_rois(
roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border)
detection_time = time() - time_detection_start
print("blob detection time (s):", detection_time)
# prune blobs in overlapping portions of sub-ROIs
time_pruning_start = time()
segments_all, df_pruning = _prune_blobs_mp(
roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels,
overlap_padding)
pruning_time = time() - time_pruning_start
print("blob pruning time (s):", pruning_time)
#print("maxes:", np.amax(segments_all, axis=0))
# get weighted mean of ratios
if df_pruning is not None:
print("\nBlob pruning ratios:")
path_pruning = "blob_ratios.csv" if save_dfs else None
df_pruning_all = df_io.data_frames_to_csv(
df_pruning, path_pruning, show=" ")
cols = df_pruning_all.columns.tolist()
blob_pruning_means = {}
if "blobs" in cols:
blobs_unpruned = df_pruning_all["blobs"]
num_blobs_unpruned = np.sum(blobs_unpruned)
for col in cols[1:]:
blob_pruning_means["mean_{}".format(col)] = [
np.sum(np.multiply(df_pruning_all[col], blobs_unpruned))
/ num_blobs_unpruned]
path_pruning_means = "blob_ratios_means.csv" if save_dfs else None
df_pruning_means = df_io.dict_to_data_frame(
blob_pruning_means, path_pruning_means, show=" ")
else:
print("no blob ratios found")
'''# report any remaining duplicates
np.set_printoptions(linewidth=500, threshold=10000000)
print("all blobs (len {}):".format(len(segments_all)))
sort = np.lexsort(
(segments_all[:, 2], segments_all[:, 1], segments_all[:, 0]))
blobs = segments_all[sort]
print(blobs)
print("checking for duplicates in all:")
print(detector.remove_duplicate_blobs(blobs, slice(0, 3)))
'''
stats_detection = None
fdbk = None
if segments_all is not None:
# remove the duplicated elements that were used for pruning
detector.replace_rel_with_abs_blob_coords(segments_all)
segments_all = detector.remove_abs_blob_coords(segments_all)
# compare detected blobs with truth blobs
# TODO: assumes ground truth is relative to any ROI offset,
# but should make customizable
if verify:
db_path_base = None
exp_name = os.path.splitext(os.path.basename(config.filename))[0]
try:
if config.truth_db is None:
# find and load truth DB based on filename and subimage
db_path_base = os.path.basename(filename_base)
print("about to verify with truth db from {}"
.format(db_path_base))
sqlite.load_truth_db(db_path_base)
if config.truth_db is not None:
# truth DB may contain multiple experiments for different
# subimages; series not included in exp name since in ROI
rois = config.truth_db.get_rois(exp_name)
if rois is None:
# exp may have been named by ROI
print("{} experiment name not found, will try with"
"ROI offset/size".format(exp_name))
exp_name = make_subimage_name(exp_name, offset, size)
rois = config.truth_db.get_rois(exp_name)
if rois is None:
raise LookupError(
"No truth set ROIs found for experiment {}, will "
"skip detection verification".format(exp_name))
print("load ROIs from exp: {}".format(exp_name))
exp_id = sqlite.insert_experiment(
config.verified_db.conn, config.verified_db.cur,
exp_name, None)
verify_tol = np.multiply(
overlap_base, settings["verify_tol_factor"])
stats_detection, fdbk = detector.verify_rois(
rois, segments_all, config.truth_db.blobs_truth,
verify_tol, config.verified_db, exp_id, config.channel)
except FileNotFoundError:
libmag.warn("Could not load truth DB from {}; "
"will not verify ROIs".format(db_path_base))
except LookupError as e:
libmag.warn(str(e))
file_time_start = time()
if config.save_subimg:
if (isinstance(config.image5d, np.memmap) and
config.image5d.filename == os.path.abspath(filename_subimg)):
# file at sub-image save path may have been opened as a memmap
# file, in which case saving would fail
libmag.warn("{} is currently open, cannot save sub-image"
.format(filename_subimg))
else:
# write sub-image, which is in ROI (3D) format
with open(filename_subimg, "wb") as f:
np.save(f, roi)
# save blobs
# TODO: only segments used; consider removing the rest except ver
outfile_blobs = open(filename_blobs, "wb")
np.savez(outfile_blobs, ver=BLOBS_NP_VER, segments=segments_all,
resolutions=config.resolutions,
basename=os.path.basename(config.filename), # only save name
offset=offset, roi_size=size) # None unless explicitly set
outfile_blobs.close()
file_save_time = time() - file_time_start
# whole image benchmarking time
times = (
[detection_time],
[pruning_time],
time() - time_start)
times_dict = {}
for key, val in zip(StackTimes, times):
times_dict[key] = val
if segments_all is None:
print("\nNo blobs detected")
else:
print("\nTotal blobs found:", len(segments_all))
detector.show_blobs_per_channel(segments_all)
print("file save time:", file_save_time)
print("\nTotal detection processing times (s):")
path_times = "stack_detection_times.csv" if save_dfs else None
df_io.dict_to_data_frame(times_dict, path_times, show=" ")
return stats_detection, fdbk, segments_all
def main(process_args_only=False, skip_dbs=False):
"""Starts the visualization GUI.
Processes command-line arguments.
Args:
process_args_only (bool): Processes command-line arguments and
returns; defaults to False.
skip_dbs (bool): True to skip loading databases; defaults to False.
"""
parser = argparse.ArgumentParser(
description="Setup environment for MagellanMapper")
# image specification arguments
parser.add_argument(
"--img",
nargs="*",
help="Main image path(s); after import, the filename is often "
"given as the original name without its extension")
parser.add_argument(
"--meta",
nargs="*",
help="Metadata path(s), which can be given as multiple files "
"corresponding to each image")
parser.add_argument("--prefix", help="Path prefix")
parser.add_argument("--suffix", help="Filename suffix")
parser.add_argument("--channel", nargs="*", type=int, help="Channel index")
parser.add_argument("--series", help="Series index")
parser.add_argument("--subimg_offset",
nargs="*",
help="Sub-image offset in x,y,z")
parser.add_argument("--subimg_size",
nargs="*",
help="Sub-image size in x,y,z")
parser.add_argument("--offset", nargs="*", help="ROI offset in x,y,z")
parser.add_argument("--size", nargs="*", help="ROI size in x,y,z")
parser.add_argument("--db", help="Database path")
parser.add_argument(
"--cpus",
help="Maximum number of CPUs/processes to use for multiprocessing "
"tasks. Use \"none\" or 0 to auto-detect this number (default).")
# task arguments
parser.add_argument("--proc",
type=str.lower,
choices=libmag.enum_names_aslist(config.ProcessTypes),
help="Image processing mode")
parser.add_argument("--register",
type=str.lower,
choices=libmag.enum_names_aslist(config.RegisterTypes),
help="Image registration task")
parser.add_argument("--df",
type=str.lower,
choices=libmag.enum_names_aslist(config.DFTasks),
help="Data frame task")
parser.add_argument("--plot_2d",
type=str.lower,
choices=libmag.enum_names_aslist(config.Plot2DTypes),
help="2D plot task; see config.Plot2DTypes")
parser.add_argument("--ec2_start",
nargs="*",
help="AWS EC2 instance start")
parser.add_argument("--ec2_list", nargs="*", help="AWS EC2 instance list")
parser.add_argument("--ec2_terminate",
nargs="*",
help="AWS EC2 instance termination")
parser.add_argument(
"--notify",
nargs="*",
help="Notification message URL, message, and attachment strings")
parser.add_argument("--grid_search",
help="Grid search hyperparameter tuning profile(s)")
# profile arguments
parser.add_argument(
"--roi_profile",
nargs="*",
help="ROI profile, which can be separated by underscores "
"for multiple profiles and given as paths to custom profiles "
"in YAML format. Multiple profile groups can be given, which "
"will each be applied to the corresponding channel. See "
"docs/settings.md for more details.")
parser.add_argument(
"--atlas_profile",
help="Atlas profile, which can be separated by underscores "
"for multiple profiles and given as paths to custom profiles "
"in YAML format. See docs/settings.md for more details.")
parser.add_argument(
"--theme",
nargs="*",
type=str.lower,
choices=libmag.enum_names_aslist(config.Themes),
help="UI theme, which can be given as multiple themes to apply "
"on top of one another")
# grouped arguments
parser.add_argument(
"--truth_db",
nargs="*",
help="Truth database; see config.TruthDB for settings and "
"config.TruthDBModes for modes")
parser.add_argument("--labels",
nargs="*",
help=_get_args_dict_help(
"Atlas labels; see config.AtlasLabels.",
config.AtlasLabels))
parser.add_argument("--transform",
nargs="*",
help=_get_args_dict_help(
"Image transformations; see config.Transforms.",
config.Transforms))
parser.add_argument(
"--reg_suffixes",
nargs="*",
help=_get_args_dict_help(
"Registered image suffixes; see config.RegSuffixes for keys "
"and config.RegNames for values", config.RegSuffixes))
parser.add_argument(
"--plot_labels",
nargs="*",
help=_get_args_dict_help(
"Plot label customizations; see config.PlotLabels ",
config.PlotLabels))
parser.add_argument(
"--set_meta",
nargs="*",
help="Set metadata values; see config.MetaKeys for settings")
# image and figure display arguments
parser.add_argument("--plane",
type=str.lower,
choices=config.PLANE,
help="Planar orientation")
parser.add_argument(
"--show",
nargs="?",
const="1",
help="If applicable, show images after completing the given task")
parser.add_argument(
"--alphas",
help="Alpha opacity levels, which can be comma-delimited for "
"multichannel images")
parser.add_argument(
"--vmin",
help="Minimum intensity levels, which can be comma-delimited "
"for multichannel images")
parser.add_argument(
"--vmax",
help="Maximum intensity levels, which can be comma-delimited "
"for multichannel images")
parser.add_argument("--seed", help="Random number generator seed")
# export arguments
parser.add_argument("--save_subimg",
action="store_true",
help="Save sub-image as separate file")
parser.add_argument("--slice", help="Slice given as start,stop,step")
parser.add_argument("--delay", help="Animation delay in ms")
parser.add_argument("--savefig", help="Extension for saved figures")
parser.add_argument("--groups",
nargs="*",
help="Group values corresponding to each image")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Verbose output to assist with debugging")
args = parser.parse_args()
if args.img is not None:
# set image file path and convert to basis for additional paths
config.filenames = args.img
config.filename = config.filenames[0]
print("Set filenames to {}, current filename {}".format(
config.filenames, config.filename))
if args.meta is not None:
# set metadata paths
config.metadata_paths = args.meta
print("Set metadata paths to", config.metadata_paths)
config.metadatas = []
for path in config.metadata_paths:
# load metadata to dictionary
md, _ = importer.load_metadata(path, assign=False)
config.metadatas.append(md)
if args.channel is not None:
# set the channels
config.channel = args.channel
print("Set channel to {}".format(config.channel))
series_list = [config.series] # list of series
if args.series is not None:
series_split = args.series.split(",")
series_list = []
for ser in series_split:
ser_split = ser.split("-")
if len(ser_split) > 1:
ser_range = np.arange(int(ser_split[0]), int(ser_split[1]) + 1)
series_list.extend(ser_range.tolist())
else:
series_list.append(int(ser_split[0]))
config.series = series_list[0]
print("Set to series_list to {}, current series {}".format(
series_list, config.series))
if args.savefig is not None:
# save figure with file type of this extension; remove leading period
config.savefig = args.savefig.lstrip(".")
print("Set savefig extension to {}".format(config.savefig))
if args.verbose:
# verbose mode, including printing longer Numpy arrays for debugging
config.verbose = args.verbose
np.set_printoptions(linewidth=200, threshold=10000)
print("Set verbose to {}".format(config.verbose))
# parse sub-image offsets and sizes;
# expects x,y,z input but stores as z,y,x by convention
if args.subimg_offset is not None:
config.subimg_offsets = _parse_coords(args.subimg_offset, True)
print("Set sub-image offsets to {} (z,y,x)".format(
config.subimg_offsets))
if args.subimg_size is not None:
config.subimg_sizes = _parse_coords(args.subimg_size, True)
print("Set sub-image sizes to {} (z,y,x)".format(config.subimg_sizes))
# parse ROI offsets and sizes, which are relative to any sub-image;
# expects x,y,z input and output
if args.offset is not None:
config.roi_offsets = _parse_coords(args.offset)
if config.roi_offsets:
config.roi_offset = config.roi_offsets[0]
print("Set ROI offsets to {}, current offset {} (x,y,z)".format(
config.roi_offsets, config.roi_offset))
if args.size is not None:
config.roi_sizes = _parse_coords(args.size)
if config.roi_sizes:
config.roi_size = config.roi_sizes[0]
print("Set ROI sizes to {}, current size {} (x,y,z)".format(
config.roi_sizes, config.roi_size))
if args.cpus is not None:
# set maximum number of CPUs
config.cpus = (None if args.cpus.lower() in ("none",
"0") else int(args.cpus))
print("Set maximum number of CPUs for multiprocessing tasks to",
config.cpus)
# set up main processing mode
if args.proc is not None:
config.proc_type = args.proc
print("processing type set to {}".format(config.proc_type))
proc_type = libmag.get_enum(config.proc_type, config.ProcessTypes)
if config.proc_type and proc_type not in config.ProcessTypes:
libmag.warn("\"{}\" processing type not found".format(
config.proc_type))
if args.set_meta is not None:
# set individual metadata values, currently used for image import
# TODO: take precedence over loaded metadata archives
config.meta_dict = args_to_dict(args.set_meta,
config.MetaKeys,
config.meta_dict,
sep_vals="|")
print("Set metadata values to {}".format(config.meta_dict))
res = config.meta_dict[config.MetaKeys.RESOLUTIONS]
if res:
# set image resolutions, taken as a single set of x,y,z and
# converting to a nested list of z,y,x
res_split = res.split(",")
if len(res_split) >= 3:
res_float = tuple(float(i) for i in res_split)[::-1]
config.resolutions = [res_float]
print("Set resolutions to {}".format(config.resolutions))
else:
res_float = None
print("Resolution ({}) should be given as 3 values (x,y,z)".
format(res))
# store single set of resolutions, similar to input
config.meta_dict[config.MetaKeys.RESOLUTIONS] = res_float
mag = config.meta_dict[config.MetaKeys.MAGNIFICATION]
if mag:
# set objective magnification
config.magnification = mag
print("Set magnification to {}".format(config.magnification))
zoom = config.meta_dict[config.MetaKeys.ZOOM]
if zoom:
# set objective zoom
config.zoom = zoom
print("Set zoom to {}".format(config.zoom))
shape = config.meta_dict[config.MetaKeys.SHAPE]
if shape:
# parse shape, storing only in dict
config.meta_dict[config.MetaKeys.SHAPE] = [
int(n) for n in shape.split(",")[::-1]
]
# set up ROI and register profiles
setup_profiles(args.roi_profile, args.atlas_profile, args.grid_search)
if args.plane is not None:
config.plane = args.plane
print("Set plane to {}".format(config.plane))
if args.save_subimg:
config.save_subimg = args.save_subimg
print("Set to save the sub-image")
if args.labels:
# set up atlas labels
setup_labels(args.labels)
if args.transform is not None:
# image transformations such as flipping, rotation
config.transform = args_to_dict(args.transform, config.Transforms,
config.transform)
print("Set transformations to {}".format(config.transform))
if args.register:
# register type to process in register module
config.register_type = args.register
print("Set register type to {}".format(config.register_type))
if args.df:
# data frame processing task
config.df_task = args.df
print("Set data frame processing task to {}".format(config.df_task))
if args.plot_2d:
# 2D plot type to process in plot_2d module
config.plot_2d_type = args.plot_2d
print("Set plot_2d type to {}".format(config.plot_2d_type))
if args.slice:
# specify a generic slice by command-line, assuming same order
# of arguments as for slice built-in function and interpreting
# "none" string as None
config.slice_vals = args.slice.split(",")
config.slice_vals = [
None if val.lower() == "none" else int(val)
for val in config.slice_vals
]
print("Set slice values to {}".format(config.slice_vals))
if args.delay:
config.delay = int(args.delay)
print("Set delay to {}".format(config.delay))
if args.show:
# show images after task is performed, if supported
config.show = _is_arg_true(args.show)
print("Set show to {}".format(config.show))
if args.groups:
config.groups = args.groups
print("Set groups to {}".format(config.groups))
if args.ec2_start is not None:
# start EC2 instances
config.ec2_start = args_with_dict(args.ec2_start)
print("Set ec2 start to {}".format(config.ec2_start))
if args.ec2_list:
# list EC2 instances
config.ec2_list = args_with_dict(args.ec2_list)
print("Set ec2 list to {}".format(config.ec2_list))
if args.ec2_terminate:
config.ec2_terminate = args.ec2_terminate
print("Set ec2 terminate to {}".format(config.ec2_terminate))
if args.notify:
notify_len = len(args.notify)
if notify_len > 0:
config.notify_url = args.notify[0]
print("Set notification URL to {}".format(config.notify_url))
if notify_len > 1:
config.notify_msg = args.notify[1]
print("Set notification message to {}".format(config.notify_msg))
if notify_len > 2:
config.notify_attach = args.notify[2]
print("Set notification attachment path to {}".format(
config.notify_attach))
if args.prefix:
config.prefix = args.prefix
print("Set path prefix to {}".format(config.prefix))
if args.suffix:
config.suffix = args.suffix
print("Set path suffix to {}".format(config.suffix))
if args.alphas:
# specify alpha levels
config.alphas = [float(val) for val in args.alphas.split(",")]
print("Set alphas to", config.alphas)
if args.vmin:
# specify vmin levels
config.vmins = [libmag.get_int(val) for val in args.vmin.split(",")]
print("Set vmins to", config.vmins)
if args.vmax:
# specify vmax levels and copy to vmax overview used for plotting
# and updated for normalization
config.vmaxs = [libmag.get_int(val) for val in args.vmax.split(",")]
config.vmax_overview = list(config.vmaxs)
print("Set vmaxs to", config.vmaxs)
if args.reg_suffixes is not None:
# specify suffixes of registered images to load
config.reg_suffixes = args_to_dict(args.reg_suffixes,
config.RegSuffixes,
config.reg_suffixes)
print("Set registered image suffixes to {}".format(
config.reg_suffixes))
if args.seed:
# specify random number generator seed
config.seed = int(args.seed)
print("Set random number generator seed to", config.seed)
if args.plot_labels is not None:
# specify general plot labels
config.plot_labels = args_to_dict(args.plot_labels, config.PlotLabels,
config.plot_labels)
print("Set plot labels to {}".format(config.plot_labels))
if args.theme is not None:
# specify themes, currently applied to Matplotlib elements
theme_names = []
for theme in args.theme:
# add theme enum if found
theme_enum = libmag.get_enum(theme, config.Themes)
if theme_enum:
config.rc_params.append(theme_enum)
theme_names.append(theme_enum.name)
print("Set to use themes to {}".format(theme_names))
# prep filename
filename_base = None
if config.filename:
filename_base = importer.filename_to_base(config.filename,
config.series)
if not skip_dbs:
setup_dbs(filename_base, args.db, args.truth_db)
# set multiprocessing start method
chunking.set_mp_start_method()
# POST-ARGUMENT PARSING
if process_args_only:
return
# if command-line driven task specified, start task and shut down
if config.register_type:
register.main()
elif config.notify_url:
notify.main()
elif config.plot_2d_type:
plot_2d.main()
elif config.df_task:
df_io.main()
elif config.grid_search_profile:
_grid_search(series_list)
elif config.ec2_list or config.ec2_start or config.ec2_terminate:
# defer importing AWS module to avoid making its dependencies
# required for MagellanMapper
from magmap.cloud import aws
aws.main()
else:
# set up image and perform any whole image processing tasks;
# do not shut down if not a command-line proc task
_process_files(series_list)
if proc_type is None or proc_type is config.ProcessTypes.LOAD:
return
shutdown()
def read_sitk_files(
filename_sitk: str,
reg_names: Optional[Union[str, Sequence[str]]] = None,
return_sitk: bool = False,
make_3d: bool = False
) -> Union["np_io.Image5d", Tuple["np_io.Image5d", sitk.Image]]:
"""Read image files through SimpleITK.
Supports identifying files based on registered suffixes and combining
multiple registered image files into a single image.
Also sets up spacing from the first loaded image in
:attr:`magmap.settings.config.resolutions` if not already set.
Args:
filename_sitk: Path to file in a format that can be read by
SimpleITK.
reg_names: Path or sequence of paths of
registered names. Can be a registered suffix or a full path.
Defaults to None to open ``filename_sitk`` as-is through
:meth:`read_sitk`.
return_sitk: True to return the loaded SimpleITK Image object.
make_3d: True to convert 2D images to 3D; defaults to False.
Returns:
``img5d``: Image5d instance with the loaded image in Numpy 5D format
(or 4D if not multi-channel, and 3D if originally 2D and ``make_3d``
is False). Associated metadata is loaded into
:module:`magmap.settings.config` attributes.
If ``return_sitk`` is True, also returns the first loaded image
in SimpleITK format.
Raises:
``FileNotFoundError`` if ``filename_sitk`` cannot be found, after
attempting to load metadata from ``filename_np``.
"""
# load image via SimpleITK
img_sitk = None
loaded_path = filename_sitk
if reg_names:
img_nps = []
if not libmag.is_seq(reg_names):
reg_names = [reg_names]
for reg_name in reg_names:
# load each registered suffix into list of images with same shape,
# keeping first image in sitk format
img, path = _load_reg_img_to_combine(filename_sitk, reg_name,
img_nps)
if img_sitk is None:
img_sitk = img
loaded_path = path
if len(img_nps) > 1:
# merge images into separate channels
img_np = np.stack(img_nps, axis=img_nps[0].ndim)
else:
img_np = img_nps[0]
else:
# load filename_sitk directly
if not os.path.exists(filename_sitk):
raise FileNotFoundError(
"could not find file {}".format(filename_sitk))
img_sitk, _ = read_sitk(filename_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if make_3d:
# convert 2D images to 3D
# TODO: consider converting img_np to 3D regardless so array in
# Image5d is at least 4D
img_sitk = _make_3d(img_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if config.resolutions is None:
# fallback to determining metadata directly from sitk file
libmag.warn(
"MagellanMapper image metadata file not loaded; will fallback to "
"{} for metadata".format(loaded_path))
config.resolutions = np.array([img_sitk.GetSpacing()[::-1]])
print("set resolutions to {}".format(config.resolutions))
# add time axis and insert into Image5d with original name
img5d = np_io.Image5d(img_np[None],
filename_sitk,
img_io=config.LoadIO.SITK)
if return_sitk:
return img5d, img_sitk
return img5d
def read_sitk_files(filename_sitk, reg_names=None, return_sitk=False):
"""Read an image file through SimpleITK and export to Numpy array format,
with support for combining multiple registered image files into a single
image.
Also sets up spacing from the first loaded image in
:attr:`magmap.settings.config.resolutions` if not already set.
Args:
filename_sitk (str): Path to file in a format that can be read by
SimpleITK.
reg_names (Union[str, List[str]]): Path or sequence of paths of
registered names. Can be a registered suffix or a full path.
Defaults to None to open ``filename_sitk`` as-is through
:meth:`read_sitk`.
return_sitk (bool): True to return the loaded SimpleITK Image object.
Returns:
:class:`numpy.ndarray`: Image array in Numpy 3D format (or 4D if
multi-channel). Associated metadata is loaded into :module:`config`
attributes.
If ``return_sitk`` is True, also returns the first loaded image
in SimpleITK format.
Raises:
``FileNotFoundError`` if ``filename_sitk`` cannot be found, after
attempting to load metadata from ``filename_np``.
"""
# load image via SimpleITK
img_sitk = None
loaded_path = filename_sitk
if reg_names:
img_nps = []
if not libmag.is_seq(reg_names):
reg_names = [reg_names]
for reg_name in reg_names:
# load each registered suffix into list of images with same shape,
# keeping first image in sitk format
img, path = _load_reg_img_to_combine(filename_sitk, reg_name,
img_nps)
if img_sitk is None:
img_sitk = img
loaded_path = path
if len(img_nps) > 1:
# merge images into separate channels
img_np = np.stack(img_nps, axis=img_nps[0].ndim)
else:
img_np = img_nps[0]
else:
# load filename_sitk directly
if not os.path.exists(filename_sitk):
raise FileNotFoundError(
"could not find file {}".format(filename_sitk))
img_sitk, _ = read_sitk(filename_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if config.resolutions is None:
# fallback to determining metadata directly from sitk file
libmag.warn(
"MagellanMapper image metadata file not loaded; will fallback to "
"{} for metadata".format(loaded_path))
config.resolutions = np.array([img_sitk.GetSpacing()[::-1]])
print("set resolutions to {}".format(config.resolutions))
if return_sitk:
return img_np, img_sitk
return img_np