def replace_vals(df, vals_from, vals_to, cols=None):
"""Replace values in a data frame for the given columns.
Args:
df (:obj:`pd.DataFrame`): Pandas data frame.
vals_from (Any): Value or sequence of values to be replaced.
vals_to (Any): Corresponding value or sequence of values to
``vals_from`` with which to replace.
cols (Union[str, List[str]]): Column name or sequence of names
to replace values; defaults to None to replace values in all
columns.
Returns:
:obj:`pd.DataFrame`: Data frame with values replaced.
"""
# convert arguments to lists
if cols is None or not libmag.is_seq(cols):
cols = [cols]
if not libmag.is_seq(vals_to):
vals_to = [vals_to]
if not libmag.is_seq(vals_from):
vals_from = [vals_from]
# parse NaN strings
vals_from = [np.nan if libmag.is_nan(v) else v for v in vals_from]
for col in cols:
# replace values in specific columns, or whole data frame if no
# columns are given
df_col = df[col] if col else df
df = df_col.replace(vals_from, vals_to)
return df
def _filter_dict(d: Dict, fn_parse_val: Callable[[Any], Any]) -> Dict:
"""Recursively filter keys and values within nested dictionaries
Args:
d: Dictionary to filter.
fn_parse_val: Function to apply to each value. Should call
this parent function if deep recursion is desired.
Returns:
Filtered dictionary.
"""
out = {}
for key, val in d.items():
if isinstance(val, dict):
# recursively filter nested dictionaries
val = fn_parse_val(val)
elif libmag.is_seq(val):
# filter each val within list
val = [fn_parse_val(v) for v in val]
else:
# filter a single val
val = fn_parse_val(val)
# filter key
key = fn_parse_val(key)
out[key] = val
return out
def pivot_with_conditions(df, index, columns, values, aggfunc="first"):
"""Pivot a data frame to columns with sub-columns for different conditions.
For example, a table of metric values for different regions within
each sample under different conditions will be reorganized to region
columns that are each split into condition sub-columns.
Args:
df (:obj:`pd.DataFrame`): Data frame to pivot.
index (str, List): Column name or sequence of columns specifying
samples, generally a sequence to later unstack.
columns (str, List): Column name or sequence of columns to pivot
into separate columns.
values (str): Column of values to move into new columns.
aggfunc (func): Aggregation function for duplicates; defaults to
"first" to take the first value.
Returns:
Tuple of the pivoted data frame and the list of pivoted columns.
"""
# use multi-level indexing; assumes that no duplicates exist for
# a given index-pivot-column combo, and if they do, simply take 1st val
df_lines = df.pivot_table(
index=index, columns=columns, values=values, aggfunc=aggfunc)
cols = df_lines.columns # may be fewer than orig
if libmag.is_seq(index) and len(index) > 1:
# move multi-index into separate sub-cols of each region and
# reset index to access all columns
df_lines = df_lines.unstack()
df_lines = df_lines.reset_index()
return df_lines, cols
def read_sitk_files(filename_sitk, reg_names=None):
"""Read files through SimpleITK and export to Numpy array format,
loading associated metadata.
Args:
filename_sitk: Path to file in a format that can be read by SimpleITK.
reg_names: Path or sequence of paths of registered names. Can
be a registered suffix or a full path. Defaults to None.
Returns:
Image array in MagellanMapper image5d format. Associated metadata will
have been loaded into module-level variables.
Raises:
``FileNotFoundError`` if ``filename_sitk`` cannot be found, after
attempting to load metadata from ``filename_np``.
"""
# load image via SimpleITK
img_sitk = None
loaded_path = filename_sitk
if reg_names:
img_nps = []
if not libmag.is_seq(reg_names):
reg_names = [reg_names]
for reg_name in reg_names:
# load each registered suffix into list of images with same shape,
# keeping first image in sitk format
img, path = _load_reg_img_to_combine(filename_sitk, reg_name,
img_nps)
if img_sitk is None:
img_sitk = img
loaded_path = path
if len(img_nps) > 1:
# merge images into separate channels
img_np = np.stack(img_nps, axis=img_nps[0].ndim)
else:
img_np = img_nps[0]
else:
# load filename_sitk directly
if not os.path.exists(filename_sitk):
raise FileNotFoundError(
"could not find file {}".format(filename_sitk))
img_sitk, _ = read_sitk(filename_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if config.resolutions is None:
# fallback to determining metadata directly from sitk file
libmag.warn(
"MagellanMapper image metadata file not loaded; will fallback to "
"{} for metadata".format(loaded_path))
config.resolutions = np.array([img_sitk.GetSpacing()[::-1]])
print("set resolutions to {}".format(config.resolutions))
return img_np
def parse_enum_val(val):
# recursively parse Enum values
if isinstance(val, dict):
val = _filter_dict(val, parse_enum_val)
elif libmag.is_seq(val):
val = [parse_enum_val(v) for v in val]
elif isinstance(val, str):
val_split = val.split(".")
if len(val_split) > 1 and val_split[0] in enums:
# replace with the corresponding Enum class
val = enums[val_split[0]][val_split[1]]
return val
def handle_extracted_plane():
# get sub-plot and hide x/y axes
ax = axs
if libmag.is_seq(ax):
ax = axs[imgi]
plot_support.hide_axes(ax)
# multiple artists can be shown at each frame by collecting
# each group of artists in a list; overlay_images returns
# a nested list containing a list for each image, which in turn
# contains a list of artists for each channel
ax_imgs = plot_support.overlay_images(ax,
self.aspect,
self.origin,
imgs,
None,
cmaps_all,
ignore_invis=True,
check_single=True)
if (colorbar is not None and len(ax_imgs) > 0
and len(ax_imgs[0]) > 0 and imgi == 0):
# add colorbar with scientific notation if outside limits
cbar = ax.figure.colorbar(ax_imgs[0][0], ax=ax, **colorbar)
plot_support.set_scinot(cbar.ax, lbls=None, units=None)
plotted_imgs[imgi] = np.array(ax_imgs).flatten()
if libmag.is_seq(text_pos) and len(text_pos) > 1:
# write plane index in axes rather than data coordinates
text = ax.text(*text_pos[:2],
"{}-plane: {}".format(
plot_support.get_plane_axis(config.plane),
self.start_planei + imgi),
transform=ax.transAxes,
color="w")
plotted_imgs[imgi] = [*plotted_imgs[imgi], text]
if scale_bar:
plot_support.add_scale_bar(ax, 1 / self.rescale, config.plane)
def convert_numpy_val(val):
# recursively convert Numpy data types to primitives
if isinstance(val, dict):
val = _filter_dict(val, convert_numpy_val)
elif libmag.is_seq(val):
# also replaces any tuples with lists, avoiding tuple flags in
# the output file for simplicity
val = [convert_numpy_val(v) for v in val]
else:
try:
val = val.item()
except AttributeError:
pass
return val
def parse_enum(d):
# recursively parse Enum keys and values within nested dictionaries
out = {}
for key, val in d.items():
if isinstance(val, dict):
# recursively parse nested dictionaries
val = parse_enum(val)
elif libmag.is_seq(val):
# parse vals within lists
val = [parse_enum_val(v) for v in val]
else:
# parse a single val
val = parse_enum_val(val)
key = parse_enum_val(key)
out[key] = val
return out
def update_max_intens_proj(self, shape, display=False):
"""Update max intensity projection planes.
Args:
shape (Union[int, Sequence[int]]): Number of planes for all
Plot Editors or sequence of plane counts in ``z,y,x``.
display (bool): True to trigger an update in the Plot Editors;
defaults to False.
"""
self._max_intens_proj = shape
is_seq = libmag.is_seq(shape)
for i, ed in enumerate(self.plot_eds.values()):
n = shape[i] if is_seq else shape
if n != ed.max_intens_proj:
ed.max_intens_proj = n
if display: ed.update_coord()
def compress_file(path_in, path_out=None):
"""Compress a file or files by tar archving and compressing with.
Assumes that ``tar`` and ``zstd`` are available shell commands.
Args:
path_in (str, List[str]): Input file path; can be a sequence of paths,
in which case the first path will determine the working directory
and the default ``path_out`` if none is given.
path_out (str): Output file path; defaults to None to use the
same path as ``path_in`` with ``.tar.zstd`` appended.
"""
tar_args = ["tar", "cfv", "-"]
if libmag.is_seq(path_in):
# set up sequence of paths, using the first path as template
path_first = path_in[0]
tar_args.extend([os.path.basename(p) for p in path_in])
else:
# set up a single path
path_first = path_in
tar_args.append(os.path.basename(path_in))
if path_out is None:
# default to using the first path for compressed file name
path_out = os.path.splitext(path_first)[0] + ".tar.zst"
# use first path for working directory
wd = os.path.dirname(path_first)
if not wd:
wd = None
# archive with tar and pipe to zstd for compression
print("Compressing \"{}\" to \"{}\"".format(path_in, path_out))
tar = subprocess.Popen(tar_args, cwd=wd, stdout=subprocess.PIPE, bufsize=0)
with open(path_out, "wb") as f:
zstd = subprocess.Popen(["pzstd", "-v"],
stdin=tar.stdout,
stdout=f,
bufsize=0)
tar.stdout.close()
# appears to work for large image with memory issues because data
# are backed by files
stderr = zstd.communicate()[1]
if stderr:
print(stderr)
def preprocess_img(image5d, preprocs, channel, out_path):
"""Pre-process an image in 3D.
Args:
image5d (:obj:`np.ndarray`): 5D array in t,z,y,x[,c].
preprocs (Union[str, list[str]]): Pre-processing tasks that will be
converted to enums in :class:`config.PreProcessKeys` to perform
in the order given.
channel (int): Channel to preprocess, or None for all channels.
out_path (str): Output base path.
Returns:
:obj:`np.ndarray`: The pre-processed image array.
"""
if preprocs is None:
print("No preprocessing tasks to perform, skipping")
return
if not libmag.is_seq(preprocs):
preprocs = [preprocs]
roi = image5d[0]
for preproc in preprocs:
# perform global pre-processing task
task = libmag.get_enum(preproc, config.PreProcessKeys)
_logger.info("Pre-processing task: %s", task)
if task is config.PreProcessKeys.SATURATE:
roi = plot_3d.saturate_roi(roi, channel=channel)
elif task is config.PreProcessKeys.DENOISE:
roi = plot_3d.denoise_roi(roi, channel)
elif task is config.PreProcessKeys.REMAP:
roi = plot_3d.remap_intensity(roi, channel)
elif task is config.PreProcessKeys.ROTATE:
roi = rotate_img(roi)
else:
_logger.warn("No preprocessing task found for: %s", preproc)
# save to new file
image5d = importer.roi_to_image5d(roi)
importer.save_np_image(image5d, out_path)
return image5d
def _mirror_label_ids(label_ids, combine=False):
"""Mirror label IDs, assuming that a "mirrored" ID is the negative
of the given ID.
Args:
label_ids (Union[int, List[int]]): Single ID or sequence of IDs.
combine (bool): True to return a list of ``label_ids`` along with
their mirrored IDs; defaults to False to return on the mirrored IDs.
Returns:
Union[int, List[int]]: A single mirrored ID if ``label_ids`` is
one ID and ``combine`` is False, or a list of IDs.
"""
if libmag.is_seq(label_ids):
mirrored = [-1 * n for n in label_ids]
if combine:
mirrored = list(label_ids).extend(mirrored)
else:
mirrored = -1 * label_ids
if combine:
mirrored = [label_ids, mirrored]
return mirrored
def load_s3_file(bucket_name, key):
"""Load a file or files in AWS S3, retrieving the metadata without
the object itself.
Args:
bucket_name (str): Name of bucket.
key (str, List[str}): Key or sequence of keys within bucket.
Returns:
dict[str, :obj:`boto3.resources.factory.s3.Object`]: Dictionary mapping
keys to their corresponding successfully loaded S3 objects.
"""
s3 = boto3.resource("s3")
bucket = s3.Bucket(bucket_name)
if libmag.is_seq(key):
# get all objects starting with common part of paths
prefix = os.path.commonprefix(key)
else:
prefix = key
key = [key]
loaded_objs = {}
objs = bucket.objects.filter(Prefix=prefix)
for obj in objs:
if obj.key in key:
# ensure an exact match with a key in the given list
try:
obj.load()
loaded_objs[obj.key] = obj
except ClientError as e:
print(e)
print("Unable to load object at bucket \"{}\", key \"{}\"".
format(bucket_name, obj.key))
print()
_show_missing_keys(bucket_name, key, loaded_objs.keys())
return loaded_objs
def pivot_with_conditions(df, index, columns, values, aggfunc="first"):
"""Pivot a data frame to columns with sub-columns for different conditions.
For example, a table of metric values for different regions within
each sample under different conditions will be reorganized to region
columns that are each split into condition sub-columns.
Args:
df (:class:`pandas.DataFrame`): Data frame to pivot.
index (Union[str, list[str]]): Column name or list of names specifying
the index for the output table.
columns (Union[str, list[str]]): Name or list of names of columns
whose values are pivoted into separate columns.
values (str): Name of column whose values are moved into the new
columns specified by ``columns``.
aggfunc (func): Aggregation function for duplicates; defaults to
"first" to take the first value.
Returns:
:class:`pandas.DataFrame`, list[str]: The pivoted data frame and
list of pivoted columns.
"""
# use multi-level indexing; assumes that no duplicates exist for
# a given index-pivot-column combo, and if they do, simply take 1st val
df_lines = df.pivot_table(index=index,
columns=columns,
values=values,
aggfunc=aggfunc)
cols = df_lines.columns # may be fewer than orig
if libmag.is_seq(index) and len(index) > 1:
# move multi-index into separate sub-cols of each region and
# reset index to access all columns
df_lines = df_lines.unstack()
df_lines = df_lines.reset_index()
return df_lines, cols
def __call__(self, x, y):
"""Get the pixel display message.
Args:
x (int): x-data coordinate.
y (int): y-data coordinate.
Returns:
str: Message showing ``x,y`` coordinates, intensity values,
and corresponding RGB label for each overlaid image at the
given location.
"""
coord = (int(y), int(x))
rgb = None
output = []
main_img_shape = self.imgs[0].shape[:2]
for i, img in enumerate(self.imgs):
# scale coordinates from axes space, based on main image, to
# given image's space to get pixel from given image
scale = np.divide(img.shape[:2], main_img_shape)
coord_img = tuple(np.multiply(coord, scale).astype(int))
if any(np.less(coord_img, 0)) or any(
np.greater_equal(coord_img, img.shape[:len(coord_img)])):
# no corresponding px for the image
px = "n/a"
else:
# get the corresponding intensity value, truncating floats
px = img[coord_img]
if i == 1:
# for the label image, get its RGB value
ax_img = self.ax_imgs[i][0]
if self.cmap_labels:
label_rgb = self.cmap_labels(
self.cmap_labels.convert_img_labels(px))
else:
label_rgb = ax_img.cmap(ax_img.norm(px))
rgb = "RGB for label {}: {}".format(
px, tuple(np.multiply(label_rgb[:3], 255).astype(int)))
if isinstance(px, float): px = "{:.4f}".format(px)
orig_coord = coord
if self.shapes:
# scale coordinates from axes space to given original image's
# space, accounting for any resizing and downsampling
shape = self.shapes
if libmag.is_seq(self.shapes[0]):
# overlaid images may have different shapes
shape = self.shapes[i]
scale = np.divide(shape[:2], main_img_shape)
orig_coord = tuple(np.multiply(coord, scale).astype(int))
if self.offset:
# shift for offset given in original image's space
off = self.offset
if libmag.is_seq(self.offset[0]):
# overlaid images may have different offsets
off = self.offset[i]
orig_coord = np.add(orig_coord, off)
output.append("Image {}: x={}, y={}, px={}".format(
i, orig_coord[1], orig_coord[0], px))
# join output message
if rgb:
output.append(rgb)
msg = "; ".join(output)
return msg
def read_sitk_files(filename_sitk, reg_names=None, return_sitk=False):
"""Read an image file through SimpleITK and export to Numpy array format,
with support for combining multiple registered image files into a single
image.
Also sets up spacing from the first loaded image in
:attr:`magmap.settings.config.resolutions` if not already set.
Args:
filename_sitk (str): Path to file in a format that can be read by
SimpleITK.
reg_names (Union[str, List[str]]): Path or sequence of paths of
registered names. Can be a registered suffix or a full path.
Defaults to None to open ``filename_sitk`` as-is through
:meth:`read_sitk`.
return_sitk (bool): True to return the loaded SimpleITK Image object.
Returns:
:class:`numpy.ndarray`: Image array in Numpy 3D format (or 4D if
multi-channel). Associated metadata is loaded into :module:`config`
attributes.
If ``return_sitk`` is True, also returns the first loaded image
in SimpleITK format.
Raises:
``FileNotFoundError`` if ``filename_sitk`` cannot be found, after
attempting to load metadata from ``filename_np``.
"""
# load image via SimpleITK
img_sitk = None
loaded_path = filename_sitk
if reg_names:
img_nps = []
if not libmag.is_seq(reg_names):
reg_names = [reg_names]
for reg_name in reg_names:
# load each registered suffix into list of images with same shape,
# keeping first image in sitk format
img, path = _load_reg_img_to_combine(filename_sitk, reg_name,
img_nps)
if img_sitk is None:
img_sitk = img
loaded_path = path
if len(img_nps) > 1:
# merge images into separate channels
img_np = np.stack(img_nps, axis=img_nps[0].ndim)
else:
img_np = img_nps[0]
else:
# load filename_sitk directly
if not os.path.exists(filename_sitk):
raise FileNotFoundError(
"could not find file {}".format(filename_sitk))
img_sitk, _ = read_sitk(filename_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if config.resolutions is None:
# fallback to determining metadata directly from sitk file
libmag.warn(
"MagellanMapper image metadata file not loaded; will fallback to "
"{} for metadata".format(loaded_path))
config.resolutions = np.array([img_sitk.GetSpacing()[::-1]])
print("set resolutions to {}".format(config.resolutions))
if return_sitk:
return img_np, img_sitk
return img_np
def delete_s3_file(bucket_name, key, hard=False, dryrun=False):
"""Delete a file on AWS S3.
Args:
bucket_name (str): Name of bucket.
key (str, List[str]): Key or sequence of keys within bucket.
hard (bool): True to delete all versions associated with the key
including any delete markers, effectively deleting the
object on S3 permanently. Defaults to False, in which case
only a delete marker will be applied if versioning is on;
without versioning, the file will be permanently deleted.
dryrun (bool): True to print paths without uploading; defaults to False.
Returns:
List[str]: Sequence of successfully deleted keys.
For versioned files, all versions of the given object must have
been deleted without error to return True.
"""
s3 = boto3.resource("s3")
bucket = s3.Bucket(bucket_name)
if libmag.is_seq(key):
# get all objects starting with common part of paths
prefix = os.path.commonprefix(key)
else:
prefix = key
key = [key]
print("Deleting selected file object(s) in bucket \"{}\", prefix \"{}\""
" (hard delete {})".format(bucket_name, prefix, hard))
deleted_keys = []
if hard:
# hard delete finds all object versions associated with the keys,
# including delete markers, and permanently deletes them
vers = bucket.object_versions.filter(Prefix=prefix)
for ver in vers:
if ver.object_key in key:
# ensure an exact match with a key in the given list
print("Permanently deleting \"{}\", versionId \"{}\"".format(
ver.object_key, ver.id))
if dryrun:
print("Deletion of \"{}\" set to dry run, so skipping".
format(ver.object_key))
else:
ver.delete()
if ver.object_key not in deleted_keys:
deleted_keys.append(ver.object_key)
else:
# default delete, which adds a delete marker for buckets with
# versioning or permanently deletes objects if no versioning
# obj = bucket.Object(key)
objs = bucket.objects.filter(Prefix=prefix)
for obj in objs:
if obj.key in key:
# ensure an exact match
try:
print("Deleting (or setting delete marker for) \"{}\"".
format(obj.key))
if dryrun:
print("Deletion of {} set to dry run, so skipping".
format(obj.key))
else:
obj.delete()
deleted_keys.append(obj.key)
except botocore.exceptions.ClientError as e:
print(e)
print("Could not find key \"{}\" to delete".format(key))
print()
_show_missing_keys(bucket_name, key, deleted_keys)
return deleted_keys
def read_sitk_files(
filename_sitk: str,
reg_names: Optional[Union[str, Sequence[str]]] = None,
return_sitk: bool = False,
make_3d: bool = False
) -> Union["np_io.Image5d", Tuple["np_io.Image5d", sitk.Image]]:
"""Read image files through SimpleITK.
Supports identifying files based on registered suffixes and combining
multiple registered image files into a single image.
Also sets up spacing from the first loaded image in
:attr:`magmap.settings.config.resolutions` if not already set.
Args:
filename_sitk: Path to file in a format that can be read by
SimpleITK.
reg_names: Path or sequence of paths of
registered names. Can be a registered suffix or a full path.
Defaults to None to open ``filename_sitk`` as-is through
:meth:`read_sitk`.
return_sitk: True to return the loaded SimpleITK Image object.
make_3d: True to convert 2D images to 3D; defaults to False.
Returns:
``img5d``: Image5d instance with the loaded image in Numpy 5D format
(or 4D if not multi-channel, and 3D if originally 2D and ``make_3d``
is False). Associated metadata is loaded into
:module:`magmap.settings.config` attributes.
If ``return_sitk`` is True, also returns the first loaded image
in SimpleITK format.
Raises:
``FileNotFoundError`` if ``filename_sitk`` cannot be found, after
attempting to load metadata from ``filename_np``.
"""
# load image via SimpleITK
img_sitk = None
loaded_path = filename_sitk
if reg_names:
img_nps = []
if not libmag.is_seq(reg_names):
reg_names = [reg_names]
for reg_name in reg_names:
# load each registered suffix into list of images with same shape,
# keeping first image in sitk format
img, path = _load_reg_img_to_combine(filename_sitk, reg_name,
img_nps)
if img_sitk is None:
img_sitk = img
loaded_path = path
if len(img_nps) > 1:
# merge images into separate channels
img_np = np.stack(img_nps, axis=img_nps[0].ndim)
else:
img_np = img_nps[0]
else:
# load filename_sitk directly
if not os.path.exists(filename_sitk):
raise FileNotFoundError(
"could not find file {}".format(filename_sitk))
img_sitk, _ = read_sitk(filename_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if make_3d:
# convert 2D images to 3D
# TODO: consider converting img_np to 3D regardless so array in
# Image5d is at least 4D
img_sitk = _make_3d(img_sitk)
img_np = sitk.GetArrayFromImage(img_sitk)
if config.resolutions is None:
# fallback to determining metadata directly from sitk file
libmag.warn(
"MagellanMapper image metadata file not loaded; will fallback to "
"{} for metadata".format(loaded_path))
config.resolutions = np.array([img_sitk.GetSpacing()[::-1]])
print("set resolutions to {}".format(config.resolutions))
# add time axis and insert into Image5d with original name
img5d = np_io.Image5d(img_np[None],
filename_sitk,
img_io=config.LoadIO.SITK)
if return_sitk:
return img5d, img_sitk
return img5d
def build_stack(self,
axs: List,
scale_bar: bool = True,
fit: bool = False) -> Optional[List]:
"""Builds a stack of Matploblit 2D images.
Uses multiprocessing to load or resize each image.
Args:
axs: Sub-plot axes.
scale_bar: True to include scale bar; defaults to True.
fit: True to fit the figure frame to the resulting image.
Returns:
:List[List[:obj:`matplotlib.image.AxesImage`]]: Nested list of
axes image objects. The first list level contains planes, and
the second level are channels within each plane.
"""
def handle_extracted_plane():
# get sub-plot and hide x/y axes
ax = axs
if libmag.is_seq(ax):
ax = axs[imgi]
plot_support.hide_axes(ax)
# multiple artists can be shown at each frame by collecting
# each group of artists in a list; overlay_images returns
# a nested list containing a list for each image, which in turn
# contains a list of artists for each channel
ax_imgs = plot_support.overlay_images(ax,
self.aspect,
self.origin,
imgs,
None,
cmaps_all,
ignore_invis=True,
check_single=True)
if (colorbar is not None and len(ax_imgs) > 0
and len(ax_imgs[0]) > 0 and imgi == 0):
# add colorbar with scientific notation if outside limits
cbar = ax.figure.colorbar(ax_imgs[0][0], ax=ax, **colorbar)
plot_support.set_scinot(cbar.ax, lbls=None, units=None)
plotted_imgs[imgi] = np.array(ax_imgs).flatten()
if libmag.is_seq(text_pos) and len(text_pos) > 1:
# write plane index in axes rather than data coordinates
text = ax.text(*text_pos[:2],
"{}-plane: {}".format(
plot_support.get_plane_axis(config.plane),
self.start_planei + imgi),
transform=ax.transAxes,
color="w")
plotted_imgs[imgi] = [*plotted_imgs[imgi], text]
if scale_bar:
plot_support.add_scale_bar(ax, 1 / self.rescale, config.plane)
# number of image types (eg atlas, labels) and corresponding planes
num_image_types = len(self.images)
if num_image_types < 1: return None
num_images = len(self.images[0])
if num_images < 1: return None
# import the images as Matplotlib artists via multiprocessing
plotted_imgs: List = [None] * num_images
img_shape = self.images[0][0].shape
target_size = np.multiply(img_shape, self.rescale).astype(int)
multichannel = self.images[0][0].ndim >= 3
if multichannel:
print("building stack for channel: {}".format(config.channel))
target_size = target_size[:-1]
# setup imshow parameters
colorbar = config.roi_profile["colorbar"]
cmaps_all = [config.cmaps, *self.cmaps_labels]
text_pos = config.plot_labels[config.PlotLabels.TEXT_POS]
StackPlaneIO.set_data(self.images)
pool_results = None
pool = None
multiprocess = self.rescale != 1
if multiprocess:
# set up multiprocessing
initializer = None
initargs = None
if not chunking.is_fork():
# set up labels image as a shared array for spawned mode
initializer, initargs = StackPlaneIO.build_pool_init(
OrderedDict([(i, img)
for i, img in enumerate(self.images)]))
pool = chunking.get_mp_pool(initializer, initargs)
pool_results = []
for i in range(num_images):
# add rotation argument if necessary
args = (i, target_size)
if pool is None:
# extract and handle without multiprocessing
imgi, imgs = self.fn_process(*args)
handle_extracted_plane()
else:
# extract plane in multiprocessing
pool_results.append(
pool.apply_async(self.fn_process, args=args))
if multiprocess:
# handle multiprocessing output
for result in pool_results:
imgi, imgs = result.get()
handle_extracted_plane()
pool.close()
pool.join()
if fit and plotted_imgs:
# fit each figure to its first available image
for ax_img in plotted_imgs:
# images may be flattened AxesImage, array of AxesImage and
# Text, or None if alpha set to 0
if libmag.is_seq(ax_img):
ax_img = ax_img[0]
if isinstance(ax_img, AxesImage):
plot_support.fit_frame_to_image(ax_img.figure,
ax_img.get_array().shape,
self.aspect)
return plotted_imgs
def imshow_multichannel(ax,
img2d,
channel,
cmaps,
aspect,
alpha=None,
vmin=None,
vmax=None,
origin=None,
interpolation=None,
norms=None,
nan_color=None,
ignore_invis=False):
"""Show multichannel 2D image with channels overlaid over one another.
Applies :attr:`config.transform` with :obj:`config.Transforms.ROTATE`
to rotate images. If not available, also checks the first element in
:attr:``config.flip`` to rotate the image by 180 degrees.
Applies :attr:`config.transform` with :obj:`config.Transforms.FLIP_HORIZ`
and :obj:`config.Transforms.FLIP_VERT` to invert images.
Args:
ax: Axes plot.
img2d: 2D image either as 2D (y, x) or 3D (y, x, channel) array.
channel: Channel to display; if None, all channels will be shown.
cmaps: List of colormaps corresponding to each channel. Colormaps
can be the names of specific maps in :mod:``config``.
aspect: Aspect ratio.
alpha (float, List[float]): Transparency level for all channels or
sequence of levels for each channel. If any value is 0, the
corresponding image will not be output. Defaults to None to use 1.
vmin (float, List[float]): Scalar or sequence of vmin levels for
all channels; defaults to None.
vmax (float, List[float]): Scalar or sequence of vmax levels for
all channels; defaults to None.
origin: Image origin; defaults to None.
interpolation: Type of interpolation; defaults to None.
norms: List of normalizations, which should correspond to ``cmaps``.
nan_color (str): String of color to use for NaN values; defaults to
None to leave these pixels empty.
ignore_invis (bool): True to give None instead of an ``AxesImage``
object that would be invisible; defaults to False.
Returns:
List of ``AxesImage`` objects.
"""
# assume that 3D array has a channel dimension
multichannel, channels = plot_3d.setup_channels(img2d, channel, 2)
img = []
num_chls = len(channels)
if alpha is None:
alpha = 1
if num_chls > 1 and not libmag.is_seq(alpha):
# if alphas not explicitly set per channel, make all channels more
# translucent at a fixed value that is higher with more channels
alpha /= np.sqrt(num_chls + 1)
# transform image based on config parameters
rotate = config.transform[config.Transforms.ROTATE]
if rotate is not None:
last_axis = img2d.ndim - 1
if multichannel:
last_axis -= 1
# use first rotation value
img2d = np.rot90(img2d, libmag.get_if_within(rotate, 0),
(last_axis - 1, last_axis))
for chl in channels:
img2d_show = img2d[..., chl] if multichannel else img2d
cmap = None if cmaps is None else cmaps[chl]
norm = None if norms is None else norms[chl]
cmap = colormaps.get_cmap(cmap)
if cmap is not None and nan_color:
# given color for masked values such as NaNs to distinguish from 0
cmap.set_bad(color=nan_color)
# get setting corresponding to the channel index, or use the value
# directly if it is a scalar
vmin_plane = libmag.get_if_within(vmin, chl)
vmax_plane = libmag.get_if_within(vmax, chl)
alpha_plane = libmag.get_if_within(alpha, chl)
img_chl = None
if not ignore_invis or alpha_plane > 0:
# skip display if alpha is 0 to avoid outputting a hidden image
# that may show up in other renderers (eg PDF viewers)
img_chl = ax.imshow(img2d_show,
cmap=cmap,
norm=norm,
aspect=aspect,
alpha=alpha_plane,
vmin=vmin_plane,
vmax=vmax_plane,
origin=origin,
interpolation=interpolation)
img.append(img_chl)
# flip horizontally or vertically by inverting axes
if config.transform[config.Transforms.FLIP_HORIZ]:
if not ax.xaxis_inverted():
ax.invert_xaxis()
if config.transform[config.Transforms.FLIP_VERT]:
inverted = ax.yaxis_inverted()
if (origin in (None, "lower") and inverted) or (origin == "upper"
and not inverted):
# invert only if inversion state is same as expected from origin
# to avoid repeated inversions with repeated calls
ax.invert_yaxis()
return img
def detect_blobs_stack(filename_base, subimg_offset, subimg_size, coloc=False):
"""Detect blobs in a full stack, such as a whole large image.
Process channels in separate sets of blocks if their profiles specify
different block sizes.
Args:
filename_base (str):
subimg_offset (Sequence[int]): Sub-image offset as ``z,y,x`` to load
from :attr:`config.image5d`; defaults to None.
subimg_size (Sequence[int]): Sub-image size as ``z,y,x`` to load
from :attr:`config.image5d`; defaults to None.
coloc (bool): True to also detect blob-colocalizations based on image
intensity; defaults to False. For match-based colocalizations,
use the ``coloc_match`` task
(:meth:`magmap.colocalizer.StackColocalizer.colocalize_stack`)
instead.
Returns:
tuple[int, int, int], str, :class:`magmap.cv.detector.Blobs`:
Combined ccuracy metrics from :class:`magmap.cv.detector.verify_rois`,
feedback message from this same function, and detected blobs across
all channels in :attr:`magmap.settings.config.channel`.
"""
channels = plot_3d.setup_channels(config.image5d, config.channel, 4)[1]
if roi_prof.ROIProfile.is_identical_settings(
[config.get_roi_profile(c) for c in channels],
roi_prof.ROIProfile.BLOCK_SIZES):
print("Will process channels together in the same blocks")
channels = [channels]
else:
print("Will process channels in separate blocks defined by their "
"profiles")
cols = ("stats", "fdbk", "blobs")
detection_out = {}
for chl in channels:
# detect blobs in each channel separately unless all channels
# are combined in a single list
if not libmag.is_seq(chl):
chl = [chl]
blobs_out = detect_blobs_blocks(
filename_base, config.image5d, subimg_offset, subimg_size,
chl, config.truth_db_mode is config.TruthDBModes.VERIFY,
not config.grid_search_profile, config.image5d_is_roi, coloc)
for col, val in zip(cols, blobs_out):
detection_out.setdefault(col, []).append(val)
print("{}\n".format("-" * 80))
stats = None
fdbk = None
blobs_all = None
if "blobs" in detection_out and detection_out["blobs"]:
# join blobs and colocalizations from all channels and save archive
blobs_all = detection_out["blobs"][0]
blobs_all.blobs = libmag.combine_arrs(
[b.blobs for b in detection_out["blobs"]
if b.blobs is not None])
print("\nTotal blobs found across channels:", len(blobs_all.blobs))
detector.show_blobs_per_channel(blobs_all.blobs)
blobs_all.colocalizations = libmag.combine_arrs(
[b.colocalizations for b in detection_out["blobs"]
if b.colocalizations is not None])
blobs_all.save_archive()
print()
# combine verification stats and feedback messages
stats = libmag.combine_arrs(
detection_out["stats"], fn=np.sum)
fdbk = "\n".join(
[f for f in detection_out["fdbk"] if f is not None])
return stats, fdbk, blobs_all
def _build_stack(ax,
images,
process_fnc,
rescale=1,
aspect=None,
origin=None,
cmaps_labels=None,
scale_bar=True,
start_planei=0):
"""Builds a stack of Matploblit 2D images.
Uses multiprocessing to load or resize each image.
Args:
images: Sequence of images. For import, each "image" is a path to
and image file. For export, each "image" is a sequence of
planes, with the first sequence assumed to an atlas,
followed by labels-based images, each consisting of
corresponding planes.
process_fnc: Function to process each image through multiprocessing,
where the function should take an index and image and return the
index and processed plane.
rescale (float): Rescale factor; defaults to 1.
cmaps_labels: Sequence of colormaps for labels-based images;
defaults to None. Length should be equal to that of
``images`` - 1.
scale_bar: True to include scale bar; defaults to True.
start_planei (int): Index of start plane, used for labeling the
plane; defaults to 0. The plane is only annotated when
:attr:`config.plot_labels[config.PlotLabels.TEXT_POS]` is given
to specify the position of the text in ``x,y`` relative to the
axes.
Returns:
:List[List[:obj:`matplotlib.image.AxesImage`]]: Nested list of
axes image objects. The first list level contains planes, and
the second level are channels within each plane.
"""
# number of image types (eg atlas, labels) and corresponding planes
num_image_types = len(images)
if num_image_types < 1: return None
num_images = len(images[0])
if num_images < 1: return None
# Matplotlib figure for building the animation
plot_support.hide_axes(ax)
# import the images as Matplotlib artists via multiprocessing
plotted_imgs = [None] * num_images
img_shape = images[0][0].shape
target_size = np.multiply(img_shape, rescale).astype(int)
multichannel = images[0][0].ndim >= 3
if multichannel:
print("building stack for channel: {}".format(config.channel))
target_size = target_size[:-1]
StackPlaneIO.set_data(images)
pool = chunking.get_mp_pool()
pool_results = []
for i in range(num_images):
# add rotation argument if necessary
pool_results.append(
pool.apply_async(process_fnc, args=(i, target_size)))
# setup imshow parameters
colorbar = config.roi_profile["colorbar"]
cmaps_all = [config.cmaps, *cmaps_labels]
img_size = None
text_pos = config.plot_labels[config.PlotLabels.TEXT_POS]
for result in pool_results:
i, imgs = result.get()
if img_size is None: img_size = imgs[0].shape
# multiple artists can be shown at each frame by collecting
# each group of artists in a list; overlay_images returns
# a nested list containing a list for each image, which in turn
# contains a list of artists for each channel
ax_imgs = plot_support.overlay_images(ax,
aspect,
origin,
imgs,
None,
cmaps_all,
ignore_invis=True,
check_single=True)
if colorbar and len(ax_imgs) > 0 and len(ax_imgs[0]) > 0:
# add colorbar with scientific notation if outside limits
cbar = ax.figure.colorbar(ax_imgs[0][0], ax=ax, shrink=0.7)
plot_support.set_scinot(cbar.ax, lbls=None, units=None)
plotted_imgs[i] = np.array(ax_imgs).flatten()
if libmag.is_seq(text_pos) and len(text_pos) > 1:
# write plane index in axes rather than data coordinates
text = ax.text(*text_pos[:2],
"{}-plane: {}".format(
plot_support.get_plane_axis(config.plane),
start_planei + i),
transform=ax.transAxes,
color="w")
plotted_imgs[i] = [*plotted_imgs[i], text]
pool.close()
pool.join()
if scale_bar:
plot_support.add_scale_bar(ax, 1 / rescale, config.plane)
return plotted_imgs
def get_region_middle(labels_ref_lookup,
label_id,
labels_img,
scaling,
both_sides=False,
incl_children=True):
"""Approximate the middle position of a region by taking the middle
value of its sorted list of coordinates.
The region's coordinate sorting prioritizes z, followed by y, etc, meaning
that the middle value will be closest to the middle of z but may fall
be slightly away from midline in the other axes if this z does not
contain y/x's around midline. Getting the coordinate at the middle
of this list rather than another coordinate midway between other values
in the region ensures that the returned coordinate will reside within
the region itself, including non-contingous regions that may be
intermixed with coordinates not part of the region.
Args:
labels_ref_lookup (Dict[int, Dict]): The labels reference lookup,
assumed to be generated by :func:`ontology.create_reverse_lookup`
to look up by ID.
label_id (int, List[int]): ID of the label to find, or sequence of IDs.
labels_img (:obj:`np.ndarray`): The registered image whose intensity
values correspond to label IDs.
scaling (:obj:`np.ndarray`): Scaling factors as a Numpy array in z,y,x
for the labels image size compared with the experiment image.
both_sides (bool, List[bool]): True to include both sides, or
sequence of booleans corresponding to ``label_id``; defaults
to False.
incl_children (bool): True to include children of ``label_id``,
False to include only ``label_id``; defaults to True.
Returns:
List[int], :obj:`np.ndarray`, List[int]: ``coord``, the middle value
of a list of all coordinates in the region at the given ID;
``img_region``, a boolean mask of the region within ``labels_img``;
and ``region_ids``, a list of the IDs included in the region.
If ``labels_ref_lookup`` is None, all values are None.
"""
if not labels_ref_lookup:
return None, None, None
# gather IDs for label, including children and opposite sides
if not libmag.is_seq(label_id):
label_id = [label_id]
if not libmag.is_seq(both_sides):
both_sides = [both_sides]
region_ids = []
for region_id, both in zip(label_id, both_sides):
if incl_children:
# add children of the label +/- both sides
region_ids.extend(
get_children_from_id(labels_ref_lookup,
region_id,
incl_parent=True,
both_sides=both))
else:
# add the label +/- its mirrored version
region_ids.append(region_id)
if both:
region_ids.append(_mirror_label_ids(region_id))
# get a list of all the region's coordinates to sort
img_region = np.isin(labels_img, region_ids)
region_coords = np.where(img_region)
#print("region_coords:\n{}".format(region_coords))
def get_middle(coords):
# recursively get value at middle of list for each axis
sort_ind = np.lexsort(coords[::-1]) # last axis is primary key
num_coords = len(sort_ind)
if num_coords > 0:
mid_ind = sort_ind[int(num_coords / 2)]
mid = coords[0][mid_ind]
if len(coords) > 1:
# shift to next axis in tuple of coords
mask = coords[0] == mid
coords = tuple(c[mask] for c in coords[1:])
return (mid, *get_middle(coords))
return (mid, )
return None
coord = None
coord_labels = get_middle(region_coords)
if coord_labels:
print("coord_labels (unscaled): {}".format(coord_labels))
print("ID at middle coord: {} (in region? {})".format(
labels_img[coord_labels], img_region[coord_labels]))
coord = tuple(np.around(coord_labels / scaling).astype(np.int))
print("coord at middle: {}".format(coord))
return coord, img_region, region_ids
