def replace_vals(df, vals_from, vals_to, cols=None): """Replace values in a data frame for the given columns. Args: df (:obj:`pd.DataFrame`): Pandas data frame. vals_from (Any): Value or sequence of values to be replaced. vals_to (Any): Corresponding value or sequence of values to ``vals_from`` with which to replace. cols (Union[str, List[str]]): Column name or sequence of names to replace values; defaults to None to replace values in all columns. Returns: :obj:`pd.DataFrame`: Data frame with values replaced. """ # convert arguments to lists if cols is None or not libmag.is_seq(cols): cols = [cols] if not libmag.is_seq(vals_to): vals_to = [vals_to] if not libmag.is_seq(vals_from): vals_from = [vals_from] # parse NaN strings vals_from = [np.nan if libmag.is_nan(v) else v for v in vals_from] for col in cols: # replace values in specific columns, or whole data frame if no # columns are given df_col = df[col] if col else df df = df_col.replace(vals_from, vals_to) return df
def _filter_dict(d: Dict, fn_parse_val: Callable[[Any], Any]) -> Dict: """Recursively filter keys and values within nested dictionaries Args: d: Dictionary to filter. fn_parse_val: Function to apply to each value. Should call this parent function if deep recursion is desired. Returns: Filtered dictionary. """ out = {} for key, val in d.items(): if isinstance(val, dict): # recursively filter nested dictionaries val = fn_parse_val(val) elif libmag.is_seq(val): # filter each val within list val = [fn_parse_val(v) for v in val] else: # filter a single val val = fn_parse_val(val) # filter key key = fn_parse_val(key) out[key] = val return out
def pivot_with_conditions(df, index, columns, values, aggfunc="first"): """Pivot a data frame to columns with sub-columns for different conditions. For example, a table of metric values for different regions within each sample under different conditions will be reorganized to region columns that are each split into condition sub-columns. Args: df (:obj:`pd.DataFrame`): Data frame to pivot. index (str, List): Column name or sequence of columns specifying samples, generally a sequence to later unstack. columns (str, List): Column name or sequence of columns to pivot into separate columns. values (str): Column of values to move into new columns. aggfunc (func): Aggregation function for duplicates; defaults to "first" to take the first value. Returns: Tuple of the pivoted data frame and the list of pivoted columns. """ # use multi-level indexing; assumes that no duplicates exist for # a given index-pivot-column combo, and if they do, simply take 1st val df_lines = df.pivot_table( index=index, columns=columns, values=values, aggfunc=aggfunc) cols = df_lines.columns # may be fewer than orig if libmag.is_seq(index) and len(index) > 1: # move multi-index into separate sub-cols of each region and # reset index to access all columns df_lines = df_lines.unstack() df_lines = df_lines.reset_index() return df_lines, cols
def read_sitk_files(filename_sitk, reg_names=None): """Read files through SimpleITK and export to Numpy array format, loading associated metadata. Args: filename_sitk: Path to file in a format that can be read by SimpleITK. reg_names: Path or sequence of paths of registered names. Can be a registered suffix or a full path. Defaults to None. Returns: Image array in MagellanMapper image5d format. Associated metadata will have been loaded into module-level variables. Raises: ``FileNotFoundError`` if ``filename_sitk`` cannot be found, after attempting to load metadata from ``filename_np``. """ # load image via SimpleITK img_sitk = None loaded_path = filename_sitk if reg_names: img_nps = [] if not libmag.is_seq(reg_names): reg_names = [reg_names] for reg_name in reg_names: # load each registered suffix into list of images with same shape, # keeping first image in sitk format img, path = _load_reg_img_to_combine(filename_sitk, reg_name, img_nps) if img_sitk is None: img_sitk = img loaded_path = path if len(img_nps) > 1: # merge images into separate channels img_np = np.stack(img_nps, axis=img_nps[0].ndim) else: img_np = img_nps[0] else: # load filename_sitk directly if not os.path.exists(filename_sitk): raise FileNotFoundError( "could not find file {}".format(filename_sitk)) img_sitk, _ = read_sitk(filename_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if config.resolutions is None: # fallback to determining metadata directly from sitk file libmag.warn( "MagellanMapper image metadata file not loaded; will fallback to " "{} for metadata".format(loaded_path)) config.resolutions = np.array([img_sitk.GetSpacing()[::-1]]) print("set resolutions to {}".format(config.resolutions)) return img_np
def parse_enum_val(val): # recursively parse Enum values if isinstance(val, dict): val = _filter_dict(val, parse_enum_val) elif libmag.is_seq(val): val = [parse_enum_val(v) for v in val] elif isinstance(val, str): val_split = val.split(".") if len(val_split) > 1 and val_split[0] in enums: # replace with the corresponding Enum class val = enums[val_split[0]][val_split[1]] return val
def handle_extracted_plane(): # get sub-plot and hide x/y axes ax = axs if libmag.is_seq(ax): ax = axs[imgi] plot_support.hide_axes(ax) # multiple artists can be shown at each frame by collecting # each group of artists in a list; overlay_images returns # a nested list containing a list for each image, which in turn # contains a list of artists for each channel ax_imgs = plot_support.overlay_images(ax, self.aspect, self.origin, imgs, None, cmaps_all, ignore_invis=True, check_single=True) if (colorbar is not None and len(ax_imgs) > 0 and len(ax_imgs[0]) > 0 and imgi == 0): # add colorbar with scientific notation if outside limits cbar = ax.figure.colorbar(ax_imgs[0][0], ax=ax, **colorbar) plot_support.set_scinot(cbar.ax, lbls=None, units=None) plotted_imgs[imgi] = np.array(ax_imgs).flatten() if libmag.is_seq(text_pos) and len(text_pos) > 1: # write plane index in axes rather than data coordinates text = ax.text(*text_pos[:2], "{}-plane: {}".format( plot_support.get_plane_axis(config.plane), self.start_planei + imgi), transform=ax.transAxes, color="w") plotted_imgs[imgi] = [*plotted_imgs[imgi], text] if scale_bar: plot_support.add_scale_bar(ax, 1 / self.rescale, config.plane)
def convert_numpy_val(val): # recursively convert Numpy data types to primitives if isinstance(val, dict): val = _filter_dict(val, convert_numpy_val) elif libmag.is_seq(val): # also replaces any tuples with lists, avoiding tuple flags in # the output file for simplicity val = [convert_numpy_val(v) for v in val] else: try: val = val.item() except AttributeError: pass return val
def parse_enum(d): # recursively parse Enum keys and values within nested dictionaries out = {} for key, val in d.items(): if isinstance(val, dict): # recursively parse nested dictionaries val = parse_enum(val) elif libmag.is_seq(val): # parse vals within lists val = [parse_enum_val(v) for v in val] else: # parse a single val val = parse_enum_val(val) key = parse_enum_val(key) out[key] = val return out
def update_max_intens_proj(self, shape, display=False): """Update max intensity projection planes. Args: shape (Union[int, Sequence[int]]): Number of planes for all Plot Editors or sequence of plane counts in ``z,y,x``. display (bool): True to trigger an update in the Plot Editors; defaults to False. """ self._max_intens_proj = shape is_seq = libmag.is_seq(shape) for i, ed in enumerate(self.plot_eds.values()): n = shape[i] if is_seq else shape if n != ed.max_intens_proj: ed.max_intens_proj = n if display: ed.update_coord()
def compress_file(path_in, path_out=None): """Compress a file or files by tar archving and compressing with. Assumes that ``tar`` and ``zstd`` are available shell commands. Args: path_in (str, List[str]): Input file path; can be a sequence of paths, in which case the first path will determine the working directory and the default ``path_out`` if none is given. path_out (str): Output file path; defaults to None to use the same path as ``path_in`` with ``.tar.zstd`` appended. """ tar_args = ["tar", "cfv", "-"] if libmag.is_seq(path_in): # set up sequence of paths, using the first path as template path_first = path_in[0] tar_args.extend([os.path.basename(p) for p in path_in]) else: # set up a single path path_first = path_in tar_args.append(os.path.basename(path_in)) if path_out is None: # default to using the first path for compressed file name path_out = os.path.splitext(path_first)[0] + ".tar.zst" # use first path for working directory wd = os.path.dirname(path_first) if not wd: wd = None # archive with tar and pipe to zstd for compression print("Compressing \"{}\" to \"{}\"".format(path_in, path_out)) tar = subprocess.Popen(tar_args, cwd=wd, stdout=subprocess.PIPE, bufsize=0) with open(path_out, "wb") as f: zstd = subprocess.Popen(["pzstd", "-v"], stdin=tar.stdout, stdout=f, bufsize=0) tar.stdout.close() # appears to work for large image with memory issues because data # are backed by files stderr = zstd.communicate()[1] if stderr: print(stderr)
def preprocess_img(image5d, preprocs, channel, out_path): """Pre-process an image in 3D. Args: image5d (:obj:`np.ndarray`): 5D array in t,z,y,x[,c]. preprocs (Union[str, list[str]]): Pre-processing tasks that will be converted to enums in :class:`config.PreProcessKeys` to perform in the order given. channel (int): Channel to preprocess, or None for all channels. out_path (str): Output base path. Returns: :obj:`np.ndarray`: The pre-processed image array. """ if preprocs is None: print("No preprocessing tasks to perform, skipping") return if not libmag.is_seq(preprocs): preprocs = [preprocs] roi = image5d[0] for preproc in preprocs: # perform global pre-processing task task = libmag.get_enum(preproc, config.PreProcessKeys) _logger.info("Pre-processing task: %s", task) if task is config.PreProcessKeys.SATURATE: roi = plot_3d.saturate_roi(roi, channel=channel) elif task is config.PreProcessKeys.DENOISE: roi = plot_3d.denoise_roi(roi, channel) elif task is config.PreProcessKeys.REMAP: roi = plot_3d.remap_intensity(roi, channel) elif task is config.PreProcessKeys.ROTATE: roi = rotate_img(roi) else: _logger.warn("No preprocessing task found for: %s", preproc) # save to new file image5d = importer.roi_to_image5d(roi) importer.save_np_image(image5d, out_path) return image5d
def _mirror_label_ids(label_ids, combine=False): """Mirror label IDs, assuming that a "mirrored" ID is the negative of the given ID. Args: label_ids (Union[int, List[int]]): Single ID or sequence of IDs. combine (bool): True to return a list of ``label_ids`` along with their mirrored IDs; defaults to False to return on the mirrored IDs. Returns: Union[int, List[int]]: A single mirrored ID if ``label_ids`` is one ID and ``combine`` is False, or a list of IDs. """ if libmag.is_seq(label_ids): mirrored = [-1 * n for n in label_ids] if combine: mirrored = list(label_ids).extend(mirrored) else: mirrored = -1 * label_ids if combine: mirrored = [label_ids, mirrored] return mirrored
def load_s3_file(bucket_name, key): """Load a file or files in AWS S3, retrieving the metadata without the object itself. Args: bucket_name (str): Name of bucket. key (str, List[str}): Key or sequence of keys within bucket. Returns: dict[str, :obj:`boto3.resources.factory.s3.Object`]: Dictionary mapping keys to their corresponding successfully loaded S3 objects. """ s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) if libmag.is_seq(key): # get all objects starting with common part of paths prefix = os.path.commonprefix(key) else: prefix = key key = [key] loaded_objs = {} objs = bucket.objects.filter(Prefix=prefix) for obj in objs: if obj.key in key: # ensure an exact match with a key in the given list try: obj.load() loaded_objs[obj.key] = obj except ClientError as e: print(e) print("Unable to load object at bucket \"{}\", key \"{}\"". format(bucket_name, obj.key)) print() _show_missing_keys(bucket_name, key, loaded_objs.keys()) return loaded_objs
def pivot_with_conditions(df, index, columns, values, aggfunc="first"): """Pivot a data frame to columns with sub-columns for different conditions. For example, a table of metric values for different regions within each sample under different conditions will be reorganized to region columns that are each split into condition sub-columns. Args: df (:class:`pandas.DataFrame`): Data frame to pivot. index (Union[str, list[str]]): Column name or list of names specifying the index for the output table. columns (Union[str, list[str]]): Name or list of names of columns whose values are pivoted into separate columns. values (str): Name of column whose values are moved into the new columns specified by ``columns``. aggfunc (func): Aggregation function for duplicates; defaults to "first" to take the first value. Returns: :class:`pandas.DataFrame`, list[str]: The pivoted data frame and list of pivoted columns. """ # use multi-level indexing; assumes that no duplicates exist for # a given index-pivot-column combo, and if they do, simply take 1st val df_lines = df.pivot_table(index=index, columns=columns, values=values, aggfunc=aggfunc) cols = df_lines.columns # may be fewer than orig if libmag.is_seq(index) and len(index) > 1: # move multi-index into separate sub-cols of each region and # reset index to access all columns df_lines = df_lines.unstack() df_lines = df_lines.reset_index() return df_lines, cols
def __call__(self, x, y): """Get the pixel display message. Args: x (int): x-data coordinate. y (int): y-data coordinate. Returns: str: Message showing ``x,y`` coordinates, intensity values, and corresponding RGB label for each overlaid image at the given location. """ coord = (int(y), int(x)) rgb = None output = [] main_img_shape = self.imgs[0].shape[:2] for i, img in enumerate(self.imgs): # scale coordinates from axes space, based on main image, to # given image's space to get pixel from given image scale = np.divide(img.shape[:2], main_img_shape) coord_img = tuple(np.multiply(coord, scale).astype(int)) if any(np.less(coord_img, 0)) or any( np.greater_equal(coord_img, img.shape[:len(coord_img)])): # no corresponding px for the image px = "n/a" else: # get the corresponding intensity value, truncating floats px = img[coord_img] if i == 1: # for the label image, get its RGB value ax_img = self.ax_imgs[i][0] if self.cmap_labels: label_rgb = self.cmap_labels( self.cmap_labels.convert_img_labels(px)) else: label_rgb = ax_img.cmap(ax_img.norm(px)) rgb = "RGB for label {}: {}".format( px, tuple(np.multiply(label_rgb[:3], 255).astype(int))) if isinstance(px, float): px = "{:.4f}".format(px) orig_coord = coord if self.shapes: # scale coordinates from axes space to given original image's # space, accounting for any resizing and downsampling shape = self.shapes if libmag.is_seq(self.shapes[0]): # overlaid images may have different shapes shape = self.shapes[i] scale = np.divide(shape[:2], main_img_shape) orig_coord = tuple(np.multiply(coord, scale).astype(int)) if self.offset: # shift for offset given in original image's space off = self.offset if libmag.is_seq(self.offset[0]): # overlaid images may have different offsets off = self.offset[i] orig_coord = np.add(orig_coord, off) output.append("Image {}: x={}, y={}, px={}".format( i, orig_coord[1], orig_coord[0], px)) # join output message if rgb: output.append(rgb) msg = "; ".join(output) return msg
def read_sitk_files(filename_sitk, reg_names=None, return_sitk=False): """Read an image file through SimpleITK and export to Numpy array format, with support for combining multiple registered image files into a single image. Also sets up spacing from the first loaded image in :attr:`magmap.settings.config.resolutions` if not already set. Args: filename_sitk (str): Path to file in a format that can be read by SimpleITK. reg_names (Union[str, List[str]]): Path or sequence of paths of registered names. Can be a registered suffix or a full path. Defaults to None to open ``filename_sitk`` as-is through :meth:`read_sitk`. return_sitk (bool): True to return the loaded SimpleITK Image object. Returns: :class:`numpy.ndarray`: Image array in Numpy 3D format (or 4D if multi-channel). Associated metadata is loaded into :module:`config` attributes. If ``return_sitk`` is True, also returns the first loaded image in SimpleITK format. Raises: ``FileNotFoundError`` if ``filename_sitk`` cannot be found, after attempting to load metadata from ``filename_np``. """ # load image via SimpleITK img_sitk = None loaded_path = filename_sitk if reg_names: img_nps = [] if not libmag.is_seq(reg_names): reg_names = [reg_names] for reg_name in reg_names: # load each registered suffix into list of images with same shape, # keeping first image in sitk format img, path = _load_reg_img_to_combine(filename_sitk, reg_name, img_nps) if img_sitk is None: img_sitk = img loaded_path = path if len(img_nps) > 1: # merge images into separate channels img_np = np.stack(img_nps, axis=img_nps[0].ndim) else: img_np = img_nps[0] else: # load filename_sitk directly if not os.path.exists(filename_sitk): raise FileNotFoundError( "could not find file {}".format(filename_sitk)) img_sitk, _ = read_sitk(filename_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if config.resolutions is None: # fallback to determining metadata directly from sitk file libmag.warn( "MagellanMapper image metadata file not loaded; will fallback to " "{} for metadata".format(loaded_path)) config.resolutions = np.array([img_sitk.GetSpacing()[::-1]]) print("set resolutions to {}".format(config.resolutions)) if return_sitk: return img_np, img_sitk return img_np
def delete_s3_file(bucket_name, key, hard=False, dryrun=False): """Delete a file on AWS S3. Args: bucket_name (str): Name of bucket. key (str, List[str]): Key or sequence of keys within bucket. hard (bool): True to delete all versions associated with the key including any delete markers, effectively deleting the object on S3 permanently. Defaults to False, in which case only a delete marker will be applied if versioning is on; without versioning, the file will be permanently deleted. dryrun (bool): True to print paths without uploading; defaults to False. Returns: List[str]: Sequence of successfully deleted keys. For versioned files, all versions of the given object must have been deleted without error to return True. """ s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) if libmag.is_seq(key): # get all objects starting with common part of paths prefix = os.path.commonprefix(key) else: prefix = key key = [key] print("Deleting selected file object(s) in bucket \"{}\", prefix \"{}\"" " (hard delete {})".format(bucket_name, prefix, hard)) deleted_keys = [] if hard: # hard delete finds all object versions associated with the keys, # including delete markers, and permanently deletes them vers = bucket.object_versions.filter(Prefix=prefix) for ver in vers: if ver.object_key in key: # ensure an exact match with a key in the given list print("Permanently deleting \"{}\", versionId \"{}\"".format( ver.object_key, ver.id)) if dryrun: print("Deletion of \"{}\" set to dry run, so skipping". format(ver.object_key)) else: ver.delete() if ver.object_key not in deleted_keys: deleted_keys.append(ver.object_key) else: # default delete, which adds a delete marker for buckets with # versioning or permanently deletes objects if no versioning # obj = bucket.Object(key) objs = bucket.objects.filter(Prefix=prefix) for obj in objs: if obj.key in key: # ensure an exact match try: print("Deleting (or setting delete marker for) \"{}\"". format(obj.key)) if dryrun: print("Deletion of {} set to dry run, so skipping". format(obj.key)) else: obj.delete() deleted_keys.append(obj.key) except botocore.exceptions.ClientError as e: print(e) print("Could not find key \"{}\" to delete".format(key)) print() _show_missing_keys(bucket_name, key, deleted_keys) return deleted_keys
def read_sitk_files( filename_sitk: str, reg_names: Optional[Union[str, Sequence[str]]] = None, return_sitk: bool = False, make_3d: bool = False ) -> Union["np_io.Image5d", Tuple["np_io.Image5d", sitk.Image]]: """Read image files through SimpleITK. Supports identifying files based on registered suffixes and combining multiple registered image files into a single image. Also sets up spacing from the first loaded image in :attr:`magmap.settings.config.resolutions` if not already set. Args: filename_sitk: Path to file in a format that can be read by SimpleITK. reg_names: Path or sequence of paths of registered names. Can be a registered suffix or a full path. Defaults to None to open ``filename_sitk`` as-is through :meth:`read_sitk`. return_sitk: True to return the loaded SimpleITK Image object. make_3d: True to convert 2D images to 3D; defaults to False. Returns: ``img5d``: Image5d instance with the loaded image in Numpy 5D format (or 4D if not multi-channel, and 3D if originally 2D and ``make_3d`` is False). Associated metadata is loaded into :module:`magmap.settings.config` attributes. If ``return_sitk`` is True, also returns the first loaded image in SimpleITK format. Raises: ``FileNotFoundError`` if ``filename_sitk`` cannot be found, after attempting to load metadata from ``filename_np``. """ # load image via SimpleITK img_sitk = None loaded_path = filename_sitk if reg_names: img_nps = [] if not libmag.is_seq(reg_names): reg_names = [reg_names] for reg_name in reg_names: # load each registered suffix into list of images with same shape, # keeping first image in sitk format img, path = _load_reg_img_to_combine(filename_sitk, reg_name, img_nps) if img_sitk is None: img_sitk = img loaded_path = path if len(img_nps) > 1: # merge images into separate channels img_np = np.stack(img_nps, axis=img_nps[0].ndim) else: img_np = img_nps[0] else: # load filename_sitk directly if not os.path.exists(filename_sitk): raise FileNotFoundError( "could not find file {}".format(filename_sitk)) img_sitk, _ = read_sitk(filename_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if make_3d: # convert 2D images to 3D # TODO: consider converting img_np to 3D regardless so array in # Image5d is at least 4D img_sitk = _make_3d(img_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if config.resolutions is None: # fallback to determining metadata directly from sitk file libmag.warn( "MagellanMapper image metadata file not loaded; will fallback to " "{} for metadata".format(loaded_path)) config.resolutions = np.array([img_sitk.GetSpacing()[::-1]]) print("set resolutions to {}".format(config.resolutions)) # add time axis and insert into Image5d with original name img5d = np_io.Image5d(img_np[None], filename_sitk, img_io=config.LoadIO.SITK) if return_sitk: return img5d, img_sitk return img5d
def build_stack(self, axs: List, scale_bar: bool = True, fit: bool = False) -> Optional[List]: """Builds a stack of Matploblit 2D images. Uses multiprocessing to load or resize each image. Args: axs: Sub-plot axes. scale_bar: True to include scale bar; defaults to True. fit: True to fit the figure frame to the resulting image. Returns: :List[List[:obj:`matplotlib.image.AxesImage`]]: Nested list of axes image objects. The first list level contains planes, and the second level are channels within each plane. """ def handle_extracted_plane(): # get sub-plot and hide x/y axes ax = axs if libmag.is_seq(ax): ax = axs[imgi] plot_support.hide_axes(ax) # multiple artists can be shown at each frame by collecting # each group of artists in a list; overlay_images returns # a nested list containing a list for each image, which in turn # contains a list of artists for each channel ax_imgs = plot_support.overlay_images(ax, self.aspect, self.origin, imgs, None, cmaps_all, ignore_invis=True, check_single=True) if (colorbar is not None and len(ax_imgs) > 0 and len(ax_imgs[0]) > 0 and imgi == 0): # add colorbar with scientific notation if outside limits cbar = ax.figure.colorbar(ax_imgs[0][0], ax=ax, **colorbar) plot_support.set_scinot(cbar.ax, lbls=None, units=None) plotted_imgs[imgi] = np.array(ax_imgs).flatten() if libmag.is_seq(text_pos) and len(text_pos) > 1: # write plane index in axes rather than data coordinates text = ax.text(*text_pos[:2], "{}-plane: {}".format( plot_support.get_plane_axis(config.plane), self.start_planei + imgi), transform=ax.transAxes, color="w") plotted_imgs[imgi] = [*plotted_imgs[imgi], text] if scale_bar: plot_support.add_scale_bar(ax, 1 / self.rescale, config.plane) # number of image types (eg atlas, labels) and corresponding planes num_image_types = len(self.images) if num_image_types < 1: return None num_images = len(self.images[0]) if num_images < 1: return None # import the images as Matplotlib artists via multiprocessing plotted_imgs: List = [None] * num_images img_shape = self.images[0][0].shape target_size = np.multiply(img_shape, self.rescale).astype(int) multichannel = self.images[0][0].ndim >= 3 if multichannel: print("building stack for channel: {}".format(config.channel)) target_size = target_size[:-1] # setup imshow parameters colorbar = config.roi_profile["colorbar"] cmaps_all = [config.cmaps, *self.cmaps_labels] text_pos = config.plot_labels[config.PlotLabels.TEXT_POS] StackPlaneIO.set_data(self.images) pool_results = None pool = None multiprocess = self.rescale != 1 if multiprocess: # set up multiprocessing initializer = None initargs = None if not chunking.is_fork(): # set up labels image as a shared array for spawned mode initializer, initargs = StackPlaneIO.build_pool_init( OrderedDict([(i, img) for i, img in enumerate(self.images)])) pool = chunking.get_mp_pool(initializer, initargs) pool_results = [] for i in range(num_images): # add rotation argument if necessary args = (i, target_size) if pool is None: # extract and handle without multiprocessing imgi, imgs = self.fn_process(*args) handle_extracted_plane() else: # extract plane in multiprocessing pool_results.append( pool.apply_async(self.fn_process, args=args)) if multiprocess: # handle multiprocessing output for result in pool_results: imgi, imgs = result.get() handle_extracted_plane() pool.close() pool.join() if fit and plotted_imgs: # fit each figure to its first available image for ax_img in plotted_imgs: # images may be flattened AxesImage, array of AxesImage and # Text, or None if alpha set to 0 if libmag.is_seq(ax_img): ax_img = ax_img[0] if isinstance(ax_img, AxesImage): plot_support.fit_frame_to_image(ax_img.figure, ax_img.get_array().shape, self.aspect) return plotted_imgs
def imshow_multichannel(ax, img2d, channel, cmaps, aspect, alpha=None, vmin=None, vmax=None, origin=None, interpolation=None, norms=None, nan_color=None, ignore_invis=False): """Show multichannel 2D image with channels overlaid over one another. Applies :attr:`config.transform` with :obj:`config.Transforms.ROTATE` to rotate images. If not available, also checks the first element in :attr:``config.flip`` to rotate the image by 180 degrees. Applies :attr:`config.transform` with :obj:`config.Transforms.FLIP_HORIZ` and :obj:`config.Transforms.FLIP_VERT` to invert images. Args: ax: Axes plot. img2d: 2D image either as 2D (y, x) or 3D (y, x, channel) array. channel: Channel to display; if None, all channels will be shown. cmaps: List of colormaps corresponding to each channel. Colormaps can be the names of specific maps in :mod:``config``. aspect: Aspect ratio. alpha (float, List[float]): Transparency level for all channels or sequence of levels for each channel. If any value is 0, the corresponding image will not be output. Defaults to None to use 1. vmin (float, List[float]): Scalar or sequence of vmin levels for all channels; defaults to None. vmax (float, List[float]): Scalar or sequence of vmax levels for all channels; defaults to None. origin: Image origin; defaults to None. interpolation: Type of interpolation; defaults to None. norms: List of normalizations, which should correspond to ``cmaps``. nan_color (str): String of color to use for NaN values; defaults to None to leave these pixels empty. ignore_invis (bool): True to give None instead of an ``AxesImage`` object that would be invisible; defaults to False. Returns: List of ``AxesImage`` objects. """ # assume that 3D array has a channel dimension multichannel, channels = plot_3d.setup_channels(img2d, channel, 2) img = [] num_chls = len(channels) if alpha is None: alpha = 1 if num_chls > 1 and not libmag.is_seq(alpha): # if alphas not explicitly set per channel, make all channels more # translucent at a fixed value that is higher with more channels alpha /= np.sqrt(num_chls + 1) # transform image based on config parameters rotate = config.transform[config.Transforms.ROTATE] if rotate is not None: last_axis = img2d.ndim - 1 if multichannel: last_axis -= 1 # use first rotation value img2d = np.rot90(img2d, libmag.get_if_within(rotate, 0), (last_axis - 1, last_axis)) for chl in channels: img2d_show = img2d[..., chl] if multichannel else img2d cmap = None if cmaps is None else cmaps[chl] norm = None if norms is None else norms[chl] cmap = colormaps.get_cmap(cmap) if cmap is not None and nan_color: # given color for masked values such as NaNs to distinguish from 0 cmap.set_bad(color=nan_color) # get setting corresponding to the channel index, or use the value # directly if it is a scalar vmin_plane = libmag.get_if_within(vmin, chl) vmax_plane = libmag.get_if_within(vmax, chl) alpha_plane = libmag.get_if_within(alpha, chl) img_chl = None if not ignore_invis or alpha_plane > 0: # skip display if alpha is 0 to avoid outputting a hidden image # that may show up in other renderers (eg PDF viewers) img_chl = ax.imshow(img2d_show, cmap=cmap, norm=norm, aspect=aspect, alpha=alpha_plane, vmin=vmin_plane, vmax=vmax_plane, origin=origin, interpolation=interpolation) img.append(img_chl) # flip horizontally or vertically by inverting axes if config.transform[config.Transforms.FLIP_HORIZ]: if not ax.xaxis_inverted(): ax.invert_xaxis() if config.transform[config.Transforms.FLIP_VERT]: inverted = ax.yaxis_inverted() if (origin in (None, "lower") and inverted) or (origin == "upper" and not inverted): # invert only if inversion state is same as expected from origin # to avoid repeated inversions with repeated calls ax.invert_yaxis() return img
def detect_blobs_stack(filename_base, subimg_offset, subimg_size, coloc=False): """Detect blobs in a full stack, such as a whole large image. Process channels in separate sets of blocks if their profiles specify different block sizes. Args: filename_base (str): subimg_offset (Sequence[int]): Sub-image offset as ``z,y,x`` to load from :attr:`config.image5d`; defaults to None. subimg_size (Sequence[int]): Sub-image size as ``z,y,x`` to load from :attr:`config.image5d`; defaults to None. coloc (bool): True to also detect blob-colocalizations based on image intensity; defaults to False. For match-based colocalizations, use the ``coloc_match`` task (:meth:`magmap.colocalizer.StackColocalizer.colocalize_stack`) instead. Returns: tuple[int, int, int], str, :class:`magmap.cv.detector.Blobs`: Combined ccuracy metrics from :class:`magmap.cv.detector.verify_rois`, feedback message from this same function, and detected blobs across all channels in :attr:`magmap.settings.config.channel`. """ channels = plot_3d.setup_channels(config.image5d, config.channel, 4)[1] if roi_prof.ROIProfile.is_identical_settings( [config.get_roi_profile(c) for c in channels], roi_prof.ROIProfile.BLOCK_SIZES): print("Will process channels together in the same blocks") channels = [channels] else: print("Will process channels in separate blocks defined by their " "profiles") cols = ("stats", "fdbk", "blobs") detection_out = {} for chl in channels: # detect blobs in each channel separately unless all channels # are combined in a single list if not libmag.is_seq(chl): chl = [chl] blobs_out = detect_blobs_blocks( filename_base, config.image5d, subimg_offset, subimg_size, chl, config.truth_db_mode is config.TruthDBModes.VERIFY, not config.grid_search_profile, config.image5d_is_roi, coloc) for col, val in zip(cols, blobs_out): detection_out.setdefault(col, []).append(val) print("{}\n".format("-" * 80)) stats = None fdbk = None blobs_all = None if "blobs" in detection_out and detection_out["blobs"]: # join blobs and colocalizations from all channels and save archive blobs_all = detection_out["blobs"][0] blobs_all.blobs = libmag.combine_arrs( [b.blobs for b in detection_out["blobs"] if b.blobs is not None]) print("\nTotal blobs found across channels:", len(blobs_all.blobs)) detector.show_blobs_per_channel(blobs_all.blobs) blobs_all.colocalizations = libmag.combine_arrs( [b.colocalizations for b in detection_out["blobs"] if b.colocalizations is not None]) blobs_all.save_archive() print() # combine verification stats and feedback messages stats = libmag.combine_arrs( detection_out["stats"], fn=np.sum) fdbk = "\n".join( [f for f in detection_out["fdbk"] if f is not None]) return stats, fdbk, blobs_all
def _build_stack(ax, images, process_fnc, rescale=1, aspect=None, origin=None, cmaps_labels=None, scale_bar=True, start_planei=0): """Builds a stack of Matploblit 2D images. Uses multiprocessing to load or resize each image. Args: images: Sequence of images. For import, each "image" is a path to and image file. For export, each "image" is a sequence of planes, with the first sequence assumed to an atlas, followed by labels-based images, each consisting of corresponding planes. process_fnc: Function to process each image through multiprocessing, where the function should take an index and image and return the index and processed plane. rescale (float): Rescale factor; defaults to 1. cmaps_labels: Sequence of colormaps for labels-based images; defaults to None. Length should be equal to that of ``images`` - 1. scale_bar: True to include scale bar; defaults to True. start_planei (int): Index of start plane, used for labeling the plane; defaults to 0. The plane is only annotated when :attr:`config.plot_labels[config.PlotLabels.TEXT_POS]` is given to specify the position of the text in ``x,y`` relative to the axes. Returns: :List[List[:obj:`matplotlib.image.AxesImage`]]: Nested list of axes image objects. The first list level contains planes, and the second level are channels within each plane. """ # number of image types (eg atlas, labels) and corresponding planes num_image_types = len(images) if num_image_types < 1: return None num_images = len(images[0]) if num_images < 1: return None # Matplotlib figure for building the animation plot_support.hide_axes(ax) # import the images as Matplotlib artists via multiprocessing plotted_imgs = [None] * num_images img_shape = images[0][0].shape target_size = np.multiply(img_shape, rescale).astype(int) multichannel = images[0][0].ndim >= 3 if multichannel: print("building stack for channel: {}".format(config.channel)) target_size = target_size[:-1] StackPlaneIO.set_data(images) pool = chunking.get_mp_pool() pool_results = [] for i in range(num_images): # add rotation argument if necessary pool_results.append( pool.apply_async(process_fnc, args=(i, target_size))) # setup imshow parameters colorbar = config.roi_profile["colorbar"] cmaps_all = [config.cmaps, *cmaps_labels] img_size = None text_pos = config.plot_labels[config.PlotLabels.TEXT_POS] for result in pool_results: i, imgs = result.get() if img_size is None: img_size = imgs[0].shape # multiple artists can be shown at each frame by collecting # each group of artists in a list; overlay_images returns # a nested list containing a list for each image, which in turn # contains a list of artists for each channel ax_imgs = plot_support.overlay_images(ax, aspect, origin, imgs, None, cmaps_all, ignore_invis=True, check_single=True) if colorbar and len(ax_imgs) > 0 and len(ax_imgs[0]) > 0: # add colorbar with scientific notation if outside limits cbar = ax.figure.colorbar(ax_imgs[0][0], ax=ax, shrink=0.7) plot_support.set_scinot(cbar.ax, lbls=None, units=None) plotted_imgs[i] = np.array(ax_imgs).flatten() if libmag.is_seq(text_pos) and len(text_pos) > 1: # write plane index in axes rather than data coordinates text = ax.text(*text_pos[:2], "{}-plane: {}".format( plot_support.get_plane_axis(config.plane), start_planei + i), transform=ax.transAxes, color="w") plotted_imgs[i] = [*plotted_imgs[i], text] pool.close() pool.join() if scale_bar: plot_support.add_scale_bar(ax, 1 / rescale, config.plane) return plotted_imgs
def get_region_middle(labels_ref_lookup, label_id, labels_img, scaling, both_sides=False, incl_children=True): """Approximate the middle position of a region by taking the middle value of its sorted list of coordinates. The region's coordinate sorting prioritizes z, followed by y, etc, meaning that the middle value will be closest to the middle of z but may fall be slightly away from midline in the other axes if this z does not contain y/x's around midline. Getting the coordinate at the middle of this list rather than another coordinate midway between other values in the region ensures that the returned coordinate will reside within the region itself, including non-contingous regions that may be intermixed with coordinates not part of the region. Args: labels_ref_lookup (Dict[int, Dict]): The labels reference lookup, assumed to be generated by :func:`ontology.create_reverse_lookup` to look up by ID. label_id (int, List[int]): ID of the label to find, or sequence of IDs. labels_img (:obj:`np.ndarray`): The registered image whose intensity values correspond to label IDs. scaling (:obj:`np.ndarray`): Scaling factors as a Numpy array in z,y,x for the labels image size compared with the experiment image. both_sides (bool, List[bool]): True to include both sides, or sequence of booleans corresponding to ``label_id``; defaults to False. incl_children (bool): True to include children of ``label_id``, False to include only ``label_id``; defaults to True. Returns: List[int], :obj:`np.ndarray`, List[int]: ``coord``, the middle value of a list of all coordinates in the region at the given ID; ``img_region``, a boolean mask of the region within ``labels_img``; and ``region_ids``, a list of the IDs included in the region. If ``labels_ref_lookup`` is None, all values are None. """ if not labels_ref_lookup: return None, None, None # gather IDs for label, including children and opposite sides if not libmag.is_seq(label_id): label_id = [label_id] if not libmag.is_seq(both_sides): both_sides = [both_sides] region_ids = [] for region_id, both in zip(label_id, both_sides): if incl_children: # add children of the label +/- both sides region_ids.extend( get_children_from_id(labels_ref_lookup, region_id, incl_parent=True, both_sides=both)) else: # add the label +/- its mirrored version region_ids.append(region_id) if both: region_ids.append(_mirror_label_ids(region_id)) # get a list of all the region's coordinates to sort img_region = np.isin(labels_img, region_ids) region_coords = np.where(img_region) #print("region_coords:\n{}".format(region_coords)) def get_middle(coords): # recursively get value at middle of list for each axis sort_ind = np.lexsort(coords[::-1]) # last axis is primary key num_coords = len(sort_ind) if num_coords > 0: mid_ind = sort_ind[int(num_coords / 2)] mid = coords[0][mid_ind] if len(coords) > 1: # shift to next axis in tuple of coords mask = coords[0] == mid coords = tuple(c[mask] for c in coords[1:]) return (mid, *get_middle(coords)) return (mid, ) return None coord = None coord_labels = get_middle(region_coords) if coord_labels: print("coord_labels (unscaled): {}".format(coord_labels)) print("ID at middle coord: {} (in region? {})".format( labels_img[coord_labels], img_region[coord_labels])) coord = tuple(np.around(coord_labels / scaling).astype(np.int)) print("coord at middle: {}".format(coord)) return coord, img_region, region_ids