def cluster_blobs(img_path, suffix=None): """Cluster blobs and save to Numpy archive. Args: img_path (str): Base path from which registered labels and blobs files will be found and output blobs file save location will be constructed. suffix (str): Suffix for ``path``; defaults to None. Returns: """ mod_path = img_path if suffix is not None: mod_path = libmag.insert_before_ext(img_path, suffix) labels_img_np = sitk_io.load_registered_img( mod_path, config.RegNames.IMG_LABELS.value) blobs = detector.Blobs().load_blobs(np_io.img_to_blobs_path(img_path)) scaling, res = np_io.find_scaling(img_path, labels_img_np.shape) if blobs is None: libmag.warn("unable to load nuclei coordinates") return # append label IDs to blobs and scale to make isotropic blobs_clus = ClusterByLabel.cluster_by_label(blobs.blobs[:, :3], labels_img_np, scaling, res) print(blobs_clus) out_path = libmag.combine_paths(mod_path, config.SUFFIX_BLOB_CLUSTERS) np.save(out_path, blobs_clus)
def animate_imgs(base_path, plotted_imgs, delay, ext=None, suffix=None): """Export to an animated image. Defaults to an animated GIF unless ``ext`` specifies otherwise. Requires ``FFMpeg`` for MP4 file format exports and ``ImageMagick`` for all other types of exports. Args: base_path (str): String from which an output path will be constructed. plotted_imgs (List[:obj:`matplotlib.image.AxesImage]): Sequence of images to include in the animation. delay (int): Delay between image display in ms. If None, the delay will defaul to 100ms. ext (str): Extension to use when saving, without the period. Defaults to None, in which case "gif" will be used. suffix (str): String to append to output path before extension; defaults to None to ignore. """ # set up animation output path and time interval if ext is None: ext = "gif" out_path = libmag.combine_paths(base_path, "animated", ext=ext) if suffix: out_path = libmag.insert_before_ext(out_path, suffix, "_") libmag.backup_file(out_path) if delay is None: delay = 100 if plotted_imgs and len(plotted_imgs[0]) > 0: fig = plotted_imgs[0][0].figure else: libmag.warn("No images available to animate") return # WORKAROUND: FFMpeg may give a "height not divisible by 2" error, fixed # by padding with a pixel # TODO: check if needed for width # TODO: account for difference in FFMpeg height and fig height for fn, size in { # fig.set_figwidth: fig.get_figwidth(), fig.set_figheight: fig.get_figheight() }.items(): if size * fig.dpi % 2 != 0: fn(size + 1. / fig.dpi) print("Padded size with", fn, fig.get_figwidth(), "to new size of", fig.get_figheight()) # generate and save animation anim = animation.ArtistAnimation(fig, plotted_imgs, interval=delay, repeat_delay=0, blit=False) try: writer = "ffmpeg" if ext == "mp4" else "imagemagick" anim.save(out_path, writer=writer) print("saved animation file to {}".format(out_path)) except ValueError as e: print(e) libmag.warn("No animation writer available for Matplotlib")
def read_sitk_files(filename_sitk, reg_names=None): """Read files through SimpleITK and export to Numpy array format, loading associated metadata. Args: filename_sitk: Path to file in a format that can be read by SimpleITK. reg_names: Path or sequence of paths of registered names. Can be a registered suffix or a full path. Defaults to None. Returns: Image array in MagellanMapper image5d format. Associated metadata will have been loaded into module-level variables. Raises: ``FileNotFoundError`` if ``filename_sitk`` cannot be found, after attempting to load metadata from ``filename_np``. """ # load image via SimpleITK img_sitk = None loaded_path = filename_sitk if reg_names: img_nps = [] if not libmag.is_seq(reg_names): reg_names = [reg_names] for reg_name in reg_names: # load each registered suffix into list of images with same shape, # keeping first image in sitk format img, path = _load_reg_img_to_combine(filename_sitk, reg_name, img_nps) if img_sitk is None: img_sitk = img loaded_path = path if len(img_nps) > 1: # merge images into separate channels img_np = np.stack(img_nps, axis=img_nps[0].ndim) else: img_np = img_nps[0] else: # load filename_sitk directly if not os.path.exists(filename_sitk): raise FileNotFoundError( "could not find file {}".format(filename_sitk)) img_sitk, _ = read_sitk(filename_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if config.resolutions is None: # fallback to determining metadata directly from sitk file libmag.warn( "MagellanMapper image metadata file not loaded; will fallback to " "{} for metadata".format(loaded_path)) config.resolutions = np.array([img_sitk.GetSpacing()[::-1]]) print("set resolutions to {}".format(config.resolutions)) return img_np
def verify_stack(filename_base, subimg_path_base, settings, segments_all, channels, overlap_base): db_path_base = os.path.basename(subimg_path_base) stats_detection = None fdbk = None try: # Truth databases are any database stored with manually # verified blobs and loaded at command-line with the # `--truth_db` flag or loaded here. While all experiments # can be stored in a single database, this verification also # supports experiments saved to separate databases in the # software root directory and named as a sub-image but with # the `sqlite.DB_SUFFIX_TRUTH` suffix. Experiments in the # database are also assumed to be named based on the full # image or the sub-image filename, without any directories. # load ROIs from previously loaded truth database or one loaded # based on sub-image filename exp_name, rois = _get_truth_db_rois( subimg_path_base, filename_base, db_path_base if config.truth_db is None else None) if rois is None: # load alternate truth database based on sub-image filename print("Loading truth ROIs from experiment:", exp_name) exp_name, rois = _get_truth_db_rois( subimg_path_base, filename_base, db_path_base) if config.truth_db is None: raise LookupError( "No truth database found for experiment {}, will " "skip detection verification".format(exp_name)) if rois is None: raise LookupError( "No truth set ROIs found for experiment {}, will " "skip detection verification".format(exp_name)) # verify each ROI and store results in a separate database exp_id = sqlite.insert_experiment( config.verified_db.conn, config.verified_db.cur, exp_name, None) verify_tol = np.multiply( overlap_base, settings["verify_tol_factor"]) stats_detection, fdbk, df_verify = verify_rois( rois, segments_all, config.truth_db.blobs_truth, verify_tol, config.verified_db, exp_id, exp_name, channels) df_io.data_frames_to_csv(df_verify, libmag.combine_paths( exp_name, "verify.csv")) except FileNotFoundError: libmag.warn("Could not load truth DB from {}; " "will not verify ROIs".format(db_path_base)) except LookupError as e: libmag.warn(str(e)) return stats_detection, fdbk
def animate_imgs(base_path, plotted_imgs, delay, ext=None, suffix=None): """Export to an animated image. Defaults to an animated GIF unless ``ext`` specifies otherwise. Requires ``FFMpeg`` for MP4 file format exports and ``ImageMagick`` for all other types of exports. Args: base_path (str): String from which an output path will be constructed. plotted_imgs (List[:obj:`matplotlib.image.AxesImage]): Sequence of images to include in the animation. delay (int): Delay between image display in ms. If None, the delay will defaul to 100ms. ext (str): Extension to use when saving, without the period. Defaults to None, in which case "gif" will be used. suffix (str): String to append to output path before extension; defaults to None to ignore. """ if ext is None: ext = "gif" out_path = libmag.combine_paths(base_path, "animated", ext=ext) if suffix: out_path = libmag.insert_before_ext(out_path, suffix, "_") libmag.backup_file(out_path) if delay is None: delay = 100 if plotted_imgs and len(plotted_imgs[0]) > 0: fig = plotted_imgs[0][0].figure else: libmag.warn("No images available to animate") return anim = animation.ArtistAnimation(fig, plotted_imgs, interval=delay, repeat_delay=0, blit=False) try: writer = "ffmpeg" if ext == "mp4" else "imagemagick" anim.save(out_path, writer=writer) print("saved animation file to {}".format(out_path)) except ValueError as e: print(e) libmag.warn("No animation writer available for Matplotlib")
def setup_images(path=None, series=None, offset=None, size=None, proc_mode=None, allow_import=True): """Sets up an image and all associated images and metadata. Paths for related files such as registered images will generally be constructed from ``path``. If :attr:`config.prefix` is set, it will be used in place of ``path`` for registered labels. Args: path (str): Path to image from which MagellanMapper-style paths will be generated. series (int): Image series number; defaults to None. offset (List[int]): Sub-image offset given in z,y,x; defaults to None. size (List[int]): Sub-image shape given in z,y,x; defaults to None. proc_mode (str): Processing mode, which should be a key in :class:`config.ProcessTypes`, case-insensitive; defaults to None. allow_import (bool): True to allow importing the image if it cannot be loaded; defaults to True. """ def add_metadata(): # override metadata set from command-line metadata args if available md = { config.MetaKeys.RESOLUTIONS: config.meta_dict[config.MetaKeys.RESOLUTIONS], config.MetaKeys.MAGNIFICATION: config.meta_dict[config.MetaKeys.MAGNIFICATION], config.MetaKeys.ZOOM: config.meta_dict[config.MetaKeys.ZOOM], config.MetaKeys.SHAPE: config.meta_dict[config.MetaKeys.SHAPE], config.MetaKeys.DTYPE: config.meta_dict[config.MetaKeys.DTYPE], } for key, val in md.items(): if val is not None: # explicitly set metadata takes precedence over extracted vals import_md[key] = val # LOAD MAIN IMAGE # reset image5d config.image5d = None config.image5d_is_roi = False load_subimage = offset is not None and size is not None config.resolutions = None # reset label images config.labels_img = None config.borders_img = None filename_base = importer.filename_to_base(path, series) subimg_base = None if load_subimage and not config.save_subimg: # load a saved sub-image file if available and not set to save one subimg_base = stack_detect.make_subimage_name(filename_base, offset, size) filename_subimg = libmag.combine_paths(subimg_base, config.SUFFIX_SUBIMG) try: # load sub-image if available config.image5d = np.load(filename_subimg, mmap_mode="r") config.image5d = importer.roi_to_image5d(config.image5d) config.image5d_is_roi = True config.image5d_io = config.LoadIO.NP print("Loaded sub-image from {} with shape {}".format( filename_subimg, config.image5d.shape)) # after loading sub-image, load original image's metadata # for essential data such as vmin/vmax; will only warn if # fails to load since metadata could be specified elsewhere _, orig_info = importer.make_filenames(path, series) print("load original image metadata from:", orig_info) importer.load_metadata(orig_info) except IOError: print("Ignored sub-image file from {} as unable to load".format( filename_subimg)) proc_type = libmag.get_enum(proc_mode, config.ProcessTypes) if proc_type in (config.ProcessTypes.LOAD, config.ProcessTypes.EXPORT_ROIS, config.ProcessTypes.EXPORT_BLOBS, config.ProcessTypes.DETECT): # load a blobs archive try: if subimg_base: try: # load blobs generated from sub-image config.blobs = load_blobs(subimg_base) except (FileNotFoundError, KeyError): # fallback to loading from full image blobs and getting # a subset, shifting them relative to sub-image offset print("Unable to load blobs file based on {}, will try " "from {}".format(subimg_base, filename_base)) config.blobs = load_blobs(filename_base) config.blobs, _ = detector.get_blobs_in_roi(config.blobs, offset, size, reverse=False) detector.shift_blob_rel_coords(config.blobs, np.multiply(offset, -1)) else: # load full image blobs config.blobs = load_blobs(filename_base) except (FileNotFoundError, KeyError) as e2: print("Unable to load blobs file") if proc_type in (config.ProcessTypes.LOAD, config.ProcessTypes.EXPORT_BLOBS): # blobs expected but not found raise e2 if path and config.image5d is None: # load or import the main image stack print("Loading main image") try: if path.endswith(sitk_io.EXTS_3D): # attempt to format supported by SimpleITK and prepend time axis config.image5d = sitk_io.read_sitk_files(path)[None] config.image5d_io = config.LoadIO.SITK else: # load or import from MagellanMapper Numpy format import_only = proc_type is config.ProcessTypes.IMPORT_ONLY if not import_only: # load previously imported image config.image5d = importer.read_file(path, series) if allow_import: # re-import over existing image or import new image if os.path.isdir(path) and all( [r is None for r in config.reg_suffixes.values()]): # import directory of single plane images to single # stack if no register suffixes are set chls, import_md = importer.setup_import_dir(path) add_metadata() prefix = config.prefix if not prefix: prefix = os.path.join( os.path.dirname(path), importer.DEFAULT_IMG_STACK_NAME) config.image5d = importer.import_planes_to_stack( chls, prefix, import_md) elif import_only or config.image5d is None: # import multi-plane image chls, import_path = importer.setup_import_multipage( path) prefix = config.prefix if config.prefix else import_path import_md = importer.setup_import_metadata( chls, config.channel, series) add_metadata() config.image5d = importer.import_multiplane_images( chls, prefix, import_md, series, channel=config.channel) config.image5d_io = config.LoadIO.NP except FileNotFoundError as e: print(e) print("Could not load {}, will fall back to any associated " "registered image".format(path)) if config.metadatas and config.metadatas[0]: # assign metadata from alternate file if given to supersede settings # for any loaded image5d # TODO: access metadata directly from given image5d's dict to allow # loading multiple image5d images simultaneously importer.assign_metadata(config.metadatas[0]) # main image is currently required since many parameters depend on it atlas_suffix = config.reg_suffixes[config.RegSuffixes.ATLAS] if atlas_suffix is None and config.image5d is None: # fallback to atlas if main image not already loaded atlas_suffix = config.RegNames.IMG_ATLAS.value print( "main image is not set, falling back to registered " "image with suffix", atlas_suffix) # use prefix to get images registered to a different image, eg a # downsampled version, or a different version of registered images path = config.prefix if config.prefix else path if path and atlas_suffix is not None: try: # will take the place of any previously loaded image5d config.image5d = sitk_io.read_sitk_files( path, reg_names=atlas_suffix)[None] config.image5d_io = config.LoadIO.SITK except FileNotFoundError as e: print(e) annotation_suffix = config.reg_suffixes[config.RegSuffixes.ANNOTATION] if annotation_suffix is not None: # load labels image, set up scaling, and load labels file try: # TODO: need to support multichannel labels images config.labels_img = sitk_io.read_sitk_files( path, reg_names=annotation_suffix) if config.image5d is not None: config.labels_scaling = importer.calc_scaling( config.image5d, config.labels_img) if config.load_labels is not None: labels_ref = ontology.load_labels_ref(config.load_labels) if isinstance(labels_ref, pd.DataFrame): # parse CSV files loaded into data frame config.labels_ref_lookup = ontology.create_lookup_pd( labels_ref) else: # parse dict from ABA JSON file config.labels_ref_lookup = ( ontology.create_aba_reverse_lookup(labels_ref)) except FileNotFoundError as e: print(e) borders_suffix = config.reg_suffixes[config.RegSuffixes.BORDERS] if borders_suffix is not None: # load borders image, which can also be another labels image try: config.borders_img = sitk_io.read_sitk_files( path, reg_names=borders_suffix) except FileNotFoundError as e: print(e) if (config.atlas_labels[config.AtlasLabels.ORIG_COLORS] and config.load_labels is not None): # load original labels image from same directory as ontology # file for consistent ID-color mapping, even if labels are missing try: config.labels_img_orig = sitk_io.load_registered_img( config.load_labels, config.RegNames.IMG_LABELS.value) except FileNotFoundError as e: print(e) libmag.warn( "could not load original labels image; colors may differ" "differ from it") load_rot90 = config.roi_profile["load_rot90"] if load_rot90 and config.image5d is not None: # rotate main image specified num of times x90deg after loading since # need to rotate images output by deep learning toolkit config.image5d = np.rot90(config.image5d, load_rot90, (2, 3)) if (config.image5d is not None and load_subimage and not config.image5d_is_roi): # crop full image to bounds of sub-image config.image5d = plot_3d.prepare_subimg(config.image5d, size, offset)[None] config.image5d_is_roi = True # add any additional image5d thresholds for multichannel images, such # as those loaded without metadata for these settings colormaps.setup_cmaps() num_channels = get_num_channels(config.image5d) config.near_max = libmag.pad_seq(config.near_max, num_channels, -1) config.near_min = libmag.pad_seq(config.near_min, num_channels, 0) config.vmax_overview = libmag.pad_seq(config.vmax_overview, num_channels) colormaps.setup_colormaps(num_channels)
def plot_knns(img_paths, suffix=None, show=False, names=None): """Plot k-nearest-neighbor distances for multiple sets of blobs, overlaying on a single plot. Args: img_paths (List[str]): Base paths from which registered labels and blobs files will be found and output blobs file save location will be constructed. suffix (str): Suffix for ``path``; defaults to None. show (bool): True to plot the distances; defaults to False. names (List[str]): Sequence of names corresponding to ``img_paths`` for the plot legend. """ cluster_settings = config.atlas_profile[profiles.RegKeys.METRICS_CLUSTER] knn_n = cluster_settings[profiles.RegKeys.KNN_N] if not knn_n: knn_n = cluster_settings[profiles.RegKeys.DBSCAN_MINPTS] - 1 print("Calculating k-nearest-neighbor distances and plotting distances " "for neighbor {}".format(knn_n)) # set up combined data frames for all samples at each zoom level df_keys = ("ov", "zoom") dfs_comb = {key: [] for key in df_keys} names_disp = names if names else [] for i, img_path in enumerate(img_paths): # load blobs associated with image mod_path = img_path if suffix is not None: mod_path = libmag.insert_before_ext(img_path, suffix) labels_img_np = sitk_io.load_registered_img( mod_path, config.RegNames.IMG_LABELS.value) blobs = detector.Blobs().load_blobs(np_io.img_to_blobs_path(img_path)) scaling, res = np_io.find_scaling(img_path, labels_img_np.shape) if blobs is None: libmag.warn("unable to load nuclei coordinates for", img_path) continue # convert to physical units and display k-nearest-neighbors for nuclei blobs_phys = np.multiply(blobs.blobs[:, :3], res) # TESTING: given the same blobs, simply shift #blobs = np.multiply(blobs[i*10000000:, :3], res) _, _, dfs = knn_dist(blobs_phys, knn_n, 2, 1000000, False) if names is None: # default to naming from filename names_disp.append(os.path.basename(mod_path)) for j, df in enumerate(dfs): dfs_comb[df_keys[j]].append(df) for key in dfs_comb: # combine data frames at each zoom level, save, and plot with # different colors for each image df = df_io.join_dfs(dfs_comb[key], "point") dist_cols = [col for col in df.columns if col.startswith("dist")] rename_cols = {col: name for col, name in zip(dist_cols, names_disp)} df = df.rename(rename_cols, axis=1) out_path = "knn_dist_combine_{}".format(key) df_io.data_frames_to_csv(df, out_path) plot_2d.plot_lines(out_path, "point", rename_cols.values(), df=df, show=show, title=config.plot_labels[config.PlotLabels.TITLE])
def detect_blobs_blocks(filename_base, image5d, offset, size, channels, verify=False, save_dfs=True, full_roi=False, coloc=False): """Detect blobs by block processing of a large image. All channels are processed in the same blocks. Args: filename_base: Base path to use file output. image5d: Large image to process as a Numpy array of t,z,y,x,[c] offset: Sub-image offset given as coordinates in z,y,x. size: Sub-image shape given in z,y,x. channels (Sequence[int]): Sequence of channels, where None detects in all channels. verify: True to verify detections against truth database; defaults to False. save_dfs: True to save data frames to file; defaults to True. full_roi (bool): True to treat ``image5d`` as the full ROI; defaults to False. coloc (bool): True to perform blob co-localizations; defaults to False. Returns: tuple[int, int, int], str, :class:`magmap.cv.detector.Blobs`: Accuracy metrics from :class:`magmap.cv.detector.verify_rois`, feedback message from this same function, and detected blobs. """ time_start = time() subimg_path_base = filename_base if size is None or offset is None: # uses the entire stack if no size or offset specified size = image5d.shape[1:4] offset = (0, 0, 0) else: # get base path for sub-image subimg_path_base = naming.make_subimage_name( filename_base, offset, size) filename_blobs = libmag.combine_paths(subimg_path_base, config.SUFFIX_BLOBS) # get ROI for given region, including all channels if full_roi: # treat the full image as the ROI roi = image5d[0] else: roi = plot_3d.prepare_subimg(image5d, offset, size) num_chls_roi = 1 if len(roi.shape) < 4 else roi.shape[3] if num_chls_roi < 2: coloc = False print("Unable to co-localize as image has only 1 channel") # prep chunking ROI into sub-ROIs with size based on segment_size, scaling # by physical units to make more independent of resolution; use profile # from first channel to be processed for block settings time_detection_start = time() settings = config.get_roi_profile(channels[0]) print("Profile for block settings:", settings[settings.NAME_KEY]) sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border, \ tol, overlap_base, overlap, overlap_padding = setup_blocks( settings, roi.shape) # TODO: option to distribute groups of sub-ROIs to different servers # for blob detection seg_rois = StackDetector.detect_blobs_sub_rois( roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border, coloc, channels) detection_time = time() - time_detection_start print("blob detection time (s):", detection_time) # prune blobs in overlapping portions of sub-ROIs time_pruning_start = time() segments_all, df_pruning = StackPruner.prune_blobs_mp( roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels, overlap_padding) pruning_time = time() - time_pruning_start print("blob pruning time (s):", pruning_time) #print("maxes:", np.amax(segments_all, axis=0)) # get weighted mean of ratios if df_pruning is not None: print("\nBlob pruning ratios:") path_pruning = "blob_ratios.csv" if save_dfs else None df_pruning_all = df_io.data_frames_to_csv( df_pruning, path_pruning, show=" ") cols = df_pruning_all.columns.tolist() blob_pruning_means = {} if "blobs" in cols: blobs_unpruned = df_pruning_all["blobs"] num_blobs_unpruned = np.sum(blobs_unpruned) for col in cols[1:]: blob_pruning_means["mean_{}".format(col)] = [ np.sum(np.multiply(df_pruning_all[col], blobs_unpruned)) / num_blobs_unpruned] path_pruning_means = "blob_ratios_means.csv" if save_dfs else None df_pruning_means = df_io.dict_to_data_frame( blob_pruning_means, path_pruning_means, show=" ") else: print("no blob ratios found") '''# report any remaining duplicates np.set_printoptions(linewidth=500, threshold=10000000) print("all blobs (len {}):".format(len(segments_all))) sort = np.lexsort( (segments_all[:, 2], segments_all[:, 1], segments_all[:, 0])) blobs = segments_all[sort] print(blobs) print("checking for duplicates in all:") print(detector.remove_duplicate_blobs(blobs, slice(0, 3))) ''' stats_detection = None fdbk = None colocs = None if segments_all is not None: # remove the duplicated elements that were used for pruning detector.replace_rel_with_abs_blob_coords(segments_all) if coloc: colocs = segments_all[:, 10:10+num_chls_roi].astype(np.uint8) # remove absolute coordinate and any co-localization columns segments_all = detector.remove_abs_blob_coords(segments_all) # compare detected blobs with truth blobs # TODO: assumes ground truth is relative to any ROI offset, # but should make customizable if verify: stats_detection, fdbk = verifier.verify_stack( filename_base, subimg_path_base, settings, segments_all, channels, overlap_base) if config.save_subimg: subimg_base_path = libmag.combine_paths( subimg_path_base, config.SUFFIX_SUBIMG) if (isinstance(config.image5d, np.memmap) and config.image5d.filename == os.path.abspath(subimg_base_path)): # file at sub-image save path may have been opened as a memmap # file, in which case saving would fail libmag.warn("{} is currently open, cannot save sub-image" .format(subimg_base_path)) else: # write sub-image, which is in ROI (3D) format with open(subimg_base_path, "wb") as f: np.save(f, roi) # store blobs in Blobs instance # TODO: consider separating into blobs and blobs metadata archives blobs = detector.Blobs( segments_all, colocalizations=colocs, path=filename_blobs) blobs.resolutions = config.resolutions blobs.basename = os.path.basename(config.filename) blobs.roi_offset = offset blobs.roi_size = size # whole image benchmarking time times = ( [detection_time], [pruning_time], time() - time_start) times_dict = {} for key, val in zip(StackTimes, times): times_dict[key] = val if segments_all is None: print("\nNo blobs detected") else: print("\nTotal blobs found:", len(segments_all)) detector.show_blobs_per_channel(segments_all) print("\nTotal detection processing times (s):") path_times = "stack_detection_times.csv" if save_dfs else None df_io.dict_to_data_frame(times_dict, path_times, show=" ") return stats_detection, fdbk, blobs
def process_cli_args(): """Parse command-line arguments. Typically stores values as :mod:`magmap.settings.config` attributes. """ parser = argparse.ArgumentParser( description="Setup environment for MagellanMapper") # image specification arguments # image path(s) specified as an optional argument; takes precedence # over positional argument parser.add_argument( "--img", nargs="*", default=None, help="Main image path(s); after import, the filename is often " "given as the original name without its extension") # alternatively specified as the first and only positional parameter # with as many arguments as desired parser.add_argument( "img_paths", nargs="*", default=None, help="Main image path(s); can also be given as --img, which takes " "precedence over this argument") parser.add_argument( "--meta", nargs="*", help="Metadata path(s), which can be given as multiple files " "corresponding to each image") parser.add_argument("--prefix", help="Path prefix") parser.add_argument("--suffix", help="Filename suffix") parser.add_argument("--channel", nargs="*", type=int, help="Channel index") parser.add_argument("--series", help="Series index") parser.add_argument("--subimg_offset", nargs="*", help="Sub-image offset in x,y,z") parser.add_argument("--subimg_size", nargs="*", help="Sub-image size in x,y,z") parser.add_argument("--offset", nargs="*", help="ROI offset in x,y,z") parser.add_argument("--size", nargs="*", help="ROI size in x,y,z") parser.add_argument("--db", help="Database path") parser.add_argument( "--cpus", help="Maximum number of CPUs/processes to use for multiprocessing " "tasks. Use \"none\" or 0 to auto-detect this number (default).") parser.add_argument( "--load", nargs="*", help="Load associated data files; see config.LoadData for settings") # task arguments parser.add_argument("--proc", type=str.lower, choices=libmag.enum_names_aslist(config.ProcessTypes), help="Image processing mode") parser.add_argument("--register", type=str.lower, choices=libmag.enum_names_aslist(config.RegisterTypes), help="Image registration task") parser.add_argument("--df", type=str.lower, choices=libmag.enum_names_aslist(config.DFTasks), help="Data frame task") parser.add_argument("--plot_2d", type=str.lower, choices=libmag.enum_names_aslist(config.Plot2DTypes), help="2D plot task; see config.Plot2DTypes") parser.add_argument("--ec2_start", nargs="*", help="AWS EC2 instance start") parser.add_argument("--ec2_list", nargs="*", help="AWS EC2 instance list") parser.add_argument("--ec2_terminate", nargs="*", help="AWS EC2 instance termination") parser.add_argument( "--notify", nargs="*", help="Notification message URL, message, and attachment strings") parser.add_argument("--grid_search", help="Grid search hyperparameter tuning profile(s)") # profile arguments parser.add_argument( "--roi_profile", nargs="*", help="ROI profile, which can be separated by underscores " "for multiple profiles and given as paths to custom profiles " "in YAML format. Multiple profile groups can be given, which " "will each be applied to the corresponding channel. See " "docs/settings.md for more details.") parser.add_argument( "--atlas_profile", help="Atlas profile, which can be separated by underscores " "for multiple profiles and given as paths to custom profiles " "in YAML format. See docs/settings.md for more details.") parser.add_argument( "--theme", nargs="*", type=str.lower, choices=libmag.enum_names_aslist(config.Themes), help="UI theme, which can be given as multiple themes to apply " "on top of one another") # grouped arguments parser.add_argument( "--truth_db", nargs="*", help="Truth database; see config.TruthDB for settings and " "config.TruthDBModes for modes") parser.add_argument("--labels", nargs="*", help=_get_args_dict_help( "Atlas labels; see config.AtlasLabels.", config.AtlasLabels)) parser.add_argument("--transform", nargs="*", help=_get_args_dict_help( "Image transformations; see config.Transforms.", config.Transforms)) parser.add_argument( "--reg_suffixes", nargs="*", help=_get_args_dict_help( "Registered image suffixes; see config.RegSuffixes for keys " "and config.RegNames for values", config.RegSuffixes)) parser.add_argument( "--plot_labels", nargs="*", help=_get_args_dict_help( "Plot label customizations; see config.PlotLabels ", config.PlotLabels)) parser.add_argument( "--set_meta", nargs="*", help="Set metadata values; see config.MetaKeys for settings") # image and figure display arguments parser.add_argument("--plane", type=str.lower, choices=config.PLANE, help="Planar orientation") parser.add_argument( "--show", nargs="?", const="1", help="If applicable, show images after completing the given task") parser.add_argument( "--alphas", help="Alpha opacity levels, which can be comma-delimited for " "multichannel images") parser.add_argument( "--vmin", help="Minimum intensity levels, which can be comma-delimited " "for multichannel images") parser.add_argument( "--vmax", help="Maximum intensity levels, which can be comma-delimited " "for multichannel images") parser.add_argument("--seed", help="Random number generator seed") # export arguments parser.add_argument("--save_subimg", action="store_true", help="Save sub-image as separate file") parser.add_argument("--slice", help="Slice given as start,stop,step") parser.add_argument("--delay", help="Animation delay in ms") parser.add_argument("--savefig", help="Extension for saved figures") parser.add_argument("--groups", nargs="*", help="Group values corresponding to each image") parser.add_argument( "-v", "--verbose", nargs="*", help=_get_args_dict_help( "Verbose output to assist with debugging; see config.Verbosity.", config.Verbosity)) # only parse recognized arguments to avoid error for unrecognized ones args, args_unknown = parser.parse_known_args() if args.verbose: # verbose mode, including printing longer Numpy arrays for debugging config.verbose = True config.verbosity = args_to_dict(args.verbose, config.Verbosity, config.verbosity) logs.update_log_level(config.logger, config.verbosity[config.Verbosity.LEVEL]) np.set_printoptions(linewidth=200, threshold=10000) _logger.info("Set verbose to {}".format(config.verbosity)) # set up logging to given file unless explicitly given an empty string log_path = config.verbosity[config.Verbosity.LOG_PATH] if log_path != "": if log_path is None: log_path = os.path.join(config.user_app_dirs.user_data_dir, "out.log") # log to file logs.add_file_handler(config.logger, log_path) # redirect standard out/error to logging sys.stdout = logs.LogWriter(config.logger.info) sys.stderr = logs.LogWriter(config.logger.error) if args.img is not None or args.img_paths: # set image file path and convert to basis for additional paths config.filenames = args.img if args.img else args.img_paths config.filename = config.filenames[0] print("Set filenames to {}, current filename {}".format( config.filenames, config.filename)) if args.meta is not None: # set metadata paths config.metadata_paths = args.meta print("Set metadata paths to", config.metadata_paths) config.metadatas = [] for path in config.metadata_paths: # load metadata to dictionary md, _ = importer.load_metadata(path, assign=False) config.metadatas.append(md) if args.channel is not None: # set the channels config.channel = args.channel print("Set channel to {}".format(config.channel)) config.series_list = [config.series] # list of series if args.series is not None: series_split = args.series.split(",") config.series_list = [] for ser in series_split: ser_split = ser.split("-") if len(ser_split) > 1: ser_range = np.arange(int(ser_split[0]), int(ser_split[1]) + 1) config.series_list.extend(ser_range.tolist()) else: config.series_list.append(int(ser_split[0])) config.series = config.series_list[0] print("Set to series_list to {}, current series {}".format( config.series_list, config.series)) if args.savefig is not None: # save figure with file type of this extension; remove leading period config.savefig = _parse_none(args.savefig.lstrip(".")) print("Set savefig extension to {}".format(config.savefig)) # parse sub-image offsets and sizes; # expects x,y,z input but stores as z,y,x by convention if args.subimg_offset is not None: config.subimg_offsets = _parse_coords(args.subimg_offset, True) print("Set sub-image offsets to {} (z,y,x)".format( config.subimg_offsets)) if args.subimg_size is not None: config.subimg_sizes = _parse_coords(args.subimg_size, True) print("Set sub-image sizes to {} (z,y,x)".format(config.subimg_sizes)) # parse ROI offsets and sizes, which are relative to any sub-image; # expects x,y,z input and output if args.offset is not None: config.roi_offsets = _parse_coords(args.offset) if config.roi_offsets: config.roi_offset = config.roi_offsets[0] print("Set ROI offsets to {}, current offset {} (x,y,z)".format( config.roi_offsets, config.roi_offset)) if args.size is not None: config.roi_sizes = _parse_coords(args.size) if config.roi_sizes: config.roi_size = config.roi_sizes[0] print("Set ROI sizes to {}, current size {} (x,y,z)".format( config.roi_sizes, config.roi_size)) if args.cpus is not None: # set maximum number of CPUs config.cpus = _parse_none(args.cpus.lower(), int) print("Set maximum number of CPUs for multiprocessing tasks to", config.cpus) if args.load is not None: # flag loading data sources with default sub-arg indicating that the # data should be loaded from a default path; otherwise, load from # path given by the sub-arg; change delimiter to allow paths with "," config.load_data = args_to_dict(args.load, config.LoadData, config.load_data, sep_vals="|", default=True) print("Set to load the data types: {}".format(config.load_data)) # set up main processing mode if args.proc is not None: config.proc_type = args.proc print("processing type set to {}".format(config.proc_type)) proc_type = libmag.get_enum(config.proc_type, config.ProcessTypes) if config.proc_type and proc_type not in config.ProcessTypes: libmag.warn("\"{}\" processing type not found".format( config.proc_type)) if args.set_meta is not None: # set individual metadata values, currently used for image import # TODO: take precedence over loaded metadata archives config.meta_dict = args_to_dict(args.set_meta, config.MetaKeys, config.meta_dict, sep_vals="|") print("Set metadata values to {}".format(config.meta_dict)) res = config.meta_dict[config.MetaKeys.RESOLUTIONS] if res: # set image resolutions, taken as a single set of x,y,z and # converting to a nested list of z,y,x res_split = res.split(",") if len(res_split) >= 3: res_float = tuple(float(i) for i in res_split)[::-1] config.resolutions = [res_float] print("Set resolutions to {}".format(config.resolutions)) else: res_float = None print("Resolution ({}) should be given as 3 values (x,y,z)". format(res)) # store single set of resolutions, similar to input config.meta_dict[config.MetaKeys.RESOLUTIONS] = res_float mag = config.meta_dict[config.MetaKeys.MAGNIFICATION] if mag: # set objective magnification config.magnification = mag print("Set magnification to {}".format(config.magnification)) zoom = config.meta_dict[config.MetaKeys.ZOOM] if zoom: # set objective zoom config.zoom = zoom print("Set zoom to {}".format(config.zoom)) shape = config.meta_dict[config.MetaKeys.SHAPE] if shape: # parse shape, storing only in dict config.meta_dict[config.MetaKeys.SHAPE] = [ int(n) for n in shape.split(",")[::-1] ] # set up ROI and register profiles setup_roi_profiles(args.roi_profile) setup_atlas_profiles(args.atlas_profile) setup_grid_search_profiles(args.grid_search) if args.plane is not None: config.plane = args.plane print("Set plane to {}".format(config.plane)) if args.save_subimg: config.save_subimg = args.save_subimg print("Set to save the sub-image") if args.labels: # set up atlas labels setup_labels(args.labels) if args.transform is not None: # image transformations such as flipping, rotation config.transform = args_to_dict(args.transform, config.Transforms, config.transform) print("Set transformations to {}".format(config.transform)) if args.register: # register type to process in register module config.register_type = args.register print("Set register type to {}".format(config.register_type)) if args.df: # data frame processing task config.df_task = args.df print("Set data frame processing task to {}".format(config.df_task)) if args.plot_2d: # 2D plot type to process in plot_2d module config.plot_2d_type = args.plot_2d print("Set plot_2d type to {}".format(config.plot_2d_type)) if args.slice: # specify a generic slice by command-line, assuming same order # of arguments as for slice built-in function and interpreting # "none" string as None config.slice_vals = args.slice.split(",") config.slice_vals = [ _parse_none(val.lower(), int) for val in config.slice_vals ] print("Set slice values to {}".format(config.slice_vals)) if args.delay: config.delay = int(args.delay) print("Set delay to {}".format(config.delay)) if args.show: # show images after task is performed, if supported config.show = _is_arg_true(args.show) print("Set show to {}".format(config.show)) if args.groups: config.groups = args.groups print("Set groups to {}".format(config.groups)) if args.ec2_start is not None: # start EC2 instances config.ec2_start = args_with_dict(args.ec2_start) print("Set ec2 start to {}".format(config.ec2_start)) if args.ec2_list: # list EC2 instances config.ec2_list = args_with_dict(args.ec2_list) print("Set ec2 list to {}".format(config.ec2_list)) if args.ec2_terminate: config.ec2_terminate = args.ec2_terminate print("Set ec2 terminate to {}".format(config.ec2_terminate)) if args.notify: notify_len = len(args.notify) if notify_len > 0: config.notify_url = args.notify[0] print("Set notification URL to {}".format(config.notify_url)) if notify_len > 1: config.notify_msg = args.notify[1] print("Set notification message to {}".format(config.notify_msg)) if notify_len > 2: config.notify_attach = args.notify[2] print("Set notification attachment path to {}".format( config.notify_attach)) if args.prefix: config.prefix = args.prefix print("Set path prefix to {}".format(config.prefix)) if args.suffix: config.suffix = args.suffix print("Set path suffix to {}".format(config.suffix)) if args.alphas: # specify alpha levels config.alphas = [float(val) for val in args.alphas.split(",")] print("Set alphas to", config.alphas) if args.vmin: # specify vmin levels config.vmins = [libmag.get_int(val) for val in args.vmin.split(",")] print("Set vmins to", config.vmins) if args.vmax: # specify vmax levels and copy to vmax overview used for plotting # and updated for normalization config.vmaxs = [libmag.get_int(val) for val in args.vmax.split(",")] config.vmax_overview = list(config.vmaxs) print("Set vmaxs to", config.vmaxs) if args.reg_suffixes is not None: # specify suffixes of registered images to load config.reg_suffixes = args_to_dict(args.reg_suffixes, config.RegSuffixes, config.reg_suffixes) print("Set registered image suffixes to {}".format( config.reg_suffixes)) if args.seed: # specify random number generator seed config.seed = int(args.seed) print("Set random number generator seed to", config.seed) if args.plot_labels is not None: # specify general plot labels config.plot_labels = args_to_dict(args.plot_labels, config.PlotLabels, config.plot_labels) print("Set plot labels to {}".format(config.plot_labels)) if args.theme is not None: # specify themes, currently applied to Matplotlib elements theme_names = [] for theme in args.theme: # add theme enum if found theme_enum = libmag.get_enum(theme, config.Themes) if theme_enum: config.rc_params.append(theme_enum) theme_names.append(theme_enum.name) print("Set to use themes to {}".format(theme_names)) # set up Matplotlib styles/themes plot_2d.setup_style() # set up application directories user_dir = config.user_app_dirs.user_data_dir if not os.path.isdir(user_dir): # make application data directory if os.path.exists(user_dir): # backup any non-directory file libmag.backup_file(user_dir) os.makedirs(user_dir) if args.db: # set main database path to user arg config.db_path = args.db print("Set database name to {}".format(config.db_path)) else: # set default path config.db_path = os.path.join(user_dir, config.db_path) if args.truth_db: # set settings for separate database of "truth blobs" config.truth_db_params = args_to_dict(args.truth_db, config.TruthDB, config.truth_db_params, sep_vals="|") mode = config.truth_db_params[config.TruthDB.MODE] config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes) libmag.printv(config.truth_db_params) print("Mapped \"{}\" truth_db mode to {}".format( mode, config.truth_db_mode)) # notify user of full args list, including unrecognized args _logger.debug(f"All command-line arguments: {sys.argv}") if args_unknown: _logger.info( f"The following command-line arguments were unrecognized and " f"ignored: {args_unknown}")
def detect_blobs_large_image(filename_base, image5d, offset, size, verify=False, save_dfs=True, full_roi=False): """Detect blobs within a large image through parallel processing of smaller chunks. Args: filename_base: Base path to use file output. image5d: Large image to process as a Numpy array of t,z,y,x,[c] offset: Sub-image offset given as coordinates in z,y,x. size: Sub-image shape given in z,y,x. verify: True to verify detections against truth database; defaults to False. save_dfs: True to save data frames to file; defaults to True. full_roi (bool): True to treat ``image5d`` as the full ROI; defaults to False. """ time_start = time() if size is None or offset is None: # uses the entire stack if no size or offset specified size = image5d.shape[1:4] offset = (0, 0, 0) else: # change base filename for ROI-based partial stack filename_base = make_subimage_name(filename_base, offset, size) filename_subimg = libmag.combine_paths(filename_base, config.SUFFIX_SUBIMG) filename_blobs = libmag.combine_paths(filename_base, config.SUFFIX_BLOBS) # get ROI for given region, including all channels if full_roi: # treat the full image as the ROI roi = image5d[0] else: roi = plot_3d.prepare_subimg(image5d, size, offset) _, channels = plot_3d.setup_channels(roi, config.channel, 3) # prep chunking ROI into sub-ROIs with size based on segment_size, scaling # by physical units to make more independent of resolution time_detection_start = time() settings = config.roi_profile # use default settings scaling_factor = detector.calc_scaling_factor() print("microsope scaling factor based on resolutions: {}" .format(scaling_factor)) denoise_size = config.roi_profile["denoise_size"] denoise_max_shape = None if denoise_size: # further subdivide each sub-ROI for local preprocessing denoise_max_shape = np.ceil( np.multiply(scaling_factor, denoise_size)).astype(int) # overlap sub-ROIs to minimize edge effects overlap_base = chunking.calc_overlap() tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int) overlap_padding = np.copy(tol) overlap = np.copy(overlap_base) exclude_border = config.roi_profile["exclude_border"] if exclude_border is not None: # exclude border to avoid blob detector edge effects, where blobs # often collect at the faces of the sub-ROI; # ensure that overlap is greater than twice the border exclusion per # axis so that no plane will be excluded from both overlapping sub-ROIs exclude_border_thresh = np.multiply(2, exclude_border) overlap_less = np.less(overlap, exclude_border_thresh) overlap[overlap_less] = exclude_border_thresh[overlap_less] excluded = np.greater(exclude_border, 0) overlap[excluded] += 1 # additional padding overlap_padding[excluded] = 0 # no need to prune past excluded border print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond " "overlap for pruning: {}, exclude borders: {}" .format(overlap, tol, overlap_padding, exclude_border)) max_pixels = np.ceil(np.multiply( scaling_factor, config.roi_profile["segment_size"])).astype(int) print("preprocessing max shape: {}, detection max pixels: {}" .format(denoise_max_shape, max_pixels)) sub_roi_slices, sub_rois_offsets = chunking.stack_splitter( roi.shape, max_pixels, overlap) # TODO: option to distribute groups of sub-ROIs to different servers # for blob detection seg_rois = detect_blobs_sub_rois( roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border) detection_time = time() - time_detection_start print("blob detection time (s):", detection_time) # prune blobs in overlapping portions of sub-ROIs time_pruning_start = time() segments_all, df_pruning = _prune_blobs_mp( roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels, overlap_padding) pruning_time = time() - time_pruning_start print("blob pruning time (s):", pruning_time) #print("maxes:", np.amax(segments_all, axis=0)) # get weighted mean of ratios if df_pruning is not None: print("\nBlob pruning ratios:") path_pruning = "blob_ratios.csv" if save_dfs else None df_pruning_all = df_io.data_frames_to_csv( df_pruning, path_pruning, show=" ") cols = df_pruning_all.columns.tolist() blob_pruning_means = {} if "blobs" in cols: blobs_unpruned = df_pruning_all["blobs"] num_blobs_unpruned = np.sum(blobs_unpruned) for col in cols[1:]: blob_pruning_means["mean_{}".format(col)] = [ np.sum(np.multiply(df_pruning_all[col], blobs_unpruned)) / num_blobs_unpruned] path_pruning_means = "blob_ratios_means.csv" if save_dfs else None df_pruning_means = df_io.dict_to_data_frame( blob_pruning_means, path_pruning_means, show=" ") else: print("no blob ratios found") '''# report any remaining duplicates np.set_printoptions(linewidth=500, threshold=10000000) print("all blobs (len {}):".format(len(segments_all))) sort = np.lexsort( (segments_all[:, 2], segments_all[:, 1], segments_all[:, 0])) blobs = segments_all[sort] print(blobs) print("checking for duplicates in all:") print(detector.remove_duplicate_blobs(blobs, slice(0, 3))) ''' stats_detection = None fdbk = None if segments_all is not None: # remove the duplicated elements that were used for pruning detector.replace_rel_with_abs_blob_coords(segments_all) segments_all = detector.remove_abs_blob_coords(segments_all) # compare detected blobs with truth blobs # TODO: assumes ground truth is relative to any ROI offset, # but should make customizable if verify: db_path_base = None exp_name = os.path.splitext(os.path.basename(config.filename))[0] try: if config.truth_db is None: # find and load truth DB based on filename and subimage db_path_base = os.path.basename(filename_base) print("about to verify with truth db from {}" .format(db_path_base)) sqlite.load_truth_db(db_path_base) if config.truth_db is not None: # truth DB may contain multiple experiments for different # subimages; series not included in exp name since in ROI rois = config.truth_db.get_rois(exp_name) if rois is None: # exp may have been named by ROI print("{} experiment name not found, will try with" "ROI offset/size".format(exp_name)) exp_name = make_subimage_name(exp_name, offset, size) rois = config.truth_db.get_rois(exp_name) if rois is None: raise LookupError( "No truth set ROIs found for experiment {}, will " "skip detection verification".format(exp_name)) print("load ROIs from exp: {}".format(exp_name)) exp_id = sqlite.insert_experiment( config.verified_db.conn, config.verified_db.cur, exp_name, None) verify_tol = np.multiply( overlap_base, settings["verify_tol_factor"]) stats_detection, fdbk = detector.verify_rois( rois, segments_all, config.truth_db.blobs_truth, verify_tol, config.verified_db, exp_id, config.channel) except FileNotFoundError: libmag.warn("Could not load truth DB from {}; " "will not verify ROIs".format(db_path_base)) except LookupError as e: libmag.warn(str(e)) file_time_start = time() if config.save_subimg: if (isinstance(config.image5d, np.memmap) and config.image5d.filename == os.path.abspath(filename_subimg)): # file at sub-image save path may have been opened as a memmap # file, in which case saving would fail libmag.warn("{} is currently open, cannot save sub-image" .format(filename_subimg)) else: # write sub-image, which is in ROI (3D) format with open(filename_subimg, "wb") as f: np.save(f, roi) # save blobs # TODO: only segments used; consider removing the rest except ver outfile_blobs = open(filename_blobs, "wb") np.savez(outfile_blobs, ver=BLOBS_NP_VER, segments=segments_all, resolutions=config.resolutions, basename=os.path.basename(config.filename), # only save name offset=offset, roi_size=size) # None unless explicitly set outfile_blobs.close() file_save_time = time() - file_time_start # whole image benchmarking time times = ( [detection_time], [pruning_time], time() - time_start) times_dict = {} for key, val in zip(StackTimes, times): times_dict[key] = val if segments_all is None: print("\nNo blobs detected") else: print("\nTotal blobs found:", len(segments_all)) detector.show_blobs_per_channel(segments_all) print("file save time:", file_save_time) print("\nTotal detection processing times (s):") path_times = "stack_detection_times.csv" if save_dfs else None df_io.dict_to_data_frame(times_dict, path_times, show=" ") return stats_detection, fdbk, segments_all
def main(process_args_only=False, skip_dbs=False): """Starts the visualization GUI. Processes command-line arguments. Args: process_args_only (bool): Processes command-line arguments and returns; defaults to False. skip_dbs (bool): True to skip loading databases; defaults to False. """ parser = argparse.ArgumentParser( description="Setup environment for MagellanMapper") # image specification arguments parser.add_argument( "--img", nargs="*", help="Main image path(s); after import, the filename is often " "given as the original name without its extension") parser.add_argument( "--meta", nargs="*", help="Metadata path(s), which can be given as multiple files " "corresponding to each image") parser.add_argument("--prefix", help="Path prefix") parser.add_argument("--suffix", help="Filename suffix") parser.add_argument("--channel", nargs="*", type=int, help="Channel index") parser.add_argument("--series", help="Series index") parser.add_argument("--subimg_offset", nargs="*", help="Sub-image offset in x,y,z") parser.add_argument("--subimg_size", nargs="*", help="Sub-image size in x,y,z") parser.add_argument("--offset", nargs="*", help="ROI offset in x,y,z") parser.add_argument("--size", nargs="*", help="ROI size in x,y,z") parser.add_argument("--db", help="Database path") parser.add_argument( "--cpus", help="Maximum number of CPUs/processes to use for multiprocessing " "tasks. Use \"none\" or 0 to auto-detect this number (default).") # task arguments parser.add_argument("--proc", type=str.lower, choices=libmag.enum_names_aslist(config.ProcessTypes), help="Image processing mode") parser.add_argument("--register", type=str.lower, choices=libmag.enum_names_aslist(config.RegisterTypes), help="Image registration task") parser.add_argument("--df", type=str.lower, choices=libmag.enum_names_aslist(config.DFTasks), help="Data frame task") parser.add_argument("--plot_2d", type=str.lower, choices=libmag.enum_names_aslist(config.Plot2DTypes), help="2D plot task; see config.Plot2DTypes") parser.add_argument("--ec2_start", nargs="*", help="AWS EC2 instance start") parser.add_argument("--ec2_list", nargs="*", help="AWS EC2 instance list") parser.add_argument("--ec2_terminate", nargs="*", help="AWS EC2 instance termination") parser.add_argument( "--notify", nargs="*", help="Notification message URL, message, and attachment strings") parser.add_argument("--grid_search", help="Grid search hyperparameter tuning profile(s)") # profile arguments parser.add_argument( "--roi_profile", nargs="*", help="ROI profile, which can be separated by underscores " "for multiple profiles and given as paths to custom profiles " "in YAML format. Multiple profile groups can be given, which " "will each be applied to the corresponding channel. See " "docs/settings.md for more details.") parser.add_argument( "--atlas_profile", help="Atlas profile, which can be separated by underscores " "for multiple profiles and given as paths to custom profiles " "in YAML format. See docs/settings.md for more details.") parser.add_argument( "--theme", nargs="*", type=str.lower, choices=libmag.enum_names_aslist(config.Themes), help="UI theme, which can be given as multiple themes to apply " "on top of one another") # grouped arguments parser.add_argument( "--truth_db", nargs="*", help="Truth database; see config.TruthDB for settings and " "config.TruthDBModes for modes") parser.add_argument("--labels", nargs="*", help=_get_args_dict_help( "Atlas labels; see config.AtlasLabels.", config.AtlasLabels)) parser.add_argument("--transform", nargs="*", help=_get_args_dict_help( "Image transformations; see config.Transforms.", config.Transforms)) parser.add_argument( "--reg_suffixes", nargs="*", help=_get_args_dict_help( "Registered image suffixes; see config.RegSuffixes for keys " "and config.RegNames for values", config.RegSuffixes)) parser.add_argument( "--plot_labels", nargs="*", help=_get_args_dict_help( "Plot label customizations; see config.PlotLabels ", config.PlotLabels)) parser.add_argument( "--set_meta", nargs="*", help="Set metadata values; see config.MetaKeys for settings") # image and figure display arguments parser.add_argument("--plane", type=str.lower, choices=config.PLANE, help="Planar orientation") parser.add_argument( "--show", nargs="?", const="1", help="If applicable, show images after completing the given task") parser.add_argument( "--alphas", help="Alpha opacity levels, which can be comma-delimited for " "multichannel images") parser.add_argument( "--vmin", help="Minimum intensity levels, which can be comma-delimited " "for multichannel images") parser.add_argument( "--vmax", help="Maximum intensity levels, which can be comma-delimited " "for multichannel images") parser.add_argument("--seed", help="Random number generator seed") # export arguments parser.add_argument("--save_subimg", action="store_true", help="Save sub-image as separate file") parser.add_argument("--slice", help="Slice given as start,stop,step") parser.add_argument("--delay", help="Animation delay in ms") parser.add_argument("--savefig", help="Extension for saved figures") parser.add_argument("--groups", nargs="*", help="Group values corresponding to each image") parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output to assist with debugging") args = parser.parse_args() if args.img is not None: # set image file path and convert to basis for additional paths config.filenames = args.img config.filename = config.filenames[0] print("Set filenames to {}, current filename {}".format( config.filenames, config.filename)) if args.meta is not None: # set metadata paths config.metadata_paths = args.meta print("Set metadata paths to", config.metadata_paths) config.metadatas = [] for path in config.metadata_paths: # load metadata to dictionary md, _ = importer.load_metadata(path, assign=False) config.metadatas.append(md) if args.channel is not None: # set the channels config.channel = args.channel print("Set channel to {}".format(config.channel)) series_list = [config.series] # list of series if args.series is not None: series_split = args.series.split(",") series_list = [] for ser in series_split: ser_split = ser.split("-") if len(ser_split) > 1: ser_range = np.arange(int(ser_split[0]), int(ser_split[1]) + 1) series_list.extend(ser_range.tolist()) else: series_list.append(int(ser_split[0])) config.series = series_list[0] print("Set to series_list to {}, current series {}".format( series_list, config.series)) if args.savefig is not None: # save figure with file type of this extension; remove leading period config.savefig = args.savefig.lstrip(".") print("Set savefig extension to {}".format(config.savefig)) if args.verbose: # verbose mode, including printing longer Numpy arrays for debugging config.verbose = args.verbose np.set_printoptions(linewidth=200, threshold=10000) print("Set verbose to {}".format(config.verbose)) # parse sub-image offsets and sizes; # expects x,y,z input but stores as z,y,x by convention if args.subimg_offset is not None: config.subimg_offsets = _parse_coords(args.subimg_offset, True) print("Set sub-image offsets to {} (z,y,x)".format( config.subimg_offsets)) if args.subimg_size is not None: config.subimg_sizes = _parse_coords(args.subimg_size, True) print("Set sub-image sizes to {} (z,y,x)".format(config.subimg_sizes)) # parse ROI offsets and sizes, which are relative to any sub-image; # expects x,y,z input and output if args.offset is not None: config.roi_offsets = _parse_coords(args.offset) if config.roi_offsets: config.roi_offset = config.roi_offsets[0] print("Set ROI offsets to {}, current offset {} (x,y,z)".format( config.roi_offsets, config.roi_offset)) if args.size is not None: config.roi_sizes = _parse_coords(args.size) if config.roi_sizes: config.roi_size = config.roi_sizes[0] print("Set ROI sizes to {}, current size {} (x,y,z)".format( config.roi_sizes, config.roi_size)) if args.cpus is not None: # set maximum number of CPUs config.cpus = (None if args.cpus.lower() in ("none", "0") else int(args.cpus)) print("Set maximum number of CPUs for multiprocessing tasks to", config.cpus) # set up main processing mode if args.proc is not None: config.proc_type = args.proc print("processing type set to {}".format(config.proc_type)) proc_type = libmag.get_enum(config.proc_type, config.ProcessTypes) if config.proc_type and proc_type not in config.ProcessTypes: libmag.warn("\"{}\" processing type not found".format( config.proc_type)) if args.set_meta is not None: # set individual metadata values, currently used for image import # TODO: take precedence over loaded metadata archives config.meta_dict = args_to_dict(args.set_meta, config.MetaKeys, config.meta_dict, sep_vals="|") print("Set metadata values to {}".format(config.meta_dict)) res = config.meta_dict[config.MetaKeys.RESOLUTIONS] if res: # set image resolutions, taken as a single set of x,y,z and # converting to a nested list of z,y,x res_split = res.split(",") if len(res_split) >= 3: res_float = tuple(float(i) for i in res_split)[::-1] config.resolutions = [res_float] print("Set resolutions to {}".format(config.resolutions)) else: res_float = None print("Resolution ({}) should be given as 3 values (x,y,z)". format(res)) # store single set of resolutions, similar to input config.meta_dict[config.MetaKeys.RESOLUTIONS] = res_float mag = config.meta_dict[config.MetaKeys.MAGNIFICATION] if mag: # set objective magnification config.magnification = mag print("Set magnification to {}".format(config.magnification)) zoom = config.meta_dict[config.MetaKeys.ZOOM] if zoom: # set objective zoom config.zoom = zoom print("Set zoom to {}".format(config.zoom)) shape = config.meta_dict[config.MetaKeys.SHAPE] if shape: # parse shape, storing only in dict config.meta_dict[config.MetaKeys.SHAPE] = [ int(n) for n in shape.split(",")[::-1] ] # set up ROI and register profiles setup_profiles(args.roi_profile, args.atlas_profile, args.grid_search) if args.plane is not None: config.plane = args.plane print("Set plane to {}".format(config.plane)) if args.save_subimg: config.save_subimg = args.save_subimg print("Set to save the sub-image") if args.labels: # set up atlas labels setup_labels(args.labels) if args.transform is not None: # image transformations such as flipping, rotation config.transform = args_to_dict(args.transform, config.Transforms, config.transform) print("Set transformations to {}".format(config.transform)) if args.register: # register type to process in register module config.register_type = args.register print("Set register type to {}".format(config.register_type)) if args.df: # data frame processing task config.df_task = args.df print("Set data frame processing task to {}".format(config.df_task)) if args.plot_2d: # 2D plot type to process in plot_2d module config.plot_2d_type = args.plot_2d print("Set plot_2d type to {}".format(config.plot_2d_type)) if args.slice: # specify a generic slice by command-line, assuming same order # of arguments as for slice built-in function and interpreting # "none" string as None config.slice_vals = args.slice.split(",") config.slice_vals = [ None if val.lower() == "none" else int(val) for val in config.slice_vals ] print("Set slice values to {}".format(config.slice_vals)) if args.delay: config.delay = int(args.delay) print("Set delay to {}".format(config.delay)) if args.show: # show images after task is performed, if supported config.show = _is_arg_true(args.show) print("Set show to {}".format(config.show)) if args.groups: config.groups = args.groups print("Set groups to {}".format(config.groups)) if args.ec2_start is not None: # start EC2 instances config.ec2_start = args_with_dict(args.ec2_start) print("Set ec2 start to {}".format(config.ec2_start)) if args.ec2_list: # list EC2 instances config.ec2_list = args_with_dict(args.ec2_list) print("Set ec2 list to {}".format(config.ec2_list)) if args.ec2_terminate: config.ec2_terminate = args.ec2_terminate print("Set ec2 terminate to {}".format(config.ec2_terminate)) if args.notify: notify_len = len(args.notify) if notify_len > 0: config.notify_url = args.notify[0] print("Set notification URL to {}".format(config.notify_url)) if notify_len > 1: config.notify_msg = args.notify[1] print("Set notification message to {}".format(config.notify_msg)) if notify_len > 2: config.notify_attach = args.notify[2] print("Set notification attachment path to {}".format( config.notify_attach)) if args.prefix: config.prefix = args.prefix print("Set path prefix to {}".format(config.prefix)) if args.suffix: config.suffix = args.suffix print("Set path suffix to {}".format(config.suffix)) if args.alphas: # specify alpha levels config.alphas = [float(val) for val in args.alphas.split(",")] print("Set alphas to", config.alphas) if args.vmin: # specify vmin levels config.vmins = [libmag.get_int(val) for val in args.vmin.split(",")] print("Set vmins to", config.vmins) if args.vmax: # specify vmax levels and copy to vmax overview used for plotting # and updated for normalization config.vmaxs = [libmag.get_int(val) for val in args.vmax.split(",")] config.vmax_overview = list(config.vmaxs) print("Set vmaxs to", config.vmaxs) if args.reg_suffixes is not None: # specify suffixes of registered images to load config.reg_suffixes = args_to_dict(args.reg_suffixes, config.RegSuffixes, config.reg_suffixes) print("Set registered image suffixes to {}".format( config.reg_suffixes)) if args.seed: # specify random number generator seed config.seed = int(args.seed) print("Set random number generator seed to", config.seed) if args.plot_labels is not None: # specify general plot labels config.plot_labels = args_to_dict(args.plot_labels, config.PlotLabels, config.plot_labels) print("Set plot labels to {}".format(config.plot_labels)) if args.theme is not None: # specify themes, currently applied to Matplotlib elements theme_names = [] for theme in args.theme: # add theme enum if found theme_enum = libmag.get_enum(theme, config.Themes) if theme_enum: config.rc_params.append(theme_enum) theme_names.append(theme_enum.name) print("Set to use themes to {}".format(theme_names)) # prep filename filename_base = None if config.filename: filename_base = importer.filename_to_base(config.filename, config.series) if not skip_dbs: setup_dbs(filename_base, args.db, args.truth_db) # set multiprocessing start method chunking.set_mp_start_method() # POST-ARGUMENT PARSING if process_args_only: return # if command-line driven task specified, start task and shut down if config.register_type: register.main() elif config.notify_url: notify.main() elif config.plot_2d_type: plot_2d.main() elif config.df_task: df_io.main() elif config.grid_search_profile: _grid_search(series_list) elif config.ec2_list or config.ec2_start or config.ec2_terminate: # defer importing AWS module to avoid making its dependencies # required for MagellanMapper from magmap.cloud import aws aws.main() else: # set up image and perform any whole image processing tasks; # do not shut down if not a command-line proc task _process_files(series_list) if proc_type is None or proc_type is config.ProcessTypes.LOAD: return shutdown()
def read_sitk_files( filename_sitk: str, reg_names: Optional[Union[str, Sequence[str]]] = None, return_sitk: bool = False, make_3d: bool = False ) -> Union["np_io.Image5d", Tuple["np_io.Image5d", sitk.Image]]: """Read image files through SimpleITK. Supports identifying files based on registered suffixes and combining multiple registered image files into a single image. Also sets up spacing from the first loaded image in :attr:`magmap.settings.config.resolutions` if not already set. Args: filename_sitk: Path to file in a format that can be read by SimpleITK. reg_names: Path or sequence of paths of registered names. Can be a registered suffix or a full path. Defaults to None to open ``filename_sitk`` as-is through :meth:`read_sitk`. return_sitk: True to return the loaded SimpleITK Image object. make_3d: True to convert 2D images to 3D; defaults to False. Returns: ``img5d``: Image5d instance with the loaded image in Numpy 5D format (or 4D if not multi-channel, and 3D if originally 2D and ``make_3d`` is False). Associated metadata is loaded into :module:`magmap.settings.config` attributes. If ``return_sitk`` is True, also returns the first loaded image in SimpleITK format. Raises: ``FileNotFoundError`` if ``filename_sitk`` cannot be found, after attempting to load metadata from ``filename_np``. """ # load image via SimpleITK img_sitk = None loaded_path = filename_sitk if reg_names: img_nps = [] if not libmag.is_seq(reg_names): reg_names = [reg_names] for reg_name in reg_names: # load each registered suffix into list of images with same shape, # keeping first image in sitk format img, path = _load_reg_img_to_combine(filename_sitk, reg_name, img_nps) if img_sitk is None: img_sitk = img loaded_path = path if len(img_nps) > 1: # merge images into separate channels img_np = np.stack(img_nps, axis=img_nps[0].ndim) else: img_np = img_nps[0] else: # load filename_sitk directly if not os.path.exists(filename_sitk): raise FileNotFoundError( "could not find file {}".format(filename_sitk)) img_sitk, _ = read_sitk(filename_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if make_3d: # convert 2D images to 3D # TODO: consider converting img_np to 3D regardless so array in # Image5d is at least 4D img_sitk = _make_3d(img_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if config.resolutions is None: # fallback to determining metadata directly from sitk file libmag.warn( "MagellanMapper image metadata file not loaded; will fallback to " "{} for metadata".format(loaded_path)) config.resolutions = np.array([img_sitk.GetSpacing()[::-1]]) print("set resolutions to {}".format(config.resolutions)) # add time axis and insert into Image5d with original name img5d = np_io.Image5d(img_np[None], filename_sitk, img_io=config.LoadIO.SITK) if return_sitk: return img5d, img_sitk return img5d
def read_sitk_files(filename_sitk, reg_names=None, return_sitk=False): """Read an image file through SimpleITK and export to Numpy array format, with support for combining multiple registered image files into a single image. Also sets up spacing from the first loaded image in :attr:`magmap.settings.config.resolutions` if not already set. Args: filename_sitk (str): Path to file in a format that can be read by SimpleITK. reg_names (Union[str, List[str]]): Path or sequence of paths of registered names. Can be a registered suffix or a full path. Defaults to None to open ``filename_sitk`` as-is through :meth:`read_sitk`. return_sitk (bool): True to return the loaded SimpleITK Image object. Returns: :class:`numpy.ndarray`: Image array in Numpy 3D format (or 4D if multi-channel). Associated metadata is loaded into :module:`config` attributes. If ``return_sitk`` is True, also returns the first loaded image in SimpleITK format. Raises: ``FileNotFoundError`` if ``filename_sitk`` cannot be found, after attempting to load metadata from ``filename_np``. """ # load image via SimpleITK img_sitk = None loaded_path = filename_sitk if reg_names: img_nps = [] if not libmag.is_seq(reg_names): reg_names = [reg_names] for reg_name in reg_names: # load each registered suffix into list of images with same shape, # keeping first image in sitk format img, path = _load_reg_img_to_combine(filename_sitk, reg_name, img_nps) if img_sitk is None: img_sitk = img loaded_path = path if len(img_nps) > 1: # merge images into separate channels img_np = np.stack(img_nps, axis=img_nps[0].ndim) else: img_np = img_nps[0] else: # load filename_sitk directly if not os.path.exists(filename_sitk): raise FileNotFoundError( "could not find file {}".format(filename_sitk)) img_sitk, _ = read_sitk(filename_sitk) img_np = sitk.GetArrayFromImage(img_sitk) if config.resolutions is None: # fallback to determining metadata directly from sitk file libmag.warn( "MagellanMapper image metadata file not loaded; will fallback to " "{} for metadata".format(loaded_path)) config.resolutions = np.array([img_sitk.GetSpacing()[::-1]]) print("set resolutions to {}".format(config.resolutions)) if return_sitk: return img_np, img_sitk return img_np