def _get_truth_db_rois(subimg_path_base, filename_base, db_path_base=None): """Get ROIs from a truth database. Args: subimg_path_base (str): Base path with sub-image. filename_base (str): Base path without sub-image to find the experiment, used only if an experiment cannot be found based on ``subimg_path_base``. db_path_base (str): Path to database to load; defaults to None to use :attr:`config.truth_db`. Returns: str, list[:class:`sqlite3.Row`]: Found experiment name and list of database ROI rows in that experiment, or None for each if the ROIs are not found. """ name = None exp_rois = None if db_path_base: # load truth DB _logger.debug("Loading truth db for verifications from '%s'", db_path_base) sqlite.load_truth_db(db_path_base) if config.truth_db is not None: # load experiment and ROIs from truth DB using the sub-image-based # name; series not included in exp name since in ROI name = sqlite.get_exp_name(subimg_path_base) _logger.debug("Loading truth ROIs from experiment '%s'", name) exp_rois = config.truth_db.get_rois(name) if exp_rois is None: # exp may have been named without sub-image old_name = name name = sqlite.get_exp_name(filename_base) _logger.debug( "'%s' experiment name not found, will try without any " "sub-image offset/size: '%s'", old_name, name) exp_rois = config.truth_db.get_rois(name) if exp_rois is None: # exp may differ from image name all together _logger.debug( "'%s' experiment name not found, will try first " "available experiment", name) exps = config.truth_db.select_experiment() if exps: name = exps[0]["name"] _logger.debug( "Loading ROIs from first available experiment: '%s'", name) exp_rois = config.truth_db.get_rois(name) if not exp_rois: _logger.warn("No matching experiments found in the truth database") return name, exp_rois
def _get_truth_db_rois(subimg_path_base, filename_base, db_path_base=None): """Get ROIs from a truth database. Args: subimg_path_base (str): Base path with sub-image. filename_base (str): Base path without sub-image to find the experiment, used only if an experiment cannot be found based on ``subimg_path_base``. db_path_base (str): Path to database to load; defaults to None to use :attr:`config.truth_db`. Returns: str, list[:class:`sqlite3.Row`]: Found experiment name and list of database ROI rows in that experiment, or None for each if the ROIs are not found. """ name = None exp_rois = None if db_path_base: # load truth DB print("Loading truth db for verifications from", db_path_base) sqlite.load_truth_db(db_path_base) if config.truth_db is not None: # load experiment and ROIs from truth DB using the sub-image-based # name; series not included in exp name since in ROI name = sqlite.get_exp_name(subimg_path_base) print("Loading truth ROIs from experiment:", name) exp_rois = config.truth_db.get_rois(name) if exp_rois is None: # exp may have been named without sub-image print("{} experiment name not found, will try without " "sub-image offset/size".format(name)) name = sqlite.get_exp_name(filename_base) exp_rois = config.truth_db.get_rois(name) return name, exp_rois
def setup_dbs(): """Set up databases for the given image file. Only sets up each database if it has not been set up already. """ # prep filename filename_base = None if config.filename: filename_base = importer.filename_to_base(config.filename, config.series) # get any user-supplied truth database path, falling back to name based # on filename or default name truth_db_path = config.truth_db_params[config.TruthDB.PATH] user_dir = config.user_app_dirs.user_data_dir truth_db_name_base = filename_base if filename_base else os.path.join( user_dir, sqlite.DB_NAME_BASE) if config.truth_db_mode is config.TruthDBModes.VIEW: # loads truth DB as a separate database in parallel with the given # editable database, with name based on filename by default unless # truth DB name explicitly given path = truth_db_path if truth_db_path else truth_db_name_base try: sqlite.load_truth_db(path) except FileNotFoundError as e: print(e) print("Could not load truth DB from current image path") elif config.truth_db_mode is config.TruthDBModes.VERIFY: if not config.verified_db: # creates a new verified DB to store all ROC results config.verified_db = sqlite.ClrDB() config.verified_db.load_db( os.path.join(user_dir, sqlite.DB_NAME_VERIFIED), True) if truth_db_path: # load truth DB path to verify against if explicitly given try: sqlite.load_truth_db(truth_db_path) except FileNotFoundError as e: print(e) print("Could not load truth DB from {}".format(truth_db_path)) elif config.truth_db_mode is config.TruthDBModes.VERIFIED: # loads verified DB as the main DB, which includes copies of truth # values with flags for whether they were detected path = os.path.join(user_dir, sqlite.DB_NAME_VERIFIED) if truth_db_path: path = truth_db_path try: config.db = sqlite.ClrDB() config.db.load_db(path) config.verified_db = config.db except FileNotFoundError as e: print(e) print("Could not load verified DB from {}".format( sqlite.DB_NAME_VERIFIED)) elif config.truth_db_mode is config.TruthDBModes.EDIT: # loads truth DB as the main database for editing rather than # loading as a truth database config.db_path = truth_db_path if not config.db_path: config.db_path = "{}{}".format( os.path.basename(truth_db_name_base), sqlite.DB_SUFFIX_TRUTH) print("Editing truth database at {}".format(config.db_path)) if config.db is None: # load the main database config.db = sqlite.ClrDB() config.db.load_db(None, False)
def detect_blobs_large_image(filename_base, image5d, offset, size, verify=False, save_dfs=True, full_roi=False): """Detect blobs within a large image through parallel processing of smaller chunks. Args: filename_base: Base path to use file output. image5d: Large image to process as a Numpy array of t,z,y,x,[c] offset: Sub-image offset given as coordinates in z,y,x. size: Sub-image shape given in z,y,x. verify: True to verify detections against truth database; defaults to False. save_dfs: True to save data frames to file; defaults to True. full_roi (bool): True to treat ``image5d`` as the full ROI; defaults to False. """ time_start = time() if size is None or offset is None: # uses the entire stack if no size or offset specified size = image5d.shape[1:4] offset = (0, 0, 0) else: # change base filename for ROI-based partial stack filename_base = make_subimage_name(filename_base, offset, size) filename_subimg = libmag.combine_paths(filename_base, config.SUFFIX_SUBIMG) filename_blobs = libmag.combine_paths(filename_base, config.SUFFIX_BLOBS) # get ROI for given region, including all channels if full_roi: # treat the full image as the ROI roi = image5d[0] else: roi = plot_3d.prepare_subimg(image5d, size, offset) _, channels = plot_3d.setup_channels(roi, config.channel, 3) # prep chunking ROI into sub-ROIs with size based on segment_size, scaling # by physical units to make more independent of resolution time_detection_start = time() settings = config.roi_profile # use default settings scaling_factor = detector.calc_scaling_factor() print("microsope scaling factor based on resolutions: {}" .format(scaling_factor)) denoise_size = config.roi_profile["denoise_size"] denoise_max_shape = None if denoise_size: # further subdivide each sub-ROI for local preprocessing denoise_max_shape = np.ceil( np.multiply(scaling_factor, denoise_size)).astype(int) # overlap sub-ROIs to minimize edge effects overlap_base = chunking.calc_overlap() tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int) overlap_padding = np.copy(tol) overlap = np.copy(overlap_base) exclude_border = config.roi_profile["exclude_border"] if exclude_border is not None: # exclude border to avoid blob detector edge effects, where blobs # often collect at the faces of the sub-ROI; # ensure that overlap is greater than twice the border exclusion per # axis so that no plane will be excluded from both overlapping sub-ROIs exclude_border_thresh = np.multiply(2, exclude_border) overlap_less = np.less(overlap, exclude_border_thresh) overlap[overlap_less] = exclude_border_thresh[overlap_less] excluded = np.greater(exclude_border, 0) overlap[excluded] += 1 # additional padding overlap_padding[excluded] = 0 # no need to prune past excluded border print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond " "overlap for pruning: {}, exclude borders: {}" .format(overlap, tol, overlap_padding, exclude_border)) max_pixels = np.ceil(np.multiply( scaling_factor, config.roi_profile["segment_size"])).astype(int) print("preprocessing max shape: {}, detection max pixels: {}" .format(denoise_max_shape, max_pixels)) sub_roi_slices, sub_rois_offsets = chunking.stack_splitter( roi.shape, max_pixels, overlap) # TODO: option to distribute groups of sub-ROIs to different servers # for blob detection seg_rois = detect_blobs_sub_rois( roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border) detection_time = time() - time_detection_start print("blob detection time (s):", detection_time) # prune blobs in overlapping portions of sub-ROIs time_pruning_start = time() segments_all, df_pruning = _prune_blobs_mp( roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels, overlap_padding) pruning_time = time() - time_pruning_start print("blob pruning time (s):", pruning_time) #print("maxes:", np.amax(segments_all, axis=0)) # get weighted mean of ratios if df_pruning is not None: print("\nBlob pruning ratios:") path_pruning = "blob_ratios.csv" if save_dfs else None df_pruning_all = df_io.data_frames_to_csv( df_pruning, path_pruning, show=" ") cols = df_pruning_all.columns.tolist() blob_pruning_means = {} if "blobs" in cols: blobs_unpruned = df_pruning_all["blobs"] num_blobs_unpruned = np.sum(blobs_unpruned) for col in cols[1:]: blob_pruning_means["mean_{}".format(col)] = [ np.sum(np.multiply(df_pruning_all[col], blobs_unpruned)) / num_blobs_unpruned] path_pruning_means = "blob_ratios_means.csv" if save_dfs else None df_pruning_means = df_io.dict_to_data_frame( blob_pruning_means, path_pruning_means, show=" ") else: print("no blob ratios found") '''# report any remaining duplicates np.set_printoptions(linewidth=500, threshold=10000000) print("all blobs (len {}):".format(len(segments_all))) sort = np.lexsort( (segments_all[:, 2], segments_all[:, 1], segments_all[:, 0])) blobs = segments_all[sort] print(blobs) print("checking for duplicates in all:") print(detector.remove_duplicate_blobs(blobs, slice(0, 3))) ''' stats_detection = None fdbk = None if segments_all is not None: # remove the duplicated elements that were used for pruning detector.replace_rel_with_abs_blob_coords(segments_all) segments_all = detector.remove_abs_blob_coords(segments_all) # compare detected blobs with truth blobs # TODO: assumes ground truth is relative to any ROI offset, # but should make customizable if verify: db_path_base = None exp_name = os.path.splitext(os.path.basename(config.filename))[0] try: if config.truth_db is None: # find and load truth DB based on filename and subimage db_path_base = os.path.basename(filename_base) print("about to verify with truth db from {}" .format(db_path_base)) sqlite.load_truth_db(db_path_base) if config.truth_db is not None: # truth DB may contain multiple experiments for different # subimages; series not included in exp name since in ROI rois = config.truth_db.get_rois(exp_name) if rois is None: # exp may have been named by ROI print("{} experiment name not found, will try with" "ROI offset/size".format(exp_name)) exp_name = make_subimage_name(exp_name, offset, size) rois = config.truth_db.get_rois(exp_name) if rois is None: raise LookupError( "No truth set ROIs found for experiment {}, will " "skip detection verification".format(exp_name)) print("load ROIs from exp: {}".format(exp_name)) exp_id = sqlite.insert_experiment( config.verified_db.conn, config.verified_db.cur, exp_name, None) verify_tol = np.multiply( overlap_base, settings["verify_tol_factor"]) stats_detection, fdbk = detector.verify_rois( rois, segments_all, config.truth_db.blobs_truth, verify_tol, config.verified_db, exp_id, config.channel) except FileNotFoundError: libmag.warn("Could not load truth DB from {}; " "will not verify ROIs".format(db_path_base)) except LookupError as e: libmag.warn(str(e)) file_time_start = time() if config.save_subimg: if (isinstance(config.image5d, np.memmap) and config.image5d.filename == os.path.abspath(filename_subimg)): # file at sub-image save path may have been opened as a memmap # file, in which case saving would fail libmag.warn("{} is currently open, cannot save sub-image" .format(filename_subimg)) else: # write sub-image, which is in ROI (3D) format with open(filename_subimg, "wb") as f: np.save(f, roi) # save blobs # TODO: only segments used; consider removing the rest except ver outfile_blobs = open(filename_blobs, "wb") np.savez(outfile_blobs, ver=BLOBS_NP_VER, segments=segments_all, resolutions=config.resolutions, basename=os.path.basename(config.filename), # only save name offset=offset, roi_size=size) # None unless explicitly set outfile_blobs.close() file_save_time = time() - file_time_start # whole image benchmarking time times = ( [detection_time], [pruning_time], time() - time_start) times_dict = {} for key, val in zip(StackTimes, times): times_dict[key] = val if segments_all is None: print("\nNo blobs detected") else: print("\nTotal blobs found:", len(segments_all)) detector.show_blobs_per_channel(segments_all) print("file save time:", file_save_time) print("\nTotal detection processing times (s):") path_times = "stack_detection_times.csv" if save_dfs else None df_io.dict_to_data_frame(times_dict, path_times, show=" ") return stats_detection, fdbk, segments_all
def setup_dbs(filename_base, db_path=None, truth_db_config=None): """Set up databases for the given image file if the given database has not been set up already. Args: filename_base (str): Image base path. db_path (str): Main database path; defaults to None to use a default path. truth_db_config (List[str]): Sequence of truth database configuration settings; defaults to None to not load truth-related databases. """ if db_path: config.db_name = db_path print("Set database name to {}".format(config.db_name)) # load "truth blobs" from separate database for viewing if truth_db_config is not None: # set the truth database mode config.truth_db_params = args_to_dict(truth_db_config, config.TruthDB, config.truth_db_params, sep_vals="|") mode = config.truth_db_params[config.TruthDB.MODE] config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes) libmag.printv(config.truth_db_params) print("Mapped \"{}\" truth_db mode to {}".format( mode, config.truth_db_mode)) truth_db_path = config.truth_db_params[config.TruthDB.PATH] truth_db_name_base = filename_base if filename_base else sqlite.DB_NAME_BASE if config.truth_db_mode is config.TruthDBModes.VIEW: # loads truth DB as a separate database in parallel with the given # editable database, with name based on filename by default unless # truth DB name explicitly given path = truth_db_path if truth_db_path else truth_db_name_base try: sqlite.load_truth_db(path) except FileNotFoundError as e: print(e) print("Could not load truth DB from current image path") elif config.truth_db_mode is config.TruthDBModes.VERIFY: if not config.verified_db: # creates a new verified DB to store all ROC results config.verified_db = sqlite.ClrDB() config.verified_db.load_db(sqlite.DB_NAME_VERIFIED, True) if truth_db_path: # load truth DB path to verify against if explicitly given try: sqlite.load_truth_db(truth_db_path) except FileNotFoundError as e: print(e) print("Could not load truth DB from {}".format(truth_db_path)) elif config.truth_db_mode is config.TruthDBModes.VERIFIED: # loads verified DB as the main DB, which includes copies of truth # values with flags for whether they were detected path = sqlite.DB_NAME_VERIFIED if truth_db_path: path = truth_db_path try: config.db = sqlite.ClrDB() config.db.load_db(path) config.verified_db = config.db except FileNotFoundError as e: print(e) print("Could not load verified DB from {}".format( sqlite.DB_NAME_VERIFIED)) elif config.truth_db_mode is config.TruthDBModes.EDIT: # loads truth DB as the main database for editing rather than # loading as a truth database config.db_name = truth_db_path if not config.db_name: config.db_name = "{}{}".format( os.path.basename(truth_db_name_base), sqlite.DB_SUFFIX_TRUTH) print("Editing truth database at {}".format(config.db_name)) if config.db is None: config.db = sqlite.ClrDB() config.db.load_db(None, False)