def _get_truth_db_rois(subimg_path_base, filename_base, db_path_base=None):
"""Get ROIs from a truth database.
Args:
subimg_path_base (str): Base path with sub-image.
filename_base (str): Base path without sub-image to find the
experiment, used only if an experiment cannot be found based on
``subimg_path_base``.
db_path_base (str): Path to database to load; defaults to None
to use :attr:`config.truth_db`.
Returns:
str, list[:class:`sqlite3.Row`]: Found experiment name and
list of database ROI rows in that experiment, or None for each
if the ROIs are not found.
"""
name = None
exp_rois = None
if db_path_base:
# load truth DB
_logger.debug("Loading truth db for verifications from '%s'",
db_path_base)
sqlite.load_truth_db(db_path_base)
if config.truth_db is not None:
# load experiment and ROIs from truth DB using the sub-image-based
# name; series not included in exp name since in ROI
name = sqlite.get_exp_name(subimg_path_base)
_logger.debug("Loading truth ROIs from experiment '%s'", name)
exp_rois = config.truth_db.get_rois(name)
if exp_rois is None:
# exp may have been named without sub-image
old_name = name
name = sqlite.get_exp_name(filename_base)
_logger.debug(
"'%s' experiment name not found, will try without any "
"sub-image offset/size: '%s'", old_name, name)
exp_rois = config.truth_db.get_rois(name)
if exp_rois is None:
# exp may differ from image name all together
_logger.debug(
"'%s' experiment name not found, will try first "
"available experiment", name)
exps = config.truth_db.select_experiment()
if exps:
name = exps[0]["name"]
_logger.debug(
"Loading ROIs from first available experiment: '%s'", name)
exp_rois = config.truth_db.get_rois(name)
if not exp_rois:
_logger.warn("No matching experiments found in the truth database")
return name, exp_rois
def _get_truth_db_rois(subimg_path_base, filename_base, db_path_base=None):
"""Get ROIs from a truth database.
Args:
subimg_path_base (str): Base path with sub-image.
filename_base (str): Base path without sub-image to find the
experiment, used only if an experiment cannot be found based on
``subimg_path_base``.
db_path_base (str): Path to database to load; defaults to None
to use :attr:`config.truth_db`.
Returns:
str, list[:class:`sqlite3.Row`]: Found experiment name and
list of database ROI rows in that experiment, or None for each
if the ROIs are not found.
"""
name = None
exp_rois = None
if db_path_base:
# load truth DB
print("Loading truth db for verifications from", db_path_base)
sqlite.load_truth_db(db_path_base)
if config.truth_db is not None:
# load experiment and ROIs from truth DB using the sub-image-based
# name; series not included in exp name since in ROI
name = sqlite.get_exp_name(subimg_path_base)
print("Loading truth ROIs from experiment:", name)
exp_rois = config.truth_db.get_rois(name)
if exp_rois is None:
# exp may have been named without sub-image
print("{} experiment name not found, will try without "
"sub-image offset/size".format(name))
name = sqlite.get_exp_name(filename_base)
exp_rois = config.truth_db.get_rois(name)
return name, exp_rois
def setup_dbs():
"""Set up databases for the given image file.
Only sets up each database if it has not been set up already.
"""
# prep filename
filename_base = None
if config.filename:
filename_base = importer.filename_to_base(config.filename,
config.series)
# get any user-supplied truth database path, falling back to name based
# on filename or default name
truth_db_path = config.truth_db_params[config.TruthDB.PATH]
user_dir = config.user_app_dirs.user_data_dir
truth_db_name_base = filename_base if filename_base else os.path.join(
user_dir, sqlite.DB_NAME_BASE)
if config.truth_db_mode is config.TruthDBModes.VIEW:
# loads truth DB as a separate database in parallel with the given
# editable database, with name based on filename by default unless
# truth DB name explicitly given
path = truth_db_path if truth_db_path else truth_db_name_base
try:
sqlite.load_truth_db(path)
except FileNotFoundError as e:
print(e)
print("Could not load truth DB from current image path")
elif config.truth_db_mode is config.TruthDBModes.VERIFY:
if not config.verified_db:
# creates a new verified DB to store all ROC results
config.verified_db = sqlite.ClrDB()
config.verified_db.load_db(
os.path.join(user_dir, sqlite.DB_NAME_VERIFIED), True)
if truth_db_path:
# load truth DB path to verify against if explicitly given
try:
sqlite.load_truth_db(truth_db_path)
except FileNotFoundError as e:
print(e)
print("Could not load truth DB from {}".format(truth_db_path))
elif config.truth_db_mode is config.TruthDBModes.VERIFIED:
# loads verified DB as the main DB, which includes copies of truth
# values with flags for whether they were detected
path = os.path.join(user_dir, sqlite.DB_NAME_VERIFIED)
if truth_db_path: path = truth_db_path
try:
config.db = sqlite.ClrDB()
config.db.load_db(path)
config.verified_db = config.db
except FileNotFoundError as e:
print(e)
print("Could not load verified DB from {}".format(
sqlite.DB_NAME_VERIFIED))
elif config.truth_db_mode is config.TruthDBModes.EDIT:
# loads truth DB as the main database for editing rather than
# loading as a truth database
config.db_path = truth_db_path
if not config.db_path:
config.db_path = "{}{}".format(
os.path.basename(truth_db_name_base), sqlite.DB_SUFFIX_TRUTH)
print("Editing truth database at {}".format(config.db_path))
if config.db is None:
# load the main database
config.db = sqlite.ClrDB()
config.db.load_db(None, False)
def detect_blobs_large_image(filename_base, image5d, offset, size,
verify=False, save_dfs=True, full_roi=False):
"""Detect blobs within a large image through parallel processing of
smaller chunks.
Args:
filename_base: Base path to use file output.
image5d: Large image to process as a Numpy array of t,z,y,x,[c]
offset: Sub-image offset given as coordinates in z,y,x.
size: Sub-image shape given in z,y,x.
verify: True to verify detections against truth database; defaults
to False.
save_dfs: True to save data frames to file; defaults to True.
full_roi (bool): True to treat ``image5d`` as the full ROI; defaults
to False.
"""
time_start = time()
if size is None or offset is None:
# uses the entire stack if no size or offset specified
size = image5d.shape[1:4]
offset = (0, 0, 0)
else:
# change base filename for ROI-based partial stack
filename_base = make_subimage_name(filename_base, offset, size)
filename_subimg = libmag.combine_paths(filename_base, config.SUFFIX_SUBIMG)
filename_blobs = libmag.combine_paths(filename_base, config.SUFFIX_BLOBS)
# get ROI for given region, including all channels
if full_roi:
# treat the full image as the ROI
roi = image5d[0]
else:
roi = plot_3d.prepare_subimg(image5d, size, offset)
_, channels = plot_3d.setup_channels(roi, config.channel, 3)
# prep chunking ROI into sub-ROIs with size based on segment_size, scaling
# by physical units to make more independent of resolution
time_detection_start = time()
settings = config.roi_profile # use default settings
scaling_factor = detector.calc_scaling_factor()
print("microsope scaling factor based on resolutions: {}"
.format(scaling_factor))
denoise_size = config.roi_profile["denoise_size"]
denoise_max_shape = None
if denoise_size:
# further subdivide each sub-ROI for local preprocessing
denoise_max_shape = np.ceil(
np.multiply(scaling_factor, denoise_size)).astype(int)
# overlap sub-ROIs to minimize edge effects
overlap_base = chunking.calc_overlap()
tol = np.multiply(overlap_base, settings["prune_tol_factor"]).astype(int)
overlap_padding = np.copy(tol)
overlap = np.copy(overlap_base)
exclude_border = config.roi_profile["exclude_border"]
if exclude_border is not None:
# exclude border to avoid blob detector edge effects, where blobs
# often collect at the faces of the sub-ROI;
# ensure that overlap is greater than twice the border exclusion per
# axis so that no plane will be excluded from both overlapping sub-ROIs
exclude_border_thresh = np.multiply(2, exclude_border)
overlap_less = np.less(overlap, exclude_border_thresh)
overlap[overlap_less] = exclude_border_thresh[overlap_less]
excluded = np.greater(exclude_border, 0)
overlap[excluded] += 1 # additional padding
overlap_padding[excluded] = 0 # no need to prune past excluded border
print("sub-ROI overlap: {}, pruning tolerance: {}, padding beyond "
"overlap for pruning: {}, exclude borders: {}"
.format(overlap, tol, overlap_padding, exclude_border))
max_pixels = np.ceil(np.multiply(
scaling_factor,
config.roi_profile["segment_size"])).astype(int)
print("preprocessing max shape: {}, detection max pixels: {}"
.format(denoise_max_shape, max_pixels))
sub_roi_slices, sub_rois_offsets = chunking.stack_splitter(
roi.shape, max_pixels, overlap)
# TODO: option to distribute groups of sub-ROIs to different servers
# for blob detection
seg_rois = detect_blobs_sub_rois(
roi, sub_roi_slices, sub_rois_offsets, denoise_max_shape, exclude_border)
detection_time = time() - time_detection_start
print("blob detection time (s):", detection_time)
# prune blobs in overlapping portions of sub-ROIs
time_pruning_start = time()
segments_all, df_pruning = _prune_blobs_mp(
roi, seg_rois, overlap, tol, sub_roi_slices, sub_rois_offsets, channels,
overlap_padding)
pruning_time = time() - time_pruning_start
print("blob pruning time (s):", pruning_time)
#print("maxes:", np.amax(segments_all, axis=0))
# get weighted mean of ratios
if df_pruning is not None:
print("\nBlob pruning ratios:")
path_pruning = "blob_ratios.csv" if save_dfs else None
df_pruning_all = df_io.data_frames_to_csv(
df_pruning, path_pruning, show=" ")
cols = df_pruning_all.columns.tolist()
blob_pruning_means = {}
if "blobs" in cols:
blobs_unpruned = df_pruning_all["blobs"]
num_blobs_unpruned = np.sum(blobs_unpruned)
for col in cols[1:]:
blob_pruning_means["mean_{}".format(col)] = [
np.sum(np.multiply(df_pruning_all[col], blobs_unpruned))
/ num_blobs_unpruned]
path_pruning_means = "blob_ratios_means.csv" if save_dfs else None
df_pruning_means = df_io.dict_to_data_frame(
blob_pruning_means, path_pruning_means, show=" ")
else:
print("no blob ratios found")
'''# report any remaining duplicates
np.set_printoptions(linewidth=500, threshold=10000000)
print("all blobs (len {}):".format(len(segments_all)))
sort = np.lexsort(
(segments_all[:, 2], segments_all[:, 1], segments_all[:, 0]))
blobs = segments_all[sort]
print(blobs)
print("checking for duplicates in all:")
print(detector.remove_duplicate_blobs(blobs, slice(0, 3)))
'''
stats_detection = None
fdbk = None
if segments_all is not None:
# remove the duplicated elements that were used for pruning
detector.replace_rel_with_abs_blob_coords(segments_all)
segments_all = detector.remove_abs_blob_coords(segments_all)
# compare detected blobs with truth blobs
# TODO: assumes ground truth is relative to any ROI offset,
# but should make customizable
if verify:
db_path_base = None
exp_name = os.path.splitext(os.path.basename(config.filename))[0]
try:
if config.truth_db is None:
# find and load truth DB based on filename and subimage
db_path_base = os.path.basename(filename_base)
print("about to verify with truth db from {}"
.format(db_path_base))
sqlite.load_truth_db(db_path_base)
if config.truth_db is not None:
# truth DB may contain multiple experiments for different
# subimages; series not included in exp name since in ROI
rois = config.truth_db.get_rois(exp_name)
if rois is None:
# exp may have been named by ROI
print("{} experiment name not found, will try with"
"ROI offset/size".format(exp_name))
exp_name = make_subimage_name(exp_name, offset, size)
rois = config.truth_db.get_rois(exp_name)
if rois is None:
raise LookupError(
"No truth set ROIs found for experiment {}, will "
"skip detection verification".format(exp_name))
print("load ROIs from exp: {}".format(exp_name))
exp_id = sqlite.insert_experiment(
config.verified_db.conn, config.verified_db.cur,
exp_name, None)
verify_tol = np.multiply(
overlap_base, settings["verify_tol_factor"])
stats_detection, fdbk = detector.verify_rois(
rois, segments_all, config.truth_db.blobs_truth,
verify_tol, config.verified_db, exp_id, config.channel)
except FileNotFoundError:
libmag.warn("Could not load truth DB from {}; "
"will not verify ROIs".format(db_path_base))
except LookupError as e:
libmag.warn(str(e))
file_time_start = time()
if config.save_subimg:
if (isinstance(config.image5d, np.memmap) and
config.image5d.filename == os.path.abspath(filename_subimg)):
# file at sub-image save path may have been opened as a memmap
# file, in which case saving would fail
libmag.warn("{} is currently open, cannot save sub-image"
.format(filename_subimg))
else:
# write sub-image, which is in ROI (3D) format
with open(filename_subimg, "wb") as f:
np.save(f, roi)
# save blobs
# TODO: only segments used; consider removing the rest except ver
outfile_blobs = open(filename_blobs, "wb")
np.savez(outfile_blobs, ver=BLOBS_NP_VER, segments=segments_all,
resolutions=config.resolutions,
basename=os.path.basename(config.filename), # only save name
offset=offset, roi_size=size) # None unless explicitly set
outfile_blobs.close()
file_save_time = time() - file_time_start
# whole image benchmarking time
times = (
[detection_time],
[pruning_time],
time() - time_start)
times_dict = {}
for key, val in zip(StackTimes, times):
times_dict[key] = val
if segments_all is None:
print("\nNo blobs detected")
else:
print("\nTotal blobs found:", len(segments_all))
detector.show_blobs_per_channel(segments_all)
print("file save time:", file_save_time)
print("\nTotal detection processing times (s):")
path_times = "stack_detection_times.csv" if save_dfs else None
df_io.dict_to_data_frame(times_dict, path_times, show=" ")
return stats_detection, fdbk, segments_all
def setup_dbs(filename_base, db_path=None, truth_db_config=None):
"""Set up databases for the given image file if the given database has
not been set up already.
Args:
filename_base (str): Image base path.
db_path (str): Main database path; defaults to None to use a default
path.
truth_db_config (List[str]): Sequence of truth database configuration
settings; defaults to None to not load truth-related databases.
"""
if db_path:
config.db_name = db_path
print("Set database name to {}".format(config.db_name))
# load "truth blobs" from separate database for viewing
if truth_db_config is not None:
# set the truth database mode
config.truth_db_params = args_to_dict(truth_db_config,
config.TruthDB,
config.truth_db_params,
sep_vals="|")
mode = config.truth_db_params[config.TruthDB.MODE]
config.truth_db_mode = libmag.get_enum(mode, config.TruthDBModes)
libmag.printv(config.truth_db_params)
print("Mapped \"{}\" truth_db mode to {}".format(
mode, config.truth_db_mode))
truth_db_path = config.truth_db_params[config.TruthDB.PATH]
truth_db_name_base = filename_base if filename_base else sqlite.DB_NAME_BASE
if config.truth_db_mode is config.TruthDBModes.VIEW:
# loads truth DB as a separate database in parallel with the given
# editable database, with name based on filename by default unless
# truth DB name explicitly given
path = truth_db_path if truth_db_path else truth_db_name_base
try:
sqlite.load_truth_db(path)
except FileNotFoundError as e:
print(e)
print("Could not load truth DB from current image path")
elif config.truth_db_mode is config.TruthDBModes.VERIFY:
if not config.verified_db:
# creates a new verified DB to store all ROC results
config.verified_db = sqlite.ClrDB()
config.verified_db.load_db(sqlite.DB_NAME_VERIFIED, True)
if truth_db_path:
# load truth DB path to verify against if explicitly given
try:
sqlite.load_truth_db(truth_db_path)
except FileNotFoundError as e:
print(e)
print("Could not load truth DB from {}".format(truth_db_path))
elif config.truth_db_mode is config.TruthDBModes.VERIFIED:
# loads verified DB as the main DB, which includes copies of truth
# values with flags for whether they were detected
path = sqlite.DB_NAME_VERIFIED
if truth_db_path: path = truth_db_path
try:
config.db = sqlite.ClrDB()
config.db.load_db(path)
config.verified_db = config.db
except FileNotFoundError as e:
print(e)
print("Could not load verified DB from {}".format(
sqlite.DB_NAME_VERIFIED))
elif config.truth_db_mode is config.TruthDBModes.EDIT:
# loads truth DB as the main database for editing rather than
# loading as a truth database
config.db_name = truth_db_path
if not config.db_name:
config.db_name = "{}{}".format(
os.path.basename(truth_db_name_base), sqlite.DB_SUFFIX_TRUTH)
print("Editing truth database at {}".format(config.db_name))
if config.db is None:
config.db = sqlite.ClrDB()
config.db.load_db(None, False)