def _parse_blob_matches(self, rows):
"""Parse blob match selection.
Args:
rows (List[:obj:`sqlite3.Row`]): Sequence of rows.
Returns:
:class:`magmap.cv.colocalizer.BlobMatch`: Blob match object.
Deprecated: 1.6.0
Use :meth:`select_blob_matches` instead.
"""
# build list of blob matches, which contain matching blobs and their
# distances, converting blob IDs to full blobs
matches = []
for row in rows:
matches.append((
self.select_blob_by_id(row["blob1"])[0],
self.select_blob_by_id(row["blob2"])[0], row["dist"]))
if len(rows) > 0:
# convert to data frame to access by named columns
df = df_io.dict_to_data_frame(rows, records_cols=rows[0].keys())
blob_matches = colocalizer.BlobMatch(
matches, df["id"], df["roi_id"], df["blob1"], df["blob2"])
else:
blob_matches = colocalizer.BlobMatch()
return blob_matches
def select_blob_matches_by_blob_id(self, row_id, blobn, blob_ids):
"""Select blob matches corresponding to the given blob IDs in the
given blob column.
Args:
row_id (int): Row ID.
blobn (int): 1 or 2 to indicate the first or second blob column,
respectively.
blob_ids (List[int]): Blob IDs.
Returns:
:class:`magmap.cv.colocalizer.BlobMatch`: Blob match object,
which is empty if not matches are found.
"""
matches = []
if isinstance(blob_ids, np.ndarray):
blob_ids = blob_ids.tolist()
max_params = 990 # max params of 999 in sqlite < v3.32.0
for i in range(len(blob_ids) // max_params + 1):
# select blob matches by block to avoid exceeding sqlite parameter
# limit
ids = blob_ids[i * max_params:(i + 1) * max_params]
ids.insert(0, row_id)
self.cur.execute(
"SELECT {}, id FROM blob_matches WHERE roi_id = ?"
"AND blob{} IN ({})".format(_COLS_BLOB_MATCHES, blobn,
",".join("?" * (len(ids) - 1))),
ids)
df = self._parse_blob_matches(self.cur.fetchall()).df
if df is not None:
matches.append(df)
if len(matches) > 0:
return colocalizer.BlobMatch(df=df_io.data_frames_to_csv(matches))
return colocalizer.BlobMatch()
def match_blobs_roi(blobs, blobs_base, offset, size, thresh, scaling,
inner_padding, resize=None):
"""Match blobs from two sets of blobs in an ROI, prioritizing the inner
portion of ROIs to avoid missing detections because of edge effects
while also adding matches between a blob in the inner ROI and another
blob in the remaining portion of the ROI.
Args:
blobs (:obj:`np.ndarray`): The blobs to be matched against
``blobs_base``, given as 2D array of
``[[z, row, column, radius, ...], ...]``.
blobs_base (:obj:`np.ndarray`): The blobs to which ``blobs`` will
be matched, in the same format as ``blobs``.
offset (List[int]): ROI offset from which to select blobs in x,y,z.
size (List[int]): ROI size in x,y,z.
thresh (float): Distance map threshold
scaling (List[float]): Scaling normalized by ``tol``.
inner_padding (List[float]): ROI padding shape.
resize (List[float]): Resize sequence retrieved from ROI profile;
defaults to None.
Returns:
:class:`numpy.ndarray`, :class:`numpy.ndarray`, list[int], list[int],
list[list[:class:`numpy.ndarray`, :class:`numpy.ndarray`, float]]:
Array of blobs from ``blobs``; corresponding array from ``blobs_base``
matching blobs in ``blobs``; offset of the inner portion of the ROI
in absolute coordinates of x,y,z; shape of this inner portion of the
ROI; and list of blob matches, each given as a list of
``blob_master, blob, distance``.
"""
# get all blobs in inner and total ROI
offset_inner = np.add(offset, inner_padding)
size_inner = np.subtract(size, inner_padding * 2)
libmag.printv(
"offset: {}, offset_inner: {}, size: {}, size_inner: {}"
.format(offset, offset_inner, size, size_inner))
blobs_roi, _ = detector.get_blobs_in_roi(blobs, offset, size)
if resize is not None:
# TODO: doesn't align with exported ROIs
padding = config.plot_labels[config.PlotLabels.PADDING]
libmag.printv("shifting blobs in ROI by offset {}, border {}"
.format(offset, padding))
blobs_roi = detector.shift_blob_rel_coords(blobs_roi, offset)
if padding:
blobs_roi = detector.shift_blob_rel_coords(blobs_roi, padding)
blobs_inner, blobs_inner_mask = detector.get_blobs_in_roi(
blobs_roi, offset_inner, size_inner)
blobs_base_roi, _ = detector.get_blobs_in_roi(blobs_base, offset, size)
blobs_base_inner, blobs_base_inner_mask = detector.get_blobs_in_roi(
blobs_base_roi, offset_inner, size_inner)
# compare blobs from inner region of ROI with all base blobs,
# prioritizing the closest matches
found, found_base, dists = find_closest_blobs_cdist(
blobs_inner, blobs_base_roi, thresh, scaling)
blobs_inner[:, 4] = 0
blobs_inner[found, 4] = 1
blobs_base_roi[blobs_base_inner_mask, 5] = 0
blobs_base_roi[found_base, 5] = 1
# add any base blobs missed in the inner ROI by comparing with
# test blobs from outer ROI
blobs_base_inner_missed = blobs_base_roi[blobs_base_roi[:, 5] == 0]
blobs_outer = blobs_roi[np.invert(blobs_inner_mask)]
found_out, found_base_out, dists_out = find_closest_blobs_cdist(
blobs_outer, blobs_base_inner_missed, thresh, scaling)
blobs_base_inner_missed[found_base_out, 5] = 1
# combine inner and outer groups
blobs_truth_inner_plus = np.concatenate(
(blobs_base_roi[blobs_base_roi[:, 5] == 1],
blobs_base_inner_missed))
blobs_outer[found_out, 4] = 1
blobs_inner_plus = np.concatenate((blobs_inner, blobs_outer[found_out]))
matches_inner = _match_blobs(
blobs_inner, blobs_base_roi, found, found_base, dists)
matches_outer = _match_blobs(
blobs_outer, blobs_base_inner_missed, found_out,
found_base_out, dists_out)
matches = colocalizer.BlobMatch([*matches_inner, *matches_outer])
if config.verbose:
'''
print("blobs_roi:\n{}".format(blobs_roi))
print("blobs_inner:\n{}".format(blobs_inner))
print("blobs_base_inner:\n{}".format(blobs_base_inner))
print("blobs_base_roi:\n{}".format(blobs_base_roi))
print("found inner:\n{}"
.format(blobs_inner[found]))
print("truth found:\n{}"
.format(blobs_base_roi[found_base]))
print("blobs_outer:\n{}".format(blobs_outer))
print("blobs_base_inner_missed:\n{}"
.format(blobs_base_inner_missed))
print("truth blobs detected by an outside blob:\n{}"
.format(blobs_base_inner_missed[found_base_out]))
print("all those outside detection blobs:\n{}"
.format(blobs_roi_extra))
print("blobs_inner_plus:\n{}".format(blobs_inner_plus))
print("blobs_truth_inner_plus:\n{}".format(blobs_truth_inner_plus))
'''
print("Closest matches found (truth, detected, distance):")
msgs = ("\n- Inner ROI:", "\n- Outer ROI:")
for msg, matches_sub in zip(msgs, (matches_inner, matches_outer)):
print(msg)
for match in matches_sub:
print(
"Blob1:", match[0][:3], "chl",
detector.get_blob_channel(match[0]), "Blob2:", match[1][:3],
"chl", detector.get_blob_channel(match[1]),
"dist:", match[2])
print()
return blobs_inner_plus, blobs_truth_inner_plus, offset_inner, size_inner, \
matches
def select_blob_matches(
self, roi_id: int, offset: Optional[Sequence[int]] = None,
shape: Optional[Sequence[int]] = None) -> "colocalizer.BlobMatch":
"""Select blob matches for the given ROI.
Args:
roi_id: ROI ID.
offset: ROI offset in ``z,y,x``; defaults to None.
shape: ROI shape in ``z,y,x``; defaults to None.
Returns:
Blob matches.
"""
_logger.debug("Selecting blob matches for ROI ID: %s", roi_id)
start = time()
# set up columns for each table
cols_matches = _specify_table_cols(
_COLS_BLOB_MATCHES + ', id', ', ', 'bm')
cols_blobs = _COLS_BLOBS + ", id"
cols_blobs1 = _specify_table_cols(cols_blobs, ', ', 'b1')
cols_blobs2 = _specify_table_cols(cols_blobs, ', ', 'b2')
# set up select statement
stmnt = (
f"SELECT {cols_matches}, "
f"{cols_blobs1}, "
f"{cols_blobs2} "
f"FROM blob_matches bm "
f"INNER JOIN blobs b1 ON bm.blob1 = b1.id "
f"INNER JOIN blobs b2 ON bm.blob2 = b2.id "
f"WHERE bm.roi_id = ?")
args = [roi_id, ]
if offset is not None and shape is not None:
# add ROI parameters
bounds = zip(offset, np.add(offset, shape))
bounds = [str(b) for bound in bounds for b in bound]
stmnt += (
" AND b1.z >= ? AND b1.z < ?"
"AND b1.y >= ? AND b1.y < ? AND b1.x >= ? AND b1.x < ?"
"AND b2.z >= ? AND b2.z < ?"
"AND b2.y >= ? AND b2.y < ? AND b2.x >= ? AND b2.x < ?")
args.extend(bounds)
args.extend(bounds)
# execute query
self.cur.execute(stmnt, args)
rows = self.cur.fetchall()
df_matches = None
if len(rows) > 0:
# convert to data frame to access by named columns
df = df_io.dict_to_data_frame(rows, records_cols=rows[0].keys())
def get_cols(col_full):
# extract column aliases
return [c.split(" ")[1] for c in col_full.split(", ")]
# extract columns for blob matches
df_matches = df[get_cols(cols_matches)]
df_matches = df_matches.rename(columns={
"bm_blob1": colocalizer.BlobMatch.Cols.BLOB1_ID.value,
"bm_blob2": colocalizer.BlobMatch.Cols.BLOB2_ID.value,
"bm_id": colocalizer.BlobMatch.Cols.MATCH_ID.value,
"bm_roi_id": colocalizer.BlobMatch.Cols.ROI_ID.value,
"bm_dist": colocalizer.BlobMatch.Cols.DIST.value,
})
# merge each set of blob columns into a single column of blob lists
cols_dict = {
colocalizer.BlobMatch.Cols.BLOB1.value: cols_blobs1,
colocalizer.BlobMatch.Cols.BLOB2.value: cols_blobs2,
}
for col, cols in cols_dict.items():
cols = get_cols(cols)[1:]
df_matches[col] = df[cols].to_numpy().tolist()
blob_matches = colocalizer.BlobMatch(df=df_matches)
_logger.debug("Finished selecting blob matches in %s s", time() - start)
return blob_matches
def select_blob_matches_by_blob_id(
self,
row_id: int,
blobn: int,
blob_ids: Sequence[int],
max_params: int = 100000
) -> "colocalizer.BlobMatch":
"""Select blob matches corresponding to the given blob IDs in the
given blob column.
Args:
row_id: Row ID.
blobn: 1 or 2 to indicate the first or second blob column,
respectively.
blob_ids: Blob IDs.
max_params: Maximum number of parameters for the `SELECT`
statements; defaults to 100000. The max is determined by
`SQLITE_MAX_VARIABLE_NUMBER` set at the sqlite3 compile
time. If this number is exceeded, this function is called
recursively with half the given `max_params`.
Returns:
Blob match object, which is empty if not matches are found.
Raises:
:meth:`sqlit3.OperationalError`: if the maximum number of
parameters is < 1.
Deprecated: 1.6.0
Use :meth:`select_blob_matches` instead.
"""
if max_params < 1:
raise sqlite3.OperationalError(
"Could not determine number of parameters for selecting blob "
"matches")
matches = []
if isinstance(blob_ids, np.ndarray):
blob_ids = blob_ids.tolist()
try:
# select matches by block to avoid exceeding sqlite parameter limit
nblocks = len(blob_ids) // max_params + 1
for i in range(nblocks):
_logger.info(
"Selecting blob matches block %s of %s", i, nblocks - 1)
ids = blob_ids[i*max_params:(i+1)*max_params]
ids.insert(0, row_id)
self.cur.execute(
f"SELECT {_COLS_BLOB_MATCHES}, id FROM blob_matches "
f"WHERE roi_id = ? AND blob{blobn} "
f"IN ({','.join('?' * (len(ids) - 1))})",
ids)
df = self._parse_blob_matches(self.cur.fetchall()).df
if df is not None:
matches.append(df)
except sqlite3.OperationalError:
# call recursively with halved number of parameters
_logger.debug(
"Exceeded max sqlite query parameters; trying with smaller "
"number")
return self.select_blob_matches_by_blob_id(
row_id, blobn, blob_ids, max_params // 2)
if len(matches) > 0:
return colocalizer.BlobMatch(df=df_io.data_frames_to_csv(matches))
return colocalizer.BlobMatch()