def test_copysegmentation_from_dvid_to_dvid_input_mask(
setup_dvid_segmentation_input, disable_auto_retry):
template_dir, config, volume, dvid_address, repo_uuid, _output_segmentation_name = setup_dvid_segmentation_input
# make sure we get a fresh output
output_segmentation_name = 'copyseg-with-input-mask-from-dvid'
config["output"]["dvid"]["segmentation-name"] = output_segmentation_name
# Add an offset, which is added to both the input volume AND the mask labels
offset = 2000
config["copysegmentation"]["add-offset-to-ids"] = offset
# Select some labels that don't extend throughout the whole volume
selected_labels = pd.unique(volume[150, 64:128, 64:128].reshape(-1))
assert 0 not in selected_labels
selected_coords = np.array(
mask_for_labels(volume, selected_labels).nonzero()).transpose()
selected_box = np.array(
[selected_coords.min(axis=0), 1 + selected_coords.max(axis=0)])
input_box = np.array(config["input"]["geometry"]["bounding-box"])[:, ::-1]
subvol_box = box_intersection(input_box, selected_box)
selected_subvol = extract_subvol(volume, subvol_box).copy()
selected_subvol = apply_mask_for_labels(selected_subvol, selected_labels)
config["copysegmentation"]["input-mask-labels"] = selected_labels.tolist()
selected_subvol = np.where(selected_subvol, selected_subvol + offset, 0)
expected_vol = np.zeros(volume.shape, np.uint64)
overwrite_subvol(expected_vol, subvol_box, selected_subvol)
setup = template_dir, config, expected_vol, dvid_address, repo_uuid, output_segmentation_name
_box_zyx, _expected_vol, _output_vol = _run_to_dvid(setup)
def test_mask_for_labels():
volume = [[0, 2, 3], [4, 5, 0]]
volume = np.asarray(volume)
masked_volume = mask_for_labels(volume, {2, 5, 9})
expected = [[0, 1, 0], [0, 1, 0]]
assert (masked_volume == expected).all()
def combine_with_output(input_brick):
output_box = input_brick.physical_box + translation_offset_zyx
output_vol = output_service.get_subvolume(output_box, scale=0)
output_vol = np.asarray(output_vol, order='C')
mask = None
if input_mask_labels:
mask = mask_for_labels(input_brick.volume,
input_mask_labels)
if output_mask_labels:
output_mask = mask_for_labels(output_vol,
output_mask_labels)
if mask is None:
mask = output_mask
else:
mask[:] &= output_mask
# Start with the complete output, then
# change voxels that fall within both masks.
output_vol[mask] = input_brick.volume[mask]
input_brick.compress()
return output_vol
def mitos_in_neighborhood(mito_roi_source, neighborhood_origin_xyz,
neighborhood_id, mito_res_scale_diff):
"""
Determine how many non-trivial mito objects overlap with the given "neighborhood object",
and return a table of their IDs and sizes.
1. Download the neighborhood mask for the given neighborhood_id.
2. Erode the neighborhood mask by 1 px (see note in the comment above).
3. Fetch the mito segmentation for the voxels within the neighborhood.
4. Fetch (from dvid) the sizes of each mito object.
5. Filter out the mitos that are smaller than the minimum size that is
actually used in our published mito analyses.
6. Just for additional info, determine how many connected components
are formed by the mito objects.
7. Return the mito IDs, sizses, and CC info as a DataFrame.
"""
# The neighborhood segmentation source
protocol, url = mito_roi_source.split('://')[-2:]
server, uuid, instance = url.split('/')
server = f'{protocol}://{server}'
origin_zyx = np.array(neighborhood_origin_xyz[::-1])
box = [origin_zyx - RADIUS, 1 + origin_zyx + RADIUS]
# Align box to the analysis scale before scaling it.
box = round_box(box, (2**ANALYSIS_SCALE))
# Scale box
box //= (2**ANALYSIS_SCALE)
neighborhood_seg = fetch_labelmap_voxels(server,
uuid,
instance,
box,
scale=ANALYSIS_SCALE)
neighborhood_mask = (neighborhood_seg == neighborhood_id)
# This is equivalent to a 1-px erosion
# See note above for why we do this.
neighborhood_mask ^= binary_edge_mask(neighborhood_mask, 'inner')
mito_seg = fetch_labelmap_voxels(*MITO_SEG,
box,
supervoxels=True,
scale=ANALYSIS_SCALE -
mito_res_scale_diff)
assert neighborhood_mask.shape == mito_seg.shape
mito_seg = np.where(neighborhood_mask, mito_seg, 0)
# The mito segmentation includes little scraps and slivers
# that were filtered out of the "real" mito set.
# Filter those scraps out of our results here.
mito_ids = set(pd.unique(mito_seg.ravel())) - {0}
mito_sizes = fetch_sizes(*MITO_SEG, [*mito_ids], supervoxels=True)
mito_sizes = mito_sizes.rename_axis('mito')
mito_sizes *= (2**mito_res_scale_diff)**3
# This is our main result: mito IDs (and their sizes)
mito_sizes = mito_sizes.loc[mito_sizes >= MIN_MITO_SIZE]
# Just for extra info, group the mitos we found into connected components.
mito_mask = mask_for_labels(mito_seg, mito_sizes.index)
mito_box = compute_nonzero_box(mito_mask)
mito_mask = extract_subvol(mito_mask, mito_box)
mito_seg = extract_subvol(mito_seg, mito_box)
mito_cc = label(mito_mask, connectivity=1)
ct = contingency_table(mito_seg, mito_cc).reset_index()
ct = ct.rename(columns={
'left': 'mito',
'right': 'cc',
'voxel_count': 'cc_size'
})
ct = ct.set_index('mito')
mito_sizes = pd.DataFrame(mito_sizes).merge(ct,
'left',
left_index=True,
right_index=True)
return mito_sizes
def _crop_body_mask_and_mito_seg(body_mask, mito_seg, mask_box, search_cfg, batch_tbars, primary_point, logger):
"""
To reduce the size of the analysis volumes during distance computation
(the most expensive step), we pre-filter out components of the body mask
don't actually contain both points of interest and mito.
If those segments don't even touch the volume edges,
then any points on those segments can be safely marked 'done'
if this is the final search config.
"""
with Timer("Filtering components and cropping", logger):
body_cc = labelMultiArrayWithBackground((body_mask != 0).view(np.uint8))
# Keep only components which contain both mito and points
tbar_points = batch_tbars[[*'zyx']].values // (2 ** search_cfg.analysis_scale)
is_in_box = (tbar_points >= mask_box[0]).all(axis=1) & (tbar_points < mask_box[1]).all(axis=1)
tbar_points = tbar_points[is_in_box]
pts_local = tbar_points - mask_box[0]
point_cc_df = batch_tbars.iloc[is_in_box][[*'zyx']].copy()
point_cc_df['cc'] = body_cc[tuple(np.transpose(pts_local))]
point_ccs = set(point_cc_df['cc'])
mito_ccs = set(pd.unique(body_cc[mito_seg != 0]))
keep_ccs = point_ccs & mito_ccs
keep_mask = mask_for_labels(body_cc, keep_ccs)
body_mask = np.where(keep_mask, body_mask, 0)
mito_seg = np.where(keep_mask, mito_seg, 0)
logger.info(f"Dropped {body_cc.max() - len(keep_ccs)} components, kept {len(keep_ccs)}")
# Also determine the set of points which should be marked as hopeless,
# due to a lack of mitos on their components.
# Hopeless points are those which reside on hopeless components.
# Hopeless components are any components that fall within the dilation
# region and still ended up without mitos.
hopeless_point_ids = []
if search_cfg.is_final and (point_ccs - mito_ccs):
with Timer("Identifying hopeless points", logger):
# Calculate the region that is subject to repairs (dilation),
# in local coordinates.
dr = search_cfg.dilation_radius_s0 // (2 ** search_cfg.analysis_scale)
buf = search_cfg.dilation_exclusion_buffer_s0 // (2 ** search_cfg.analysis_scale)
buf += max(1, dr)
R = search_cfg.radius_s0 // (2 ** search_cfg.analysis_scale)
orig_box = np.array([primary_point - R, primary_point + R + 1])
inner_box = orig_box + np.array([buf, -buf])[:, None]
inner_box = box_intersection(mask_box, inner_box)
inner_box = inner_box - mask_box[0]
inner_vol = body_cc[box_to_slicing(*inner_box)]
inner_ccs = set(pd.unique(inner_vol.ravel())) - {0}
# Overwrite body_cc, we don't need it for anything else after this.
body_cc[box_to_slicing(*inner_box)] = 0
outer_ccs = set(body_cc.ravel())
hopeless_ccs = (inner_ccs - outer_ccs) - mito_ccs # noqa
hopeless_point_ids = point_cc_df.query('cc in @hopeless_ccs').index
# Shrink the volume bounding box to encompass only the
# non-zero portion of the filtered body mask.
nz_box = compute_nonzero_box(keep_mask)
if not nz_box.any():
return None, None, nz_box, hopeless_point_ids
body_mask = body_mask[box_to_slicing(*nz_box)]
mito_seg = mito_seg[box_to_slicing(*nz_box)]
mask_box = mask_box[0] + nz_box
return body_mask, mito_seg, mask_box, hopeless_point_ids
def select_hulls_for_mito_bodies(mito_body_ct,
mito_bodies_mask,
mito_binary,
body_seg,
hull_masks,
seed_bodies,
box,
scale,
viewer=None,
res0=8,
progress=False):
mito_bodies_mito_seg = np.where(mito_bodies_mask & mito_binary, body_seg,
0)
nonmito_body_seg = np.where(mito_bodies_mask, 0, body_seg)
hull_cc_overlap_stats = []
for hull_cc, (mask_box, mask) in tqdm_proxy(hull_masks.items(),
disable=not progress):
mbms = mito_bodies_mito_seg[box_to_slicing(*mask_box)]
masked_hull_cc_bodies = np.where(mask, mbms, 0)
# Faster to check for any non-zero values at all before trying to count them.
# This early check saves a lot of time in practice.
if not masked_hull_cc_bodies.any():
continue
# This hull was generated from a particular seed body (non-mito body).
# If it accidentally overlaps with any other non-mito bodies,
# then delete those voxels from the hull.
# If that causes the hull to become split apart into multiple connected components,
# then keep only the component(s) which overlap the seed body.
seed_body = seed_bodies[hull_cc]
nmbs = nonmito_body_seg[box_to_slicing(*mask_box)]
other_bodies = set(pd.unique(nmbs[mask])) - {0, seed_body}
if other_bodies:
# Keep only the voxels on mito bodies or on the
# particular non-mito body for this hull (the "seed body").
mbm = mito_bodies_mask[box_to_slicing(*mask_box)]
mask[:] &= (mbm | (nmbs == seed_body))
mask = vigra.taggedView(mask, 'zyx')
mask_cc = vigra.analysis.labelMultiArrayWithBackground(
mask.view(np.uint8))
if mask_cc.max() > 1:
mask_ct = contingency_table(mask_cc, nmbs).reset_index()
keep_ccs = mask_ct['left'].loc[(mask_ct['left'] != 0) &
(mask_ct['right'] == seed_body)]
mask[:] = mask_for_labels(mask_cc, keep_ccs)
mito_bodies, counts = np.unique(masked_hull_cc_bodies,
return_counts=True)
overlaps = pd.DataFrame({
'mito_body': mito_bodies,
'overlap': counts,
'hull_cc': hull_cc,
'hull_size': mask.sum(),
'hull_body': seed_body
})
hull_cc_overlap_stats.append(overlaps)
if len(hull_cc_overlap_stats) == 0:
logger.warning("Could not find any matches for any mito bodies!")
mito_body_ct['hull_body'] = np.uint64(0)
return mito_body_ct
hull_cc_overlap_stats = pd.concat(hull_cc_overlap_stats, ignore_index=True)
hull_cc_overlap_stats = hull_cc_overlap_stats.query(
'mito_body != 0').copy()
# Aggregate the stats for each body and the hull bodies it overlaps with,
# Select the hull_body with the most overlap, or in the case of ties, the hull body that is largest overall.
# (Ties are probably more common in the event that two hulls completely encompass a small mito body.)
hull_body_overlap_stats = hull_cc_overlap_stats.groupby(
['mito_body', 'hull_body'])[['overlap', 'hull_size']].sum()
hull_body_overlap_stats = hull_body_overlap_stats.sort_values(
['mito_body', 'overlap', 'hull_size'], ascending=False)
hull_body_overlap_stats = hull_body_overlap_stats.reset_index()
mito_hull_selections = (hull_body_overlap_stats.drop_duplicates(
'mito_body').set_index('mito_body')['hull_body'])
mito_body_ct = mito_body_ct.merge(mito_hull_selections,
'left',
left_index=True,
right_index=True)
mito_body_ct['hull_body'] = mito_body_ct['hull_body'].fillna(0)
dtypes = {col: np.float32 for col in mito_body_ct.columns}
dtypes['hull_body'] = np.uint64
mito_body_ct = mito_body_ct.astype(dtypes)
if viewer:
assert mito_hull_selections.index.dtype == mito_hull_selections.values.dtype == np.uint64
mito_hull_mapper = LabelMapper(mito_hull_selections.index.values,
mito_hull_selections.values)
remapped_body_seg = mito_hull_mapper.apply(body_seg, True)
remapped_body_seg = apply_mask_for_labels(remapped_body_seg,
mito_hull_selections.values)
update_seg_layer(viewer, 'altered-bodies', remapped_body_seg, scale,
box)
# Show the final hull masks (after erasure of non-target bodies)
assert sorted(hull_masks.keys()) == [*range(1, 1 + len(hull_masks))]
hull_cc_overlap_stats = hull_cc_overlap_stats.sort_values('hull_size')
hull_seg = np.zeros_like(remapped_body_seg)
for row in hull_cc_overlap_stats.itertuples():
mask_box, mask = hull_masks[row.hull_cc]
view = hull_seg[box_to_slicing(*mask_box)]
view[:] = np.where(mask, row.hull_body, view)
update_seg_layer(viewer, 'final-hull-seg', hull_seg, scale, box)
return mito_body_ct
def identify_mito_bodies(body_seg,
mito_binary,
box,
scale,
halo,
body_seg_dvid_src=None,
viewer=None,
res0=8,
resource_mgr_client=None):
# Identify segments that are mostly mito
ct = contingency_table(body_seg,
mito_binary).reset_index().rename(columns={
'left': 'body',
'right': 'is_mito'
})
ct = ct.pivot(index='body', columns='is_mito',
values='voxel_count').fillna(0).rename(columns={
0: 'non_mito',
1: 'mito'
})
if 'mito' not in ct or 'non_mito' not in ct:
# Nothing to do if there aren't any mito voxels
return None, None, None
ct[['mito', 'non_mito']] *= ((2**scale)**3)
ct['body_size_local'] = ct.eval('mito+non_mito')
ct['mito_frac_local'] = ct.eval('mito/body_size_local')
ct = ct.sort_values('mito_frac_local', ascending=False)
# Also compute the halo vs. non-halo sizes of every body.
central_box = (box - box[0]) + [[halo, halo, halo], [-halo, -halo, -halo]]
central_body_seg = body_seg[box_to_slicing(*central_box)]
central_sizes = (pd.Series(central_body_seg.ravel(
'K')).value_counts().rename('body_size_central').rename_axis('body'))
central_mask = np.ones(central_box[1] - central_box[0], bool)
update_mask_layer(viewer, 'central-box', central_mask, scale,
central_box + box[0])
ct = ct.merge(central_sizes, 'left', on='body').fillna(0)
ct['halo_size'] = ct.eval('body_size_local - body_size_central')
ct = ct.query('body != 0')
# Immediately drop bodies that reside only in the halo
ct = ct.query('body_size_central > 0').copy()
# For the bodies that MIGHT pass the mito threshold (based on their local size)
# fetch their global size, if a dvid source was provided.
# If not, we'll just use the local size, which is less accurate but
# faster since we've already got it.
if body_seg_dvid_src is None:
ct['body_size'] = ct['body_size_local']
else:
local_mito_bodies = ct.query(
'mito_frac_local >= @MITO_EDGE_FRAC').index
if resource_mgr_client is None:
body_sizes = fetch_sizes(*body_seg_dvid_src,
local_mito_bodies).rename('body_size')
else:
with resource_mgr_client.access_context(body_seg_dvid_src[0], True,
1, 1):
body_sizes = fetch_sizes(*body_seg_dvid_src,
local_mito_bodies).rename('body_size')
ct = ct.merge(body_sizes, 'left', on='body')
# Due to downsampling effects, bodies can be larger at scale-1 than at scale-0, especially for tiny volumes.
ct['mito_frac_global_vol'] = np.minimum(ct.eval('mito/body_size'), 1.0)
# Calculate the proportion of mito edge pixels
body_edges = np.where(edge_mask(body_seg, 'both'), body_seg, np.uint64(0))
edge_ct = contingency_table(
body_edges, mito_binary).reset_index().rename(columns={
'left': 'body',
'right': 'is_mito'
})
edge_ct = edge_ct.pivot(index='body',
columns='is_mito',
values='voxel_count').fillna(0).rename(columns={
0: 'non_mito',
1: 'mito'
})
# Surface area scales with square of resolution, not cube
edge_ct[['mito', 'non_mito']] *= ((2**scale)**2)
edge_ct['body_size_local'] = edge_ct.eval('mito+non_mito')
edge_ct['mito_frac_local'] = edge_ct.eval('mito/body_size_local')
edge_ct = edge_ct.sort_values('mito_frac_local', ascending=False)
edge_ct = edge_ct.query('body != 0')
full_ct = ct.merge(edge_ct, 'inner', on='body', suffixes=['_vol', '_edge'])
q = ("body_size < @MAX_MITO_FRAGMENT_VOL"
" and mito_frac_global_vol >= @MITO_VOL_FRAC"
" and mito_frac_local_edge >= @MITO_EDGE_FRAC")
filtered_ct = full_ct.query(q)
mito_bodies = filtered_ct.index
mito_bodies_mask = mask_for_labels(body_seg, mito_bodies)
update_mask_layer(viewer, 'mito-bodies-mask', mito_bodies_mask, scale, box,
res0)
if len(filtered_ct) == 0:
return None, None, None
return mito_bodies, mito_bodies_mask, filtered_ct.copy()