def _adjacency_euclidean_distance(segmentation):
graph = RAG(segmentation, connectivity=connectivity)
# Initialize the node's data for computing their centroids.
for n in graph:
graph.node[n].update({
'count': 0,
'centroid': np.zeros((2), dtype=np.float32)
})
# Run through each segmentation pixel and add the pixel's coordinates
# to the centroid data.
for index in np.ndindex(segmentation.shape):
current = segmentation[index]
graph.node[current]['count'] += 1
graph.node[current]['centroid'] += index
# Centroid is the sum of all pixel coordinates / pixel count.
for n in graph:
graph.node[n]['centroid'] = (graph.node[n]['centroid'] /
graph.node[n]['count'])
# Run through each edge and calculate the euclidian distance based on
# the two node's centroids.
for n1, n2, d in graph.edges_iter(data=True):
diff = graph.node[n1]['centroid'] - graph.node[n2]['centroid']
d['weight'] = np.linalg.norm(diff)
# Return graph as adjacency matrix.
return nx.to_numpy_matrix(graph, dtype=np.float32)
def rag_solidity(labels, connectivity=2):
graph = RAG()
# The footprint is constructed in such a way that the first
# element in the array being passed to _add_edge_filter is
# the central value.
fp = ndi.generate_binary_structure(labels.ndim, connectivity)
for d in range(fp.ndim):
fp = fp.swapaxes(0, d)
fp[0, ...] = 0
fp = fp.swapaxes(0, d)
# For example
# if labels.ndim = 2 and connectivity = 1
# fp = [[0,0,0],
# [0,1,1],
# [0,1,0]]
#
# if labels.ndim = 2 and connectivity = 2
# fp = [[0,0,0],
# [0,1,1],
# [0,1,1]]
ndi.generic_filter(labels,
function=_add_edge_filter,
footprint=fp,
mode='nearest',
output=np.zeros(labels.shape, dtype=np.uint8),
extra_arguments=(graph, ))
# remove bg_label
# graph.remove_node(-1)
graph.remove_node(0)
for n in graph:
mask = (labels == n)
solidity = 1. * mask.sum() / convex_hull_image(mask).sum()
graph.node[n].update({
'labels': [n],
'solidity': solidity,
'mask': mask
})
for x, y, d in graph.edges_iter(data=True):
new_mask = np.logical_or(graph.node[x]['mask'], graph.node[y]['mask'])
new_solidity = 1. * new_mask.sum() / convex_hull_image(new_mask).sum()
org_solidity = np.mean(
[graph.node[x]['solidity'], graph.node[y]['solidity']])
d['weight'] = org_solidity / new_solidity
return graph
def rag_solidity(labels, connectivity=2):
graph = RAG()
# The footprint is constructed in such a way that the first
# element in the array being passed to _add_edge_filter is
# the central value.
fp = ndi.generate_binary_structure(labels.ndim, connectivity)
for d in range(fp.ndim):
fp = fp.swapaxes(0, d)
fp[0, ...] = 0
fp = fp.swapaxes(0, d)
# For example
# if labels.ndim = 2 and connectivity = 1
# fp = [[0,0,0],
# [0,1,1],
# [0,1,0]]
#
# if labels.ndim = 2 and connectivity = 2
# fp = [[0,0,0],
# [0,1,1],
# [0,1,1]]
ndi.generic_filter(
labels,
function=_add_edge_filter,
footprint=fp,
mode='nearest',
output=np.zeros(labels.shape, dtype=np.uint8),
extra_arguments=(graph,))
# remove bg_label
# graph.remove_node(-1)
graph.remove_node(0)
for n in graph:
mask = (labels == n)
solidity = 1. * mask.sum() / convex_hull_image(mask).sum()
graph.node[n].update({'labels': [n],
'solidity': solidity,
'mask': mask})
for x, y, d in graph.edges_iter(data=True):
new_mask = np.logical_or(graph.node[x]['mask'], graph.node[y]['mask'])
new_solidity = 1. * new_mask.sum() / convex_hull_image(new_mask).sum()
org_solidity = np.mean([graph.node[x]['solidity'],
graph.node[y]['solidity']])
d['weight'] = org_solidity / new_solidity
return graph
def __initialize_rag(self):
if self.rag is None:
self.__log_info("Extracting RAG from fragments...")
self.rag = RAG(self.fragments, connectivity=2)
self.__log_info("RAG contains %d nodes and %d edges",
len(self.rag.nodes()), len(self.rag.edges()))
self.__log_info("Computing LSDs for initial fragments...")
dims = len(self.segmentation.shape)
for u in self.rag.nodes():
self.__log_debug("Initializing node %d", u)
data = self.rag.node[u]
if 'roi' not in data:
bbs = find_objects(self.fragments == u)
if len(bbs) == 0:
data['roi'] = None
else:
assert len(bbs) == 1
roi = self.__slice_to_roi(bbs[0])
data['roi'] = roi
data['score'] = self.__compute_node_score(u)
if 'labels' not in data:
data['labels'] = [u] # needed by scikit
else:
assert u in data['labels'], (
"Labels list of a node has to contain the node itself.")
self.__log_debug("Node %d: %s", u, data)
self.__log_info("Scoring initial edges...")
for (u, v) in self.rag.edges():
self.__log_debug("Initializing edge (%d, %d)", u, v)
score = self.__score_merge(u, v)
self.rag[u][v]['weight'] = score['weight']
def __init__(self, data_source, data_meta_path, graph_path):
self.ROI = data_source
with open(data_meta_path, 'r') as f:
self.ROI_META = json.load(f)
self.labeled_supervoxels = self.ROI['watershed_tot'][:, :, :]
self.N_vertices = len(np.unique(self.labeled_supervoxels))
self.data = []
for leaf in self.ROI_META:
self.data.append(self.ROI[leaf][:, :, :])
print('creating initial graph')
self.G = RAG(label_image=self.labeled_supervoxels)
print('adding node attributes')
x = 0
for i in self.G.nodes:
indices = np.where(self.labeled_supervoxels == i)
self.G.nodes[i]['cost'] = self.compute_vertex_cost(indices)
self.G.nodes[i]['label'] = -1
#self.G.nodes[i]['neighbors'] = []
if (x % 100 == 0):
print(x)
x += 1
self.nodes = self.G.nodes
self.edges = self.G.edges
print('saving graph')
nx.write_gpickle(self.G, graph_path)
def _initialize_graph(rx, ry, data, model):
'''Initializes the Region Adjacency Graph (RAG).'''
# Define merging decision function
mdf = lambda vect:model.predict_proba(np.atleast_2d(vect))[0,1]
# Initialize RAG
xmap, ymap = get_xymaps(rx, ry)
segments = np.arange(rx*ry).reshape((rx,ry))
rag = RAG(segments)
# Initialize nodes
data_reshaped = data.reshape((rx,ry,data.shape[1]))
for n in rag:
rag.nodes[n].update({'labels': [n]})
for index in np.ndindex(segments.shape):
current = segments[index]
rag.nodes[current]['count'] = 1
rag.nodes[current]['master'] = data_reshaped[index]
rag.nodes[current]['xpos'] = xmap[current]
rag.nodes[current]['ypos'] = ymap[current]
# Initialize edges
edge_heap = []
for n1, n2, d in rag.edges(data=True):
master_x = rag.nodes[n1]['master']
master_y = rag.nodes[n2]['master']
weight = mdf(_vector_similarity(master_x, master_y))
# Push the edge into the heap
heap_item = [weight, n1, n2, (weight < 0.5)]
d['heap item'] = heap_item
heapq.heappush(edge_heap, heap_item)
return rag, edge_heap, segments
def _merging_function(self, graph: graph.RAG, src: int, dst: int) -> None:
graph.nodes[dst]["x"] += graph.nodes[src]["x"]
graph.nodes[dst]["N"] += graph.nodes[src]["N"]
graph.nodes[dst]["mean"] = graph.nodes[dst]["x"] / \
graph.nodes[dst]["N"]
graph.nodes[dst]["mean"] = graph.nodes[dst]["mean"] / np.linalg.norm(
graph.nodes[dst]["mean"])
graph.nodes[dst]["y"] = np.vstack(
(graph.nodes[dst]["y"], graph.nodes[src]["y"]))
graph.nodes[dst]["r"] = self._color_features_per_channel(
graph.nodes[dst]["y"][:, 0])
graph.nodes[dst]["g"] = self._color_features_per_channel(
graph.nodes[dst]["y"][:, 1])
graph.nodes[dst]["b"] = self._color_features_per_channel(
graph.nodes[dst]["y"][:, 2])
graph.nodes[dst]["r"] = graph.nodes[dst]["r"] / np.linalg.norm(
graph.nodes[dst]["r"])
graph.nodes[dst]["g"] = graph.nodes[dst]["r"] / np.linalg.norm(
graph.nodes[dst]["g"])
graph.nodes[dst]["b"] = graph.nodes[dst]["r"] / np.linalg.norm(
graph.nodes[dst]["b"])
def _adjacency(segmentation):
graph = RAG(segmentation, connectivity=connectivity)
# Simply return the unweighted adjacency matrix of the computed graph.
return nx.to_numpy_matrix(graph, dtype=np.float32)
def rag_histograms(image,
labels,
connectivity=2,
gt=None,
bins=5,
method='opencv'):
# histogram distance question:
# https://stackoverflow.com/questions/6499491/comparing-two-histograms
# http://www.pyimagesearch.com/2014/07/14/3-ways-compare-histograms-using-opencv-python/
"""Compute the Region Adjacency Graph using histogram distance.
----------
image : ndarray, shape(M, N, [..., P,] 3)
Input image.
labels : ndarray, shape(M, N, [..., P,])
The labelled image. This should have one dimension less than
`image`. If `image` has dimensions `(M, N, 3)` `labels` should have
dimensions `(M, N)`.
Returns
-------
out : RAG
The region adjacency graph.
"""
graph = RAG(labels, connectivity=connectivity)
for n in graph:
region = (labels == n)
if method == 'opencv':
hist = cv2.calcHist([image], [0, 1, 2], region.astype(np.uint8),
[bins, bins, bins], [0., 1., 0., 1., 0., 1.])
cv2.normalize(hist, hist)
hist = hist.flatten()
elif method == 'channelcat':
crop = image[region, :].reshape(-1, 3)
red, rbins = np.histogram(crop[:, 0],
bins=bins[0],
density=True,
range=(0., 1.))
green, gbins = np.histogram(crop[:, 1],
bins=bins[1],
density=True,
range=(0., 1.))
blue, bbins = np.histogram(crop[:, 2],
bins=bins[2],
density=True,
range=(0., 1.))
hist = np.hstack((red, green, blue))
elif method == 'channelcat2':
crop = image[region, :].reshape(-1, 3)
red = cv2.calcHist([image], [0], region.astype(np.uint8),
[bins[0]], [0., 1.])
green = cv2.calcHist([image], [1], region.astype(np.uint8),
[bins[1]], [0., 1.])
blue = cv2.calcHist([image], [2], region.astype(np.uint8),
[bins[2]], [0., 1.])
hist = np.vstack((red, green, blue))
cv2.normalize(hist, hist)
hist = hist.flatten()
graph.node[n].update({'labels': [n], 'feat': hist})
if gt is not None:
# majority vote for superpixel label
(values, counts) = np.unique(gt[region], return_counts=True)
ind = np.argmax(counts)
graph.node[n]['gt'] = values[ind]
for x, y, d in graph.edges_iter(data=True):
diff = graph.node[x]['feat'] - graph.node[y]['feat']
diff = np.abs(diff)
d['weight'] = diff
return graph
class LsdAgglomeration(object):
'''Create a local shape descriptor agglomerator.
Args:
fragments (``np.ndarray``):
Initial fragments.
target_lsds (``np.ndarray``):
The local shape descriptors to match.
lsd_extractor (``LsdExtractor``):
The local shape descriptor object used to compute the difference
between the segmentation and the target LSDs.
voxel_size (``tuple`` of ``int``, optional):
The voxel size of ``fragments``. Defaults to 1.
rag (`class:Rag`, optional):
A custom region adjacency graph (RAG) to agglomerate on. If not
given, a RAG will be extracted from ``fragments``.
'''
def __init__(self,
fragments,
target_lsds,
lsd_extractor,
voxel_size=None,
rag=None,
log_prefix=''):
self.segmentation = np.array(fragments)
self.lsds = np.zeros_like(target_lsds)
self.fragments = fragments
self.target_lsds = target_lsds
self.lsd_extractor = lsd_extractor
self.rag = rag
self.context = lsd_extractor.get_context()
self.log_prefix = log_prefix
if voxel_size is None:
self.voxel_size = (1, ) * len(fragments.shape)
else:
self.voxel_size = voxel_size
self.__initialize_rag()
def merge_until(self, threshold, max_merges=-1):
'''Merge until the given threshold. Since edges are scored by how much
they decrease the distance to ``target_lsds``, a threshold of 0 should
be optimal.
Returns the merge history.'''
self.__log_info("Merging until %f...", threshold)
merge_func = lambda _, src, dst: self.__merge_nodes(src, dst)
weight_func = lambda _g, _s, u, v: self.__score_merge(u, v)
merge_history = merge_hierarchical(self.fragments,
self.rag,
thresh=threshold,
rag_copy=False,
in_place_merge=True,
merge_func=merge_func,
weight_func=weight_func,
max_merges=max_merges,
return_segmenation=False)
self.__log_info("Finished merging")
return merge_history
def get_segmentation(self):
'''Return the segmentation obtained so far by calls to
``merge_until``.'''
return self.segmentation
def get_lsds(self):
'''Return the local shape descriptors corresponding to the current
segmentation.'''
return self.lsds
def __initialize_rag(self):
if self.rag is None:
self.__log_info("Extracting RAG from fragments...")
self.rag = RAG(self.fragments, connectivity=2)
self.__log_info("RAG contains %d nodes and %d edges",
len(self.rag.nodes()), len(self.rag.edges()))
self.__log_info("Computing LSDs for initial fragments...")
dims = len(self.segmentation.shape)
for u in self.rag.nodes():
self.__log_debug("Initializing node %d", u)
data = self.rag.node[u]
if 'roi' not in data:
bbs = find_objects(self.fragments == u)
if len(bbs) == 0:
data['roi'] = None
else:
assert len(bbs) == 1
roi = self.__slice_to_roi(bbs[0])
data['roi'] = roi
data['score'] = self.__compute_node_score(u)
if 'labels' not in data:
data['labels'] = [u] # needed by scikit
else:
assert u in data['labels'], (
"Labels list of a node has to contain the node itself.")
self.__log_debug("Node %d: %s", u, data)
self.__log_info("Scoring initial edges...")
for (u, v) in self.rag.edges():
self.__log_debug("Initializing edge (%d, %d)", u, v)
score = self.__score_merge(u, v)
self.rag[u][v]['weight'] = score['weight']
def __score_merge(self, u, v):
'''Callback for merge_hierarchical, called to get the weight of a new
edge.'''
weight = self.__compute_edge_score(u, v)
self.__log_debug("Scoring merge between %d and %d with %f", u, v,
weight)
return {'weight': weight}
def __compute_node_score(self, u):
'''Compute the LSDs score for a node.
The node score is the sum of squared differences between the node LSDs
and the target LSDs.
This also stores the node's LSDs in self.lsds.
'''
# get ROI
roi = self.rag.node[u]['roi']
# node is not part of volume
if roi is None:
return 0
# get slice of segmentation for roi
segmentation = self.segmentation[roi.to_slices()]
# get LSDs for u
lsds = self.lsd_extractor.get_descriptors(segmentation,
labels=[u],
voxel_size=self.voxel_size)
# subtract from target LSDs
u_mask = segmentation == u
lsds_slice = (slice(None), ) + roi.to_slices()
diff = self.target_lsds[lsds_slice] - lsds
diff[:, u_mask == 0] = 0
# update LSDs for u
self.lsds[lsds_slice][:, u_mask] = lsds[:, u_mask]
return np.sum(diff**2)
def __merge_nodes(self, u, v):
'''Merge node u into v.
This does not change the graph (this is taken care of by the
hierarchical agglomeration).
This updates the segmentation (u is replaced by v), the LSDs of the
current segmentaion, the ROI of v, and computes the new score for v.
'''
self.__merge_segmentation(u, v)
(change_roi, context_roi) = self.__get_lsds_edge_rois(u, v)
# get slice of segmentation for context_roi (no copy, we want to keep
# the changes made)
segmentation = self.segmentation[context_roi.to_slices()]
# slices to cut change ROI from LSDs
lsds_slice = (slice(None), ) + change_roi.to_slices()
# change ROI relative to context ROI
change_in_context_roi = change_roi - context_roi.get_begin()
# get LSDs for (u + v)
lsds_merged = self.lsd_extractor.get_descriptors(
segmentation,
roi=change_in_context_roi,
labels=[v],
voxel_size=self.voxel_size)
# update LSDs (only where segmentation == v)
v_mask = segmentation[change_in_context_roi.to_slices()] == v
self.lsds[lsds_slice][:, v_mask] = lsds_merged[:, v_mask]
# set the ROI of v to the union of u and v
roi_u = self.rag.node[u]['roi']
roi_v = self.rag.node[v]['roi']
self.rag.node[v]['roi'] = roi_u.union(roi_v)
# update node score
self.rag.node[v]['score'] = (self.rag.node[v]['score'] +
self.rag.node[u]['score'] +
self.rag[u][v]['weight'])
self.__log_info("Merged %d into %d with score %f", u, v,
self.rag[u][v]['weight'])
self.__log_debug(" -> merge fragments %s and %s",
self.rag.node[u]['labels'],
self.rag.node[v]['labels'])
self.__log_debug("Updated score of %d (merged with %d) to %f", u, v,
self.rag.node[v]['score'])
def __merge_segmentation(self, u, v):
'''Replace u with v in segmentation.'''
segmentation_u = self.segmentation[self.rag.node[u]['roi'].to_slices()]
segmentation_u[segmentation_u == u] = v
def __compute_edge_score(self, u, v):
'''Compute the LSDs score for an edge.
The edge score is by how much the incident node scores would improve
when merged (negative if the score decreases). More formally, it is:
s(u + v) - (s(u) + s(v))
where s(.) is the score of a node and (u + v) is a node obtained from
merging u and v.
'''
(change_roi, context_roi) = self.__get_lsds_edge_rois(u, v)
if change_roi is None:
return 0
# get slice of segmentation for context_roi (make a copy, since we
# change it later)
segmentation = self.segmentation[context_roi.to_slices()]
segmentation = np.array(segmentation)
# slices to cut change ROI from LSDs
lsds_slice = (slice(None), ) + change_roi.to_slices()
# change ROI relative to context ROI
change_in_context_roi = change_roi - context_roi.get_begin()
# mask for voxels in u and v for change ROI
not_uv_mask = np.logical_not(
np.isin(segmentation[change_in_context_roi.to_slices()], [u, v]))
# get s(u) + s(v)
lsds_separate = self.lsds[lsds_slice]
diff = self.target_lsds[lsds_slice] - lsds_separate
diff[:, not_uv_mask] = 0
score_separate = np.sum(diff**2)
# mark u as v in segmentation
segmentation[segmentation == u] = v
# get s(u + v)
lsds_merged = self.lsd_extractor.get_descriptors(
segmentation,
roi=change_in_context_roi,
labels=[v],
voxel_size=self.voxel_size)
diff = self.target_lsds[lsds_slice] - lsds_merged
diff[:, not_uv_mask] = 0
score_merged = np.sum(diff**2)
assert lsds_separate.shape == lsds_merged.shape
self.__log_debug("Edge score for (%d, %d) is %f - %f = %f", u, v,
score_merged, score_separate,
score_merged - score_separate)
return score_merged - score_separate
def __get_lsds_edge_rois(self, u, v):
'''Get two ROIs (change_roi, context_roi).
change_roi bounds the regions in which LSDs are affected by a merge of
u and v.
context_roi is a superset of change_roi and bounds the region that
needs to be considered to compute LSDs in change_roi.
'''
# get node ROIs
roi_u = self.rag.node[u]['roi']
roi_v = self.rag.node[v]['roi']
# nodes that are not part of the volume have no change_roi
if roi_u is None or roi_v is None:
return (None, None)
# the ROI of the complete volume
total_roi = gp.Roi((0, ) * len(self.segmentation.shape),
self.segmentation.shape)
# the context used by the shape descriptor in voxels
context = tuple(
int(math.ceil(c / vs))
for c, vs in zip(self.context, self.voxel_size))
# grow the node ROIs by context and ensure they are still within the
# total ROI
roi_u_grown = roi_u.grow(context, context)
roi_v_grown = roi_v.grow(context, context)
roi_u_grown = roi_u_grown.intersect(total_roi)
roi_v_grown = roi_v_grown.intersect(total_roi)
# LSDs have to be computed and compared to target only within the
# intersection of the grown node ROIs (other parts of u and v are not
# affected by the merge, due to finite context)
change_roi = roi_u_grown.intersect(roi_v_grown)
# we can further restric the compute ROI to the union of the node ROIs,
# since voxels outside of the nodes do not contribute, either
change_roi = change_roi.intersect(roi_u.union(roi_v))
if change_roi.empty():
self.__log_warning(
"change ROI between %s and %s is empty: u=%s, v=%s, "
"u_grown=%s, v_grown=%s", u, v, roi_u, roi_v, roi_u_grown,
roi_v_grown)
return (None, None)
# the context we need to compute LSDs in change_roi
context_roi = change_roi.grow(context, context)
# this can again be limited to the union of the node ROIs
context_roi = context_roi.intersect(roi_u.union(roi_v))
# finally, ensure that we deliver multiples of the downsampling factor
# used by the lsd_extractor
dims = change_roi.dims()
change_roi = change_roi.snap_to_grid(
(self.lsd_extractor.downsample, ) * dims)
context_roi = context_roi.snap_to_grid(
(self.lsd_extractor.downsample, ) * dims)
return (change_roi, context_roi)
def __slice_to_roi(self, slices):
offset = tuple(s.start for s in slices)
shape = tuple(s.stop - s.start for s in slices)
roi = gp.Roi(offset, shape)
roi = roi.snap_to_grid((self.lsd_extractor.downsample, ) * roi.dims())
return roi
def __log_debug(self, message, *args):
logger.debug(self.log_prefix + message, *args)
def __log_info(self, message, *args):
logger.info(self.log_prefix + message, *args)
def __log_warning(self, message, *args):
logger.warning(self.log_prefix + message, *args)