def compute_nh(self, graph_depth):
# load the graph
graph_path = os.path.join(self.tmp_folder, 'graph.n5', 'graph')
graph = ndist.loadAsUndirectedGraph(graph_path)
# load the node labels
node_label_path = os.path.join(self.tmp_folder, 'node_labels.n5')
node_label_key = 'node_labels'
node_labels = z5py.File(node_label_path)[node_label_key][:]
self.assertEqual(len(node_labels), graph.numberOfNodes)
# run bfs up to depth 4 to get complete lifted nh
lifted_graph = nlmc.liftedMulticutObjective(graph)
lifted_graph.insertLiftedEdgesBfs(graph_depth)
nh = lifted_graph.liftedUvIds()
# filter by nodes which have a node labeling
node_ids = np.arange(len(node_labels))
nodes_with_label = node_ids[node_labels != 0]
nh_mask = np.in1d(nh, nodes_with_label).reshape(nh.shape)
nh_mask = nh_mask.all(axis=1)
nh = nh[nh_mask]
# need to lexsort - awkward in numpy ...
nh = nh[np.lexsort(np.rot90(nh))]
return nh
def load_graph(self, graph_path, graph_key):
graph_ds = z5py.File(graph_path)[graph_key]
n_edges = graph_ds.attrs['numberOfEdges']
graph = ndist.loadAsUndirectedGraph(os.path.join(
graph_path, graph_key))
self.assertEqual(n_edges, graph.numberOfEdges)
return graph
def test_lifted_nh(self):
import cluster_tools.lifted_features.sparse_lifted_neighborhood as nh_tasks
graph = ndist.loadAsUndirectedGraph(self.output_path, self.graph_key)
n_nodes = graph.numberOfNodes
node_labels = np.ones(n_nodes, dtype='uint64')
node_label_path = os.path.join(self.tmp_folder, 'node_labels.n5')
node_label_key = 'node_labels'
with z5py.File(node_label_path) as f:
ds = f.require_dataset(node_label_key, shape=node_labels.shape,
dtype=node_labels.dtype,
chunks=(1000,), compression='gzip')
ds[:] = node_labels
base_name = "SparseLiftedNeighborhood"
name = base_name + self.get_target_name()
task = getattr(nh_tasks, name)
# try different graph depth and different number of threads !
graph_depth = 3
out_key = 'lifted_nh'
t = task(tmp_folder=self.tmp_folder, config_dir=self.config_folder, max_jobs=1,
graph_path=self.output_path, graph_key=self.graph_key,
node_label_path=node_label_path, node_label_key=node_label_key,
output_path=self.output_path, output_key=out_key,
prefix='', nh_graph_depth=graph_depth)
ret = luigi.build([t], local_scheduler=True)
self.assertTrue(ret)
self.check_result(graph_depth, node_label_path, node_label_key)
def test_lifted_nh(self):
graph_path = os.path.join(self.tmp_folder, 'graph.n5')
graph_key = 'graph'
graph_config = GraphWorkflow.get_config()['initial_sub_graphs']
graph_config["ignore_label"] = False
with open(
os.path.join(self.config_folder, 'initial_sub_graphs.config'),
'w') as f:
json.dump(graph_config, f)
task_graph = GraphWorkflow(tmp_folder=self.tmp_folder,
config_dir=self.config_folder,
max_jobs=8,
target='local',
input_path=self.input_path,
input_key=self.ws_key,
graph_path=graph_path,
output_key=graph_key)
ret = luigi.build([task_graph], local_scheduler=True)
self.assertTrue(ret)
graph = ndist.loadAsUndirectedGraph(os.path.join(
graph_path, graph_key))
n_nodes = graph.numberOfNodes
node_labels = np.ones(n_nodes, dtype='uint64')
node_label_path = os.path.join(self.tmp_folder, 'node_labels.n5')
node_label_key = 'node_labels'
with z5py.File(node_label_path) as f:
ds = f.require_dataset(node_label_key,
shape=node_labels.shape,
dtype=node_labels.dtype,
chunks=(1000, ),
compression='gzip')
ds[:] = node_labels
# TODO try different graph depth and different number of threads !
graph_depth = 3
out_path = os.path.join(self.tmp_folder, 'lifted_nh.h5')
out_key = 'lifted_nh'
task_nh = SparseLiftedNeighborhoodLocal(
tmp_folder=self.tmp_folder,
config_dir=self.config_folder,
max_jobs=1,
dependency=task_graph,
graph_path=graph_path,
graph_key=graph_key,
node_label_path=node_label_path,
node_label_key=node_label_key,
output_path=out_path,
output_key=out_key,
prefix='',
nh_graph_depth=graph_depth)
ret = luigi.build([task_nh], local_scheduler=True)
self.assertTrue(ret)
self._check_result(graph_depth)
def _check_result(self):
# check shapes
with z5py.File(self.input_path) as f:
seg = f[self.input_key][:]
shape = seg.shape
with z5py.File(self.output_path) as f:
shape_ = tuple(f.attrs['shape'])
self.assertEqual(shape, shape_)
# check graph
# compute nifty rag
rag = nrag.gridRag(seg, numberOfLabels=int(seg.max()) + 1)
# load the graph
graph = ndist.loadAsUndirectedGraph(
os.path.join(self.output_path, 'graph'))
self.assertEqual(rag.numberOfNodes, graph.numberOfNodes)
self.assertEqual(rag.numberOfEdges, graph.numberOfEdges)
self.assertTrue(np.allclose(rag.uvIds(), graph.uvIds()))
def test_extraction(self):
# load complete graph
graph = ndist.loadAsUndirectedGraph(self.path_to_graph)
# TODO test for more block-ids
block_ids = list(range(64))
for block_id in block_ids:
block_path = self.graph_block_prefix + str(block_id)
assert os.path.exists(block_path), block_path
nodes = ndist.loadNodes(block_path)
if len(nodes) == 1:
continue
# get the subgraph from in-memory graph
inner_edges_a, outer_edges_a, graph_a = graph.extractSubgraphFromNodes(
nodes)
# get the subgraph from disc
inner_edges_b, outer_edges_b, uvs_b = ndist.extractSubgraphFromNodes(
nodes, self.path_to_nodes, self.graph_block_prefix)
# tests for equality
n_nodes_a = graph_a.numberOfNodes
uvs_a = graph_a.uvIds()
n_nodes_b = int(uvs_b.max() + 1)
# test graph
self.assertEqual(n_nodes_a, n_nodes_b)
self.assertEqual(uvs_a.shape, uvs_b.shape)
self.assertTrue((uvs_a == uvs_b).all())
# test edge ids
self.assertEqual(inner_edges_a.shape, inner_edges_b.shape)
self.assertTrue((inner_edges_a == inner_edges_b).all())
self.assertEqual(outer_edges_a.shape, outer_edges_b.shape)
self.assertTrue((outer_edges_a == outer_edges_b).all())
def load_graph(self, graph_path, graph_key):
graph = ndist.loadAsUndirectedGraph(os.path.join(
graph_path, graph_key))
return graph