def agglomerate_wsdt(thresh=.1, size_thresh=500):
f = z5py.File('/home/papec/Work/neurodata_hdd/scotts_blocks/data_test_small.n5')
affs = 1. - f['predictions/full_affs'][:3, :]
affs_xy = np.mean(affs[1:3], axis=0)
affs_z = affs[0]
wsdt = cseg.DTWatershed(0.2, 1.6)
ws, max_id = wsdt(affs_xy)
rag = nrag.gridRagStacked2D(ws.astype('uint32'),
numberOfLabels=int(max_id + 1),
dtype='uint32')
features_z = nrag.accumulateEdgeStandardFeatures(rag, affs_z, keepZOnly=True, zDirection=2)[1]
features_z = features_z[:, 0]
edge_offset = rag.totalNumberOfInSliceEdges
edge_sizes = rag.edgeLengths()[edge_offset:]
uvs = rag.uvIds()[edge_offset:]
assert len(features_z) == len(uvs)
# TODO filter by edge overlap as well !
merge_edges = np.logical_and(features_z < thresh, edge_sizes > size_thresh)
merge_nodes = uvs[merge_edges]
ufd = nifty.ufd.ufd(rag.numberOfNodes)
ufd.merge(merge_nodes)
node_labels = ufd.elementLabeling()
ws_merged = nrag.projectScalarNodeDataToPixels(rag, node_labels)
raw = f['gray'][:]
view([raw, affs.transpose((1, 2, 3, 0)), ws, ws_merged],
['raw', 'affs', 'ws', 'ws-merged'])
def testStandardAccumulation(self):
seg, val = self.makeToyData()
rag = nrag.gridRagStacked2D(seg)
# test the different z accumulations
for zDir in (0, 1, 2):
feats = nrag.accumulateEdgeStandardFeatures(rag, val, zDir)
self.assertEqual(len(feats), rag.numberOfEdges)
self.checkToyFeats(feats, zDir)
def testStandardAccumulation(self):
seg, val = self.makeToyData()
rag = nrag.gridRagStacked2D(seg)
# test the different z accumulations
for zDir in (0, 1, 2):
feats = nrag.accumulateEdgeStandardFeatures(rag,
val,
zDir)
self.assertEqual(len(feats), rag.numberOfEdges)
self.checkToyFeats(feats, zDir)
def ignore_label_test_test(self, accumulation_function):
labels_with_ignore = self.make_labels_with_ignore(self.shape)
rag = nrag.gridRagStacked2D(labels_with_ignore,
numberOfLabels=labels_with_ignore.max() +
1,
ignoreLabel=0,
numberOfThreads=1)
n_edges_xy = rag.totalNumberOfInSliceEdges
n_edges_z = rag.totalNumberOfInBetweenSliceEdges
# test complete accumulation
print("Complete Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
numberOfThreads=1)
self.check_features(feats_xy, n_edges_xy)
self.check_features(feats_z, n_edges_z)
print("... passed")
# test xy-feature accumulation
print("Complete XY Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepXYOnly=True,
numberOfThreads=-1)
self.check_features(feats_xy, n_edges_xy)
self.assertEqual(len(feats_z), 1)
print("... passed")
# test z-feature accumulation for all 3 directions
print("Complete Z Accumulations ...")
for z_direction in (0, 1, 2):
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepZOnly=True,
zDirection=z_direction,
numberOfThreads=-1)
self.assertEqual(len(feats_xy), 1)
self.check_features(feats_z, n_edges_z)
print("... passed")
def ignore_label_test_test(self, accumulation_function):
labels_with_ignore = self.make_labels_with_ignore(self.shape)
rag = nrag.gridRagStacked2D(labels_with_ignore,
numberOfLabels=labels_with_ignore.max() + 1,
ignoreLabel=0,
numberOfThreads=1)
n_edges_xy = rag.totalNumberOfInSliceEdges
n_edges_z = rag.totalNumberOfInBetweenSliceEdges
# test complete accumulation
print("Complete Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
numberOfThreads=1)
self.check_features(feats_xy, n_edges_xy)
self.check_features(feats_z, n_edges_z)
print("... passed")
# test xy-feature accumulation
print("Complete XY Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepXYOnly=True,
numberOfThreads=-1)
self.check_features(feats_xy, n_edges_xy)
self.assertEqual(len(feats_z), 1)
print("... passed")
# test z-feature accumulation for all 3 directions
print("Complete Z Accumulations ...")
for z_direction in (0, 1, 2):
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepZOnly=True,
zDirection=z_direction,
numberOfThreads=-1)
self.assertEqual(len(feats_xy), 1)
self.check_features(feats_z, n_edges_z)
print("... passed")
def accumulation_in_core_test(self, accumulation_function):
rag = nrag.gridRagStacked2D(self.labels,
numberOfLabels=self.n_labels,
numberOfThreads=-1)
n_edges_xy = rag.totalNumberOfInSliceEdges
n_edges_z = rag.totalNumberOfInBetweenSliceEdges
# test complete accumulation
# print("Complete Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
numberOfThreads=-1)
self.check_features(feats_xy, n_edges_xy)
self.check_features(feats_z, n_edges_z)
# print("... passed")
# test xy-feature accumulation
# print("Complete XY Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepXYOnly=True,
numberOfThreads=-1)
self.check_features(feats_xy, n_edges_xy)
self.assertEqual(len(feats_z), 1)
# print("... passed")
# test z-feature accumulation for all 3 directions
# print("Complete Z Accumulations ...")
for z_direction in (0, 1, 2):
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepZOnly=True,
zDirection=z_direction,
numberOfThreads=-1)
self.assertEqual(len(feats_xy), 1)
self.check_features(feats_z, n_edges_z)
def accumulation_in_core_test(self, accumulation_function):
rag = nrag.gridRagStacked2D(self.labels,
numberOfLabels=self.n_labels,
numberOfThreads=-1)
n_edges_xy = rag.totalNumberOfInSliceEdges
n_edges_z = rag.totalNumberOfInBetweenSliceEdges
# test complete accumulation
# print("Complete Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
numberOfThreads=-1)
self.check_features(feats_xy, n_edges_xy)
self.check_features(feats_z, n_edges_z)
# print("... passed")
# test xy-feature accumulation
# print("Complete XY Accumulation ...")
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepXYOnly=True,
numberOfThreads=-1)
self.check_features(feats_xy, n_edges_xy)
self.assertEqual(len(feats_z), 1)
# print("... passed")
# test z-feature accumulation for all 3 directions
# print("Complete Z Accumulations ...")
for z_direction in (0, 1, 2):
feats_xy, feats_z = accumulation_function(rag,
self.data,
keepZOnly=True,
zDirection=z_direction,
numberOfThreads=-1)
self.assertEqual(len(feats_xy), 1)
self.check_features(feats_z, n_edges_z)
def test_in_vs_out_of_core(self):
accumulation_function = nrag.accumulateEdgeStandardFeatures
n_feats = 9
import z5py
import nifty.z5
###############
# get features with out of core calculation
label_path = os.path.join(self.tmp_dir, 'labels.n5')
f_labels = z5py.File(label_path, use_zarr_format=False)
dsl = f_labels.create_dataset('data',
dtype='uint32',
shape=self.shape,
chunks=(1, 25, 25),
compression='raw')
dsl[:] = self.labels
rag_ooc = nrag.gridRagStacked2DZ5(nifty.z5.datasetWrapper(
'uint32', label_path, 'data'),
numberOfLabels=self.n_labels,
numberOfThreads=1)
data_path = os.path.join(self.tmp_dir, 'data.n5')
f_data = z5py.File(data_path, use_zarr_format=False)
dsd = f_data.create_dataset('data',
dtype='float32',
shape=self.shape,
chunks=(1, 25, 25),
compression='raw')
dsd[:] = self.data
n_edges_xy = rag_ooc.totalNumberOfInSliceEdges
n_edges_z = rag_ooc.totalNumberOfInBetweenSliceEdges
def open_features(keep_xy=False, keep_z=False):
p_xy = os.path.join(self.tmp_dir, 'xy.n5')
p_z = os.path.join(self.tmp_dir, 'z.n5')
f_xy = z5py.File(p_xy, use_zarr_format=False)
f_z = z5py.File(p_z, use_zarr_format=False)
f_xy.create_dataset('data',
dtype='float32',
shape=(1 if keep_z else n_edges_xy, n_feats),
chunks=(1 if keep_z else 500, n_feats),
compression='raw')
f_z.create_dataset('data',
dtype='float32',
shape=(1 if keep_xy else n_edges_z, n_feats),
chunks=(1 if keep_xy else 500, n_feats),
compression='raw')
return p_xy, p_z
def load_features(p_xy, p_z):
xy_feats = z5py.File(p_xy)['data'][:]
z_feats = z5py.File(p_z)['data'][:]
rmtree(p_xy)
rmtree(p_z)
return xy_feats, z_feats
path_xy, path_z = open_features()
# test complete accumulation
accumulation_function(rag_ooc,
nifty.z5.datasetWrapper('float32', data_path,
'data'),
nifty.z5.datasetWrapper('float32', path_xy,
'data'),
nifty.z5.datasetWrapper('float32', path_z,
'data'),
numberOfThreads=-1)
feats_xy_ooc, feats_z_ooc = load_features(path_xy, path_z)
###############
# get features with in core calculation
rag = nrag.gridRagStacked2D(self.labels,
numberOfLabels=self.n_labels,
numberOfThreads=1)
# test complete accumulation
feats_xy, feats_z = accumulation_function(rag,
self.data,
numberOfThreads=1)
self.assertEqual(feats_xy.shape, feats_xy_ooc.shape)
self.assertTrue(np.allclose(feats_xy, feats_xy_ooc))
self.assertEqual(feats_z.shape, feats_z_ooc.shape)
self.assertTrue(np.allclose(feats_z, feats_z_ooc))
def test_in_vs_out_of_core(self):
accumulation_function = nrag.accumulateEdgeStandardFeatures
n_feats = 9
import z5py
import nifty.z5
###############
# get features with out of core calculation
label_path = os.path.join(self.tmp_dir, 'labels.n5')
f_labels = z5py.File(label_path, use_zarr_format=False)
dsl = f_labels.create_dataset('data',
dtype='uint32',
shape=self.shape,
chunks=(1, 25, 25),
compression='raw')
dsl[:] = self.labels
rag_ooc = nrag.gridRagStacked2DZ5(nifty.z5.datasetWrapper('uint32',
os.path.join(label_path,
'data')),
numberOfLabels=self.n_labels,
numberOfThreads=1)
data_path = os.path.join(self.tmp_dir, 'data.n5')
f_data = z5py.File(data_path, use_zarr_format=False)
dsd = f_data.create_dataset('data',
dtype='float32',
shape=self.shape,
chunks=(1, 25, 25),
compression='raw')
dsd[:] = self.data
n_edges_xy = rag_ooc.totalNumberOfInSliceEdges
n_edges_z = rag_ooc.totalNumberOfInBetweenSliceEdges
def open_features(keep_xy=False, keep_z=False):
p_xy = os.path.join(self.tmp_dir, 'xy.n5')
p_z = os.path.join(self.tmp_dir, 'z.n5')
f_xy = z5py.File(p_xy, use_zarr_format=False)
f_z = z5py.File(p_z, use_zarr_format=False)
f_xy.create_dataset('data',
dtype='float32',
shape=(1 if keep_z else n_edges_xy, n_feats),
chunks=(1 if keep_z else 500, n_feats),
compression='raw')
f_z.create_dataset('data',
dtype='float32',
shape=(1 if keep_xy else n_edges_z, n_feats),
chunks=(1 if keep_xy else 500, n_feats),
compression='raw')
return p_xy, p_z
def load_features(p_xy, p_z):
xy_feats = z5py.File(p_xy)['data'][:]
z_feats = z5py.File(p_z)['data'][:]
rmtree(p_xy)
rmtree(p_z)
return xy_feats, z_feats
path_xy, path_z = open_features()
# test complete accumulation
accumulation_function(rag_ooc,
nifty.z5.datasetWrapper('float32',
os.path.join(data_path, 'data')),
nifty.z5.datasetWrapper('float32',
os.path.join(path_xy, 'data')),
nifty.z5.datasetWrapper('float32',
os.path.join(path_z, 'data')),
numberOfThreads=-11)
feats_xy_ooc, feats_z_ooc = load_features(path_xy, path_z)
###############
# get features with in core calculation
rag = nrag.gridRagStacked2D(self.labels,
numberOfLabels=self.n_labels,
numberOfThreads=1)
# test complete accumulation
feats_xy, feats_z = accumulation_function(rag,
self.data,
numberOfThreads=1)
self.assertEqual(feats_xy.shape, feats_xy_ooc.shape)
self.assertTrue(np.allclose(feats_xy, feats_xy_ooc))
self.assertEqual(feats_z.shape, feats_z_ooc.shape)
self.assertTrue(np.allclose(feats_z, feats_z_ooc))