def test_relabel_components(self):
path = Path(self.path_to("test_swc_source.swc"))
# write test swc
self._write_swc(path, self._toy_swc_points())
# read arrays
swc = PointsKey("SWC")
source = SwcFileSource(path, swc)
with build(source):
batch = source.request_batch(
BatchRequest(
{swc: PointsSpec(roi=Roi((0, 1, 0), (11, 10, 1)))}))
temp_g = nx.DiGraph()
for point_id, point in batch.points[swc].data.items():
temp_g.add_node(point.point_id, label_id=point.label_id)
if (point.parent_id != -1 and point.parent_id != point.point_id
and point.parent_id in batch.points[swc].data):
temp_g.add_edge(point.point_id, point.parent_id)
previous_label = None
ccs = list(nx.weakly_connected_components(temp_g))
self.assertEqual(len(ccs), 3)
for cc in ccs:
self.assertEqual(len(cc), 10)
label = None
for point_id in cc:
if label is None:
label = temp_g.nodes[point_id]["label_id"]
self.assertNotEqual(label, previous_label)
self.assertEqual(temp_g.nodes[point_id]["label_id"], label)
previous_label = label
def test_read_single_swc(self):
path = Path(self.path_to("test_swc_source.swc"))
# write test swc
self._write_swc(path, self._toy_swc_points())
# read arrays
swc = PointsKey("SWC")
source = SwcFileSource(path, swc)
with build(source):
batch = source.request_batch(
BatchRequest(
{swc: PointsSpec(roi=Roi((0, 0, 0), (11, 11, 1)))}))
for point in self._toy_swc_points():
self.assertCountEqual(
point.location,
batch.points[swc].data[point.point_id].location)
def test_create_boundary_nodes(self):
path = Path(self.path_to("test_swc_source.swc"))
# write test swc
self._write_swc(path, self._toy_swc_points(),
{"resolution": np.array([2, 2, 2])})
# read arrays
swc = PointsKey("SWC")
source = SwcFileSource(path, swc)
with build(source):
batch = source.request_batch(
BatchRequest({swc: PointsSpec(roi=Roi((0, 5, 0), (1, 3, 1)))}))
temp_g = nx.DiGraph()
for point_id, point in batch.points[swc].data.items():
temp_g.add_node(point.point_id,
label_id=point.label_id,
location=point.location)
if (point.parent_id != -1 and point.parent_id != point.point_id
and point.parent_id in batch.points[swc].data):
temp_g.add_edge(point.point_id, point.parent_id)
else:
root = point.point_id
current = root
expected_path = [
tuple(np.array([0.0, 5.0, 0.0])),
tuple(np.array([0.0, 6.0, 0.0])),
tuple(np.array([0.0, 7.0, 0.0])),
]
expected_node_ids = [0, 1, 2]
path = []
node_ids = []
while current is not None:
node_ids.append(current)
path.append(tuple(temp_g.nodes[current]["location"]))
predecessors = list(temp_g._pred[current].keys())
current = predecessors[0] if len(predecessors) == 1 else None
self.assertCountEqual(path, expected_path)
self.assertCountEqual(node_ids, expected_node_ids)
def test_overlap(self):
path = Path(self.path_to("test_swc_sources"))
path.mkdir(parents=True, exist_ok=True)
# write test swc
for i in range(3):
self._write_swc(
path / "{}.swc".format(i),
self._toy_swc_points(),
{"offset": np.array([0, 0, 0])},
)
# read arrays
swc = PointsKey("SWC")
source = SwcFileSource(path, swc)
with build(source):
batch = source.request_batch(
BatchRequest(
{swc: PointsSpec(roi=Roi((0, 0, 0), (11, 11, 1)))}))
temp_g = nx.DiGraph()
for point_id, point in batch.points[swc].data.items():
temp_g.add_node(point.point_id, label_id=point.label_id)
if (point.parent_id != -1 and point.parent_id != point.point_id
and point.parent_id in batch.points[swc].data):
temp_g.add_edge(point.point_id, point.parent_id)
previous_label = None
ccs = list(nx.weakly_connected_components(temp_g))
self.assertEqual(len(ccs), 3)
for cc in ccs:
self.assertEqual(len(cc), 41)
label = None
for point_id in cc:
if label is None:
label = temp_g.nodes[point_id]["label_id"]
self.assertNotEqual(label, previous_label)
self.assertEqual(temp_g.nodes[point_id]["label_id"], label)
previous_label = label
def test_recenter():
path = Path(self.path_to("test_swc_source.swc"))
# write test swc
self._write_swc(path, self._toy_swc_points())
# read arrays
swc_source = PointsKey("SWC_SOURCE")
labels_source = ArrayKey("LABELS_SOURCE")
img_source = ArrayKey("IMG_SOURCE")
img_swc = PointsKey("IMG_SWC")
label_swc = PointsKey("LABEL_SWC")
imgs = ArrayKey("IMGS")
labels = ArrayKey("LABELS")
points_a = PointsKey("SKELETON_A")
points_b = PointsKey("SKELETON_B")
img_a = ArrayKey("VOLUME_A")
img_b = ArrayKey("VOLUME_B")
labels_a = ArrayKey("LABELS_A")
labels_b = ArrayKey("LABELS_B")
# Get points from test swc
swc_file_source = SwcFileSource(
path, [swc_source], [PointsSpec(roi=Roi((-10, -10, -10), (31, 31, 31)))]
)
# Create an artificial image source by rasterizing the points
image_source = (
SwcFileSource(
path, [img_swc], [PointsSpec(roi=Roi((-10, -10, -10), (31, 31, 31)))]
)
+ RasterizeSkeleton(
points=img_swc,
array=img_source,
array_spec=ArraySpec(
interpolatable=True, dtype=np.uint32, voxel_size=Coordinate((1, 1, 1))
),
)
+ BinarizeLabels(labels=img_source, labels_binary=imgs)
+ GrowLabels(array=imgs, radius=0)
)
# Create an artificial label source by rasterizing the points
label_source = (
SwcFileSource(
path, [label_swc], [PointsSpec(roi=Roi((-10, -10, -10), (31, 31, 31)))]
)
+ RasterizeSkeleton(
points=label_swc,
array=labels_source,
array_spec=ArraySpec(
interpolatable=True, dtype=np.uint32, voxel_size=Coordinate((1, 1, 1))
),
)
+ BinarizeLabels(labels=labels_source, labels_binary=labels)
+ GrowLabels(array=labels, radius=1)
)
skeleton = tuple()
skeleton += (
(swc_file_source, image_source, label_source)
+ MergeProvider()
+ RandomLocation(ensure_nonempty=swc_source, ensure_centered=True)
)
pipeline = (
skeleton
+ GetNeuronPair(
point_source=swc_source,
array_source=imgs,
label_source=labels,
points=(points_a, points_b),
arrays=(img_a, img_b),
labels=(labels_a, labels_b),
seperate_by=4,
shift_attempts=100,
)
+ Recenter(points_a, img_a, max_offset=4)
+ Recenter(points_b, img_b, max_offset=4)
)
request = BatchRequest()
data_shape = 9
request.add(points_a, Coordinate((data_shape, data_shape, data_shape)))
request.add(points_b, Coordinate((data_shape, data_shape, data_shape)))
request.add(img_a, Coordinate((data_shape, data_shape, data_shape)))
request.add(img_b, Coordinate((data_shape, data_shape, data_shape)))
request.add(labels_a, Coordinate((data_shape, data_shape, data_shape)))
request.add(labels_b, Coordinate((data_shape, data_shape, data_shape)))
with build(pipeline):
batch = pipeline.request_batch(request)
data_a = batch[img_a].data
assert data_a[tuple(np.array(data_a.shape) // 2)] == 1
data_a = np.pad(data_a, (1,), "constant", constant_values=(0,))
data_b = batch[img_b].data
assert data_b[tuple(np.array(data_b.shape) // 2)] == 1
data_b = np.pad(data_b, (1,), "constant", constant_values=(0,))
data_c = data_a + data_b
data = np.array((data_a, data_b, data_c))
for _, point in batch[points_a].data.items():
assert (
data[(0,) + tuple(int(x) + 1 for x in point.location)] == 1
), "data at {} is not 1, its {}".format(
point.location, data[(0,) + tuple(int(x) for x in point.location)]
)
for _, point in batch[points_b].data.items():
assert (
data[(1,) + tuple(int(x) + 1 for x in point.location)] == 1
), "data at {} is not 1".format(point.location)
def test_two_disjoint_lines_softmask(self):
LABEL_RADIUS = 3
RAW_RADIUS = 3
# exagerated to show problem
BLEND_SMOOTHNESS = 10
bb = Roi(Coordinate([0, 0, 0]), ([256, 256, 256]))
voxel_size = Coordinate([1, 1, 1])
swc_files = ("test_line_a.swc", "test_line_b.swc")
swc_paths = tuple(
Path(self.path_to(file_name)) for file_name in swc_files)
# create two lines seperated by a given distance and write them to swc files
intercepts, slopes = self._get_line_pair(roi=bb, dist=3 * LABEL_RADIUS)
for intercept, slope, swc_path in zip(intercepts, slopes, swc_paths):
swc_points = self._get_points(intercept, slope, bb)
self._write_swc(swc_path, swc_points)
# create swc sources
fused = ArrayKey("FUSED")
fused_labels = ArrayKey("FUSED_LABELS")
swc_key_names = ("SWC_A", "SWC_B")
labels_key_names = ("LABELS_A", "LABELS_B")
raw_key_names = ("RAW_A", "RAW_B")
swc_keys = tuple(PointsKey(name) for name in swc_key_names)
labels_keys = tuple(ArrayKey(name) for name in labels_key_names)
raw_keys = tuple(ArrayKey(name) for name in raw_key_names)
# add request
request = BatchRequest()
request.add(fused, bb.get_shape())
request.add(fused_labels, bb.get_shape())
request.add(labels_keys[0], bb.get_shape())
request.add(labels_keys[1], bb.get_shape())
request.add(raw_keys[0], bb.get_shape())
request.add(raw_keys[1], bb.get_shape())
request.add(swc_keys[0], bb.get_shape())
request.add(swc_keys[1], bb.get_shape())
# data source for swc a
data_sources_a = tuple()
data_sources_a = (data_sources_a + SwcFileSource(
swc_paths[0], swc_keys[0], PointsSpec(roi=bb)) + RasterizeSkeleton(
points=swc_keys[0],
array=labels_keys[0],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=LABEL_RADIUS,
) + RasterizeSkeleton(
points=swc_keys[0],
array=raw_keys[0],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=RAW_RADIUS,
))
# data source for swc b
data_sources_b = tuple()
data_sources_b = (data_sources_b + SwcFileSource(
swc_paths[1], swc_keys[1], PointsSpec(roi=bb)) + RasterizeSkeleton(
points=swc_keys[1],
array=labels_keys[1],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=LABEL_RADIUS,
) + RasterizeSkeleton(
points=swc_keys[1],
array=raw_keys[1],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=RAW_RADIUS,
))
data_sources = tuple([data_sources_a, data_sources_b
]) + MergeProvider()
pipeline = data_sources + FusionAugment(
raw_keys[0],
raw_keys[1],
labels_keys[0],
labels_keys[1],
fused,
fused_labels,
blend_mode="labels_mask",
blend_smoothness=BLEND_SMOOTHNESS,
num_blended_objects=0,
)
with build(pipeline):
batch = pipeline.request_batch(request)
fused_data = batch[fused].data
fused_data = np.pad(fused_data, (1, ),
"constant",
constant_values=(0, ))
a_data = batch[raw_keys[0]].data
a_data = np.pad(a_data, (1, ), "constant", constant_values=(0, ))
b_data = batch[raw_keys[1]].data
b_data = np.pad(b_data, (1, ), "constant", constant_values=(0, ))
all_data = np.zeros((5, ) + fused_data.shape)
all_data[0, :, :, :] = fused_data
all_data[1, :, :, :] = a_data + b_data
all_data[2, :, :, :] = fused_data - a_data - b_data
all_data[3, :, :, :] = a_data
all_data[4, :, :, :] = b_data
# Uncomment to visualize problem
if imported_volshow:
volshow(all_data)
# input("Press enter when you are done viewing the data: ")
diff = np.linalg.norm(fused_data - a_data - b_data)
self.assertAlmostEqual(diff, 0)
def test_two_disjoin_lines_intensity(self):
# This is worryingly slow for such a small volume (256**3) and only 2
# straight lines for skeletons.
LABEL_RADIUS = 3
RAW_RADIUS = 3
BLEND_SMOOTHNESS = 3
bb = Roi(Coordinate([0, 0, 0]), ([256, 256, 256]))
voxel_size = Coordinate([1, 1, 1])
swc_files = ("test_line_a.swc", "test_line_b.swc")
swc_paths = tuple(
Path(self.path_to(file_name)) for file_name in swc_files)
# create two lines seperated by a given distance and write them to swc files
intercepts, slopes = self._get_line_pair(roi=bb, dist=3 * LABEL_RADIUS)
for intercept, slope, swc_path in zip(intercepts, slopes, swc_paths):
swc_points = self._get_points(intercept, slope, bb)
self._write_swc(swc_path, swc_points)
# create swc sources
fused = ArrayKey("FUSED")
fused_labels = ArrayKey("FUSED_LABELS")
swc_key_names = ("SWC_A", "SWC_B")
labels_key_names = ("LABELS_A", "LABELS_B")
raw_key_names = ("RAW_A", "RAW_B")
swc_keys = tuple(PointsKey(name) for name in swc_key_names)
labels_keys = tuple(ArrayKey(name) for name in labels_key_names)
raw_keys = tuple(ArrayKey(name) for name in raw_key_names)
# add request
request = BatchRequest()
request.add(fused, bb.get_shape())
request.add(fused_labels, bb.get_shape())
request.add(labels_keys[0], bb.get_shape())
request.add(labels_keys[1], bb.get_shape())
request.add(raw_keys[0], bb.get_shape())
request.add(raw_keys[1], bb.get_shape())
request.add(swc_keys[0], bb.get_shape())
request.add(swc_keys[1], bb.get_shape())
# data source for swc a
data_sources_a = tuple()
data_sources_a = (data_sources_a + SwcFileSource(
swc_paths[0], swc_keys[0], PointsSpec(roi=bb)) + RasterizeSkeleton(
points=swc_keys[0],
array=labels_keys[0],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=LABEL_RADIUS,
) + RasterizeSkeleton(
points=swc_keys[0],
array=raw_keys[0],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=RAW_RADIUS,
))
# data source for swc b
data_sources_b = tuple()
data_sources_b = (data_sources_b + SwcFileSource(
swc_paths[1], swc_keys[1], PointsSpec(roi=bb)) + RasterizeSkeleton(
points=swc_keys[1],
array=labels_keys[1],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=LABEL_RADIUS,
) + RasterizeSkeleton(
points=swc_keys[1],
array=raw_keys[1],
array_spec=ArraySpec(interpolatable=False,
dtype=np.uint32,
voxel_size=voxel_size),
radius=RAW_RADIUS,
))
data_sources = tuple([data_sources_a, data_sources_b
]) + MergeProvider()
pipeline = data_sources + FusionAugment(
raw_keys[0],
raw_keys[1],
labels_keys[0],
labels_keys[1],
fused,
fused_labels,
blend_mode="intensity",
blend_smoothness=BLEND_SMOOTHNESS,
num_blended_objects=0,
)
with build(pipeline):
batch = pipeline.request_batch(request)
fused_data = batch[fused].data
fused_data = np.pad(fused_data, (1, ),
"constant",
constant_values=(0, ))
a_data = batch[raw_keys[0]].data
a_data = np.pad(a_data, (1, ), "constant", constant_values=(0, ))
b_data = batch[raw_keys[1]].data
b_data = np.pad(b_data, (1, ), "constant", constant_values=(0, ))
diff = np.linalg.norm(fused_data - a_data - b_data)
self.assertAlmostEqual(diff, 0)
def test_rasterize_speed(self):
# This is worryingly slow for such a small volume (256**3) and only 2
# straight lines for skeletons.
LABEL_RADIUS = 3
bb = Roi(Coordinate([0, 0, 0]), ([256, 256, 256]))
voxel_size = Coordinate([1, 1, 1])
swc_files = ("test_line_a.swc", "test_line_b.swc")
swc_paths = tuple(Path(self.path_to(file_name)) for file_name in swc_files)
# create two lines seperated by a given distance and write them to swc files
intercepts, slopes = self._get_line_pair(roi=bb, dist=3 * LABEL_RADIUS)
for intercept, slope, swc_path in zip(intercepts, slopes, swc_paths):
swc_points = self._get_points(intercept, slope, bb)
self._write_swc(swc_path, swc_points)
# create swc sources
swc_key_names = ("SWC_A", "SWC_B")
labels_key_names = ("LABELS_A", "LABELS_B")
swc_keys = tuple(PointsKey(name) for name in swc_key_names)
labels_keys = tuple(ArrayKey(name) for name in labels_key_names)
# add request
request = BatchRequest()
request.add(labels_keys[0], bb.get_shape())
request.add(labels_keys[1], bb.get_shape())
request.add(swc_keys[0], bb.get_shape())
request.add(swc_keys[1], bb.get_shape())
# data source for swc a
data_sources_a = tuple()
data_sources_a = (
data_sources_a
+ SwcFileSource(swc_paths[0], swc_keys[0], PointsSpec(roi=bb))
+ RasterizeSkeleton(
points=swc_keys[0],
array=labels_keys[0],
array_spec=ArraySpec(
interpolatable=False, dtype=np.uint32, voxel_size=voxel_size
),
radius=LABEL_RADIUS,
)
)
# data source for swc b
data_sources_b = tuple()
data_sources_b = (
data_sources_b
+ SwcFileSource(swc_paths[1], swc_keys[1], PointsSpec(roi=bb))
+ RasterizeSkeleton(
points=swc_keys[1],
array=labels_keys[1],
array_spec=ArraySpec(
interpolatable=False, dtype=np.uint32, voxel_size=voxel_size
),
radius=LABEL_RADIUS,
)
)
data_sources = tuple([data_sources_a, data_sources_b]) + MergeProvider()
pipeline = data_sources
t1 = time.time()
with build(pipeline):
batch = pipeline.request_batch(request)
a_data = batch[labels_keys[0]].data
a_data = np.pad(a_data, (1,), "constant", constant_values=(0,))
b_data = batch[labels_keys[1]].data
b_data = np.pad(b_data, (1,), "constant", constant_values=(0,))
t2 = time.time()
self.assertLess(t2 - t1, 0.1)