def _misalign(self, num_sections):
"""
:param num_sections: number of sections
:return: (shifts, slips)
"""
slips = [Coordinate((0, 0))] * num_sections
shifts = [Coordinate((0, 0))] * num_sections
for z in range(num_sections):
r = np.random.random()
if r <= self.prob_slip:
slips[z] = self._random_offset()
elif r <= self.prob_slip + self.prob_shift:
offset = self._random_offset()
for zp in range(z, num_sections):
shifts[zp] += offset
logger.debug("misaligning sections with " + str(shifts))
return shifts, slips
def test_ensure_center_non_zero(self):
path = Path(self.path_to("test_swc_source.swc"))
# write test swc
self._write_swc(path, self._toy_swc_points().to_nx_graph())
# read arrays
swc = PointsKey("SWC")
img = ArrayKey("IMG")
pipeline = (SwcFileSource(
path, [swc], [PointsSpec(roi=Roi((0, 0, 0), (11, 11, 11)))]) +
RandomLocation(ensure_nonempty=swc, ensure_centered=True) +
RasterizeSkeleton(
points=swc,
array=img,
array_spec=ArraySpec(
interpolatable=False,
dtype=np.uint32,
voxel_size=Coordinate((1, 1, 1)),
),
))
request = BatchRequest()
request.add(img, Coordinate((5, 5, 5)))
request.add(swc, Coordinate((5, 5, 5)))
with build(pipeline):
batch = pipeline.request_batch(request)
data = batch[img].data
g = batch[swc]
assert g.num_vertices() > 0
self.assertNotEqual(data[tuple(np.array(data.shape) // 2)], 0)
def test_embedding_pipeline(
tmpdir, aux_task, blend_mode, fusion_pipeline, train_embedding, snapshot_every
):
setup_config = DEFAULT_CONFIG
setup_config["FUSION_PIPELINE"] = fusion_pipeline
setup_config["TRAIN_EMBEDDING"] = train_embedding
setup_config["SNAPSHOT_EVERY"] = snapshot_every
setup_config["TENSORBOARD_LOG_DIR"] = tmpdir
setup_config["SNAPSHOT_DIR"] = tmpdir
setup_config["SNAPSHOT_FILE_NAME"] = "test_snapshot"
setup_config["MATCHING_FAILURES_DIR"] = None
setup_config["BLEND_MODE"] = blend_mode
setup_config["AUX_TASK"] = aux_task
voxel_size = Coordinate(setup_config["VOXEL_SIZE"])
output_size = Coordinate(setup_config["OUTPUT_SHAPE"]) * voxel_size
input_size = Coordinate(setup_config["INPUT_SHAPE"]) * voxel_size
pipeline, raw, output, inputs = embedding_pipeline(
setup_config, get_test_data_sources
)
request = BatchRequest()
request.add(raw, input_size)
request.add(output, output_size)
for key in inputs:
request.add(key, output_size)
with build(pipeline):
batch = pipeline.request_batch(request)
assert output in batch
assert raw in batch
def test_nodes():
initial_locations = {
1: np.array([1, 1, 1], dtype=np.float32),
2: np.array([500, 500, 500], dtype=np.float32),
3: np.array([550, 550, 550], dtype=np.float32),
}
replacement_locations = {
1: np.array([0, 0, 0], dtype=np.float32),
2: np.array([50, 50, 50], dtype=np.float32),
3: np.array([55, 55, 55], dtype=np.float32),
}
nodes = [
Node(id=id, location=location)
for id, location in initial_locations.items()
]
edges = [Edge(1, 2), Edge(2, 3)]
spec = GraphSpec(roi=Roi(Coordinate([-500, -500, -500]),
Coordinate([1500, 1500, 1500])))
graph = Graph(nodes, edges, spec)
for node in graph.nodes:
node.location = replacement_locations[node.id]
for node in graph.nodes:
assert all(np.isclose(node.location, replacement_locations[node.id]))
def process(self, batch, request):
points = batch.points[self.point_source]
array = batch.arrays[self.array_source]
center = array.spec.roi.get_begin() + array.spec.roi.get_shape() / 2
closest = None
for point in points.data.values():
closest = (closest if closest is not None
and np.linalg.norm(center - closest) <
np.linalg.norm(center - Coordinate(point.location)) else
Coordinate(point.location))
logger.debug("center: {}".format(center))
direction = closest - center
logger.debug("direction: {}".format(direction))
# Shift and crop points and array
points, array = self._shift_and_crop(
points,
array,
direction=direction,
output_roi=request[self.point_source].roi,
)
batch.points[self.point_source] = points
batch.arrays[self.array_source] = array
return batch
def test_6_neighborhood():
# array keys
graph = GraphKey("GRAPH")
neighborhood = ArrayKey("NEIGHBORHOOD")
neighborhood_mask = ArrayKey("NEIGHBORHOOD_MASK")
distance = 1
pipeline = TestSource(graph) + Neighborhood(
graph,
neighborhood,
neighborhood_mask,
distance,
array_specs={
neighborhood: ArraySpec(voxel_size=Coordinate((1, 1, 1))),
neighborhood_mask: ArraySpec(voxel_size=Coordinate((1, 1, 1))),
},
k=6,
)
request = BatchRequest()
request[neighborhood] = ArraySpec(roi=Roi((0, 0, 0), (10, 10, 10)))
request[neighborhood_mask] = ArraySpec(roi=Roi((0, 0, 0), (10, 10, 10)))
with build(pipeline):
batch = pipeline.request_batch(request)
n_data = batch[neighborhood].data
n_mask = batch[neighborhood_mask].data
masked_ind = list(
set([(0, i, 0) for i in range(10) if i not in [0, 4]] +
[(i, 5, 0)
for i in range(10)] + [(i, 4, 0)
for i in range(10) if i not in [0]]))
assert all(n_mask[tuple(zip(*masked_ind))]
), f"expected {masked_ind} but saw {np.where(n_mask==1)}"
def test_transpose():
voxel_size = Coordinate((20, 20))
graph_key = GraphKey("GRAPH")
array_key = ArrayKey("ARRAY")
graph = Graph(
[Node(id=1, location=np.array([450, 550]))],
[],
GraphSpec(roi=Roi((100, 200), (800, 600))),
)
data = np.zeros([40, 30])
data[17, 17] = 1
array = Array(
data, ArraySpec(roi=Roi((100, 200), (800, 600)),
voxel_size=voxel_size))
default_pipeline = (
(GraphSource(graph_key, graph), ArraySource(array_key, array)) +
MergeProvider() + SimpleAugment(
mirror_only=[], transpose_only=[0, 1], transpose_probs=[0, 0]))
transpose_pipeline = (
(GraphSource(graph_key, graph), ArraySource(array_key, array)) +
MergeProvider() + SimpleAugment(
mirror_only=[], transpose_only=[0, 1], transpose_probs=[1, 1]))
request = BatchRequest()
request[graph_key] = GraphSpec(roi=Roi((400, 500), (200, 300)))
request[array_key] = ArraySpec(roi=Roi((400, 500), (200, 300)))
with build(default_pipeline):
expected_location = [450, 550]
batch = default_pipeline.request_batch(request)
assert len(list(batch[graph_key].nodes)) == 1
node = list(batch[graph_key].nodes)[0]
assert all(np.isclose(node.location, expected_location))
node_voxel_index = Coordinate(
(node.location - batch[array_key].spec.roi.get_offset()) /
voxel_size)
assert (
batch[array_key].data[node_voxel_index] == 1
), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}"
with build(transpose_pipeline):
expected_location = [410, 590]
batch = transpose_pipeline.request_batch(request)
assert len(list(batch[graph_key].nodes)) == 1
node = list(batch[graph_key].nodes)[0]
assert all(np.isclose(node.location, expected_location))
node_voxel_index = Coordinate(
(node.location - batch[array_key].spec.roi.get_offset()) /
voxel_size)
assert (
batch[array_key].data[node_voxel_index] == 1
), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}"
def test_realistic_valid_examples(example, use_gurobi):
penalty_attr = "penalty"
location_attr = "location"
example_dir = Path(__file__).parent / "mouselight_examples" / "valid" / example
consensus = PointsKey("CONSENSUS")
skeletonization = PointsKey("SKELETONIZATION")
matched = PointsKey("MATCHED")
matched_with_fallback = PointsKey("MATCHED_WITH_FALLBACK")
inf_roi = Roi(Coordinate((None,) * 3), Coordinate((None,) * 3))
request = BatchRequest()
request[matched] = PointsSpec(roi=inf_roi)
request[matched_with_fallback] = PointsSpec(roi=inf_roi)
request[consensus] = PointsSpec(roi=inf_roi)
pipeline = (
(
GraphSource(example_dir / "graph.obj", [skeletonization]),
GraphSource(example_dir / "tree.obj", [consensus]),
)
+ MergeProvider()
+ TopologicalMatcher(
skeletonization,
consensus,
matched,
expected_edge_len=10,
match_distance_threshold=76,
max_gap_crossing=48,
use_gurobi=use_gurobi,
location_attr=location_attr,
penalty_attr=penalty_attr,
)
+ TopologicalMatcher(
skeletonization,
consensus,
matched_with_fallback,
expected_edge_len=10,
match_distance_threshold=76,
max_gap_crossing=48,
use_gurobi=use_gurobi,
location_attr=location_attr,
penalty_attr=penalty_attr,
with_fallback=True,
)
)
with build(pipeline):
batch = pipeline.request_batch(request)
consensus_ccs = list(batch[consensus].connected_components)
matched_with_fallback_ccs = list(batch[matched_with_fallback].connected_components)
matched_ccs = list(batch[matched].connected_components)
assert len(matched_ccs) == len(consensus_ccs)
def test_crop(self):
g = Graph(self.nodes, self.edges, self.spec)
sub_g = g.crop(Roi(Coordinate([1, 1, 1]), Coordinate([3, 3, 3])))
self.assertEqual(g.spec.roi, self.spec.roi)
self.assertEqual(sub_g.spec.roi,
Roi(Coordinate([1, 1, 1]), Coordinate([3, 3, 3])))
sub_g.spec.directed = False
self.assertTrue(g.spec.directed)
self.assertFalse(sub_g.spec.directed)
def get_test_data_sources(setup_config):
input_shape = Coordinate(setup_config["INPUT_SHAPE"])
voxel_size = Coordinate(setup_config["VOXEL_SIZE"])
input_size = input_shape * voxel_size
micron_scale = voxel_size[0]
# New array keys
# Note: These are intended to be requested with size input_size
raw = ArrayKey("RAW")
matched = GraphKey("MATCHED")
nonempty_placeholder = GraphKey("NONEMPTY")
labels = ArrayKey("LABELS")
ensure_nonempty = matched
data_sources = ((
TestImageSource(
array=raw,
array_specs={
raw:
ArraySpec(interpolatable=True,
voxel_size=voxel_size,
dtype=np.uint16)
},
size=input_size * 3,
voxel_size=voxel_size,
),
TestPointSource(
points=[matched, nonempty_placeholder],
directed=False,
size=input_size * 3,
num_points=333,
),
) + MergeProvider() + RandomLocation(
ensure_nonempty=ensure_nonempty,
ensure_centered=True,
point_balance_radius=10 * micron_scale,
) + RasterizeSkeleton(
points=matched,
array=labels,
array_spec=ArraySpec(
interpolatable=False, voxel_size=voxel_size, dtype=np.uint64),
) + Normalize(raw))
return (
data_sources,
raw,
labels,
nonempty_placeholder,
matched,
)
def __init__(self):
self.dtype = float
self.__vertices = [
Node(id=1, location=np.array([1, 1, 1], dtype=self.dtype)),
Node(id=2, location=np.array([500, 500, 500], dtype=self.dtype)),
Node(id=3, location=np.array([550, 550, 550], dtype=self.dtype)),
]
self.__edges = [Edge(1, 2), Edge(2, 3)]
self.__spec = GraphSpec(roi=Roi(Coordinate([-500, -500, -500]),
Coordinate([1500, 1500, 1500])))
self.graph = Graph(self.__vertices, self.__edges, self.__spec)
def test_3d(self):
test_graph = GraphKey("TEST_GRAPH")
graph_spec = GraphSpec(roi=Roi((0, 0, 0), (5, 5, 5)))
test_array = ArrayKey("TEST_ARRAY")
array_spec = ArraySpec(
roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1))
)
test_array2 = ArrayKey("TEST_ARRAY2")
array2_spec = ArraySpec(
roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1))
)
snapshot_request = BatchRequest()
snapshot_request.add(test_graph, Coordinate((5, 5, 5)))
pipeline = ExampleSource(
[test_graph, test_array, test_array2], [graph_spec, array_spec, array2_spec]
) + Snapshot(
{
test_graph: "graphs/graph",
test_array: "volumes/array",
test_array2: "volumes/array2",
},
output_dir=str(self.test_dir),
every=2,
additional_request=snapshot_request,
output_filename="snapshot.hdf",
)
snapshot_file_path = Path(self.test_dir, "snapshot.hdf")
with build(pipeline):
request = BatchRequest()
roi = Roi((0, 0, 0), (5, 5, 5))
request[test_array] = ArraySpec(roi=roi)
request[test_array2] = ArraySpec(roi=roi)
pipeline.request_batch(request)
assert snapshot_file_path.exists()
f = h5py.File(snapshot_file_path)
assert f["volumes/array"] is not None
assert f["graphs/graph-ids"] is not None
snapshot_file_path.unlink()
pipeline.request_batch(request)
assert not snapshot_file_path.exists()
def test_get_line_pair(self):
dist = 3
intercepts, slopes = self._get_line_pair(roi=Roi(
Coordinate([0, 0, 0]), Coordinate([10, 10, 10])),
dist=dist)
a, b = intercepts
# check that the line connecting the closest points is perp to both slopes
self.assertAlmostEqual(np.linalg.norm(np.dot(b - a, slopes[0])), 0)
self.assertAlmostEqual(np.linalg.norm(np.dot(b - a, slopes[1])), 0)
# check the intercepts are the expected distance
self.assertAlmostEqual(
np.linalg.norm(intercepts[1] - intercepts[0]) - dist, 0)
def test_pipeline3(self):
array_key = ArrayKey("TEST_ARRAY")
points_key = GraphKey("TEST_POINTS")
voxel_size = Coordinate((1, 1))
spec = ArraySpec(voxel_size=voxel_size, interpolatable=True)
hdf5_source = Hdf5Source(self.fake_data_file, {array_key: "testdata"},
array_specs={array_key: spec})
csv_source = CsvPointsSource(
self.fake_points_file,
points_key,
GraphSpec(roi=Roi(shape=Coordinate((100, 100)), offset=(0, 0))),
)
request = BatchRequest()
shape = Coordinate((60, 60))
request.add(array_key, shape, voxel_size=Coordinate((1, 1)))
request.add(points_key, shape)
shift_node = ShiftAugment(prob_slip=0.2,
prob_shift=0.2,
sigma=5,
shift_axis=0)
pipeline = ((hdf5_source, csv_source) + MergeProvider() +
RandomLocation(ensure_nonempty=points_key) + shift_node)
with build(pipeline) as b:
request = b.request_batch(request)
# print(request[points_key])
target_vals = [
self.fake_data[point[0]][point[1]] for point in self.fake_points
]
result_data = request[array_key].data
result_points = list(request[points_key].nodes)
result_vals = [
result_data[int(point.location[0])][int(point.location[1])]
for point in result_points
]
for result_val in result_vals:
self.assertTrue(
result_val in target_vals,
msg=
"result value {} at points {} not in target values {} at points {}"
.format(
result_val,
list(result_points),
target_vals,
self.fake_points,
),
)
def _get_source_roi(self, transformation):
dims = transformation.shape[0]
# get bounding box of needed data for transformation
bb_min = Coordinate(
int(math.floor(transformation[d].min())) for d in range(dims))
bb_max = Coordinate(
int(math.ceil(transformation[d].max())) + 1 for d in range(dims))
# create roi sufficiently large to feed transformation
source_roi = Roi(bb_min, bb_max - bb_min)
return source_roi
def process(self, batch, request):
gt = batch[self.gt]
voxel_size = self.spec[self.neighborhood].voxel_size
request_roi = request[self.neighborhood].roi
if voxel_size is None:
voxel_size = Coordinate((1, ) * len(gt.spec.roi.get_shape()))
neighborhood = np.zeros(
(self.k, ) + (request_roi.get_shape() / voxel_size),
dtype=np.float32)
neighborhood_mask = np.zeros((request_roi.get_shape() / voxel_size),
dtype=int)
k_neighborhood_offsets = self.get_neighborhood_offsets()
for i, connected_component in enumerate(gt.connected_components):
component_id = i + 1
node_locations = [
gt.node(node_id).location for node_id in connected_component
]
component_kdtree = cKDTree(node_locations)
for node_id in connected_component:
node = gt.node(node_id)
location = node.location
if request_roi.contains(location):
query_points = k_neighborhood_offsets + location
query_neighbors = component_kdtree.query(query_points,
1)[0]
voxel_index = Coordinate(
(location - request_roi.get_offset()) // voxel_size)
neighborhood[(slice(None), ) +
voxel_index] = (query_neighbors /
self.distance)
if neighborhood_mask[voxel_index] == 0:
neighborhood_mask[voxel_index] = component_id
elif neighborhood_mask[voxel_index] != component_id:
neighborhood_mask[voxel_index] = -1
neighborhood_mask = neighborhood_mask > 0
outputs = Batch()
neighborhood_spec = self.array_specs[self.neighborhood].copy()
neighborhood_spec.roi = request_roi
outputs[self.neighborhood] = Array(neighborhood, neighborhood_spec)
neighborhood_mask_spec = self.array_specs[
self.neighborhood_mask].copy()
neighborhood_mask_spec.roi = request_roi
outputs[self.neighborhood_mask] = Array(neighborhood_mask > 0,
neighborhood_mask_spec)
return outputs
def visualize_embedding_pipeline(fusion_pipeline, train_embedding):
setup_config = DEFAULT_CONFIG
setup_config["FUSION_PIPELINE"] = fusion_pipeline
setup_config["TRAIN_EMBEDDING"] = train_embedding
voxel_size = Coordinate(setup_config["VOXEL_SIZE"])
output_size = Coordinate(setup_config["OUTPUT_SHAPE"]) * voxel_size
input_size = Coordinate(setup_config["INPUT_SHAPE"]) * voxel_size
pipeline, raw, output = embedding_pipeline(setup_config,
get_test_data_sources)
request = BatchRequest()
request.add(raw, input_size)
request.add(output, output_size)
with build(pipeline):
pipeline.request_batch(request)
visualize_hdf5(Path("snapshots/snapshot_1.hdf"), tuple(voxel_size))
def provide(self, request: BatchRequest) -> Batch:
"""
First request points with specific seeds, then request the rest if
valid points are found.
"""
logger.debug(f"growing request by {self._get_growth()}")
has_component = False
while not has_component:
base_seed, add_seed, direction, prepare_profiling_stats = self.get_valid_seeds(
request)
timing_prepare = Timing(self, "prepare")
timing_prepare.start()
request_base = self.prepare(request, base_seed, -direction)
if add_seed is not None:
request_add = self.prepare(request, add_seed, direction)
else:
request_add = None
logger.debug(f"No add_request needed!")
timing_prepare.stop()
base = self.upstream_provider.request_batch(request_base)
if request_add is not None:
add = self.upstream_provider.request_batch(request_add)
else:
add = self._empty_copy(base)
has_component = True
timing_process = Timing(self, "process")
timing_process.start()
base = self.process(base, Coordinate([0, 0, 0]), request=request)
add = self.process(add, -Coordinate([0, 0, 0]), request=request)
batch = self.merge_batches(base, add)
timing_process.stop()
batch.profiling_stats.merge_with(prepare_profiling_stats)
batch.profiling_stats.add(timing_prepare)
batch.profiling_stats.add(timing_process)
return batch
def test_without_placeholder(self):
test_labels = ArrayKey("TEST_LABELS")
test_points = GraphKey("TEST_POINTS")
pipeline = (
PointTestSource3D() + RandomLocation(ensure_nonempty=test_points) +
ElasticAugment([10, 10, 10], [0.1, 0.1, 0.1], [0, 2.0 * math.pi]) +
Snapshot(
{test_labels: "volumes/labels"},
output_dir=self.path_to(),
output_filename="elastic_augment_test{id}-{iteration}.hdf",
))
with build(pipeline):
for i in range(2):
request_size = Coordinate((40, 40, 40))
request_a = BatchRequest(random_seed=i)
request_a.add(test_points, request_size)
request_b = BatchRequest(random_seed=i)
request_b.add(test_points, request_size)
request_b.add(test_labels, request_size)
# No array to provide a voxel size to ElasticAugment
with pytest.raises(PipelineRequestError):
pipeline.request_batch(request_a)
batch_b = pipeline.request_batch(request_b)
self.assertIn(test_labels, batch_b)
def test_placeholder(self):
test_labels = ArrayKey("TEST_LABELS")
test_points = GraphKey("TEST_POINTS")
pipeline = (
PointTestSource3D() + RandomLocation(ensure_nonempty=test_points) +
ElasticAugment([10, 10, 10], [0.1, 0.1, 0.1], [0, 2.0 * math.pi]) +
Snapshot(
{test_labels: "volumes/labels"},
output_dir=self.path_to(),
output_filename="elastic_augment_test{id}-{iteration}.hdf",
))
with build(pipeline):
for i in range(2):
request_size = Coordinate((40, 40, 40))
request_a = BatchRequest(random_seed=i)
request_a.add(test_points, request_size)
request_a.add(test_labels, request_size, placeholder=True)
request_b = BatchRequest(random_seed=i)
request_b.add(test_points, request_size)
request_b.add(test_labels, request_size)
batch_a = pipeline.request_batch(request_a)
batch_b = pipeline.request_batch(request_b)
points_a = batch_a[test_points].nodes
points_b = batch_b[test_points].nodes
for a, b in zip(points_a, points_b):
assert all(np.isclose(a.location, b.location))
def setup(self):
self.voxel_size = Coordinate(
min(axis) for axis in zip(*[
array_spec.voxel_size
for array_spec in self.spec.array_specs.values()
]))
self.spatial_dims = self.voxel_size.dims
def __init__(self):
self.voxel_size = Coordinate((40, 4, 4))
self.nodes = [
# corners
Node(id=1, location=np.array((-200, -200, -200))),
Node(id=2, location=np.array((-200, -200, 199))),
Node(id=3, location=np.array((-200, 199, -200))),
Node(id=4, location=np.array((-200, 199, 199))),
Node(id=5, location=np.array((199, -200, -200))),
Node(id=6, location=np.array((199, -200, 199))),
Node(id=7, location=np.array((199, 199, -200))),
Node(id=8, location=np.array((199, 199, 199))),
# center
Node(id=9, location=np.array((0, 0, 0))),
Node(id=10, location=np.array((-1, -1, -1))),
]
self.graph_spec = GraphSpec(roi=Roi((-100, -100, -100), (300, 300, 300)))
self.array_spec = ArraySpec(
roi=Roi((-200, -200, -200), (400, 400, 400)), voxel_size=self.voxel_size
)
self.graph = Graph(self.nodes, [], self.graph_spec)
def __init__(self,
z_resolution,
prob_slip=0,
prob_shift=0,
max_misalign=(0, 0),
ignore_keys_for_slip=(),
seed=None):
super(BatchFilter, self).__init__()
self.z_resolution = Coordinate((z_resolution, ))
self.prob_slip = prob_slip
self.prob_shift = prob_shift
self.max_misalign = max_misalign
self.ignore_keys_for_slip = ignore_keys_for_slip
self.seed = seed
logger.debug(
'initialized with parameters '
'prob_slip=%f '
'prob_shift=%f '
'max_misalign=%s '
'ignore_keys_for_slip=%s '
'seed=%d', self.prob_slip, self.prob_shift, self.max_misalign,
self.ignore_keys_for_slip, self.seed)
self.translations = {}
self.target_rois = {}
def setup(self):
self._read_points()
logger.debug("Locations: %s", self.locations)
if self.points_spec is not None:
self.provides(self.points, self.points_spec)
return
min_bb = Coordinate(np.floor(np.amin(self.locations, 0)))
max_bb = Coordinate(np.ceil(np.amax(self.locations, 0)) + 1)
roi = Roi(min_bb, max_bb - min_bb)
self.provides(self.points, PointsSpec(roi=roi))
def test_shift_points5(self):
data = [
Node(id=0, location=np.array([3, 0])),
Node(id=1, location=np.array([3, 2])),
Node(id=2, location=np.array([3, 4])),
Node(id=3, location=np.array([3, 6])),
Node(id=4, location=np.array([3, 8])),
]
spec = GraphSpec(Roi(offset=(0, 0), shape=(15, 10)))
points = Graph(data, [], spec)
request_roi = Roi(offset=(3, 0), shape=(9, 10))
shift_array = np.array([[3, 0], [-3, 0], [0, 0], [-3, 0], [3, 0]],
dtype=int)
lcm_voxel_size = Coordinate((3, 2))
shifted_data = [
Node(id=0, location=np.array([6, 0])),
Node(id=2, location=np.array([3, 4])),
Node(id=4, location=np.array([6, 8])),
]
result = ShiftAugment.shift_points(
points,
request_roi,
shift_array,
shift_axis=1,
lcm_voxel_size=lcm_voxel_size,
)
# print("test 4", result.data, shifted_data)
self.assertTrue(self.points_equal(result.nodes, shifted_data))
self.assertTrue(result.spec == GraphSpec(request_roi))
def test_merge_basics(self):
voxel_size = (1, 1, 1)
GraphKey("PRESYN")
ArrayKey("GT_LABELS")
graphsource = GraphTestSource(voxel_size)
arraysource = ArrayTestSoure(voxel_size)
pipeline = (graphsource, arraysource) + MergeProvider() + RandomLocation()
window_request = Coordinate((50, 50, 50))
with build(pipeline):
# Check basic merging.
request = BatchRequest()
request.add((GraphKeys.PRESYN), window_request)
request.add((ArrayKeys.GT_LABELS), window_request)
batch_res = pipeline.request_batch(request)
self.assertTrue(ArrayKeys.GT_LABELS in batch_res.arrays)
self.assertTrue(GraphKeys.PRESYN in batch_res.graphs)
# Check that request of only one source also works.
request = BatchRequest()
request.add((GraphKeys.PRESYN), window_request)
batch_res = pipeline.request_batch(request)
self.assertFalse(ArrayKeys.GT_LABELS in batch_res.arrays)
self.assertTrue(GraphKeys.PRESYN in batch_res.graphs)
# Check that it fails, when having two sources that provide the same type.
arraysource2 = ArrayTestSoure(voxel_size)
pipeline_fail = (arraysource, arraysource2) + MergeProvider() + RandomLocation()
with self.assertRaises(PipelineSetupError):
with build(pipeline_fail):
pass
def _create_rotation_transformation(shape, angle, subsample=1, voxel_size=None):
dims = len(shape)
subsample_shape = tuple(max(1,int(s/subsample)) for s in shape)
control_points = (2,)*dims
if voxel_size is None:
voxel_size = Coordinate((1,) * dims)
# map control points to world coordinates
control_point_scaling_factor = tuple(float(s-1) * vs for s, vs in zip(shape, voxel_size))
# rotate control points
center = np.array([0.5*(d-1)*vs for d, vs in zip(shape, voxel_size)])
# print("Creating rotation transformation with:")
# print("\tangle : " + str(angle))
# print("\tcenter: " + str(center))
control_point_offsets = np.zeros((dims,) + control_points, dtype=np.float32)
for control_point in np.ndindex(control_points):
point = np.array(control_point)*control_point_scaling_factor
center_offset = np.array([p-c for c,p in zip(center, point)], dtype=np.float32)
rotated_offset = np.array(center_offset)
rotated_offset[-2:] = _rotate(center_offset[-2:], angle)
displacement = rotated_offset - center_offset
control_point_offsets[(slice(None),) + control_point] += displacement
return augment.upscale_transformation(control_point_offsets, subsample_shape)
def test_prepare1(self):
key = ArrayKey("TEST_ARRAY")
spec = ArraySpec(voxel_size=Coordinate((1, 1)), interpolatable=True)
hdf5_source = Hdf5Source(self.fake_data_file, {key: "testdata"},
array_specs={key: spec})
request = BatchRequest()
shape = Coordinate((3, 3))
request.add(key, shape, voxel_size=Coordinate((1, 1)))
shift_node = ShiftAugment(sigma=1, shift_axis=0)
with build((hdf5_source + shift_node)):
shift_node.prepare(request)
self.assertTrue(shift_node.ndim == 2)
self.assertTrue(shift_node.shift_sigmas == tuple([0.0, 1.0]))
def _create_identity_transformation(shape, voxel_size=None, offset=None, subsample=1):
dims = len(shape)
if voxel_size is None:
voxel_size = Coordinate((1,) * dims)
if offset is None:
offset = Coordinate((0,) * dims)
subsample_shape = tuple(max(1,int(s/subsample)) for s in shape)
step_width = tuple(float(shape[d]-1)/(subsample_shape[d]-1) if subsample_shape[d] > 1 else 1 for d in range(dims))
step_width = tuple(s*vs for s, vs in zip(step_width, voxel_size))
axis_ranges = (
np.arange(subsample_shape[d], dtype=np.float32)*step_width[d] + offset[d]
for d in range(dims)
)
return np.array(np.meshgrid(*axis_ranges, indexing='ij'), dtype=np.float32)
def test_pipeline2(self):
key = ArrayKey("TEST_ARRAY")
spec = ArraySpec(voxel_size=Coordinate((3, 1)), interpolatable=True)
hdf5_source = Hdf5Source(self.fake_data_file, {key: "testdata"},
array_specs={key: spec})
request = BatchRequest()
shape = Coordinate((3, 3))
request.add(key, shape, voxel_size=Coordinate((3, 1)))
shift_node = ShiftAugment(prob_slip=0.2,
prob_shift=0.2,
sigma=1,
shift_axis=0)
with build((hdf5_source + shift_node)) as b:
b.request_batch(request)
