def test_shift_points5(self):
data = [
Node(id=0, location=np.array([3, 0])),
Node(id=1, location=np.array([3, 2])),
Node(id=2, location=np.array([3, 4])),
Node(id=3, location=np.array([3, 6])),
Node(id=4, location=np.array([3, 8])),
]
spec = GraphSpec(Roi(offset=(0, 0), shape=(15, 10)))
points = Graph(data, [], spec)
request_roi = Roi(offset=(3, 0), shape=(9, 10))
shift_array = np.array([[3, 0], [-3, 0], [0, 0], [-3, 0], [3, 0]],
dtype=int)
lcm_voxel_size = Coordinate((3, 2))
shifted_data = [
Node(id=0, location=np.array([6, 0])),
Node(id=2, location=np.array([3, 4])),
Node(id=4, location=np.array([6, 8])),
]
result = ShiftAugment.shift_points(
points,
request_roi,
shift_array,
shift_axis=1,
lcm_voxel_size=lcm_voxel_size,
)
# print("test 4", result.data, shifted_data)
self.assertTrue(self.points_equal(result.nodes, shifted_data))
self.assertTrue(result.spec == GraphSpec(request_roi))
def test_transpose():
voxel_size = Coordinate((20, 20))
graph_key = GraphKey("GRAPH")
array_key = ArrayKey("ARRAY")
graph = Graph(
[Node(id=1, location=np.array([450, 550]))],
[],
GraphSpec(roi=Roi((100, 200), (800, 600))),
)
data = np.zeros([40, 30])
data[17, 17] = 1
array = Array(
data, ArraySpec(roi=Roi((100, 200), (800, 600)),
voxel_size=voxel_size))
default_pipeline = (
(GraphSource(graph_key, graph), ArraySource(array_key, array)) +
MergeProvider() + SimpleAugment(
mirror_only=[], transpose_only=[0, 1], transpose_probs=[0, 0]))
transpose_pipeline = (
(GraphSource(graph_key, graph), ArraySource(array_key, array)) +
MergeProvider() + SimpleAugment(
mirror_only=[], transpose_only=[0, 1], transpose_probs=[1, 1]))
request = BatchRequest()
request[graph_key] = GraphSpec(roi=Roi((400, 500), (200, 300)))
request[array_key] = ArraySpec(roi=Roi((400, 500), (200, 300)))
with build(default_pipeline):
expected_location = [450, 550]
batch = default_pipeline.request_batch(request)
assert len(list(batch[graph_key].nodes)) == 1
node = list(batch[graph_key].nodes)[0]
assert all(np.isclose(node.location, expected_location))
node_voxel_index = Coordinate(
(node.location - batch[array_key].spec.roi.get_offset()) /
voxel_size)
assert (
batch[array_key].data[node_voxel_index] == 1
), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}"
with build(transpose_pipeline):
expected_location = [410, 590]
batch = transpose_pipeline.request_batch(request)
assert len(list(batch[graph_key].nodes)) == 1
node = list(batch[graph_key].nodes)[0]
assert all(np.isclose(node.location, expected_location))
node_voxel_index = Coordinate(
(node.location - batch[array_key].spec.roi.get_offset()) /
voxel_size)
assert (
batch[array_key].data[node_voxel_index] == 1
), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}"
def __init__(self, graph):
self.graph = graph
self.graph_spec = GraphSpec(roi=Roi((-10, -10, -10), (30, 30, 30)),
directed=False)
self.component_1_nodes = [
Node(i, np.array([0, i, 0])) for i in range(10)
] + [Node(i + 10, np.array([i, 5, 0])) for i in range(10)]
self.component_1_edges = ([Edge(i, i + 1) for i in range(9)] +
[Edge(i + 10, i + 11)
for i in range(9)] + [Edge(5, 15)])
self.component_2_nodes = [
Node(i + 20, np.array([i, 4, 0])) for i in range(10)
]
self.component_2_edges = [Edge(i + 20, i + 21) for i in range(9)]
def setup(self):
roi = Roi(Coordinate([0] * len(self.size)), self.size)
for points_key in self.points:
self.provides(points_key, GraphSpec(roi=roi,
directed=self.directed))
k = min(self.size)
nodes = [
Node(id=i, location=np.array([i * k / self.num_points] * 3))
for i in range(self.num_points)
]
edges = [Edge(i, i + 1) for i in range(self.num_points - 1)]
self.graph = Graph(nodes, edges,
GraphSpec(roi=roi, directed=self.directed))
class TestSource(BatchProvider):
def __init__(self, graph):
self.graph = graph
self.graph_spec = GraphSpec(roi=Roi((-10, -10, -10), (30, 30, 30)),
directed=False)
self.component_1_nodes = [
Node(i, np.array([0, i, 0])) for i in range(10)
] + [Node(i + 10, np.array([i, 5, 0])) for i in range(10)]
self.component_1_edges = ([Edge(i, i + 1) for i in range(9)] +
[Edge(i + 10, i + 11)
for i in range(9)] + [Edge(5, 15)])
self.component_2_nodes = [
Node(i + 20, np.array([i, 4, 0])) for i in range(10)
]
self.component_2_edges = [Edge(i + 20, i + 21) for i in range(9)]
def setup(self):
self.provides(self.graph, self.graph_spec)
def provide(self, request):
outputs = Batch()
spec = self.graph_spec.copy()
spec.roi = request[self.graph].roi
outputs[self.graph] = Graph(
self.component_1_nodes + self.component_2_nodes,
self.component_1_edges + self.component_2_edges,
spec,
)
return outputs
def __init__(self):
self.graph = Graph(
[Node(id=1, location=np.array([50, 70, 100]))],
[],
GraphSpec(roi=Roi((-200, -200, -200), (400, 400, 478))),
)
def test_nodes():
initial_locations = {
1: np.array([1, 1, 1], dtype=np.float32),
2: np.array([500, 500, 500], dtype=np.float32),
3: np.array([550, 550, 550], dtype=np.float32),
}
replacement_locations = {
1: np.array([0, 0, 0], dtype=np.float32),
2: np.array([50, 50, 50], dtype=np.float32),
3: np.array([55, 55, 55], dtype=np.float32),
}
nodes = [
Node(id=id, location=location)
for id, location in initial_locations.items()
]
edges = [Edge(1, 2), Edge(2, 3)]
spec = GraphSpec(roi=Roi(Coordinate([-500, -500, -500]),
Coordinate([1500, 1500, 1500])))
graph = Graph(nodes, edges, spec)
for node in graph.nodes:
node.location = replacement_locations[node.id]
for node in graph.nodes:
assert all(np.isclose(node.location, replacement_locations[node.id]))
def provide(self, request):
batch = Batch()
roi_points = request[GraphKeys.TEST_POINTS].roi
roi_array = request[ArrayKeys.TEST_LABELS].roi
roi_voxel = roi_array // self.spec[ArrayKeys.TEST_LABELS].voxel_size
data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32)
data[:, ::2] = 100
for node in self.points:
loc = self.point_to_voxel(roi_array, node.location)
data[loc] = node.id
spec = self.spec[ArrayKeys.TEST_LABELS].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec)
points = []
for node in self.points:
if roi_points.contains(node.location):
points.append(node)
batch.graphs[GraphKeys.TEST_POINTS] = Graph(points, [],
GraphSpec(roi=roi_points))
return batch
def __init__(self):
self.graph = Graph([
Node(1, np.array([1, 1, 1])),
Node(2, np.array([500, 500, 500])),
Node(3, np.array([550, 550, 550])),
], [], GraphSpec(roi=Roi((-500, -500, -500), (1500, 1500, 1500))))
def __init__(self):
self.voxel_size = Coordinate((40, 4, 4))
self.nodes = [
# corners
Node(id=1, location=np.array((-200, -200, -200))),
Node(id=2, location=np.array((-200, -200, 199))),
Node(id=3, location=np.array((-200, 199, -200))),
Node(id=4, location=np.array((-200, 199, 199))),
Node(id=5, location=np.array((199, -200, -200))),
Node(id=6, location=np.array((199, -200, 199))),
Node(id=7, location=np.array((199, 199, -200))),
Node(id=8, location=np.array((199, 199, 199))),
# center
Node(id=9, location=np.array((0, 0, 0))),
Node(id=10, location=np.array((-1, -1, -1))),
]
self.graph_spec = GraphSpec(roi=Roi((-100, -100, -100), (300, 300, 300)))
self.array_spec = ArraySpec(
roi=Roi((-200, -200, -200), (400, 400, 400)), voxel_size=self.voxel_size
)
self.graph = Graph(self.nodes, [], self.graph_spec)
def test_with_edge(self):
graph_with_edge = GraphKey("TEST_GRAPH_WITH_EDGE")
array_with_edge = ArrayKey("RASTERIZED_EDGE")
pipeline = GraphTestSourceWithEdge() + RasterizeGraph(
GraphKeys.TEST_GRAPH_WITH_EDGE,
ArrayKeys.RASTERIZED_EDGE,
ArraySpec(voxel_size=(1, 1, 1)),
settings=RasterizationSettings(0.5),
)
with build(pipeline):
request = BatchRequest()
roi = Roi((0, 0, 0), (10, 10, 10))
request[GraphKeys.TEST_GRAPH_WITH_EDGE] = GraphSpec(roi=roi)
request[ArrayKeys.RASTERIZED_EDGE] = ArraySpec(roi=roi)
batch = pipeline.request_batch(request)
rasterized = batch.arrays[ArrayKeys.RASTERIZED_EDGE].data
assert (
rasterized.sum() == 10
), f"rasterized has ones at: {np.where(rasterized==1)}"
def test_relabel_components(self):
path = Path(self.path_to("test_swc_source.swc"))
# write test swc
self._write_swc(path, self._toy_swc_points().to_nx_graph())
# read arrays
swc = GraphKey("SWC")
source = SwcFileSource(path, [swc])
with build(source):
batch = source.request_batch(
BatchRequest({swc:
GraphSpec(roi=Roi((0, 1, 5), (11, 10, 1)))}))
temp_g = batch.points[swc]
temp_g.relabel_connected_components()
previous_label = None
ccs = list(temp_g.connected_components)
self.assertEqual(len(ccs), 3)
for cc in ccs:
self.assertEqual(len(cc), 10)
label = None
for point_id in cc:
if label is None:
label = temp_g.node(point_id).attrs["component"]
self.assertNotEqual(label, previous_label)
self.assertEqual(
temp_g.node(point_id).attrs["component"], label)
previous_label = label
def provide(self, request):
batch = Batch()
if GraphKeys.TEST_POINTS in request:
roi_points = request[GraphKeys.TEST_POINTS].roi
contained_points = []
for point in self.points:
if roi_points.contains(point.location):
contained_points.append(copy.deepcopy(point))
batch[GraphKeys.TEST_POINTS] = Graph(contained_points, [],
GraphSpec(roi=roi_points))
if ArrayKeys.TEST_LABELS in request:
roi_array = request[ArrayKeys.TEST_LABELS].roi
roi_voxel = roi_array // self.spec[
ArrayKeys.TEST_LABELS].voxel_size
data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32)
data[:, ::2] = 100
for point in self.points:
loc = self.point_to_voxel(roi_array, point.location)
data[loc] = point.id
spec = self.spec[ArrayKeys.TEST_LABELS].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec)
return batch
def test_mirror(self):
test_graph = GraphKey("TEST_GRAPH")
pipeline = TestSource() + SimpleAugment(
mirror_only=[0, 1, 2], transpose_only=[]
)
request = BatchRequest()
request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=Roi((0, 20, 33), (100, 100, 120)))
possible_loc = [[50, 49], [70, 29], [100, 86]]
with build(pipeline):
seen_mirrored = False
for i in range(100):
batch = pipeline.request_batch(request)
assert len(list(batch[GraphKeys.TEST_GRAPH].nodes)) == 1
node = list(batch[GraphKeys.TEST_GRAPH].nodes)[0]
logging.debug(node.location)
assert all(
[
node.location[dim] in possible_loc[dim]
for dim in range(3)
]
)
seen_mirrored = seen_mirrored or any(
[node.location[dim] == possible_loc[dim][1] for dim in range(3)]
)
assert Roi((0, 20, 33), (100, 100, 120)).contains(batch[GraphKeys.TEST_GRAPH].spec.roi)
assert batch[GraphKeys.TEST_GRAPH].spec.roi.contains(node.location)
assert seen_mirrored
def test_square(self):
test_graph = GraphKey("TEST_GRAPH")
test_array1 = ArrayKey("TEST_ARRAY1")
test_array2 = ArrayKey("TEST_ARRAY2")
pipeline = ((ArrayTestSource(), TestSource()) + MergeProvider() +
Pad(test_array1, None) + Pad(test_array2, None) + Pad(test_graph, None)
+ SimpleAugment(
mirror_only=[1,2], transpose_only=[1,2]
))
request = BatchRequest()
request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=Roi((0, 50, 65), (100, 100, 100)))
request[ArrayKeys.TEST_ARRAY1] = ArraySpec(roi=Roi((0, 0, 15), (100, 200, 200)))
request[ArrayKeys.TEST_ARRAY2] = ArraySpec(roi=Roi((0, 50, 65), (100, 100, 100)))
with build(pipeline):
for i in range(100):
batch = pipeline.request_batch(request)
assert len(list(batch[GraphKeys.TEST_GRAPH].nodes)) == 1
for (array_key, array) in batch.arrays.items():
assert batch.arrays[array_key].data.shape == batch.arrays[array_key].spec.roi.get_shape()
def provide(self, request):
batch = Batch()
graph_roi = request[GraphKeys.PRESYN].roi
batch.graphs[GraphKeys.PRESYN] = Graph([], [],
GraphSpec(roi=graph_roi))
return batch
def __init__(self):
self.graph = Graph(
[Node(id=1, location=np.array([500, 500, 500]))],
[],
GraphSpec(roi=Roi((0, 0, 0), (1000, 1000, 1000))),
)
def setup(self):
self.provides(
ArrayKeys.RAW,
ArraySpec(roi=Roi((200, 20, 20), (1800, 180, 180)), voxel_size=(20, 2, 2)),
)
self.provides(
GraphKeys.PRESYN, GraphSpec(roi=Roi((200, 20, 20), (1800, 180, 180)))
)
def test_shift_points2(self):
data = [Node(id=1, location=np.array([0, 1]))]
spec = GraphSpec(Roi(offset=(0, 0), shape=(5, 5)))
points = Graph(data, [], spec)
request_roi = Roi(offset=(0, 1), shape=(5, 3))
shift_array = np.array([[0, 0], [0, -1], [0, 0], [0, 0], [0, 1]],
dtype=int)
lcm_voxel_size = Coordinate((1, 1))
result = ShiftAugment.shift_points(
points,
request_roi,
shift_array,
shift_axis=0,
lcm_voxel_size=lcm_voxel_size,
)
# print("test 2", result.data, data)
self.assertTrue(self.points_equal(result.nodes, data))
self.assertTrue(result.spec == GraphSpec(request_roi))
def setup(self):
self.provides(
ArrayKeys.TEST_LABELS,
ArraySpec(roi=Roi((200, 20, 20), (1800, 180, 180)),
voxel_size=(20, 2, 2)),
)
self.provides(GraphKeys.TEST_GRAPH,
GraphSpec(roi=Roi((200, 20, 20), (1800, 180, 180))))
def test_3d_basics(self):
test_labels = ArrayKey("TEST_LABELS")
test_points = GraphKey("TEST_POINTS")
test_raster = ArrayKey("TEST_RASTER")
pipeline = (
PointTestSource3D() + ElasticAugment(
[10, 10, 10],
[0.1, 0.1, 0.1],
# [0, 0, 0], # no jitter
[0, 2.0 * math.pi],
) + RasterizeGraph(
test_points,
test_raster,
settings=RasterizationSettings(radius=2, mode="peak"),
) + Snapshot(
{
test_labels: "volumes/labels",
test_raster: "volumes/raster"
},
dataset_dtypes={test_raster: np.float32},
output_dir=self.path_to(),
output_filename="elastic_augment_test{id}-{iteration}.hdf",
))
for _ in range(5):
with build(pipeline):
request_roi = Roi((-20, -20, -20), (40, 40, 40))
request = BatchRequest()
request[test_labels] = ArraySpec(roi=request_roi)
request[test_points] = GraphSpec(roi=request_roi)
request[test_raster] = ArraySpec(roi=request_roi)
batch = pipeline.request_batch(request)
labels = batch[test_labels]
points = batch[test_points]
# the point at (0, 0, 0) should not have moved
self.assertTrue(points.contains(0))
labels_data_roi = (
labels.spec.roi -
labels.spec.roi.get_begin()) / labels.spec.voxel_size
# points should have moved together with the voxels
for point in points.nodes:
loc = point.location - labels.spec.roi.get_begin()
loc = loc / labels.spec.voxel_size
loc = Coordinate(int(round(x)) for x in loc)
if labels_data_roi.contains(loc):
self.assertEqual(labels.data[loc], point.id)
def provide(self, request):
batch = Batch()
# have the pixels encode their position
if ArrayKeys.RAW in request:
# the z,y,x coordinates of the ROI
roi = request[ArrayKeys.RAW].roi
roi_voxel = roi // self.spec[ArrayKeys.RAW].voxel_size
meshgrids = np.meshgrid(
range(roi_voxel.get_begin()[0], roi_voxel.get_end()[0]),
range(roi_voxel.get_begin()[1], roi_voxel.get_end()[1]),
range(roi_voxel.get_begin()[2], roi_voxel.get_end()[2]),
indexing="ij",
)
data = meshgrids[0] + meshgrids[1] + meshgrids[2]
spec = self.spec[ArrayKeys.RAW].copy()
spec.roi = roi
batch.arrays[ArrayKeys.RAW] = Array(data, spec)
if ArrayKeys.GT_LABELS in request:
roi = request[ArrayKeys.GT_LABELS].roi
roi_voxel_shape = (
roi // self.spec[ArrayKeys.GT_LABELS].voxel_size
).get_shape()
data = np.ones(roi_voxel_shape)
data[roi_voxel_shape[0] // 2 :, roi_voxel_shape[1] // 2 :, :] = 2
data[roi_voxel_shape[0] // 2 :, -(roi_voxel_shape[1] // 2) :, :] = 3
spec = self.spec[ArrayKeys.GT_LABELS].copy()
spec.roi = roi
batch.arrays[ArrayKeys.GT_LABELS] = Array(data, spec)
if GraphKeys.PRESYN in request:
data_presyn, data_postsyn = self.__get_pre_and_postsyn_locations(
roi=request[GraphKeys.PRESYN].roi
)
elif GraphKeys.POSTSYN in request:
data_presyn, data_postsyn = self.__get_pre_and_postsyn_locations(
roi=request[GraphKeys.POSTSYN].roi
)
voxel_size_points = self.spec[ArrayKeys.RAW].voxel_size
for (graph_key, spec) in request.graph_specs.items():
if graph_key == GraphKeys.PRESYN:
data = data_presyn
if graph_key == GraphKeys.POSTSYN:
data = data_postsyn
batch.graphs[graph_key] = Graph(
list(data.values()), [], GraphSpec(spec.roi)
)
return batch
def test_filter_components():
raw = GraphKey("RAW")
pipeline = TestSource() + FilterComponents(raw, 100,
Coordinate((10, 10, 10)))
request_no_fallback = BatchRequest()
request_no_fallback[raw] = GraphSpec(roi=Roi((0, 0, 0), (20, 20, 20)))
with build(pipeline):
batch = pipeline.request_batch(request_no_fallback)
assert raw in batch
assert len(list(batch[raw].connected_components)) == 1
request_fallback = BatchRequest()
request_fallback[raw] = GraphSpec(roi=Roi((20, 20, 20), (20, 20, 20)))
with build(pipeline):
batch = pipeline.request_batch(request_fallback)
assert raw in batch
assert len(list(batch[raw].connected_components)) == 0
def __init__(self):
self.dtype = float
self.__vertices = [
Node(id=1, location=np.array([1, 1, 1], dtype=self.dtype)),
Node(id=2, location=np.array([500, 500, 500], dtype=self.dtype)),
Node(id=3, location=np.array([550, 550, 550], dtype=self.dtype)),
]
self.__edges = [Edge(1, 2), Edge(2, 3)]
self.__spec = GraphSpec(roi=Roi(Coordinate([-500, -500, -500]),
Coordinate([1500, 1500, 1500])))
self.graph = Graph(self.__vertices, self.__edges, self.__spec)
def prepare_points(self,
request: BatchRequest) -> Tuple[BatchRequest, int]:
"""
Request either point_source or nonempty_placeholder
"""
growth = self._get_growth()
seed = self.seed
dps = BatchRequest(random_seed=seed)
point_key = (self.point_source if self.nonempty_placeholder is None
else self.nonempty_placeholder)
if any([points in request for points in self.points]):
dps[point_key] = request.points_specs.get(self.points[0],
request[self.points[1]])
elif any([array in request for array in self.arrays]):
dps[point_key] = GraphSpec(roi=request.array_specs.get(
self.arrays[0], request[self.arrays[1]]).roi)
elif any([labels in request for labels in self.labels]):
dps[point_key] = GraphSpec(roi=request.array_specs.get(
self.labels[0], request[self.labels[1]]).roi)
else:
raise ValueError(
"One of the following must be requested: {}, {}, {}".format(
self.points, self.arrays, self.labels))
dps[point_key].roi = dps[point_key].roi.grow(growth, growth)
dps[self.array_source] = copy.deepcopy(request[self.arrays[0]])
dps[self.array_source].roi = dps[self.array_source].roi.grow(
growth, growth)
dps[self.array_source].placeholder = True
dps[self.label_source] = copy.deepcopy(request[self.labels[0]])
dps[self.label_source].roi = dps[self.label_source].roi.grow(
growth, growth)
dps[self.label_source].placeholder = True
return dps, seed
def test_3d(self):
test_graph = GraphKey("TEST_GRAPH")
graph_spec = GraphSpec(roi=Roi((0, 0, 0), (5, 5, 5)))
test_array = ArrayKey("TEST_ARRAY")
array_spec = ArraySpec(
roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1))
)
test_array2 = ArrayKey("TEST_ARRAY2")
array2_spec = ArraySpec(
roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1))
)
snapshot_request = BatchRequest()
snapshot_request.add(test_graph, Coordinate((5, 5, 5)))
pipeline = ExampleSource(
[test_graph, test_array, test_array2], [graph_spec, array_spec, array2_spec]
) + Snapshot(
{
test_graph: "graphs/graph",
test_array: "volumes/array",
test_array2: "volumes/array2",
},
output_dir=str(self.test_dir),
every=2,
additional_request=snapshot_request,
output_filename="snapshot.hdf",
)
snapshot_file_path = Path(self.test_dir, "snapshot.hdf")
with build(pipeline):
request = BatchRequest()
roi = Roi((0, 0, 0), (5, 5, 5))
request[test_array] = ArraySpec(roi=roi)
request[test_array2] = ArraySpec(roi=roi)
pipeline.request_batch(request)
assert snapshot_file_path.exists()
f = h5py.File(snapshot_file_path)
assert f["volumes/array"] is not None
assert f["graphs/graph-ids"] is not None
snapshot_file_path.unlink()
pipeline.request_batch(request)
assert not snapshot_file_path.exists()
def test_multi_transpose(self):
test_graph = GraphKey("TEST_GRAPH")
test_array1 = ArrayKey("TEST_ARRAY1")
test_array2 = ArrayKey("TEST_ARRAY2")
point = np.array([50, 70, 100])
transpose_dims = [0, 1, 2]
pipeline = (ArrayTestSource(),
ExampleSource()) + MergeProvider() + SimpleAugment(
mirror_only=[], transpose_only=transpose_dims)
request = BatchRequest()
offset = (0, 20, 33)
request[GraphKeys.TEST_GRAPH] = GraphSpec(
roi=Roi(offset, (100, 100, 120)))
request[ArrayKeys.TEST_ARRAY1] = ArraySpec(
roi=Roi((0, 0, 0), (100, 200, 300)))
request[ArrayKeys.TEST_ARRAY2] = ArraySpec(
roi=Roi((0, 100, 250), (100, 100, 50)))
# Create all possible permurations of our transpose dims
transpose_combinations = list(permutations(transpose_dims, 3))
possible_loc = np.zeros((len(transpose_combinations), 3))
# Transpose points in all possible ways
for i, comb in enumerate(transpose_combinations):
possible_loc[i] = point[np.array(comb)]
with build(pipeline):
seen_transposed = False
seen_node = True
for i in range(100):
batch = pipeline.request_batch(request)
if len(list(batch[GraphKeys.TEST_GRAPH].nodes)) == 1:
seen_node = True
node = list(batch[GraphKeys.TEST_GRAPH].nodes)[0]
assert node.location in possible_loc
seen_transposed = seen_transposed or any(
[node.location[dim] != point[dim] for dim in range(3)])
assert Roi((0, 20, 33), (100, 100, 120)).contains(
batch[GraphKeys.TEST_GRAPH].spec.roi)
assert batch[GraphKeys.TEST_GRAPH].spec.roi.contains(
node.location)
for (array_key, array) in batch.arrays.items():
assert batch.arrays[array_key].data.shape == batch.arrays[
array_key].spec.roi.get_shape()
assert seen_transposed
assert seen_node
def setup(self):
self.provides(
ArrayKeys.RAW,
ArraySpec(roi=Roi((20000, 2000, 2000), (2000, 200, 200)),
voxel_size=(20, 2, 2)))
self.provides(
ArrayKeys.GT_LABELS,
ArraySpec(roi=Roi((20100, 2010, 2010), (1800, 180, 180)),
voxel_size=(20, 2, 2)))
self.provides(
GraphKeys.GT_GRAPH,
GraphSpec(roi=Roi((None, None, None), (None, None, None)), ))
def test_pipeline3(self):
array_key = ArrayKey("TEST_ARRAY")
points_key = GraphKey("TEST_POINTS")
voxel_size = Coordinate((1, 1))
spec = ArraySpec(voxel_size=voxel_size, interpolatable=True)
hdf5_source = Hdf5Source(self.fake_data_file, {array_key: "testdata"},
array_specs={array_key: spec})
csv_source = CsvPointsSource(
self.fake_points_file,
points_key,
GraphSpec(roi=Roi(shape=Coordinate((100, 100)), offset=(0, 0))),
)
request = BatchRequest()
shape = Coordinate((60, 60))
request.add(array_key, shape, voxel_size=Coordinate((1, 1)))
request.add(points_key, shape)
shift_node = ShiftAugment(prob_slip=0.2,
prob_shift=0.2,
sigma=5,
shift_axis=0)
pipeline = ((hdf5_source, csv_source) + MergeProvider() +
RandomLocation(ensure_nonempty=points_key) + shift_node)
with build(pipeline) as b:
request = b.request_batch(request)
# print(request[points_key])
target_vals = [
self.fake_data[point[0]][point[1]] for point in self.fake_points
]
result_data = request[array_key].data
result_points = list(request[points_key].nodes)
result_vals = [
result_data[int(point.location[0])][int(point.location[1])]
for point in result_points
]
for result_val in result_vals:
self.assertTrue(
result_val in target_vals,
msg=
"result value {} at points {} not in target values {} at points {}"
.format(
result_val,
list(result_points),
target_vals,
self.fake_points,
),
)
def __init__(self):
self.voxel_size = Coordinate((1, 1, 1))
self.nodes = [
# corners
Node(id=1, location=np.array((0, 4, 4))),
Node(id=2, location=np.array((9, 4, 4)))
]
self.edges = [
Edge(1, 2)
]
self.graph_spec = GraphSpec(roi=Roi((0, 0, 0), (10, 10, 10)))
self.graph = Graph(self.nodes, self.edges, self.graph_spec)
