def provide(self, request: BatchRequest) -> Batch:
if self.array in request:
spec = request[self.array].copy()
spec.dtype = self.array_specs[self.array].dtype
spec.voxel_size = self.voxel_size
roi = spec.roi
assert (a % b == 0
for a, b in zip(roi.get_shape(), spec.voxel_size))
assert (a // b < 500
for a, b in zip(roi.get_shape(), spec.voxel_size))
array_start = roi.get_begin() / spec.voxel_size
array_size = roi.get_shape() / spec.voxel_size
array = np.zeros((2, ) + tuple(array_size), dtype=spec.dtype)
for x, y, z in itertools.product(
*
[range(a, a + b) for a, b in zip(array_start, array_size)]):
i = x - array_start[0]
j = y - array_start[1]
k = z - array_start[2]
if (abs(x * spec.voxel_size[0] - y * spec.voxel_size[1]) <=
max(spec.voxel_size)
and abs(x * spec.voxel_size[0] -
z * spec.voxel_size[2]) <= max(spec.voxel_size)
and
abs(y * spec.voxel_size[1] - z * spec.voxel_size[2]) <=
max(spec.voxel_size)):
array[:, int(i), int(j), int(k)] = 255
batch = Batch()
batch[self.array] = Array(array, spec)
return batch
else:
return Batch()
def merge_batches(self, base: Batch, add: Batch) -> Batch:
combined = Batch()
combined.profiling_stats.merge_with(base.profiling_stats)
combined.profiling_stats.merge_with(add.profiling_stats)
base_map = {
self.point_source: self.points[0],
self.array_source: self.arrays[0],
self.label_source: self.labels[0],
**{k: v[0]
for k, v in self.extra_keys.items()},
}
for key, value in base.items():
combined[base_map.get(key, key)] = value
add_map = {
self.point_source: self.points[1],
self.array_source: self.arrays[1],
self.label_source: self.labels[1],
**{k: v[1]
for k, v in self.extra_keys.items()},
}
for key, value in add.items():
combined[add_map.get(key, key)] = value
return combined
def provide(self, request):
batch = Batch()
# have the pixels encode their position
for (array_key, spec) in request.array_specs.items():
roi = spec.roi
for d in range(3):
assert roi.get_begin(
)[d] % 4 == 0, "roi %s does not align with voxels"
data_roi = roi / 4
# the z,y,x coordinates of the ROI
meshgrids = np.meshgrid(
range(data_roi.get_begin()[0],
data_roi.get_end()[0]),
range(data_roi.get_begin()[1],
data_roi.get_end()[1]),
range(data_roi.get_begin()[2],
data_roi.get_end()[2]),
indexing="ij",
)
data = meshgrids[0] + meshgrids[1] + meshgrids[2]
spec = self.spec[array_key].copy()
spec.roi = roi
batch.arrays[array_key] = Array(data, spec)
return batch
def provide(self, request):
batch = Batch()
# have the pixels encode their position
if ArrayKeys.RAW in request:
# the z,y,x coordinates of the ROI
roi = request[ArrayKeys.RAW].roi
roi_voxel = roi // self.spec[ArrayKeys.RAW].voxel_size
meshgrids = np.meshgrid(
range(roi_voxel.get_begin()[0], roi_voxel.get_end()[0]),
range(roi_voxel.get_begin()[1], roi_voxel.get_end()[1]),
range(roi_voxel.get_begin()[2], roi_voxel.get_end()[2]),
indexing="ij",
)
data = meshgrids[0] + meshgrids[1] + meshgrids[2]
spec = self.spec[ArrayKeys.RAW].copy()
spec.roi = roi
batch.arrays[ArrayKeys.RAW] = Array(data, spec)
if ArrayKeys.GT_LABELS in request:
roi = request[ArrayKeys.GT_LABELS].roi
roi_voxel_shape = (
roi // self.spec[ArrayKeys.GT_LABELS].voxel_size
).get_shape()
data = np.ones(roi_voxel_shape)
data[roi_voxel_shape[0] // 2 :, roi_voxel_shape[1] // 2 :, :] = 2
data[roi_voxel_shape[0] // 2 :, -(roi_voxel_shape[1] // 2) :, :] = 3
spec = self.spec[ArrayKeys.GT_LABELS].copy()
spec.roi = roi
batch.arrays[ArrayKeys.GT_LABELS] = Array(data, spec)
if GraphKeys.PRESYN in request:
data_presyn, data_postsyn = self.__get_pre_and_postsyn_locations(
roi=request[GraphKeys.PRESYN].roi
)
elif GraphKeys.POSTSYN in request:
data_presyn, data_postsyn = self.__get_pre_and_postsyn_locations(
roi=request[GraphKeys.POSTSYN].roi
)
voxel_size_points = self.spec[ArrayKeys.RAW].voxel_size
for (graph_key, spec) in request.graph_specs.items():
if graph_key == GraphKeys.PRESYN:
data = data_presyn
if graph_key == GraphKeys.POSTSYN:
data = data_postsyn
batch.graphs[graph_key] = Graph(
list(data.values()), [], GraphSpec(spec.roi)
)
return batch
def provide(self, request):
batch = Batch()
graph_roi = request[GraphKeys.PRESYN].roi
batch.graphs[GraphKeys.PRESYN] = Graph([], [],
GraphSpec(roi=graph_roi))
return batch
def test_get_total_roi_nonspatial_array():
raw = ArrayKey('RAW')
nonspatial = ArrayKey('NONSPATIAL')
voxel_size = Coordinate((1, 2))
roi = Roi((100, 200), (20, 20))
raw_spec = ArraySpec(roi=roi, voxel_size=voxel_size)
nonspatial_spec = ArraySpec(nonspatial=True)
batch = Batch()
batch[raw] = Array(data=np.zeros((20, 10)), spec=raw_spec)
batch[nonspatial] = Array(data=np.zeros((2, 3)), spec=nonspatial_spec)
assert batch.get_total_roi() == roi
def provide(self, request):
batch = Batch()
roi = request[GraphKeys.TEST_POINTS].roi
batch[GraphKeys.TEST_POINTS] = self.graph.crop(roi).trim(roi)
return batch
def provide(self, request):
batch = Batch()
if GraphKeys.TEST_POINTS in request:
roi_points = request[GraphKeys.TEST_POINTS].roi
contained_points = []
for point in self.points:
if roi_points.contains(point.location):
contained_points.append(copy.deepcopy(point))
batch[GraphKeys.TEST_POINTS] = Graph(contained_points, [],
GraphSpec(roi=roi_points))
if ArrayKeys.TEST_LABELS in request:
roi_array = request[ArrayKeys.TEST_LABELS].roi
roi_voxel = roi_array // self.spec[
ArrayKeys.TEST_LABELS].voxel_size
data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32)
data[:, ::2] = 100
for point in self.points:
loc = self.point_to_voxel(roi_array, point.location)
data[loc] = point.id
spec = self.spec[ArrayKeys.TEST_LABELS].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec)
return batch
def provide(self, request):
batch = Batch()
if PointsKeys.TEST_POINTS in request:
roi_points = request[PointsKeys.TEST_POINTS].roi
points = {}
for i, point in self.points.items():
if roi_points.contains(point.location):
points[i] = copy.deepcopy(point)
batch.points[PointsKeys.TEST_POINTS] = Points(
points, PointsSpec(roi=roi_points)
)
if ArrayKeys.TEST_LABELS in request:
roi_array = request[ArrayKeys.TEST_LABELS].roi
roi_voxel = roi_array // self.spec[ArrayKeys.TEST_LABELS].voxel_size
data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32)
data[:, ::2] = 100
for i, point in self.points.items():
loc = self.point_to_voxel(roi_array, point.location)
data[loc] = i
spec = self.spec[ArrayKeys.TEST_LABELS].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec)
return batch
def provide(self, request):
batch = Batch()
roi_points = request[GraphKeys.TEST_POINTS].roi
roi_array = request[ArrayKeys.TEST_LABELS].roi
roi_voxel = roi_array // self.spec[ArrayKeys.TEST_LABELS].voxel_size
data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32)
data[:, ::2] = 100
for node in self.points:
loc = self.point_to_voxel(roi_array, node.location)
data[loc] = node.id
spec = self.spec[ArrayKeys.TEST_LABELS].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec)
points = []
for node in self.points:
if roi_points.contains(node.location):
points.append(node)
batch.graphs[GraphKeys.TEST_POINTS] = Graph(points, [],
GraphSpec(roi=roi_points))
return batch
def provide(self, request):
timing = Timing(self)
timing.start()
min_bb = request[self.points].roi.get_begin()
max_bb = request[self.points].roi.get_end()
logger.debug(
"CSV points source got request for %s",
request[self.points].roi)
point_filter = np.ones((self.locations.shape[0],), dtype=np.bool)
for d in range(self.locations.shape[1]):
point_filter = np.logical_and(point_filter,
self.locations[:, d] >= min_bb[d])
point_filter = np.logical_and(point_filter,
self.locations[:, d] < max_bb[d])
points_data = self._get_points(point_filter)
logger.debug("Points data: %s", points_data)
logger.debug("Type of point: %s", type(list(points_data.values())[0]))
points_spec = PointsSpec(roi=request[self.points].roi.copy())
batch = Batch()
batch.points[self.points] = Points(points_data, points_spec)
timing.stop()
batch.profiling_stats.add(timing)
return batch
def process(self, batch, request):
outputs = Batch()
g = batch[self.graph].to_nx_graph()
branch_points = [n for n in g.nodes if g.degree(n) > 2]
for branch_point in branch_points:
if g.is_directed():
successors = list(g.successors(branch_point))
predecessors = list(g.predecessors(branch_point))
lowest = min(successors + predecessors)
for successor in successors:
if successor != lowest:
g.remove_edge(branch_point, successor)
for predecessor in predecessors:
if predecessor != lowest:
g.remove_edge(predecessor, branch_point)
else:
neighbors = sorted(list(g.neighbors(branch_point)))
for neighbor in neighbors[1:]:
g.remove_edge(branch_point, neighbor)
outputs[self.graph] = Graph.from_nx_graph(
g, batch[self.graph].spec.copy())
return outputs
def _empty_copy(self, base: Batch):
add = Batch()
for key, array in base.arrays.items():
add[key] = Array(np.zeros_like(array.data),
spec=copy.deepcopy(array.spec))
for key, points in base.points.items():
add[key] = Graph([], [], spec=copy.deepcopy(points.spec))
return add
def provide(self, request):
# print("ScanTestSource: Got request " + str(request))
batch = Batch()
# have the pixels encode their position
for (array_key, spec) in request.array_specs.items():
roi = spec.roi
roi_voxel = roi // self.spec[array_key].voxel_size
# print("ScanTestSource: Adding " + str(array_key))
# the z,y,x coordinates of the ROI
meshgrids = np.meshgrid(range(roi_voxel.get_begin()[0],
roi_voxel.get_end()[0]),
range(roi_voxel.get_begin()[1],
roi_voxel.get_end()[1]),
range(roi_voxel.get_begin()[2],
roi_voxel.get_end()[2]),
indexing='ij')
data = meshgrids[0] + meshgrids[1] + meshgrids[2]
# print("Roi is: " + str(roi))
spec = self.spec[array_key].copy()
spec.roi = roi
batch.arrays[array_key] = Array(data, spec)
for graph_key, spec in request.graph_specs.items():
# node at x, y, z if x%100==0, y%10==0, z%10==0
nodes = []
start = spec.roi.get_begin() - tuple(
x % s for x, s in zip(spec.roi.get_begin(), [100, 10, 10]))
for i, j, k in itertools.product(*[
range(a, b, s) for a, b, s in zip(
start, spec.roi.get_end(), [100, 10, 10])
]):
location = np.array([i, j, k])
if spec.roi.contains(location):
nodes.append(
Node(id=coordinate_to_id(i, j, k), location=location))
batch.graphs[graph_key] = Graph(nodes, [], spec)
return batch
def process(self, batch, request):
g = batch[self.graph].to_nx_graph()
assert batch[self.graph].spec.roi.get_shape() == self.read_size
logger.debug(
f"{self.name()} got graph with {g.number_of_nodes()} nodes, and "
f"{g.number_of_edges()} edges!")
write_roi = batch[self.graph].spec.roi.grow(-self.context,
-self.context)
cc_func = (nx.connected_components
if not g.is_directed() else nx.weakly_connected_components)
for cc in cc_func(g):
contained_nodes = [
n for n in cc if write_roi.contains(g.nodes[n]["location"])
]
if len(contained_nodes) == 0:
continue
else:
cc_id = min(contained_nodes)
cc_subgraph = g.subgraph(cc)
# total edge length of this connected component in this write_roi
total_edge_len = 0
for u, v in cc_subgraph.edges:
u_loc = cc_subgraph.nodes[u]["location"]
v_loc = cc_subgraph.nodes[v]["location"]
edge_len = np.linalg.norm(u_loc - v_loc)
if write_roi.contains(u_loc) and write_roi.contains(v_loc):
total_edge_len += edge_len
elif write_roi.contains(u_loc) or write_roi.contains(
v_loc):
total_edge_len += edge_len / 2
for u in contained_nodes:
attrs = cc_subgraph.nodes[u]
attrs[self.component_attr] = int(cc_id)
attrs[self.size_attr] = float(total_edge_len)
count = 0
for node, attrs in g.nodes.items():
if write_roi.contains(attrs["location"]):
assert self.component_attr in attrs
count += 1
logger.debug(
f"{self.name()} updated component id of {count} nodes in write_roi"
)
outputs = Batch()
outputs[self.graph] = Graph.from_nx_graph(
g, batch[self.graph].spec.copy())
return outputs
def provide(self, request):
roi_array = request[ArrayKeys.GT_LABELS].roi
data = np.zeros(roi_array.get_shape() /
self.spec[ArrayKeys.GT_LABELS].voxel_size)
batch = Batch()
spec = self.spec[ArrayKeys.GT_LABELS].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.GT_LABELS] = Array(data, spec)
return batch
def provide(self, request):
outputs = Batch()
spec = self.graph_spec.copy()
spec.roi = request[self.graph].roi
outputs[self.graph] = Graph(
self.component_1_nodes + self.component_2_nodes,
self.component_1_edges + self.component_2_edges,
spec,
)
return outputs
def provide(self, request):
batch = Batch()
graph_roi = request[GraphKeys.TEST_GRAPH_WITH_EDGE].roi
batch.graphs[GraphKeys.TEST_GRAPH_WITH_EDGE] = self.graph.crop(graph_roi).trim(
graph_roi
)
return batch
def provide(self, request):
batch = Batch()
roi = request[GraphKeys.TEST_GRAPH].roi
sub_graph = self.graph.crop(roi)
batch[GraphKeys.TEST_GRAPH] = sub_graph
return batch
def provide(self, request: BatchRequest) -> Batch:
batch = Batch()
for points_key in self.points:
if points_key in request:
spec = request[points_key].copy()
subgraph = self.graph.crop(roi=spec.roi)
subgraph.relabel_connected_components()
batch[points_key] = subgraph
return batch
def process(self, batch, request):
outputs = Batch()
# make sure a ROI is what we requested
b_roi = request[ArrayKeys.C].roi.grow(self.context, self.context)
assert batch[ArrayKeys.B].spec.roi == b_roi
# add C to batch
c = batch[ArrayKeys.B].crop(request[ArrayKeys.C].roi)
outputs[ArrayKeys.C] = c
return outputs
def provide(self, request):
batch = Batch()
roi1 = request[ArrayKeys.TEST_ARRAY1].roi
roi2 = request[ArrayKeys.TEST_ARRAY2].roi
batch[ArrayKeys.TEST_ARRAY1] = self.array.crop(roi1)
batch[ArrayKeys.TEST_ARRAY2] = self.array.crop(roi2)
return batch
def provide(self, request: BatchRequest) -> Batch:
batch = Batch()
for points_key in self.points:
if points_key in request:
spec = request[points_key].copy()
subgraph = self.graph.crop(roi=spec.roi, copy=True)
batch[points_key] = GraphPoints._from_graph(
subgraph, spec.copy())
return batch
def process(self, batch, request):
outputs = Batch()
g = batch[self.graph].to_nx_graph()
for node, attrs in list(g.nodes.items()):
if attrs[self.size_attr] < self.size_threshold:
g.remove_node(node)
outputs[self.graph] = Graph.from_nx_graph(
g, batch[self.graph].spec.copy())
return outputs
def provide(self, request):
batch = Batch()
spec = request[ArrayKeys.RAW].copy()
spec.voxel_size = self.voxel_size
size = spec.roi.get_shape() / spec.voxel_size
offset = spec.roi.get_offset() / spec.voxel_size
slce = tuple(slice(o, o + s) for o, s in zip(offset, size))
batch.arrays[ArrayKeys.RAW] = Array(data=self.data[slce], spec=spec)
return batch
def provide(self, request):
batch = Batch()
spec = self.spec[ArrayKeys.A]
x = np.array(list(range(17)), dtype=np.float32).reshape([17, 1])
x = x + x.T
batch.arrays[ArrayKeys.A] = Array(x,
spec).crop(request[ArrayKeys.A].roi)
return batch
def provide(self, request):
data = np.zeros(
request[ArrayKeys.RAW].roi.get_shape() /
self.spec[ArrayKeys.RAW].voxel_size,
dtype=np.uint8,
)
spec = copy.deepcopy(self.spec[ArrayKeys.RAW])
spec.roi = request[ArrayKeys.RAW].roi
batch = Batch()
batch.arrays[ArrayKeys.RAW] = Array(data, spec)
return batch
def provide(self, request):
batch = Batch()
spec = request[self.array].copy()
spec.voxel_size = self.voxel_size
data = np.zeros(request[self.array].roi.get_shape() / self.voxel_size)
if request.array_specs[self.array].roi.contains((0, 0, 0)):
data[:] = 1
batch.arrays[self.array] = Array(data=data, spec=spec)
return batch
def process(self,
batch: Batch,
direction: Coordinate,
request: BatchRequest,
inplace=True) -> Batch:
if not inplace:
batch = copy.deepcopy(batch)
logger.debug("processing")
# Shift and crop points and array
return_roi = self._return_roi(request)
for source_key, point_set in batch.points.items():
point_set = self._shift_and_crop_points(point_set, direction,
return_roi)
batch.points[source_key] = point_set
for source_key, array_set in batch.arrays.items():
array_set = self._shift_and_crop_array(array_set, direction,
return_roi)
batch.arrays[source_key] = array_set
return batch
def process(self, batch, request):
gt = batch[self.gt]
voxel_size = self.spec[self.neighborhood].voxel_size
request_roi = request[self.neighborhood].roi
if voxel_size is None:
voxel_size = Coordinate((1, ) * len(gt.spec.roi.get_shape()))
neighborhood = np.zeros(
(self.k, ) + (request_roi.get_shape() / voxel_size),
dtype=np.float32)
neighborhood_mask = np.zeros((request_roi.get_shape() / voxel_size),
dtype=int)
k_neighborhood_offsets = self.get_neighborhood_offsets()
for i, connected_component in enumerate(gt.connected_components):
component_id = i + 1
node_locations = [
gt.node(node_id).location for node_id in connected_component
]
component_kdtree = cKDTree(node_locations)
for node_id in connected_component:
node = gt.node(node_id)
location = node.location
if request_roi.contains(location):
query_points = k_neighborhood_offsets + location
query_neighbors = component_kdtree.query(query_points,
1)[0]
voxel_index = Coordinate(
(location - request_roi.get_offset()) // voxel_size)
neighborhood[(slice(None), ) +
voxel_index] = (query_neighbors /
self.distance)
if neighborhood_mask[voxel_index] == 0:
neighborhood_mask[voxel_index] = component_id
elif neighborhood_mask[voxel_index] != component_id:
neighborhood_mask[voxel_index] = -1
neighborhood_mask = neighborhood_mask > 0
outputs = Batch()
neighborhood_spec = self.array_specs[self.neighborhood].copy()
neighborhood_spec.roi = request_roi
outputs[self.neighborhood] = Array(neighborhood, neighborhood_spec)
neighborhood_mask_spec = self.array_specs[
self.neighborhood_mask].copy()
neighborhood_mask_spec.roi = request_roi
outputs[self.neighborhood_mask] = Array(neighborhood_mask > 0,
neighborhood_mask_spec)
return outputs
