def provide(self, request: BatchRequest) -> Batch: if self.array in request: spec = request[self.array].copy() spec.dtype = self.array_specs[self.array].dtype spec.voxel_size = self.voxel_size roi = spec.roi assert (a % b == 0 for a, b in zip(roi.get_shape(), spec.voxel_size)) assert (a // b < 500 for a, b in zip(roi.get_shape(), spec.voxel_size)) array_start = roi.get_begin() / spec.voxel_size array_size = roi.get_shape() / spec.voxel_size array = np.zeros((2, ) + tuple(array_size), dtype=spec.dtype) for x, y, z in itertools.product( * [range(a, a + b) for a, b in zip(array_start, array_size)]): i = x - array_start[0] j = y - array_start[1] k = z - array_start[2] if (abs(x * spec.voxel_size[0] - y * spec.voxel_size[1]) <= max(spec.voxel_size) and abs(x * spec.voxel_size[0] - z * spec.voxel_size[2]) <= max(spec.voxel_size) and abs(y * spec.voxel_size[1] - z * spec.voxel_size[2]) <= max(spec.voxel_size)): array[:, int(i), int(j), int(k)] = 255 batch = Batch() batch[self.array] = Array(array, spec) return batch else: return Batch()
def merge_batches(self, base: Batch, add: Batch) -> Batch: combined = Batch() combined.profiling_stats.merge_with(base.profiling_stats) combined.profiling_stats.merge_with(add.profiling_stats) base_map = { self.point_source: self.points[0], self.array_source: self.arrays[0], self.label_source: self.labels[0], **{k: v[0] for k, v in self.extra_keys.items()}, } for key, value in base.items(): combined[base_map.get(key, key)] = value add_map = { self.point_source: self.points[1], self.array_source: self.arrays[1], self.label_source: self.labels[1], **{k: v[1] for k, v in self.extra_keys.items()}, } for key, value in add.items(): combined[add_map.get(key, key)] = value return combined
def provide(self, request): batch = Batch() # have the pixels encode their position for (array_key, spec) in request.array_specs.items(): roi = spec.roi for d in range(3): assert roi.get_begin( )[d] % 4 == 0, "roi %s does not align with voxels" data_roi = roi / 4 # the z,y,x coordinates of the ROI meshgrids = np.meshgrid( range(data_roi.get_begin()[0], data_roi.get_end()[0]), range(data_roi.get_begin()[1], data_roi.get_end()[1]), range(data_roi.get_begin()[2], data_roi.get_end()[2]), indexing="ij", ) data = meshgrids[0] + meshgrids[1] + meshgrids[2] spec = self.spec[array_key].copy() spec.roi = roi batch.arrays[array_key] = Array(data, spec) return batch
def provide(self, request): batch = Batch() # have the pixels encode their position if ArrayKeys.RAW in request: # the z,y,x coordinates of the ROI roi = request[ArrayKeys.RAW].roi roi_voxel = roi // self.spec[ArrayKeys.RAW].voxel_size meshgrids = np.meshgrid( range(roi_voxel.get_begin()[0], roi_voxel.get_end()[0]), range(roi_voxel.get_begin()[1], roi_voxel.get_end()[1]), range(roi_voxel.get_begin()[2], roi_voxel.get_end()[2]), indexing="ij", ) data = meshgrids[0] + meshgrids[1] + meshgrids[2] spec = self.spec[ArrayKeys.RAW].copy() spec.roi = roi batch.arrays[ArrayKeys.RAW] = Array(data, spec) if ArrayKeys.GT_LABELS in request: roi = request[ArrayKeys.GT_LABELS].roi roi_voxel_shape = ( roi // self.spec[ArrayKeys.GT_LABELS].voxel_size ).get_shape() data = np.ones(roi_voxel_shape) data[roi_voxel_shape[0] // 2 :, roi_voxel_shape[1] // 2 :, :] = 2 data[roi_voxel_shape[0] // 2 :, -(roi_voxel_shape[1] // 2) :, :] = 3 spec = self.spec[ArrayKeys.GT_LABELS].copy() spec.roi = roi batch.arrays[ArrayKeys.GT_LABELS] = Array(data, spec) if GraphKeys.PRESYN in request: data_presyn, data_postsyn = self.__get_pre_and_postsyn_locations( roi=request[GraphKeys.PRESYN].roi ) elif GraphKeys.POSTSYN in request: data_presyn, data_postsyn = self.__get_pre_and_postsyn_locations( roi=request[GraphKeys.POSTSYN].roi ) voxel_size_points = self.spec[ArrayKeys.RAW].voxel_size for (graph_key, spec) in request.graph_specs.items(): if graph_key == GraphKeys.PRESYN: data = data_presyn if graph_key == GraphKeys.POSTSYN: data = data_postsyn batch.graphs[graph_key] = Graph( list(data.values()), [], GraphSpec(spec.roi) ) return batch
def provide(self, request): batch = Batch() graph_roi = request[GraphKeys.PRESYN].roi batch.graphs[GraphKeys.PRESYN] = Graph([], [], GraphSpec(roi=graph_roi)) return batch
def test_get_total_roi_nonspatial_array(): raw = ArrayKey('RAW') nonspatial = ArrayKey('NONSPATIAL') voxel_size = Coordinate((1, 2)) roi = Roi((100, 200), (20, 20)) raw_spec = ArraySpec(roi=roi, voxel_size=voxel_size) nonspatial_spec = ArraySpec(nonspatial=True) batch = Batch() batch[raw] = Array(data=np.zeros((20, 10)), spec=raw_spec) batch[nonspatial] = Array(data=np.zeros((2, 3)), spec=nonspatial_spec) assert batch.get_total_roi() == roi
def provide(self, request): batch = Batch() roi = request[GraphKeys.TEST_POINTS].roi batch[GraphKeys.TEST_POINTS] = self.graph.crop(roi).trim(roi) return batch
def provide(self, request): batch = Batch() if GraphKeys.TEST_POINTS in request: roi_points = request[GraphKeys.TEST_POINTS].roi contained_points = [] for point in self.points: if roi_points.contains(point.location): contained_points.append(copy.deepcopy(point)) batch[GraphKeys.TEST_POINTS] = Graph(contained_points, [], GraphSpec(roi=roi_points)) if ArrayKeys.TEST_LABELS in request: roi_array = request[ArrayKeys.TEST_LABELS].roi roi_voxel = roi_array // self.spec[ ArrayKeys.TEST_LABELS].voxel_size data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32) data[:, ::2] = 100 for point in self.points: loc = self.point_to_voxel(roi_array, point.location) data[loc] = point.id spec = self.spec[ArrayKeys.TEST_LABELS].copy() spec.roi = roi_array batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec) return batch
def provide(self, request): batch = Batch() if PointsKeys.TEST_POINTS in request: roi_points = request[PointsKeys.TEST_POINTS].roi points = {} for i, point in self.points.items(): if roi_points.contains(point.location): points[i] = copy.deepcopy(point) batch.points[PointsKeys.TEST_POINTS] = Points( points, PointsSpec(roi=roi_points) ) if ArrayKeys.TEST_LABELS in request: roi_array = request[ArrayKeys.TEST_LABELS].roi roi_voxel = roi_array // self.spec[ArrayKeys.TEST_LABELS].voxel_size data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32) data[:, ::2] = 100 for i, point in self.points.items(): loc = self.point_to_voxel(roi_array, point.location) data[loc] = i spec = self.spec[ArrayKeys.TEST_LABELS].copy() spec.roi = roi_array batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec) return batch
def provide(self, request): batch = Batch() roi_points = request[GraphKeys.TEST_POINTS].roi roi_array = request[ArrayKeys.TEST_LABELS].roi roi_voxel = roi_array // self.spec[ArrayKeys.TEST_LABELS].voxel_size data = np.zeros(roi_voxel.get_shape(), dtype=np.uint32) data[:, ::2] = 100 for node in self.points: loc = self.point_to_voxel(roi_array, node.location) data[loc] = node.id spec = self.spec[ArrayKeys.TEST_LABELS].copy() spec.roi = roi_array batch.arrays[ArrayKeys.TEST_LABELS] = Array(data, spec=spec) points = [] for node in self.points: if roi_points.contains(node.location): points.append(node) batch.graphs[GraphKeys.TEST_POINTS] = Graph(points, [], GraphSpec(roi=roi_points)) return batch
def provide(self, request): timing = Timing(self) timing.start() min_bb = request[self.points].roi.get_begin() max_bb = request[self.points].roi.get_end() logger.debug( "CSV points source got request for %s", request[self.points].roi) point_filter = np.ones((self.locations.shape[0],), dtype=np.bool) for d in range(self.locations.shape[1]): point_filter = np.logical_and(point_filter, self.locations[:, d] >= min_bb[d]) point_filter = np.logical_and(point_filter, self.locations[:, d] < max_bb[d]) points_data = self._get_points(point_filter) logger.debug("Points data: %s", points_data) logger.debug("Type of point: %s", type(list(points_data.values())[0])) points_spec = PointsSpec(roi=request[self.points].roi.copy()) batch = Batch() batch.points[self.points] = Points(points_data, points_spec) timing.stop() batch.profiling_stats.add(timing) return batch
def process(self, batch, request): outputs = Batch() g = batch[self.graph].to_nx_graph() branch_points = [n for n in g.nodes if g.degree(n) > 2] for branch_point in branch_points: if g.is_directed(): successors = list(g.successors(branch_point)) predecessors = list(g.predecessors(branch_point)) lowest = min(successors + predecessors) for successor in successors: if successor != lowest: g.remove_edge(branch_point, successor) for predecessor in predecessors: if predecessor != lowest: g.remove_edge(predecessor, branch_point) else: neighbors = sorted(list(g.neighbors(branch_point))) for neighbor in neighbors[1:]: g.remove_edge(branch_point, neighbor) outputs[self.graph] = Graph.from_nx_graph( g, batch[self.graph].spec.copy()) return outputs
def _empty_copy(self, base: Batch): add = Batch() for key, array in base.arrays.items(): add[key] = Array(np.zeros_like(array.data), spec=copy.deepcopy(array.spec)) for key, points in base.points.items(): add[key] = Graph([], [], spec=copy.deepcopy(points.spec)) return add
def provide(self, request): # print("ScanTestSource: Got request " + str(request)) batch = Batch() # have the pixels encode their position for (array_key, spec) in request.array_specs.items(): roi = spec.roi roi_voxel = roi // self.spec[array_key].voxel_size # print("ScanTestSource: Adding " + str(array_key)) # the z,y,x coordinates of the ROI meshgrids = np.meshgrid(range(roi_voxel.get_begin()[0], roi_voxel.get_end()[0]), range(roi_voxel.get_begin()[1], roi_voxel.get_end()[1]), range(roi_voxel.get_begin()[2], roi_voxel.get_end()[2]), indexing='ij') data = meshgrids[0] + meshgrids[1] + meshgrids[2] # print("Roi is: " + str(roi)) spec = self.spec[array_key].copy() spec.roi = roi batch.arrays[array_key] = Array(data, spec) for graph_key, spec in request.graph_specs.items(): # node at x, y, z if x%100==0, y%10==0, z%10==0 nodes = [] start = spec.roi.get_begin() - tuple( x % s for x, s in zip(spec.roi.get_begin(), [100, 10, 10])) for i, j, k in itertools.product(*[ range(a, b, s) for a, b, s in zip( start, spec.roi.get_end(), [100, 10, 10]) ]): location = np.array([i, j, k]) if spec.roi.contains(location): nodes.append( Node(id=coordinate_to_id(i, j, k), location=location)) batch.graphs[graph_key] = Graph(nodes, [], spec) return batch
def process(self, batch, request): g = batch[self.graph].to_nx_graph() assert batch[self.graph].spec.roi.get_shape() == self.read_size logger.debug( f"{self.name()} got graph with {g.number_of_nodes()} nodes, and " f"{g.number_of_edges()} edges!") write_roi = batch[self.graph].spec.roi.grow(-self.context, -self.context) cc_func = (nx.connected_components if not g.is_directed() else nx.weakly_connected_components) for cc in cc_func(g): contained_nodes = [ n for n in cc if write_roi.contains(g.nodes[n]["location"]) ] if len(contained_nodes) == 0: continue else: cc_id = min(contained_nodes) cc_subgraph = g.subgraph(cc) # total edge length of this connected component in this write_roi total_edge_len = 0 for u, v in cc_subgraph.edges: u_loc = cc_subgraph.nodes[u]["location"] v_loc = cc_subgraph.nodes[v]["location"] edge_len = np.linalg.norm(u_loc - v_loc) if write_roi.contains(u_loc) and write_roi.contains(v_loc): total_edge_len += edge_len elif write_roi.contains(u_loc) or write_roi.contains( v_loc): total_edge_len += edge_len / 2 for u in contained_nodes: attrs = cc_subgraph.nodes[u] attrs[self.component_attr] = int(cc_id) attrs[self.size_attr] = float(total_edge_len) count = 0 for node, attrs in g.nodes.items(): if write_roi.contains(attrs["location"]): assert self.component_attr in attrs count += 1 logger.debug( f"{self.name()} updated component id of {count} nodes in write_roi" ) outputs = Batch() outputs[self.graph] = Graph.from_nx_graph( g, batch[self.graph].spec.copy()) return outputs
def provide(self, request): roi_array = request[ArrayKeys.GT_LABELS].roi data = np.zeros(roi_array.get_shape() / self.spec[ArrayKeys.GT_LABELS].voxel_size) batch = Batch() spec = self.spec[ArrayKeys.GT_LABELS].copy() spec.roi = roi_array batch.arrays[ArrayKeys.GT_LABELS] = Array(data, spec) return batch
def provide(self, request): outputs = Batch() spec = self.graph_spec.copy() spec.roi = request[self.graph].roi outputs[self.graph] = Graph( self.component_1_nodes + self.component_2_nodes, self.component_1_edges + self.component_2_edges, spec, ) return outputs
def provide(self, request): batch = Batch() graph_roi = request[GraphKeys.TEST_GRAPH_WITH_EDGE].roi batch.graphs[GraphKeys.TEST_GRAPH_WITH_EDGE] = self.graph.crop(graph_roi).trim( graph_roi ) return batch
def provide(self, request): batch = Batch() roi = request[GraphKeys.TEST_GRAPH].roi sub_graph = self.graph.crop(roi) batch[GraphKeys.TEST_GRAPH] = sub_graph return batch
def provide(self, request: BatchRequest) -> Batch: batch = Batch() for points_key in self.points: if points_key in request: spec = request[points_key].copy() subgraph = self.graph.crop(roi=spec.roi) subgraph.relabel_connected_components() batch[points_key] = subgraph return batch
def process(self, batch, request): outputs = Batch() # make sure a ROI is what we requested b_roi = request[ArrayKeys.C].roi.grow(self.context, self.context) assert batch[ArrayKeys.B].spec.roi == b_roi # add C to batch c = batch[ArrayKeys.B].crop(request[ArrayKeys.C].roi) outputs[ArrayKeys.C] = c return outputs
def provide(self, request): batch = Batch() roi1 = request[ArrayKeys.TEST_ARRAY1].roi roi2 = request[ArrayKeys.TEST_ARRAY2].roi batch[ArrayKeys.TEST_ARRAY1] = self.array.crop(roi1) batch[ArrayKeys.TEST_ARRAY2] = self.array.crop(roi2) return batch
def provide(self, request: BatchRequest) -> Batch: batch = Batch() for points_key in self.points: if points_key in request: spec = request[points_key].copy() subgraph = self.graph.crop(roi=spec.roi, copy=True) batch[points_key] = GraphPoints._from_graph( subgraph, spec.copy()) return batch
def process(self, batch, request): outputs = Batch() g = batch[self.graph].to_nx_graph() for node, attrs in list(g.nodes.items()): if attrs[self.size_attr] < self.size_threshold: g.remove_node(node) outputs[self.graph] = Graph.from_nx_graph( g, batch[self.graph].spec.copy()) return outputs
def provide(self, request): batch = Batch() spec = request[ArrayKeys.RAW].copy() spec.voxel_size = self.voxel_size size = spec.roi.get_shape() / spec.voxel_size offset = spec.roi.get_offset() / spec.voxel_size slce = tuple(slice(o, o + s) for o, s in zip(offset, size)) batch.arrays[ArrayKeys.RAW] = Array(data=self.data[slce], spec=spec) return batch
def provide(self, request): batch = Batch() spec = self.spec[ArrayKeys.A] x = np.array(list(range(17)), dtype=np.float32).reshape([17, 1]) x = x + x.T batch.arrays[ArrayKeys.A] = Array(x, spec).crop(request[ArrayKeys.A].roi) return batch
def provide(self, request): data = np.zeros( request[ArrayKeys.RAW].roi.get_shape() / self.spec[ArrayKeys.RAW].voxel_size, dtype=np.uint8, ) spec = copy.deepcopy(self.spec[ArrayKeys.RAW]) spec.roi = request[ArrayKeys.RAW].roi batch = Batch() batch.arrays[ArrayKeys.RAW] = Array(data, spec) return batch
def provide(self, request): batch = Batch() spec = request[self.array].copy() spec.voxel_size = self.voxel_size data = np.zeros(request[self.array].roi.get_shape() / self.voxel_size) if request.array_specs[self.array].roi.contains((0, 0, 0)): data[:] = 1 batch.arrays[self.array] = Array(data=data, spec=spec) return batch
def process(self, batch: Batch, direction: Coordinate, request: BatchRequest, inplace=True) -> Batch: if not inplace: batch = copy.deepcopy(batch) logger.debug("processing") # Shift and crop points and array return_roi = self._return_roi(request) for source_key, point_set in batch.points.items(): point_set = self._shift_and_crop_points(point_set, direction, return_roi) batch.points[source_key] = point_set for source_key, array_set in batch.arrays.items(): array_set = self._shift_and_crop_array(array_set, direction, return_roi) batch.arrays[source_key] = array_set return batch
def process(self, batch, request): gt = batch[self.gt] voxel_size = self.spec[self.neighborhood].voxel_size request_roi = request[self.neighborhood].roi if voxel_size is None: voxel_size = Coordinate((1, ) * len(gt.spec.roi.get_shape())) neighborhood = np.zeros( (self.k, ) + (request_roi.get_shape() / voxel_size), dtype=np.float32) neighborhood_mask = np.zeros((request_roi.get_shape() / voxel_size), dtype=int) k_neighborhood_offsets = self.get_neighborhood_offsets() for i, connected_component in enumerate(gt.connected_components): component_id = i + 1 node_locations = [ gt.node(node_id).location for node_id in connected_component ] component_kdtree = cKDTree(node_locations) for node_id in connected_component: node = gt.node(node_id) location = node.location if request_roi.contains(location): query_points = k_neighborhood_offsets + location query_neighbors = component_kdtree.query(query_points, 1)[0] voxel_index = Coordinate( (location - request_roi.get_offset()) // voxel_size) neighborhood[(slice(None), ) + voxel_index] = (query_neighbors / self.distance) if neighborhood_mask[voxel_index] == 0: neighborhood_mask[voxel_index] = component_id elif neighborhood_mask[voxel_index] != component_id: neighborhood_mask[voxel_index] = -1 neighborhood_mask = neighborhood_mask > 0 outputs = Batch() neighborhood_spec = self.array_specs[self.neighborhood].copy() neighborhood_spec.roi = request_roi outputs[self.neighborhood] = Array(neighborhood, neighborhood_spec) neighborhood_mask_spec = self.array_specs[ self.neighborhood_mask].copy() neighborhood_mask_spec.roi = request_roi outputs[self.neighborhood_mask] = Array(neighborhood_mask > 0, neighborhood_mask_spec) return outputs