def __getitem__(self, index): request = gp.BatchRequest() # request.add(self.raw_0, (1, ) + tuple(self.shape)) # request.add(self.raw_0, (1, ) + tuple(self.shape)) direction = random.randrange(0, self.max_direction) offset = offset_from_direction(direction, max_direction=self.max_direction, distance=self.distance) if self.upsample is not None: ref0, ref1 = self.raw_0_us, self.raw_1_us else: ref0, ref1 = self.raw_0, self.raw_1 request[ref0] = gp.Roi((0, 0, 0), (1, 256, 256)) request[ref1] = gp.Roi((0, ) + offset, (1, 256, 256)) batch = self.pipeline.request_batch(request) out0 = batch[ref0].data out1 = batch[ref1].data raw_stack = np.concatenate((out0, out1), axis=0) shift_direction = np.array(direction, dtype=np.int64) return raw_stack, shift_direction
def process(self, batch, request): outputs = gp.Batch() gt_graph = nx.Graph() mst_graph = nx.Graph() for block, block_specs in self.specs.items(): ground_truth_key = block_specs["ground_truth"][0] mst_key = block_specs["mst_pred"][0] block_gt_graph = batch[ground_truth_key].to_nx_graph( ).to_undirected() block_mst_graph = batch[mst_key].to_nx_graph().to_undirected() gt_graph = nx.disjoint_union(gt_graph, block_gt_graph) mst_graph = nx.disjoint_union(mst_graph, block_mst_graph) for node, attrs in gt_graph.nodes.items(): attrs["id"] = node for node, attrs in mst_graph.nodes.items(): attrs["id"] = node outputs[self.gt] = gp.Graph.from_nx_graph( gt_graph, gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3), directed=False)) outputs[self.mst] = gp.Graph.from_nx_graph( mst_graph, gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3), directed=False), ) return outputs
def build_source(self): data = daisy.open_ds(filename, key) if self.time_window is None: source_roi = gp.Roi(data.roi.get_offset(), data.roi.get_shape()) else: offs = list(data.roi.get_offset()) offs[1] += self.time_window[0] sh = list(data.roi.get_shape()) offs[1] = self.time_window[1] - self.time_window[0] source_roi = gp.Roi(tuple(offs), tuple(sh)) voxel_size = gp.Coordinate(data.voxel_size) return gp.ZarrSource(filename, { self.raw_0: key, self.raw_1: key }, array_specs={ self.raw_0: gp.ArraySpec( roi=source_roi, voxel_size=voxel_size, interpolatable=True), self.raw_1: gp.ArraySpec( roi=source_roi, voxel_size=voxel_size, interpolatable=True) })
def test_shift_points5(self): data = { 0: gp.Point([3, 0]), 1: gp.Point([3, 2]), 2: gp.Point([3, 4]), 3: gp.Point([3, 6]), 4: gp.Point([3, 8]) } spec = gp.PointsSpec(gp.Roi(offset=(0, 0), shape=(15, 10))) points = gp.Points(data, spec) request_roi = gp.Roi(offset=(3, 0), shape=(9, 10)) shift_array = np.array([[3, 0], [-3, 0], [0, 0], [-3, 0], [3, 0]], dtype=int) lcm_voxel_size = gp.Coordinate((3, 2)) shifted_data = { 0: gp.Point([6, 0]), 2: gp.Point([3, 4]), 4: gp.Point([6, 8]) } result = gp.ShiftAugment.shift_points(points, request_roi, shift_array, shift_axis=1, lcm_voxel_size=lcm_voxel_size) # print("test 4", result.data, shifted_data) self.assertTrue(self.points_equal(result.data, shifted_data)) self.assertTrue(result.spec == gp.PointsSpec(request_roi))
def test_compute_upstream_roi2(self): request_roi = gp.Roi(offset=(0, 0), shape=(5, 10)) sub_shift_array = np.array([[2, 0], [-1, 0], [5, 0], [-2, 0], [0, 0]], dtype=int) upstream_roi = gp.Roi(offset=(-5, 0), shape=(12, 10)) result = gp.ShiftAugment.compute_upstream_roi(request_roi, sub_shift_array) self.assertTrue(upstream_roi == result)
def setup(self): self.provides( gp.ArrayKeys.M_PRED, gp.ArraySpec(roi=gp.Roi((0, 0, 0), (200, 200, 200)), voxel_size=self.voxel_size, interpolatable=False)) self.provides( gp.ArrayKeys.D_PRED, gp.ArraySpec(roi=gp.Roi((0, 0, 0), (200, 200, 200)), voxel_size=self.voxel_size, interpolatable=False))
def __init__(self, filename, key, density=None, channels=0, shape=(16, 256, 256), time_window=None, add_sparse_mosaic_channel=True, random_rot=False): self.filename = filename self.key = key self.shape = shape self.density = density self.raw = gp.ArrayKey('RAW_0') self.add_sparse_mosaic_channel = add_sparse_mosaic_channel self.random_rot = random_rot self.channels = channels data = daisy.open_ds(filename, key) if time_window is None: source_roi = gp.Roi(data.roi.get_offset(), data.roi.get_shape()) else: offs = list(data.roi.get_offset()) offs[1] += time_window[0] sh = list(data.roi.get_shape()) offs[1] = time_window[1] - time_window[0] source_roi = gp.Roi(tuple(offs), tuple(sh)) voxel_size = gp.Coordinate(data.voxel_size) self.pipeline = gp.ZarrSource( filename, { self.raw: key }, array_specs={ self.raw: gp.ArraySpec( roi=source_roi, voxel_size=voxel_size, interpolatable=True) }) + gp.RandomLocation() + IntensityDiffFilter(self.raw, 0, min_distance=0.1, channels=Slice(None)) # add augmentations self.pipeline = self.pipeline + gp.ElasticAugment([40, 40], [2, 2], [0, math.pi / 2.0], prob_slip=-1, spatial_dims=2) self.pipeline.setup() np.random.seed(os.getpid() + int(time.time()))
def test_get_sub_shift_array2(self): total_roi = gp.Roi(offset=(0, 0), shape=(6, 6)) item_roi = gp.Roi(offset=(1, 2), shape=(3, 3)) shift_array = np.arange(12).reshape(6, 2).astype(int) shift_axis = 0 lcm_voxel_size = gp.Coordinate((1, 1)) sub_shift_array = np.array([[2, 3], [4, 5], [6, 7]], dtype=int) result = gp.ShiftAugment.get_sub_shift_array(total_roi, item_roi, shift_array, shift_axis, lcm_voxel_size) self.assertTrue(np.array_equal(result, sub_shift_array))
def setup(self): self.provides( self.raw, gp.ArraySpec(roi=gp.Roi((0, 0), (1000, 1000)), dtype=np.uint8, interpolatable=True, voxel_size=(1, 1))) self.provides( self.gt, gp.ArraySpec(roi=gp.Roi((0, 0), (1000, 1000)), dtype=np.uint64, interpolatable=False, voxel_size=(1, 1)))
def setup(self): if str(self.snapshot_file).endswith(".h5") or str(self.snapshot_file).endswith( ".hdf" ): data = h5py.File(self.snapshot_file, "r") elif str(self.snapshot_file).endswith(".zarr"): data = zarr.open(self.snapshot_file, "r") for key, path in self.datasets.items(): if isinstance(key, gp.ArrayKey): try: x = data[path] except KeyError: raise KeyError(f"Could not find {path}") spec = self.spec_from_dataset(x) self.provides(key, spec) elif isinstance(key, gp.GraphKey): try: locations = data[f"{path}-locations"] except KeyError: raise KeyError(f"Could not find {path}-locations") spec = gp.GraphSpec( gp.Roi((None,) * len(locations[0]), (None,) * len(locations[0])), directed=self.directed.get(key), ) self.provides(key, spec)
def setup(self): # provide points in an infinite ROI self.graph_spec = gp.GraphSpec( roi=gp.Roi(offset=(0, ) * self.dims, shape=(None, ) * self.dims)) self.provides(self.graph_key, self.graph_spec)
def setup(self): self.ndims = self.data.shape[1] if self.points_spec is not None: self.provides(self.points, self.points_spec) elif isinstance(self.points, gp.ArrayKey): self.provides(self.points, gp.ArraySpec(voxel_size=((1, )))) elif isinstance(self.points, gp.GraphKey): print(self.ndims) min_bb = gp.Coordinate( np.floor(np.amin(self.data[:, :self.ndims], 0))) max_bb = gp.Coordinate( np.ceil(np.amax(self.data[:, :self.ndims], 0)) + 1) roi = gp.Roi(min_bb, max_bb - min_bb) logger.debug(f"Bounding Box: {roi}") self.provides(self.points, gp.GraphSpec(roi=roi)) if self.labels is not None: assert isinstance(self.labels, gp.ArrayKey), \ f"Label key must be an ArrayKey, \ was given {type(self.labels)}" if self.labels_spec is not None: self.provides(self.labels, self.labels_spec) else: self.provides(self.labels, gp.ArraySpec(voxel_size=((1, ))))
def __init__( self, gt, mst, output, roi=None, details=None, location_attr="location", small_component_threshold=10000, comatch_threshold=4000, edit_distance_node_spacing=5000, edge_threshold_attr="distance", num_thresholds=5, threshold_range=(0, 1), connectivity=None, output_graph=None, ): self.mst = mst self.gt = gt self.output = output self.roi = roi if self.roi is None: self.roi = gp.Roi((None, ) * 3, (None, ) * 3) self.location_attr = location_attr self.small_component_threshold = small_component_threshold self.comatch_threshold = comatch_threshold self.edit_distance_node_spacing = edit_distance_node_spacing self.edge_threshold_attr = edge_threshold_attr self.num_thresholds = num_thresholds self.threshold_range = threshold_range self.details = details self.connectivity = connectivity self.output_graph = output_graph
def test_shift_points2(self): data = {1: gp.Point([0, 1])} spec = gp.PointsSpec(gp.Roi(offset=(0, 0), shape=(5, 5))) points = gp.Points(data, spec) request_roi = gp.Roi(offset=(0, 1), shape=(5, 3)) shift_array = np.array([[0, 0], [0, -1], [0, 0], [0, 0], [0, 1]], dtype=int) lcm_voxel_size = gp.Coordinate((1, 1)) result = gp.ShiftAugment.shift_points(points, request_roi, shift_array, shift_axis=0, lcm_voxel_size=lcm_voxel_size) # print("test 2", result.data, data) self.assertTrue(self.points_equal(result.data, data)) self.assertTrue(result.spec == gp.PointsSpec(request_roi))
def setup(self): self.provides( self.array_key, gp.ArraySpec(roi=gp.Roi(offset=gp.Coordinate( (-10000, -10000, -10000)), shape=gp.Coordinate( (20000, 20000, 20000))), voxel_size=(1, 1, 1)))
def __slice_to_roi(self, slices): offset = tuple(s.start for s in slices) shape = tuple(s.stop - s.start for s in slices) roi = gp.Roi(offset, shape) roi = roi.snap_to_grid((self.lsd_extractor.downsample, ) * roi.dims()) return roi
def setup(self): # remove selected dim from provided specs for key, upstream_spec in self.get_upstream_provider().spec.items(): spec = upstream_spec.copy() spec.roi = gp.Roi(self.__remove_dim(spec.roi.get_begin()), self.__remove_dim(spec.roi.get_shape())) spec.voxel_size = self.__remove_dim(spec.voxel_size) self.spec[key] = spec
def test_context(self): d_pred = gp.ArrayKeys.D_PRED m_pred = gp.ArrayKeys.M_PRED presyn = gp.PointsKeys.PRESYN postsyn = gp.PointsKeys.POSTSYN outdir = tempfile.mkdtemp() voxel_size = gp.Coordinate((10, 10, 10)) size = ((200, 200, 200)) # Check whether the score of the entire cube is measured, although # cube of borderpoint partially outside request ROI. context = 40 shape = gp.Coordinate(size) / voxel_size m_predar = np.zeros(shape, dtype=np.float32) outsidepoint = gp.Coordinate((13, 13, 13)) borderpoint = (4, 4, 4) m_predar[3:5, 3:5, 3:5] = 1 m_predar[outsidepoint] = 1 d_predar = np.ones((3, shape[0], shape[1], shape[2])) * 0 pipeline = (TestSource(m_predar, d_predar, voxel_size=voxel_size) + ExtractSynapses(m_pred, d_pred, presyn, postsyn, out_dir=outdir, settings=parameters, context=context) + gp.PrintProfilingStats()) request = gp.BatchRequest() roi = gp.Roi((40, 40, 40), (80, 80, 80)) request[presyn] = gp.PointsSpec(roi=roi) request[postsyn] = gp.PointsSpec(roi=roi) with gp.build(pipeline): batch = pipeline.request_batch(request) synapsefile = os.path.join(outdir, "40", "40", "40.npz") with np.load(synapsefile) as data: data = dict(data) self.assertTrue(len(data['ids']) == 1) self.assertEqual(data['scores'][0], 2.0**3) # Size of the cube. for ii in range(len(voxel_size)): self.assertEqual(data['positions'][0][0][ii], borderpoint[ii] * voxel_size[ii]) for ii in range(len(voxel_size)): self.assertEqual(data['positions'][0][1][ii], borderpoint[ii] * voxel_size[ii] + 0) shutil.rmtree(outdir)
def prepare(self, request): if self.array not in request: return request[self.array].roi = gp.Roi( self.__remove_dim(request[self.array].roi.get_begin()), self.__remove_dim(request[self.array].roi.get_shape())) if request[self.array].voxel_size is not None: request[self.array].voxel_size = self.__remove_dim( request[self.array].voxel_size)
def setup(self): upstream_spec = self.get_upstream_provider().spec[self.array] spec = upstream_spec.copy() if spec.roi is not None: spec.roi = gp.Roi(self.__insert_dim(spec.roi.get_begin(), 0), self.__insert_dim(spec.roi.get_shape(), 1)) if spec.voxel_size is not None: spec.voxel_size = self.__insert_dim(spec.voxel_size, 1) self.spec[self.array] = spec self.updates(self.array, self.spec[self.array])
def prepare(self, request): upstream_spec = self.get_upstream_provider().spec # add a new dim for key, spec in request.items(): upstream_voxel_size = upstream_spec[key].voxel_size v = upstream_voxel_size[self.dim] spec.roi = gp.Roi(self.__insert_dim(spec.roi.get_begin(), 0), self.__insert_dim(spec.roi.get_shape(), v)) if spec.voxel_size is not None: spec.voxel_size = self.__insert_dim(spec.voxel_size, v)
def process(self, batch, request): if self.array not in request: return array = batch[self.array] array.spec.roi = gp.Roi( self.__insert_dim(array.spec.roi.get_begin(), 0), self.__insert_dim(array.spec.roi.get_shape(), 1)) array.spec.voxel_size = self.__insert_dim(array.spec.voxel_size, 1) array.data = array.data[:, :, np.newaxis, :, :]
def test_output_basics(self): d_pred = gp.ArrayKeys.D_PRED m_pred = gp.ArrayKeys.M_PRED presyn = gp.PointsKeys.PRESYN postsyn = gp.PointsKeys.POSTSYN voxel_size = gp.Coordinate((10, 10, 10)) size = ((200, 200, 200)) context = 40 shape = gp.Coordinate(size) / voxel_size m_predar = np.zeros(shape, dtype=np.float32) insidepoint = gp.Coordinate((10, 10, 10)) outsidepoint = gp.Coordinate((15, 15, 15)) m_predar[insidepoint] = 1 m_predar[outsidepoint] = 1 d_predar = np.ones((3, shape[0], shape[1], shape[2])) * 10 outdir = tempfile.mkdtemp() pipeline = (TestSource(m_predar, d_predar, voxel_size=voxel_size) + ExtractSynapses(m_pred, d_pred, presyn, postsyn, out_dir=outdir, settings=parameters, context=context)) request = gp.BatchRequest() roi = gp.Roi((40, 40, 40), (80, 80, 80)) request[presyn] = gp.PointsSpec(roi=roi) request[postsyn] = gp.PointsSpec(roi=roi) with gp.build(pipeline): batch = pipeline.request_batch(request) print(outdir, "outdir") synapsefile = os.path.join(outdir, "40", "40", "40.npz") with np.load(synapsefile) as data: data = dict(data) self.assertTrue(len(data['ids']) == 1) self.assertEqual(data['scores'][0], 1.0) # Size of the cube. for ii in range(len(voxel_size)): self.assertEqual(data['positions'][0][1][ii], insidepoint[ii] * voxel_size[ii]) for ii in range(len(voxel_size)): self.assertEqual(data['positions'][0][0][ii], insidepoint[ii] * voxel_size[ii] + 10) shutil.rmtree(outdir)
def prepare(self, request): if self.key not in request: return request[self.key].roi = gp.Roi( self.__insert_dim(request[self.key].roi.get_begin(), 0), self.__insert_dim(request[self.key].roi.get_shape(), 1)) if isinstance(self.key, gp.ArrayKey): if request[self.key].voxel_size is not None: request[self.key].voxel_size = self.__insert_dim( request[self.key].voxel_size, 1)
def process(self, batch, request): if self.key not in batch: return if isinstance(self.key, gp.ArrayKey): data = batch[self.key].data shape = data.shape roi = batch[self.key].spec.roi assert shape[-roi.dims()] == 1, "Channel to delete must be size 1," \ "but given shape " + str(shape) shape = self.__remove_dim(shape, len(shape) - roi.dims()) batch[self.key].data = data.reshape(shape) batch[self.key].spec.roi = gp.Roi( self.__remove_dim(roi.get_begin()), self.__remove_dim(roi.get_shape())) batch[self.key].spec.voxel_size = \ self.__remove_dim(batch[self.key].spec.voxel_size) if isinstance(self.key, gp.GraphKey): roi = batch[self.key].spec.roi batch[self.key].spec.roi = gp.Roi( self.__remove_dim(roi.get_begin()), self.__remove_dim(roi.get_shape())) graph = gp.Graph([], [], spec=batch[self.key].spec) for node in batch[self.key].nodes: print(node) new_node = gp.Node(node.id, node.location[1:], temporary=node.temporary, attrs=node.attrs) graph.add_node(new_node) print(node) print(graph.spec.roi) print(list(graph.nodes)) batch[self.key] = graph
def setup(self): self.enable_autoskip() all_rois = [] for block, block_specs in self.specs.items(): ground_truth = block_specs["ground_truth"] mst_pred = block_specs["mst_pred"] for key, spec in [ground_truth, mst_pred]: current_spec = self.spec[key].copy() current_spec.roi = spec.roi self.updates(key, current_spec) all_rois.append(current_spec.roi) self.total_roi = all_rois[0] for roi in all_rois[1:]: self.total_roi = self.total_roi.union(roi) self.provides( self.mst, gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3), directed=False)) self.provides( self.gt, gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3), directed=False))
def setup(self): upstream_spec = self.get_upstream_provider().spec[self.key] spec = upstream_spec.copy() if spec.roi is not None: spec.roi = gp.Roi(self.__remove_dim(spec.roi.get_begin()), self.__remove_dim(spec.roi.get_shape())) if isinstance(self.key, gp.ArrayKey): if spec.voxel_size is not None: spec.voxel_size = self.__remove_dim(spec.voxel_size) self.spec[self.key] = spec self.updates(self.key, self.spec[self.key])
def __init__(self, voxel_size): self.voxel_size = gp.Coordinate(voxel_size) self.roi = gp.Roi((0, 0, 0), (10, 10, 10)) * self.voxel_size self.raw = gp.ArrayKey("RAW") self.labels = gp.ArrayKey("LABELS") self.array_spec_raw = gp.ArraySpec(roi=self.roi, voxel_size=self.voxel_size, dtype='uint8', interpolatable=True) self.array_spec_labels = gp.ArraySpec(roi=self.roi, voxel_size=self.voxel_size, dtype='uint64', interpolatable=False)
def test_pipeline3(self): array_key = gp.ArrayKey("TEST_ARRAY") points_key = gp.PointsKey("TEST_POINTS") voxel_size = gp.Coordinate((1, 1)) spec = gp.ArraySpec(voxel_size=voxel_size, interpolatable=True) hdf5_source = gp.Hdf5Source(self.fake_data_file, {array_key: 'testdata'}, array_specs={array_key: spec}) csv_source = gp.CsvPointsSource( self.fake_points_file, points_key, gp.PointsSpec( roi=gp.Roi(shape=gp.Coordinate((100, 100)), offset=(0, 0)))) request = gp.BatchRequest() shape = gp.Coordinate((60, 60)) request.add(array_key, shape, voxel_size=gp.Coordinate((1, 1))) request.add(points_key, shape) shift_node = gp.ShiftAugment(prob_slip=0.2, prob_shift=0.2, sigma=5, shift_axis=0) pipeline = ((hdf5_source, csv_source) + gp.MergeProvider() + gp.RandomLocation(ensure_nonempty=points_key) + shift_node) with gp.build(pipeline) as b: request = b.request_batch(request) # print(request[points_key]) target_vals = [ self.fake_data[point[0]][point[1]] for point in self.fake_points ] result_data = request[array_key].data result_points = request[points_key].data result_vals = [ result_data[int(point.location[0])][int(point.location[1])] for point in result_points.values() ] for result_val in result_vals: self.assertTrue( result_val in target_vals, msg= "result value {} at points {} not in target values {} at points {}" .format(result_val, list(result_points.values()), target_vals, self.fake_points))
def process(self, batch, request): for key, array in batch.arrays.items(): # remove first dim array.spec.roi = gp.Roi( self.__remove_dim(array.spec.roi.get_begin()), self.__remove_dim(array.spec.roi.get_shape())) array.spec.voxel_size = self.__remove_dim(array.spec.voxel_size) assert array.data.shape[self.dim] == 1, \ "Squash for dim %d requires that the array %s has size 1 in " \ "that dim, but array shape is %s" % ( self.dim, key, array.data.shape) array.data = array.data.reshape( self.__remove_dim(array.data.shape))