def test_voxel_size(self): locations = [[0, 0, 0], [91, 20, 20], [42, 24, 57]] pipeline = ( ExampleSourceSpecifiedLocation(roi=Roi((0, 0, 0), (100, 100, 100)), voxel_size=(5, 2, 2)) + SpecifiedLocation( locations, choose_randomly=False, extra_data=None, jitter=None)) with build(pipeline): batch = pipeline.request_batch( BatchRequest({ ArrayKeys.RAW: ArraySpec(roi=Roi((0, 0, 0), (20, 20, 20))) })) # first locations is skipped # second should start at [80/5, 10/2, 10/2] = [16, 5, 5] self.assertEqual(batch.arrays[ArrayKeys.RAW].data[0, 0, 0], 40255) batch = pipeline.request_batch( BatchRequest({ ArrayKeys.RAW: ArraySpec(roi=Roi((0, 0, 0), (20, 20, 20))) })) # third should start at [30/5, 14/2, 48/2] = [6, 7, 23] self.assertEqual(batch.arrays[ArrayKeys.RAW].data[0, 0, 0], 15374)
def test_square(self): test_graph = GraphKey("TEST_GRAPH") test_array1 = ArrayKey("TEST_ARRAY1") test_array2 = ArrayKey("TEST_ARRAY2") pipeline = ((ArrayTestSource(), TestSource()) + MergeProvider() + Pad(test_array1, None) + Pad(test_array2, None) + Pad(test_graph, None) + SimpleAugment( mirror_only=[1,2], transpose_only=[1,2] )) request = BatchRequest() request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=Roi((0, 50, 65), (100, 100, 100))) request[ArrayKeys.TEST_ARRAY1] = ArraySpec(roi=Roi((0, 0, 15), (100, 200, 200))) request[ArrayKeys.TEST_ARRAY2] = ArraySpec(roi=Roi((0, 50, 65), (100, 100, 100))) with build(pipeline): for i in range(100): batch = pipeline.request_batch(request) assert len(list(batch[GraphKeys.TEST_GRAPH].nodes)) == 1 for (array_key, array) in batch.arrays.items(): assert batch.arrays[array_key].data.shape == batch.arrays[array_key].spec.roi.get_shape()
def test_with_edge(self): graph_with_edge = GraphKey("TEST_GRAPH_WITH_EDGE") array_with_edge = ArrayKey("RASTERIZED_EDGE") pipeline = GraphTestSourceWithEdge() + RasterizeGraph( GraphKeys.TEST_GRAPH_WITH_EDGE, ArrayKeys.RASTERIZED_EDGE, ArraySpec(voxel_size=(1, 1, 1)), settings=RasterizationSettings(0.5), ) with build(pipeline): request = BatchRequest() roi = Roi((0, 0, 0), (10, 10, 10)) request[GraphKeys.TEST_GRAPH_WITH_EDGE] = GraphSpec(roi=roi) request[ArrayKeys.RASTERIZED_EDGE] = ArraySpec(roi=roi) batch = pipeline.request_batch(request) rasterized = batch.arrays[ArrayKeys.RASTERIZED_EDGE].data assert ( rasterized.sum() == 10 ), f"rasterized has ones at: {np.where(rasterized==1)}"
def test_output(self): a = ArrayKey("A") b = ArrayKey("B") source_a = TestSourceRandomLocation(a) source_b = TestSourceRandomLocation(b) pipeline = (source_a, source_b) + \ MergeProvider() + CustomRandomLocation() with build(pipeline): for i in range(10): batch = pipeline.request_batch( BatchRequest({ a: ArraySpec(roi=Roi((0, 0, 0), (20, 20, 20))), b: ArraySpec(roi=Roi((0, 0, 0), (20, 20, 20))) })) self.assertTrue(np.sum(batch.arrays[a].data) > 0) self.assertTrue(np.sum(batch.arrays[b].data) > 0) # Request a ROI with the same shape as the entire ROI full_roi_a = Roi((0, 0, 0), source_a.roi.get_shape()) full_roi_b = Roi((0, 0, 0), source_b.roi.get_shape()) batch = pipeline.request_batch( BatchRequest({ a: ArraySpec(roi=full_roi_a), b: ArraySpec(roi=full_roi_b) }))
def test_precache(self): logging.getLogger("gunpowder.torch.nodes.predict").setLevel( logging.INFO) a = ArrayKey("A") pred = ArrayKey("PRED") model = ExampleModel() reference_request = BatchRequest() reference_request[a] = ArraySpec(roi=Roi((0, 0), (7, 7))) reference_request[pred] = ArraySpec(roi=Roi((1, 1), (5, 5))) source = ExampleTorchTrain2DSource() predict = Predict( model=model, inputs={"a": a}, outputs={0: pred}, array_specs={pred: ArraySpec()}, ) pipeline = source + predict + PreCache(cache_size=3, num_workers=2) request = BatchRequest({ a: ArraySpec(roi=Roi((0, 0), (17, 17))), pred: ArraySpec(roi=Roi((0, 0), (15, 15))), }) # train for a couple of iterations with build(pipeline): batch = pipeline.request_batch(request) assert pred in batch
def test_6_neighborhood(): # array keys graph = GraphKey("GRAPH") neighborhood = ArrayKey("NEIGHBORHOOD") neighborhood_mask = ArrayKey("NEIGHBORHOOD_MASK") distance = 1 pipeline = TestSource(graph) + Neighborhood( graph, neighborhood, neighborhood_mask, distance, array_specs={ neighborhood: ArraySpec(voxel_size=Coordinate((1, 1, 1))), neighborhood_mask: ArraySpec(voxel_size=Coordinate((1, 1, 1))), }, k=6, ) request = BatchRequest() request[neighborhood] = ArraySpec(roi=Roi((0, 0, 0), (10, 10, 10))) request[neighborhood_mask] = ArraySpec(roi=Roi((0, 0, 0), (10, 10, 10))) with build(pipeline): batch = pipeline.request_batch(request) n_data = batch[neighborhood].data n_mask = batch[neighborhood_mask].data masked_ind = list( set([(0, i, 0) for i in range(10) if i not in [0, 4]] + [(i, 5, 0) for i in range(10)] + [(i, 4, 0) for i in range(10) if i not in [0]])) assert all(n_mask[tuple(zip(*masked_ind))] ), f"expected {masked_ind} but saw {np.where(n_mask==1)}"
def setup(self): spec_1 = self.spec[self.channel_1] spec_2 = self.spec[self.channel_2] assert spec_1.voxel_size == spec_2.voxel_size,\ "Channels must have same voxel size" roi = spec_1.roi.intersect(spec_2.roi) spec = ArraySpec() spec.roi = roi spec.voxel_size = spec_1.voxel_size self.provides(self.output, spec)
def test_transpose(): voxel_size = Coordinate((20, 20)) graph_key = GraphKey("GRAPH") array_key = ArrayKey("ARRAY") graph = Graph( [Node(id=1, location=np.array([450, 550]))], [], GraphSpec(roi=Roi((100, 200), (800, 600))), ) data = np.zeros([40, 30]) data[17, 17] = 1 array = Array( data, ArraySpec(roi=Roi((100, 200), (800, 600)), voxel_size=voxel_size)) default_pipeline = ( (GraphSource(graph_key, graph), ArraySource(array_key, array)) + MergeProvider() + SimpleAugment( mirror_only=[], transpose_only=[0, 1], transpose_probs=[0, 0])) transpose_pipeline = ( (GraphSource(graph_key, graph), ArraySource(array_key, array)) + MergeProvider() + SimpleAugment( mirror_only=[], transpose_only=[0, 1], transpose_probs=[1, 1])) request = BatchRequest() request[graph_key] = GraphSpec(roi=Roi((400, 500), (200, 300))) request[array_key] = ArraySpec(roi=Roi((400, 500), (200, 300))) with build(default_pipeline): expected_location = [450, 550] batch = default_pipeline.request_batch(request) assert len(list(batch[graph_key].nodes)) == 1 node = list(batch[graph_key].nodes)[0] assert all(np.isclose(node.location, expected_location)) node_voxel_index = Coordinate( (node.location - batch[array_key].spec.roi.get_offset()) / voxel_size) assert ( batch[array_key].data[node_voxel_index] == 1 ), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}" with build(transpose_pipeline): expected_location = [410, 590] batch = transpose_pipeline.request_batch(request) assert len(list(batch[graph_key].nodes)) == 1 node = list(batch[graph_key].nodes)[0] assert all(np.isclose(node.location, expected_location)) node_voxel_index = Coordinate( (node.location - batch[array_key].spec.roi.get_offset()) / voxel_size) assert ( batch[array_key].data[node_voxel_index] == 1 ), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}"
def test_3d_basics(self): test_labels = ArrayKey("TEST_LABELS") test_points = GraphKey("TEST_POINTS") test_raster = ArrayKey("TEST_RASTER") pipeline = ( PointTestSource3D() + ElasticAugment( [10, 10, 10], [0.1, 0.1, 0.1], # [0, 0, 0], # no jitter [0, 2.0 * math.pi], ) + RasterizeGraph( test_points, test_raster, settings=RasterizationSettings(radius=2, mode="peak"), ) + Snapshot( { test_labels: "volumes/labels", test_raster: "volumes/raster" }, dataset_dtypes={test_raster: np.float32}, output_dir=self.path_to(), output_filename="elastic_augment_test{id}-{iteration}.hdf", )) for _ in range(5): with build(pipeline): request_roi = Roi((-20, -20, -20), (40, 40, 40)) request = BatchRequest() request[test_labels] = ArraySpec(roi=request_roi) request[test_points] = GraphSpec(roi=request_roi) request[test_raster] = ArraySpec(roi=request_roi) batch = pipeline.request_batch(request) labels = batch[test_labels] points = batch[test_points] # the point at (0, 0, 0) should not have moved self.assertTrue(points.contains(0)) labels_data_roi = ( labels.spec.roi - labels.spec.roi.get_begin()) / labels.spec.voxel_size # points should have moved together with the voxels for point in points.nodes: loc = point.location - labels.spec.roi.get_begin() loc = loc / labels.spec.voxel_size loc = Coordinate(int(round(x)) for x in loc) if labels_data_roi.contains(loc): self.assertEqual(labels.data[loc], point.id)
def test_output(self): meta_base = self.path_to('tf_graph') ArrayKey('A') ArrayKey('B') ArrayKey('C') ArrayKey('GRADIENT_A') # create model meta graph file and get input/output names (a, b, c, optimizer, loss) = self.create_meta_graph(meta_base) source = ExampleTensorflowTrainSource() train = Train( meta_base, optimizer=optimizer, loss=loss, inputs={a: ArrayKeys.A, b: ArrayKeys.B}, outputs={c: ArrayKeys.C}, gradients={a: ArrayKeys.GRADIENT_A}, save_every=100) pipeline = source + train request = BatchRequest({ ArrayKeys.A: ArraySpec(roi=Roi((0, 0), (2, 2))), ArrayKeys.B: ArraySpec(roi=Roi((0, 0), (2, 2))), ArrayKeys.C: ArraySpec(roi=Roi((0, 0), (2, 2))), ArrayKeys.GRADIENT_A: ArraySpec(roi=Roi((0, 0), (2, 2))), }) # train for a couple of iterations with build(pipeline): batch = pipeline.request_batch(request) self.assertAlmostEqual(batch.loss, 9.8994951) gradient_a = batch.arrays[ArrayKeys.GRADIENT_A].data self.assertTrue(gradient_a[0, 0] < gradient_a[0, 1]) self.assertTrue(gradient_a[0, 1] < gradient_a[1, 0]) self.assertTrue(gradient_a[1, 0] < gradient_a[1, 1]) for i in range(200-1): loss1 = batch.loss batch = pipeline.request_batch(request) loss2 = batch.loss self.assertLess(loss2, loss1) # resume training with build(pipeline): for i in range(100): loss1 = batch.loss batch = pipeline.request_batch(request) loss2 = batch.loss self.assertLess(loss2, loss1)
def get_test_data_sources(setup_config): input_shape = Coordinate(setup_config["INPUT_SHAPE"]) voxel_size = Coordinate(setup_config["VOXEL_SIZE"]) input_size = input_shape * voxel_size micron_scale = voxel_size[0] # New array keys # Note: These are intended to be requested with size input_size raw = ArrayKey("RAW") matched = GraphKey("MATCHED") nonempty_placeholder = GraphKey("NONEMPTY") labels = ArrayKey("LABELS") ensure_nonempty = matched data_sources = (( TestImageSource( array=raw, array_specs={ raw: ArraySpec(interpolatable=True, voxel_size=voxel_size, dtype=np.uint16) }, size=input_size * 3, voxel_size=voxel_size, ), TestPointSource( points=[matched, nonempty_placeholder], directed=False, size=input_size * 3, num_points=333, ), ) + MergeProvider() + RandomLocation( ensure_nonempty=ensure_nonempty, ensure_centered=True, point_balance_radius=10 * micron_scale, ) + RasterizeSkeleton( points=matched, array=labels, array_spec=ArraySpec( interpolatable=False, voxel_size=voxel_size, dtype=np.uint64), ) + Normalize(raw)) return ( data_sources, raw, labels, nonempty_placeholder, matched, )
def test_multi_transpose(self): test_graph = GraphKey("TEST_GRAPH") test_array1 = ArrayKey("TEST_ARRAY1") test_array2 = ArrayKey("TEST_ARRAY2") point = np.array([50, 70, 100]) transpose_dims = [0, 1, 2] pipeline = (ArrayTestSource(), ExampleSource()) + MergeProvider() + SimpleAugment( mirror_only=[], transpose_only=transpose_dims) request = BatchRequest() offset = (0, 20, 33) request[GraphKeys.TEST_GRAPH] = GraphSpec( roi=Roi(offset, (100, 100, 120))) request[ArrayKeys.TEST_ARRAY1] = ArraySpec( roi=Roi((0, 0, 0), (100, 200, 300))) request[ArrayKeys.TEST_ARRAY2] = ArraySpec( roi=Roi((0, 100, 250), (100, 100, 50))) # Create all possible permurations of our transpose dims transpose_combinations = list(permutations(transpose_dims, 3)) possible_loc = np.zeros((len(transpose_combinations), 3)) # Transpose points in all possible ways for i, comb in enumerate(transpose_combinations): possible_loc[i] = point[np.array(comb)] with build(pipeline): seen_transposed = False seen_node = True for i in range(100): batch = pipeline.request_batch(request) if len(list(batch[GraphKeys.TEST_GRAPH].nodes)) == 1: seen_node = True node = list(batch[GraphKeys.TEST_GRAPH].nodes)[0] assert node.location in possible_loc seen_transposed = seen_transposed or any( [node.location[dim] != point[dim] for dim in range(3)]) assert Roi((0, 20, 33), (100, 100, 120)).contains( batch[GraphKeys.TEST_GRAPH].spec.roi) assert batch[GraphKeys.TEST_GRAPH].spec.roi.contains( node.location) for (array_key, array) in batch.arrays.items(): assert batch.arrays[array_key].data.shape == batch.arrays[ array_key].spec.roi.get_shape() assert seen_transposed assert seen_node
def test_3d(self): test_graph = GraphKey("TEST_GRAPH") graph_spec = GraphSpec(roi=Roi((0, 0, 0), (5, 5, 5))) test_array = ArrayKey("TEST_ARRAY") array_spec = ArraySpec( roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1)) ) test_array2 = ArrayKey("TEST_ARRAY2") array2_spec = ArraySpec( roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1)) ) snapshot_request = BatchRequest() snapshot_request.add(test_graph, Coordinate((5, 5, 5))) pipeline = ExampleSource( [test_graph, test_array, test_array2], [graph_spec, array_spec, array2_spec] ) + Snapshot( { test_graph: "graphs/graph", test_array: "volumes/array", test_array2: "volumes/array2", }, output_dir=str(self.test_dir), every=2, additional_request=snapshot_request, output_filename="snapshot.hdf", ) snapshot_file_path = Path(self.test_dir, "snapshot.hdf") with build(pipeline): request = BatchRequest() roi = Roi((0, 0, 0), (5, 5, 5)) request[test_array] = ArraySpec(roi=roi) request[test_array2] = ArraySpec(roi=roi) pipeline.request_batch(request) assert snapshot_file_path.exists() f = h5py.File(snapshot_file_path) assert f["volumes/array"] is not None assert f["graphs/graph-ids"] is not None snapshot_file_path.unlink() pipeline.request_batch(request) assert not snapshot_file_path.exists()
def setup(self): self.provides( ArrayKeys.A, ArraySpec(roi=Roi((0, 0, 0), (1000, 1000, 1000)), voxel_size=(4, 4, 4)), ) self.provides( ArrayKeys.B, ArraySpec(roi=Roi((0, 0, 0), (1000, 1000, 1000)), voxel_size=(4, 4, 4)), )
def setup(self): spec = ArraySpec( roi=Roi((0, 0), (2, 2)), dtype=np.float32, interpolatable=True, voxel_size=(1, 1), ) self.provides(ArrayKeys.A, spec) self.provides(ArrayKeys.B, spec) spec = ArraySpec(nonspatial=True) self.provides(ArrayKeys.C, spec)
def setup(self): self.provides( ArrayKeys.RAW, ArraySpec(roi=Roi((20000, 2000, 2000), (2000, 200, 200)), voxel_size=(20, 2, 2))) self.provides( ArrayKeys.GT_LABELS, ArraySpec(roi=Roi((20100, 2010, 2010), (1800, 180, 180)), voxel_size=(20, 2, 2))) self.provides( GraphKeys.GT_GRAPH, GraphSpec(roi=Roi((None, None, None), (None, None, None)), ))
def test_mismatched_voxel_multiples(): """ Ensure we don't shift by half a voxel when transposing 2 axes. If voxel_size = [2, 2], and we transpose array of shape [4, 6]: center = total_roi.get_center() -> [2, 3] # Get distance from center, then transpose dist_to_center = center - roi.get_offset() -> [2, 3] dist_to_center = transpose(dist_to_center) -> [3, 2] # Using the transposed distance to center, get the offset. new_offset = center - dist_to_center -> [-1, 1] shape = transpose(shape) -> [6, 4] original = ((0, 0), (4, 6)) transposed = ((-1, 1), (6, 4)) This result is what we would expect from tranposing, but no longer fits the voxel grid. dist_to_center should be limited to multiples of the lcm_voxel_size. instead we should get: original = ((0, 0), (4, 6)) transposed = ((0, 0), (6, 4)) """ test_array = ArrayKey("TEST_ARRAY") data = np.zeros([3, 3]) data[ 2, 1] = 1 # voxel has Roi((4, 2) (2, 2)). Contained in Roi((0, 0), (6, 4)). at 2, 1 source = ArraySource( test_array, Array( data, ArraySpec(roi=Roi((0, 0), (6, 6)), voxel_size=(2, 2)), ), ) pipeline = source + SimpleAugment( mirror_only=[], transpose_only=[0, 1], transpose_probs={(1, 0): 1}) with build(pipeline): request = BatchRequest() request[test_array] = ArraySpec(roi=Roi((0, 0), (4, 6))) batch = pipeline.request_batch(request) data = batch[test_array].data assert data[1, 2] == 1, f"{data}"
def process_function(): scheduler = Client() worker_id = scheduler.context.worker_id num_workers = scheduler.context.num_workers gpu = actor_id_to_gpu_mapping(worker_id) os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu) _logger.info("Worker %d uses gpu %d with %d workers", worker_id, gpu, num_workers) _logger.info("Environment:") for name in os.environ.keys(): _logger.info(' %s:%s', name, os.environ[name]) # _logger.info("GPU is available: %s", tf.test.is_gpu_available()) # from tensorflow.python.client import device_lib # def get_available_gpus(): # local_device_protos = device_lib.list_local_devices() # return [x.name for x in local_device_protos if x.device_type == 'GPU'] # available_gpus = get_available_gpus() # for gpu in available_gpus: # print("Worker %d sees gpus %s" % (actor_id, available_gpus)) import tensorflow as tf with tf.device('/gpu:%d' % 0): from gunpowder import ArrayKey, ArraySpec, build, BatchRequest, DaisyRequestBlocks _RAW = ArrayKey('RAW') roi_map = { ArrayKey('OUTPUT_%d' % i): 'write_roi' for i in range(len(outputs)) } roi_map[_RAW] = 'read_roi' reference = BatchRequest() reference[_RAW] = ArraySpec(roi=None, voxel_size=input_voxel_size) for i in range(len(outputs)): reference[ArrayKey('OUTPUT_%d' % i)] = ArraySpec( roi=None, voxel_size=output_voxel_size) pipeline = pipeline_factory() pipeline += DaisyRequestBlocks(reference=reference, roi_map=roi_map, num_workers=num_cpu_workers) with build(pipeline): pipeline.request_batch(BatchRequest())
def test_ensure_center_non_zero(self): path = Path(self.path_to("test_swc_source.swc")) # write test swc self._write_swc(path, self._toy_swc_points().to_nx_graph()) # read arrays swc = PointsKey("SWC") img = ArrayKey("IMG") pipeline = (SwcFileSource( path, [swc], [PointsSpec(roi=Roi((0, 0, 0), (11, 11, 11)))]) + RandomLocation(ensure_nonempty=swc, ensure_centered=True) + RasterizeSkeleton( points=swc, array=img, array_spec=ArraySpec( interpolatable=False, dtype=np.uint32, voxel_size=Coordinate((1, 1, 1)), ), )) request = BatchRequest() request.add(img, Coordinate((5, 5, 5))) request.add(swc, Coordinate((5, 5, 5))) with build(pipeline): batch = pipeline.request_batch(request) data = batch[img].data g = batch[swc] assert g.num_vertices() > 0 self.assertNotEqual(data[tuple(np.array(data.shape) // 2)], 0)
def setup(self): self.provides( ArrayKeys.GT_LABELS, ArraySpec(roi=Roi((0, 0, 0), (100, 100, 100)), voxel_size=self.voxel_size), )
def run_augmentations( data_providers, roi, keys=(), augmentations=(), voxel_size=lambda key: None): request = BatchRequest() for key in keys: request[key] = ArraySpec(roi(key).snap_to_grid(voxel_size(key)), voxel_size=voxel_size(key)) logger.debug('Requesting batch with request %s', request) data_sources = tuple(provider for provider in data_providers) snapshot = SnapshotAsDict() pipeline = data_sources + RandomProvider() + snapshot for augmentation in augmentations: pipeline += augmentation with build(pipeline) as b: logging.info("submitting request %s", request) batch = b.request_batch(request) logger.debug("Got snapshots from request %s: %s", request, snapshot.snapshots) return batch, snapshot.snapshots[0]
def prepare(self, request: BatchRequest): deps = BatchRequest() upstream_dependencies = { self.embeddings: self.spec[self.embeddings], self.mask: self.spec[self.mask], } downstream_request = {self.mst: request[self.mst]} upstream_dependencies = ProviderSpec(array_specs=upstream_dependencies, graph_specs=downstream_request) upstream_roi = upstream_dependencies.get_common_roi() deps[self.embeddings] = ArraySpec(roi=upstream_roi) deps[self.mask] = ArraySpec(roi=upstream_roi) return deps
def test_get_total_roi_nonspatial_array(): raw = ArrayKey('RAW') nonspatial = ArrayKey('NONSPATIAL') voxel_size = Coordinate((1, 2)) roi = Roi((100, 200), (20, 20)) raw_spec = ArraySpec(roi=roi, voxel_size=voxel_size) nonspatial_spec = ArraySpec(nonspatial=True) batch = Batch() batch[raw] = Array(data=np.zeros((20, 10)), spec=raw_spec) batch[nonspatial] = Array(data=np.zeros((2, 3)), spec=nonspatial_spec) assert batch.get_total_roi() == roi
def __init__(self): self.voxel_size = Coordinate((40, 4, 4)) self.nodes = [ # corners Node(id=1, location=np.array((-200, -200, -200))), Node(id=2, location=np.array((-200, -200, 199))), Node(id=3, location=np.array((-200, 199, -200))), Node(id=4, location=np.array((-200, 199, 199))), Node(id=5, location=np.array((199, -200, -200))), Node(id=6, location=np.array((199, -200, 199))), Node(id=7, location=np.array((199, 199, -200))), Node(id=8, location=np.array((199, 199, 199))), # center Node(id=9, location=np.array((0, 0, 0))), Node(id=10, location=np.array((-1, -1, -1))), ] self.graph_spec = GraphSpec(roi=Roi((-100, -100, -100), (300, 300, 300))) self.array_spec = ArraySpec( roi=Roi((-200, -200, -200), (400, 400, 400)), voxel_size=self.voxel_size ) self.graph = Graph(self.nodes, [], self.graph_spec)
def make_data_provider(provider_string): data_providers = [] # data_dir = '/groups/saalfeld/home/hanslovskyp/experiments/quasi-isotropic/data/realigned' # file_pattern = '*merged*fixed-offset-fixed-mask.h5' pattern = provider_string.split(':')[0] paths = {**DEFAULT_PATHS} paths.update(**{entry.split('=')[0].lower() : entry.split('=')[1] for entry in provider_string.split(':')[1:]}) for data in glob.glob(pattern): h5_source = Hdf5Source( data, datasets={ RAW_KEY: paths['raw'], GT_LABELS_KEY: paths['labels'], GT_MASK_KEY: paths['mask'] }, array_specs={ GT_MASK_KEY: ArraySpec(interpolatable=False) } ) data_providers.append(h5_source) return tuple(data_providers)
def setup(self): spec = ArraySpec( roi=Roi((0, 0), (17, 17)), dtype=np.float32, interpolatable=True, voxel_size=(1, 1), ) self.provides(ArrayKeys.A, spec)
def prepare(self, request): deps = BatchRequest() deps[self.dense_mst] = request[self.mst].copy() deps[self.mst] = request[self.mst].copy() deps[self.embeddings] = ArraySpec(roi=request[self.mst].roi) return deps
def factory(): from gunpowder import ArrayKey, Pad, Normalize, IntensityScaleShift, ArraySpec from gunpowder.tensorflow import Predict from gunpowder.nodes.hdf5like_source_base import Hdf5LikeSource from gunpowder.nodes.hdf5like_write_base import Hdf5LikeWrite from gunpowder.coordinate import Coordinate from gunpowder.compat import ensure_str from fuse import Z5Source, Z5Write _RAW = ArrayKey('RAW') output_dataset_names = { ArrayKey('OUTPUT_%d' % i): ds for i, (ds, _) in enumerate(outputs) } output_tensor_to_key = { tensor: ArrayKey('OUTPUT_%d' % i) for i, (_, tensor) in enumerate(outputs) } output_array_specs = { ArrayKey('OUTPUT_%d' % i): ArraySpec(voxel_size=output_voxel_size) for i in range(len(outputs)) } input_source = Z5Source( input_container, datasets={_RAW: input[0]}, array_specs={_RAW: ArraySpec(voxel_size=input_voxel_size)}) output_write = Z5Write(output_filename=output_filename, output_dir=output_dir, dataset_names=output_dataset_names, compression_type=output_compression_type) return \ input_source + \ Normalize(_RAW) + \ Pad(_RAW, size=None) + \ IntensityScaleShift(_RAW, 2, -1) + \ Predict( weight_graph, inputs={input[1]: _RAW}, outputs=output_tensor_to_key, graph=meta_graph, array_specs=output_array_specs) + \ output_write
def test_impossible(self): a = ArrayKey("A") b = ArrayKey("B") source_a = TestSourceRandomLocation(a) source_b = TestSourceRandomLocation(b) pipeline = (source_a, source_b) + \ MergeProvider() + CustomRandomLocation() with build(pipeline): with self.assertRaises(AssertionError): batch = pipeline.request_batch( BatchRequest({ a: ArraySpec(roi=Roi((0, 0, 0), (200, 20, 20))), b: ArraySpec(roi=Roi((1000, 100, 100), (220, 22, 22))), }))
def prepare(self, request): assert ArrayKeys.C in request dependencies = BatchRequest() dependencies[ArrayKeys.B] = ArraySpec(request[ArrayKeys.C].roi.grow( self.context, self.context)) return dependencies