def _misalign(self, num_sections): """ :param num_sections: number of sections :return: (shifts, slips) """ slips = [Coordinate((0, 0))] * num_sections shifts = [Coordinate((0, 0))] * num_sections for z in range(num_sections): r = np.random.random() if r <= self.prob_slip: slips[z] = self._random_offset() elif r <= self.prob_slip + self.prob_shift: offset = self._random_offset() for zp in range(z, num_sections): shifts[zp] += offset logger.debug("misaligning sections with " + str(shifts)) return shifts, slips
def test_ensure_center_non_zero(self): path = Path(self.path_to("test_swc_source.swc")) # write test swc self._write_swc(path, self._toy_swc_points().to_nx_graph()) # read arrays swc = PointsKey("SWC") img = ArrayKey("IMG") pipeline = (SwcFileSource( path, [swc], [PointsSpec(roi=Roi((0, 0, 0), (11, 11, 11)))]) + RandomLocation(ensure_nonempty=swc, ensure_centered=True) + RasterizeSkeleton( points=swc, array=img, array_spec=ArraySpec( interpolatable=False, dtype=np.uint32, voxel_size=Coordinate((1, 1, 1)), ), )) request = BatchRequest() request.add(img, Coordinate((5, 5, 5))) request.add(swc, Coordinate((5, 5, 5))) with build(pipeline): batch = pipeline.request_batch(request) data = batch[img].data g = batch[swc] assert g.num_vertices() > 0 self.assertNotEqual(data[tuple(np.array(data.shape) // 2)], 0)
def test_embedding_pipeline( tmpdir, aux_task, blend_mode, fusion_pipeline, train_embedding, snapshot_every ): setup_config = DEFAULT_CONFIG setup_config["FUSION_PIPELINE"] = fusion_pipeline setup_config["TRAIN_EMBEDDING"] = train_embedding setup_config["SNAPSHOT_EVERY"] = snapshot_every setup_config["TENSORBOARD_LOG_DIR"] = tmpdir setup_config["SNAPSHOT_DIR"] = tmpdir setup_config["SNAPSHOT_FILE_NAME"] = "test_snapshot" setup_config["MATCHING_FAILURES_DIR"] = None setup_config["BLEND_MODE"] = blend_mode setup_config["AUX_TASK"] = aux_task voxel_size = Coordinate(setup_config["VOXEL_SIZE"]) output_size = Coordinate(setup_config["OUTPUT_SHAPE"]) * voxel_size input_size = Coordinate(setup_config["INPUT_SHAPE"]) * voxel_size pipeline, raw, output, inputs = embedding_pipeline( setup_config, get_test_data_sources ) request = BatchRequest() request.add(raw, input_size) request.add(output, output_size) for key in inputs: request.add(key, output_size) with build(pipeline): batch = pipeline.request_batch(request) assert output in batch assert raw in batch
def test_nodes(): initial_locations = { 1: np.array([1, 1, 1], dtype=np.float32), 2: np.array([500, 500, 500], dtype=np.float32), 3: np.array([550, 550, 550], dtype=np.float32), } replacement_locations = { 1: np.array([0, 0, 0], dtype=np.float32), 2: np.array([50, 50, 50], dtype=np.float32), 3: np.array([55, 55, 55], dtype=np.float32), } nodes = [ Node(id=id, location=location) for id, location in initial_locations.items() ] edges = [Edge(1, 2), Edge(2, 3)] spec = GraphSpec(roi=Roi(Coordinate([-500, -500, -500]), Coordinate([1500, 1500, 1500]))) graph = Graph(nodes, edges, spec) for node in graph.nodes: node.location = replacement_locations[node.id] for node in graph.nodes: assert all(np.isclose(node.location, replacement_locations[node.id]))
def process(self, batch, request): points = batch.points[self.point_source] array = batch.arrays[self.array_source] center = array.spec.roi.get_begin() + array.spec.roi.get_shape() / 2 closest = None for point in points.data.values(): closest = (closest if closest is not None and np.linalg.norm(center - closest) < np.linalg.norm(center - Coordinate(point.location)) else Coordinate(point.location)) logger.debug("center: {}".format(center)) direction = closest - center logger.debug("direction: {}".format(direction)) # Shift and crop points and array points, array = self._shift_and_crop( points, array, direction=direction, output_roi=request[self.point_source].roi, ) batch.points[self.point_source] = points batch.arrays[self.array_source] = array return batch
def test_6_neighborhood(): # array keys graph = GraphKey("GRAPH") neighborhood = ArrayKey("NEIGHBORHOOD") neighborhood_mask = ArrayKey("NEIGHBORHOOD_MASK") distance = 1 pipeline = TestSource(graph) + Neighborhood( graph, neighborhood, neighborhood_mask, distance, array_specs={ neighborhood: ArraySpec(voxel_size=Coordinate((1, 1, 1))), neighborhood_mask: ArraySpec(voxel_size=Coordinate((1, 1, 1))), }, k=6, ) request = BatchRequest() request[neighborhood] = ArraySpec(roi=Roi((0, 0, 0), (10, 10, 10))) request[neighborhood_mask] = ArraySpec(roi=Roi((0, 0, 0), (10, 10, 10))) with build(pipeline): batch = pipeline.request_batch(request) n_data = batch[neighborhood].data n_mask = batch[neighborhood_mask].data masked_ind = list( set([(0, i, 0) for i in range(10) if i not in [0, 4]] + [(i, 5, 0) for i in range(10)] + [(i, 4, 0) for i in range(10) if i not in [0]])) assert all(n_mask[tuple(zip(*masked_ind))] ), f"expected {masked_ind} but saw {np.where(n_mask==1)}"
def test_transpose(): voxel_size = Coordinate((20, 20)) graph_key = GraphKey("GRAPH") array_key = ArrayKey("ARRAY") graph = Graph( [Node(id=1, location=np.array([450, 550]))], [], GraphSpec(roi=Roi((100, 200), (800, 600))), ) data = np.zeros([40, 30]) data[17, 17] = 1 array = Array( data, ArraySpec(roi=Roi((100, 200), (800, 600)), voxel_size=voxel_size)) default_pipeline = ( (GraphSource(graph_key, graph), ArraySource(array_key, array)) + MergeProvider() + SimpleAugment( mirror_only=[], transpose_only=[0, 1], transpose_probs=[0, 0])) transpose_pipeline = ( (GraphSource(graph_key, graph), ArraySource(array_key, array)) + MergeProvider() + SimpleAugment( mirror_only=[], transpose_only=[0, 1], transpose_probs=[1, 1])) request = BatchRequest() request[graph_key] = GraphSpec(roi=Roi((400, 500), (200, 300))) request[array_key] = ArraySpec(roi=Roi((400, 500), (200, 300))) with build(default_pipeline): expected_location = [450, 550] batch = default_pipeline.request_batch(request) assert len(list(batch[graph_key].nodes)) == 1 node = list(batch[graph_key].nodes)[0] assert all(np.isclose(node.location, expected_location)) node_voxel_index = Coordinate( (node.location - batch[array_key].spec.roi.get_offset()) / voxel_size) assert ( batch[array_key].data[node_voxel_index] == 1 ), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}" with build(transpose_pipeline): expected_location = [410, 590] batch = transpose_pipeline.request_batch(request) assert len(list(batch[graph_key].nodes)) == 1 node = list(batch[graph_key].nodes)[0] assert all(np.isclose(node.location, expected_location)) node_voxel_index = Coordinate( (node.location - batch[array_key].spec.roi.get_offset()) / voxel_size) assert ( batch[array_key].data[node_voxel_index] == 1 ), f"Node at {np.where(batch[array_key].data == 1)} not {node_voxel_index}"
def test_realistic_valid_examples(example, use_gurobi): penalty_attr = "penalty" location_attr = "location" example_dir = Path(__file__).parent / "mouselight_examples" / "valid" / example consensus = PointsKey("CONSENSUS") skeletonization = PointsKey("SKELETONIZATION") matched = PointsKey("MATCHED") matched_with_fallback = PointsKey("MATCHED_WITH_FALLBACK") inf_roi = Roi(Coordinate((None,) * 3), Coordinate((None,) * 3)) request = BatchRequest() request[matched] = PointsSpec(roi=inf_roi) request[matched_with_fallback] = PointsSpec(roi=inf_roi) request[consensus] = PointsSpec(roi=inf_roi) pipeline = ( ( GraphSource(example_dir / "graph.obj", [skeletonization]), GraphSource(example_dir / "tree.obj", [consensus]), ) + MergeProvider() + TopologicalMatcher( skeletonization, consensus, matched, expected_edge_len=10, match_distance_threshold=76, max_gap_crossing=48, use_gurobi=use_gurobi, location_attr=location_attr, penalty_attr=penalty_attr, ) + TopologicalMatcher( skeletonization, consensus, matched_with_fallback, expected_edge_len=10, match_distance_threshold=76, max_gap_crossing=48, use_gurobi=use_gurobi, location_attr=location_attr, penalty_attr=penalty_attr, with_fallback=True, ) ) with build(pipeline): batch = pipeline.request_batch(request) consensus_ccs = list(batch[consensus].connected_components) matched_with_fallback_ccs = list(batch[matched_with_fallback].connected_components) matched_ccs = list(batch[matched].connected_components) assert len(matched_ccs) == len(consensus_ccs)
def test_crop(self): g = Graph(self.nodes, self.edges, self.spec) sub_g = g.crop(Roi(Coordinate([1, 1, 1]), Coordinate([3, 3, 3]))) self.assertEqual(g.spec.roi, self.spec.roi) self.assertEqual(sub_g.spec.roi, Roi(Coordinate([1, 1, 1]), Coordinate([3, 3, 3]))) sub_g.spec.directed = False self.assertTrue(g.spec.directed) self.assertFalse(sub_g.spec.directed)
def get_test_data_sources(setup_config): input_shape = Coordinate(setup_config["INPUT_SHAPE"]) voxel_size = Coordinate(setup_config["VOXEL_SIZE"]) input_size = input_shape * voxel_size micron_scale = voxel_size[0] # New array keys # Note: These are intended to be requested with size input_size raw = ArrayKey("RAW") matched = GraphKey("MATCHED") nonempty_placeholder = GraphKey("NONEMPTY") labels = ArrayKey("LABELS") ensure_nonempty = matched data_sources = (( TestImageSource( array=raw, array_specs={ raw: ArraySpec(interpolatable=True, voxel_size=voxel_size, dtype=np.uint16) }, size=input_size * 3, voxel_size=voxel_size, ), TestPointSource( points=[matched, nonempty_placeholder], directed=False, size=input_size * 3, num_points=333, ), ) + MergeProvider() + RandomLocation( ensure_nonempty=ensure_nonempty, ensure_centered=True, point_balance_radius=10 * micron_scale, ) + RasterizeSkeleton( points=matched, array=labels, array_spec=ArraySpec( interpolatable=False, voxel_size=voxel_size, dtype=np.uint64), ) + Normalize(raw)) return ( data_sources, raw, labels, nonempty_placeholder, matched, )
def __init__(self): self.dtype = float self.__vertices = [ Node(id=1, location=np.array([1, 1, 1], dtype=self.dtype)), Node(id=2, location=np.array([500, 500, 500], dtype=self.dtype)), Node(id=3, location=np.array([550, 550, 550], dtype=self.dtype)), ] self.__edges = [Edge(1, 2), Edge(2, 3)] self.__spec = GraphSpec(roi=Roi(Coordinate([-500, -500, -500]), Coordinate([1500, 1500, 1500]))) self.graph = Graph(self.__vertices, self.__edges, self.__spec)
def test_3d(self): test_graph = GraphKey("TEST_GRAPH") graph_spec = GraphSpec(roi=Roi((0, 0, 0), (5, 5, 5))) test_array = ArrayKey("TEST_ARRAY") array_spec = ArraySpec( roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1)) ) test_array2 = ArrayKey("TEST_ARRAY2") array2_spec = ArraySpec( roi=Roi((0, 0, 0), (5, 5, 5)), voxel_size=Coordinate((1, 1, 1)) ) snapshot_request = BatchRequest() snapshot_request.add(test_graph, Coordinate((5, 5, 5))) pipeline = ExampleSource( [test_graph, test_array, test_array2], [graph_spec, array_spec, array2_spec] ) + Snapshot( { test_graph: "graphs/graph", test_array: "volumes/array", test_array2: "volumes/array2", }, output_dir=str(self.test_dir), every=2, additional_request=snapshot_request, output_filename="snapshot.hdf", ) snapshot_file_path = Path(self.test_dir, "snapshot.hdf") with build(pipeline): request = BatchRequest() roi = Roi((0, 0, 0), (5, 5, 5)) request[test_array] = ArraySpec(roi=roi) request[test_array2] = ArraySpec(roi=roi) pipeline.request_batch(request) assert snapshot_file_path.exists() f = h5py.File(snapshot_file_path) assert f["volumes/array"] is not None assert f["graphs/graph-ids"] is not None snapshot_file_path.unlink() pipeline.request_batch(request) assert not snapshot_file_path.exists()
def test_get_line_pair(self): dist = 3 intercepts, slopes = self._get_line_pair(roi=Roi( Coordinate([0, 0, 0]), Coordinate([10, 10, 10])), dist=dist) a, b = intercepts # check that the line connecting the closest points is perp to both slopes self.assertAlmostEqual(np.linalg.norm(np.dot(b - a, slopes[0])), 0) self.assertAlmostEqual(np.linalg.norm(np.dot(b - a, slopes[1])), 0) # check the intercepts are the expected distance self.assertAlmostEqual( np.linalg.norm(intercepts[1] - intercepts[0]) - dist, 0)
def test_pipeline3(self): array_key = ArrayKey("TEST_ARRAY") points_key = GraphKey("TEST_POINTS") voxel_size = Coordinate((1, 1)) spec = ArraySpec(voxel_size=voxel_size, interpolatable=True) hdf5_source = Hdf5Source(self.fake_data_file, {array_key: "testdata"}, array_specs={array_key: spec}) csv_source = CsvPointsSource( self.fake_points_file, points_key, GraphSpec(roi=Roi(shape=Coordinate((100, 100)), offset=(0, 0))), ) request = BatchRequest() shape = Coordinate((60, 60)) request.add(array_key, shape, voxel_size=Coordinate((1, 1))) request.add(points_key, shape) shift_node = ShiftAugment(prob_slip=0.2, prob_shift=0.2, sigma=5, shift_axis=0) pipeline = ((hdf5_source, csv_source) + MergeProvider() + RandomLocation(ensure_nonempty=points_key) + shift_node) with build(pipeline) as b: request = b.request_batch(request) # print(request[points_key]) target_vals = [ self.fake_data[point[0]][point[1]] for point in self.fake_points ] result_data = request[array_key].data result_points = list(request[points_key].nodes) result_vals = [ result_data[int(point.location[0])][int(point.location[1])] for point in result_points ] for result_val in result_vals: self.assertTrue( result_val in target_vals, msg= "result value {} at points {} not in target values {} at points {}" .format( result_val, list(result_points), target_vals, self.fake_points, ), )
def _get_source_roi(self, transformation): dims = transformation.shape[0] # get bounding box of needed data for transformation bb_min = Coordinate( int(math.floor(transformation[d].min())) for d in range(dims)) bb_max = Coordinate( int(math.ceil(transformation[d].max())) + 1 for d in range(dims)) # create roi sufficiently large to feed transformation source_roi = Roi(bb_min, bb_max - bb_min) return source_roi
def process(self, batch, request): gt = batch[self.gt] voxel_size = self.spec[self.neighborhood].voxel_size request_roi = request[self.neighborhood].roi if voxel_size is None: voxel_size = Coordinate((1, ) * len(gt.spec.roi.get_shape())) neighborhood = np.zeros( (self.k, ) + (request_roi.get_shape() / voxel_size), dtype=np.float32) neighborhood_mask = np.zeros((request_roi.get_shape() / voxel_size), dtype=int) k_neighborhood_offsets = self.get_neighborhood_offsets() for i, connected_component in enumerate(gt.connected_components): component_id = i + 1 node_locations = [ gt.node(node_id).location for node_id in connected_component ] component_kdtree = cKDTree(node_locations) for node_id in connected_component: node = gt.node(node_id) location = node.location if request_roi.contains(location): query_points = k_neighborhood_offsets + location query_neighbors = component_kdtree.query(query_points, 1)[0] voxel_index = Coordinate( (location - request_roi.get_offset()) // voxel_size) neighborhood[(slice(None), ) + voxel_index] = (query_neighbors / self.distance) if neighborhood_mask[voxel_index] == 0: neighborhood_mask[voxel_index] = component_id elif neighborhood_mask[voxel_index] != component_id: neighborhood_mask[voxel_index] = -1 neighborhood_mask = neighborhood_mask > 0 outputs = Batch() neighborhood_spec = self.array_specs[self.neighborhood].copy() neighborhood_spec.roi = request_roi outputs[self.neighborhood] = Array(neighborhood, neighborhood_spec) neighborhood_mask_spec = self.array_specs[ self.neighborhood_mask].copy() neighborhood_mask_spec.roi = request_roi outputs[self.neighborhood_mask] = Array(neighborhood_mask > 0, neighborhood_mask_spec) return outputs
def visualize_embedding_pipeline(fusion_pipeline, train_embedding): setup_config = DEFAULT_CONFIG setup_config["FUSION_PIPELINE"] = fusion_pipeline setup_config["TRAIN_EMBEDDING"] = train_embedding voxel_size = Coordinate(setup_config["VOXEL_SIZE"]) output_size = Coordinate(setup_config["OUTPUT_SHAPE"]) * voxel_size input_size = Coordinate(setup_config["INPUT_SHAPE"]) * voxel_size pipeline, raw, output = embedding_pipeline(setup_config, get_test_data_sources) request = BatchRequest() request.add(raw, input_size) request.add(output, output_size) with build(pipeline): pipeline.request_batch(request) visualize_hdf5(Path("snapshots/snapshot_1.hdf"), tuple(voxel_size))
def provide(self, request: BatchRequest) -> Batch: """ First request points with specific seeds, then request the rest if valid points are found. """ logger.debug(f"growing request by {self._get_growth()}") has_component = False while not has_component: base_seed, add_seed, direction, prepare_profiling_stats = self.get_valid_seeds( request) timing_prepare = Timing(self, "prepare") timing_prepare.start() request_base = self.prepare(request, base_seed, -direction) if add_seed is not None: request_add = self.prepare(request, add_seed, direction) else: request_add = None logger.debug(f"No add_request needed!") timing_prepare.stop() base = self.upstream_provider.request_batch(request_base) if request_add is not None: add = self.upstream_provider.request_batch(request_add) else: add = self._empty_copy(base) has_component = True timing_process = Timing(self, "process") timing_process.start() base = self.process(base, Coordinate([0, 0, 0]), request=request) add = self.process(add, -Coordinate([0, 0, 0]), request=request) batch = self.merge_batches(base, add) timing_process.stop() batch.profiling_stats.merge_with(prepare_profiling_stats) batch.profiling_stats.add(timing_prepare) batch.profiling_stats.add(timing_process) return batch
def test_without_placeholder(self): test_labels = ArrayKey("TEST_LABELS") test_points = GraphKey("TEST_POINTS") pipeline = ( PointTestSource3D() + RandomLocation(ensure_nonempty=test_points) + ElasticAugment([10, 10, 10], [0.1, 0.1, 0.1], [0, 2.0 * math.pi]) + Snapshot( {test_labels: "volumes/labels"}, output_dir=self.path_to(), output_filename="elastic_augment_test{id}-{iteration}.hdf", )) with build(pipeline): for i in range(2): request_size = Coordinate((40, 40, 40)) request_a = BatchRequest(random_seed=i) request_a.add(test_points, request_size) request_b = BatchRequest(random_seed=i) request_b.add(test_points, request_size) request_b.add(test_labels, request_size) # No array to provide a voxel size to ElasticAugment with pytest.raises(PipelineRequestError): pipeline.request_batch(request_a) batch_b = pipeline.request_batch(request_b) self.assertIn(test_labels, batch_b)
def test_placeholder(self): test_labels = ArrayKey("TEST_LABELS") test_points = GraphKey("TEST_POINTS") pipeline = ( PointTestSource3D() + RandomLocation(ensure_nonempty=test_points) + ElasticAugment([10, 10, 10], [0.1, 0.1, 0.1], [0, 2.0 * math.pi]) + Snapshot( {test_labels: "volumes/labels"}, output_dir=self.path_to(), output_filename="elastic_augment_test{id}-{iteration}.hdf", )) with build(pipeline): for i in range(2): request_size = Coordinate((40, 40, 40)) request_a = BatchRequest(random_seed=i) request_a.add(test_points, request_size) request_a.add(test_labels, request_size, placeholder=True) request_b = BatchRequest(random_seed=i) request_b.add(test_points, request_size) request_b.add(test_labels, request_size) batch_a = pipeline.request_batch(request_a) batch_b = pipeline.request_batch(request_b) points_a = batch_a[test_points].nodes points_b = batch_b[test_points].nodes for a, b in zip(points_a, points_b): assert all(np.isclose(a.location, b.location))
def setup(self): self.voxel_size = Coordinate( min(axis) for axis in zip(*[ array_spec.voxel_size for array_spec in self.spec.array_specs.values() ])) self.spatial_dims = self.voxel_size.dims
def __init__(self): self.voxel_size = Coordinate((40, 4, 4)) self.nodes = [ # corners Node(id=1, location=np.array((-200, -200, -200))), Node(id=2, location=np.array((-200, -200, 199))), Node(id=3, location=np.array((-200, 199, -200))), Node(id=4, location=np.array((-200, 199, 199))), Node(id=5, location=np.array((199, -200, -200))), Node(id=6, location=np.array((199, -200, 199))), Node(id=7, location=np.array((199, 199, -200))), Node(id=8, location=np.array((199, 199, 199))), # center Node(id=9, location=np.array((0, 0, 0))), Node(id=10, location=np.array((-1, -1, -1))), ] self.graph_spec = GraphSpec(roi=Roi((-100, -100, -100), (300, 300, 300))) self.array_spec = ArraySpec( roi=Roi((-200, -200, -200), (400, 400, 400)), voxel_size=self.voxel_size ) self.graph = Graph(self.nodes, [], self.graph_spec)
def __init__(self, z_resolution, prob_slip=0, prob_shift=0, max_misalign=(0, 0), ignore_keys_for_slip=(), seed=None): super(BatchFilter, self).__init__() self.z_resolution = Coordinate((z_resolution, )) self.prob_slip = prob_slip self.prob_shift = prob_shift self.max_misalign = max_misalign self.ignore_keys_for_slip = ignore_keys_for_slip self.seed = seed logger.debug( 'initialized with parameters ' 'prob_slip=%f ' 'prob_shift=%f ' 'max_misalign=%s ' 'ignore_keys_for_slip=%s ' 'seed=%d', self.prob_slip, self.prob_shift, self.max_misalign, self.ignore_keys_for_slip, self.seed) self.translations = {} self.target_rois = {}
def setup(self): self._read_points() logger.debug("Locations: %s", self.locations) if self.points_spec is not None: self.provides(self.points, self.points_spec) return min_bb = Coordinate(np.floor(np.amin(self.locations, 0))) max_bb = Coordinate(np.ceil(np.amax(self.locations, 0)) + 1) roi = Roi(min_bb, max_bb - min_bb) self.provides(self.points, PointsSpec(roi=roi))
def test_shift_points5(self): data = [ Node(id=0, location=np.array([3, 0])), Node(id=1, location=np.array([3, 2])), Node(id=2, location=np.array([3, 4])), Node(id=3, location=np.array([3, 6])), Node(id=4, location=np.array([3, 8])), ] spec = GraphSpec(Roi(offset=(0, 0), shape=(15, 10))) points = Graph(data, [], spec) request_roi = Roi(offset=(3, 0), shape=(9, 10)) shift_array = np.array([[3, 0], [-3, 0], [0, 0], [-3, 0], [3, 0]], dtype=int) lcm_voxel_size = Coordinate((3, 2)) shifted_data = [ Node(id=0, location=np.array([6, 0])), Node(id=2, location=np.array([3, 4])), Node(id=4, location=np.array([6, 8])), ] result = ShiftAugment.shift_points( points, request_roi, shift_array, shift_axis=1, lcm_voxel_size=lcm_voxel_size, ) # print("test 4", result.data, shifted_data) self.assertTrue(self.points_equal(result.nodes, shifted_data)) self.assertTrue(result.spec == GraphSpec(request_roi))
def test_merge_basics(self): voxel_size = (1, 1, 1) GraphKey("PRESYN") ArrayKey("GT_LABELS") graphsource = GraphTestSource(voxel_size) arraysource = ArrayTestSoure(voxel_size) pipeline = (graphsource, arraysource) + MergeProvider() + RandomLocation() window_request = Coordinate((50, 50, 50)) with build(pipeline): # Check basic merging. request = BatchRequest() request.add((GraphKeys.PRESYN), window_request) request.add((ArrayKeys.GT_LABELS), window_request) batch_res = pipeline.request_batch(request) self.assertTrue(ArrayKeys.GT_LABELS in batch_res.arrays) self.assertTrue(GraphKeys.PRESYN in batch_res.graphs) # Check that request of only one source also works. request = BatchRequest() request.add((GraphKeys.PRESYN), window_request) batch_res = pipeline.request_batch(request) self.assertFalse(ArrayKeys.GT_LABELS in batch_res.arrays) self.assertTrue(GraphKeys.PRESYN in batch_res.graphs) # Check that it fails, when having two sources that provide the same type. arraysource2 = ArrayTestSoure(voxel_size) pipeline_fail = (arraysource, arraysource2) + MergeProvider() + RandomLocation() with self.assertRaises(PipelineSetupError): with build(pipeline_fail): pass
def _create_rotation_transformation(shape, angle, subsample=1, voxel_size=None): dims = len(shape) subsample_shape = tuple(max(1,int(s/subsample)) for s in shape) control_points = (2,)*dims if voxel_size is None: voxel_size = Coordinate((1,) * dims) # map control points to world coordinates control_point_scaling_factor = tuple(float(s-1) * vs for s, vs in zip(shape, voxel_size)) # rotate control points center = np.array([0.5*(d-1)*vs for d, vs in zip(shape, voxel_size)]) # print("Creating rotation transformation with:") # print("\tangle : " + str(angle)) # print("\tcenter: " + str(center)) control_point_offsets = np.zeros((dims,) + control_points, dtype=np.float32) for control_point in np.ndindex(control_points): point = np.array(control_point)*control_point_scaling_factor center_offset = np.array([p-c for c,p in zip(center, point)], dtype=np.float32) rotated_offset = np.array(center_offset) rotated_offset[-2:] = _rotate(center_offset[-2:], angle) displacement = rotated_offset - center_offset control_point_offsets[(slice(None),) + control_point] += displacement return augment.upscale_transformation(control_point_offsets, subsample_shape)
def test_prepare1(self): key = ArrayKey("TEST_ARRAY") spec = ArraySpec(voxel_size=Coordinate((1, 1)), interpolatable=True) hdf5_source = Hdf5Source(self.fake_data_file, {key: "testdata"}, array_specs={key: spec}) request = BatchRequest() shape = Coordinate((3, 3)) request.add(key, shape, voxel_size=Coordinate((1, 1))) shift_node = ShiftAugment(sigma=1, shift_axis=0) with build((hdf5_source + shift_node)): shift_node.prepare(request) self.assertTrue(shift_node.ndim == 2) self.assertTrue(shift_node.shift_sigmas == tuple([0.0, 1.0]))
def _create_identity_transformation(shape, voxel_size=None, offset=None, subsample=1): dims = len(shape) if voxel_size is None: voxel_size = Coordinate((1,) * dims) if offset is None: offset = Coordinate((0,) * dims) subsample_shape = tuple(max(1,int(s/subsample)) for s in shape) step_width = tuple(float(shape[d]-1)/(subsample_shape[d]-1) if subsample_shape[d] > 1 else 1 for d in range(dims)) step_width = tuple(s*vs for s, vs in zip(step_width, voxel_size)) axis_ranges = ( np.arange(subsample_shape[d], dtype=np.float32)*step_width[d] + offset[d] for d in range(dims) ) return np.array(np.meshgrid(*axis_ranges, indexing='ij'), dtype=np.float32)
def test_pipeline2(self): key = ArrayKey("TEST_ARRAY") spec = ArraySpec(voxel_size=Coordinate((3, 1)), interpolatable=True) hdf5_source = Hdf5Source(self.fake_data_file, {key: "testdata"}, array_specs={key: spec}) request = BatchRequest() shape = Coordinate((3, 3)) request.add(key, shape, voxel_size=Coordinate((3, 1))) shift_node = ShiftAugment(prob_slip=0.2, prob_shift=0.2, sigma=1, shift_axis=0) with build((hdf5_source + shift_node)) as b: b.request_batch(request)