def setup_dvid_segmentation_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo # Start with a low-res map of the test data # and scale it up 16x to achieve a 128-cube _ = 0 # 0 1 2 3 4 5 6 7 volume_layout = [[ [_, _, _, _, _, 6, 6, 6], # 0 [_, 1, 1, 2, 2, _, 6, _], # 1 [_, 1, 1, 2, 2, _, _, _], # 2 [_, 1, 1, 2, 8, _, 7, 7], # 3 [_, 1, 1, 2, 8, _, 7, 7], # 4 [_, _, _, _, _, _, _, _], # 5 [_, 4, 4, 4, _, _, _, _], # 6 [_, 3, 3, 3, _, 5, 9, _] ]] # 7 # 0 1 2 3 4 5 6 7 lowres_volume = np.zeros((4, 8, 8), np.uint64) lowres_volume[:] = volume_layout volume = upsample(lowres_volume, 16) assert volume.shape == (64, 128, 128) input_segmentation_name = 'findadjacencies-input' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), volume) template_dir = tempfile.mkdtemp(suffix="findadjacencies-from-dvid") config_text = textwrap.dedent(f"""\ workflow-name: findadjacencies cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [128,64,64] findadjacencies: output-table: output.csv find-closest-using-scale: 0 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, volume, dvid_address, repo_uuid
def setup_tsv_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-decimatemeshes-input' test_volume, object_boxes, object_sizes = create_test_segmentation() create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0,0,0), test_volume, downres=True, noindexing=False) tsv_name = 'segmentation-decimatemeshes-tsv' create_tarsupervoxel_instance(dvid_address, repo_uuid, tsv_name, input_segmentation_name, '.drc') # Post supervoxel meshes meshes = Mesh.from_label_volume(test_volume, progress=False) meshes_data = {f"{label}.drc": mesh.serialize(fmt='drc') for label, mesh in meshes.items()} post_load(dvid_address, repo_uuid, tsv_name, meshes_data) # Merge two of the objects (100 and 300) post_merge(dvid_address, repo_uuid, input_segmentation_name, 100, [300]) object_boxes[100] = box_union(object_boxes[100], object_boxes[300]) del object_boxes[300] object_sizes[100] += object_sizes[300] del object_sizes[300] meshes[100] = Mesh.concatenate_meshes((meshes[100], meshes[300])) del meshes[300] return dvid_address, repo_uuid, tsv_name, object_boxes, object_sizes, meshes
def setup_segmentation_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-stitchedmesh-input' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3) def place_test_object(label, corner): corner = np.array(corner) label_vol = create_test_object().astype(np.uint64) label_vol *= label corners = np.array(list(ndrange((0,0,0), label_vol.shape, (64,64,64)))) # Drop it away from (0,0,0) to make sure the workflow handles arbitrary locations corners += corner blockwise_vol = view_as_blocks(label_vol, (64,64,64)) blocks = blockwise_vol.reshape(-1,64,64,64) # post to dvid post_labelarray_blocks(dvid_address, repo_uuid, input_segmentation_name, corners, blocks, downres=True, noindexing=False) return np.array([corner, corner + label_vol.shape]) object_boxes = [] for label, corner in zip([100,200,300], [(256,256,256), (512,512,512), (1024,1024,1024)]): box = place_test_object(label, corner) object_boxes.append( box ) return dvid_address, repo_uuid, input_segmentation_name, object_boxes
def test_post_hierarchical_cleaves(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup uuid = post_branch(dvid_server, dvid_repo, 'segmentation-post_hierarchical_cleaves', '') instance_info = dvid_server, uuid, 'segmentation-post_hierarchical_cleaves' create_labelmap_instance(*instance_info) svs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] groups = [1, 1, 2, 2, 3, 3, 3, 3, 3, 4] svs = np.asarray(svs, np.uint64) # Post some supervoxels in multiple blocks, just to prove that post_hierarchical_cleaves() # doesn't assume that the labelindex has the same length as the mapping. sv_vol = np.zeros((128, 64, 64), np.uint64) sv_vol[0, 0, :len(svs)] = svs sv_vol[64, 0, 0:len(svs):2] = svs[::2] post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol) post_merge(*instance_info, 1, svs[1:]) group_mapping = pd.Series(index=svs, data=groups) final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping) assert (fetch_mapping(*instance_info, svs) == final_table['body'].values).all() assert (final_table.drop_duplicates( ['group']) == final_table.drop_duplicates(['group', 'body'])).all().all() assert (final_table.drop_duplicates( ['body']) == final_table.drop_duplicates(['group', 'body'])).all().all() # Since the mapping included all supervoxels in the body, # the last group is left with the original label. assert final_table.iloc[-1]['body'] == 1 # Now merge them all together and try again, but leave # two supevoxels out of the groups this time. merges = set(pd.unique(final_table['body'].values)) - set([1]) post_merge(*instance_info, 1, list(merges)) group_mapping = pd.Series(index=svs[:-2], data=groups[:-2]) final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping) assert len( final_table.query('body == 1') ) == 0, "Did not expect any of the groups to retain the original body ID!" assert (fetch_mapping(*instance_info, svs[:-2]) == final_table['body'].values).all() assert (final_table.drop_duplicates( ['group']) == final_table.drop_duplicates(['group', 'body'])).all().all() assert (final_table.drop_duplicates( ['body']) == final_table.drop_duplicates(['group', 'body'])).all().all() assert (fetch_mapping(*instance_info, svs[-2:]) == 1).all()
def test_fetch_sparsevol_coarse_via_labelindex(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup # Create a labelmap volume with 3 blocks. # # Supervoxels are arranged like this: # # | 1 2 | 3 4 | 5 6 | # # After merging [2,3,4,5], bodies will be: # # | 1 2 | 2 4 | 5 6 | # vol_shape = (64, 64, 256) sv_vol = np.zeros(vol_shape, np.uint64) sv_vol[:, :, 0:32] = 1 sv_vol[:, :, 32:64] = 2 sv_vol[:, :, 64:96] = 3 sv_vol[:, :, 96:128] = 4 sv_vol[:, :, 128:160] = 5 sv_vol[:, :, 160:192] = 6 instance_info = dvid_server, dvid_repo, 'segmentation-test-sparsevol-coarse' create_labelmap_instance(*instance_info) post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol) post_merge(*instance_info, 2, [3, 4, 5]) body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 2, method='protobuf') expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2) assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist()) body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 2, method='pandas') expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2) assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist()) sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 3, supervoxels=True, method='protobuf') expected_sv_svc = fetch_sparsevol_coarse(*instance_info, 3, supervoxels=True) assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist()) sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 3, supervoxels=True, method='pandas') expected_sv_svc = fetch_sparsevol_coarse(*instance_info, 3, supervoxels=True) assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist())
def setup_segmentation_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-createmeshes-input' test_volume, object_boxes, object_sizes = create_test_segmentation() create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), test_volume, downres=True, noindexing=False) return dvid_address, repo_uuid, input_segmentation_name, object_boxes, object_sizes
def _create_segmentation_instance(self, volume_config): """ Create a segmentation volume in DVID according to the given configuration. In DVID, segmentation instances are stored via a special instance type, 'labelmap', which has several features, including built-in multiscale support, supervoxel-to-body mapping, and sparse retrieval of body locations. """ if self.instance_name in fetch_repo_instances(self.server, self.uuid): logger.info( f"'{self.instance_name}' already exists, skipping creation") return settings = volume_config["dvid"]["creation-settings"] block_width = volume_config["geometry"]["block-width"] pyramid_depth = settings["max-scale"] if pyramid_depth == -1: pyramid_depth = choose_pyramid_depth(self.bounding_box_zyx, 512) if settings["compression"] != DvidInstanceCreationSettingsSchema[ "properties"]["compression"]["default"]: raise RuntimeError( "Alternative compression methods are not permitted on labelmap instances. " "Please remove the 'compression' setting from your config.") if settings["background"] != 0: raise RuntimeError( "Labelmap instances do not support custom background values. " "Please remove 'background' from your config.") create_labelmap_instance(self.server, self.uuid, self.instance_name, settings["versioned"], settings["tags"], block_width, settings["voxel-size"], settings["voxel-units"], settings["enable-index"], pyramid_depth) # Workaround for https://github.com/janelia-flyem/dvid/issues/344 # Write an empty block to the first and last blocks to set the # bounding-box in DVID now, without any concurrency issues. empty_block = np.zeros((64, 64, 64), np.uint64) first_block_start, last_block_stop = round_box(self.bounding_box_zyx, 64, 'out') last_block_start = last_block_stop - 64 self.write_subvolume(empty_block, first_block_start) self.write_subvolume(empty_block, last_block_start)
def test_fetch_mutations(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup uuid = post_branch(dvid_server, dvid_repo, 'segmentation-fetch_mutations', '') instance = 'segmentation-fetch_mutations' create_labelmap_instance(dvid_server, uuid, instance) voxels = np.zeros((64, 64, 64), dtype=np.uint64) voxels[0, 0, :10] = [*range(1, 11)] post_labelmap_voxels(dvid_server, uuid, instance, (0, 0, 0), voxels) post_merge(dvid_server, uuid, instance, 1, [2, 3, 4]) post_merge(dvid_server, uuid, instance, 5, [6, 7, 8]) post_commit(dvid_server, uuid, '') uuid2 = post_newversion(dvid_server, uuid, '') post_merge(dvid_server, uuid2, instance, 9, [10]) post_merge(dvid_server, uuid2, instance, 1, [5, 10]) mut_df = fetch_mutations(dvid_server, uuid2, instance, dag_filter='leaf-only') assert len(mut_df) == 2 assert (mut_df['uuid'] == uuid2).all() assert (mut_df['action'] == 'merge').all() assert (mut_df['target_body'] == [9, 1]).all() mut_df = fetch_mutations(dvid_server, uuid2, instance, dag_filter='leaf-and-parents') assert len(mut_df) == 4 assert (mut_df['uuid'] == [uuid, uuid, uuid2, uuid2]).all() assert (mut_df['action'] == 'merge').all() assert (mut_df['target_body'] == [1, 5, 9, 1]).all()
def test_copysegmentation_from_hdf5_to_dvid_output_mask( setup_hdf5_segmentation_input, disable_auto_retry): template_dir, config, input_volume, dvid_address, repo_uuid, _output_segmentation_name = setup_hdf5_segmentation_input # make sure we get a fresh output output_segmentation_name = 'copyseg-with-output-mask' config["output"]["dvid"]["segmentation-name"] = output_segmentation_name output_volume = np.zeros((256, 256, 256), np.uint64) mask = np.zeros((256, 256, 256), dtype=bool) masked_labels = [5, 10, 15, 20] # Start with an output that is striped (but along on block boundaries) for label, (z_start, z_stop) in enumerate(zip(range(0, 250, 10), range(10, 260, 10))): output_volume[z_start:z_stop] = label if label in masked_labels: mask[z_start:z_stop] = True # We expect the output to remain unchanged except in the masked voxels. expected_vol = np.where(mask, input_volume, output_volume) # make sure we get a fresh output output_segmentation_name = 'copyseg-with-output-mask' config["output"]["dvid"]["segmentation-name"] = output_segmentation_name config["copysegmentation"]["output-mask-labels"] = masked_labels max_scale = config["copysegmentation"]["pyramid-depth"] create_labelmap_instance(dvid_address, repo_uuid, output_segmentation_name, max_scale=max_scale) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), output_volume) setup = template_dir, config, expected_vol, dvid_address, repo_uuid, output_segmentation_name _box_zyx, _expected_vol, _output_vol = _run_to_dvid(setup)
def setup_connectedcomponents_dvid(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo _ = 0 volume_layout = [[_, _, _, _, _, _, _, _], [_, _, _, _, _, 4, _, _], [_, 1, 1, 1, _, _, 1, 1], [_, 1, _, _, _, _, _, _], [_, 1, _, _, _, _, _, _], [_, 1, _, 2, 2, 2, 2, _], [_, _, _, _, _, _, _, _], [_, 3, _, _, _, 3, _, 1]] lowres_volume = np.zeros((4, 8, 8), np.uint64) lowres_volume[:] = volume_layout volume = upsample(lowres_volume, 16) assert volume.shape == (64, 128, 128) input_segmentation_name = 'cc-input' output_segmentation_name = 'cc-output' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), volume) # Post data to the output -- don't leave it empty, # or we run into 'maxlabel' issues related to dvid issue #284 # https://github.com/janelia-flyem/dvid/issues/284 create_labelmap_instance(dvid_address, repo_uuid, output_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), volume) config_text = textwrap.dedent(f"""\ workflow-name: connectedcomponents cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,64] bounding-box: [[0,0,0], [128,128,64]] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true geometry: {{}} # Auto-set from input connectedcomponents: orig-max-label: 4 halo: 1 subset-labels: [1,2,4] # Not 3 compute-block-statistics: true log-relabeled-objects: true """) template_dir = tempfile.mkdtemp(suffix="connectedcomponents-template") with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) yaml = YAML() yaml.default_flow_style = False with open(f"{template_dir}/workflow.yaml", 'w') as f: yaml.dump(config, f) return template_dir, config, volume, dvid_address, repo_uuid, output_segmentation_name
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo # Since the same UUID is re-used for each test case, # this counter is a little hack used to make sure the segmentation # has a unique name each time, so that previous test cases don't # affect subsequent test casess. global test_case_counter test_case_counter += 1 # Normally the MaskSegmentation workflow is used to update # a segmentation instance from a parent uuid to a child uuid. # But for this test, we'll simulate that by writing to two # different instances in the same uuid. input_segmentation_name = f'masksegmentation-input-{test_case_counter}' output_segmentation_name = f'masksegmentation-output-from-dvid-{test_case_counter}' # Agglomerate some supervoxels into bodies # Choose supervoxels that intersect three Z-planes at 64, 128, 192 svs_1 = np.unique(random_segmentation[64]) svs_2 = np.unique(random_segmentation[128]) svs_3 = np.unique(random_segmentation[192]) for instance in (input_segmentation_name, output_segmentation_name): create_labelmap_instance(dvid_address, repo_uuid, instance, max_scale=MAX_SCALE) # Start with an empty mapping (the repo/instance are re-used for each test case) post_labelmap_voxels(dvid_address, repo_uuid, instance, (0, 0, 0), random_segmentation, downres=True) post_merge(dvid_address, repo_uuid, instance, svs_1[0], svs_1[1:]) post_merge(dvid_address, repo_uuid, instance, svs_2[0], svs_2[1:]) post_merge(dvid_address, repo_uuid, instance, svs_3[0], svs_3[1:]) # Create an ROI to test with -- a sphere with scale-5 resolution shape_s5 = np.array(random_segmentation.shape) // 2**5 midpoint_s5 = shape_s5 / 2 radius = midpoint_s5.min() coords_s5 = ndindex_array(*shape_s5) distances = np.sqrt(np.sum((coords_s5 - midpoint_s5)**2, axis=1)) keep = (distances < radius) coords_s5 = coords_s5[keep, :] roi_ranges = runlength_encode_to_ranges(coords_s5) roi_name = 'masksegmentation-test-roi' try: create_instance(dvid_address, repo_uuid, roi_name, 'roi') except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): pass post_roi(dvid_address, repo_uuid, roi_name, roi_ranges) roi_mask_s5 = np.zeros(shape_s5, dtype=bool) roi_mask_s5[(*coords_s5.transpose(), )] = True template_dir = tempfile.mkdtemp( suffix="masksegmentation-from-dvid-template") config_text = textwrap.dedent(f"""\ workflow-name: masksegmentation cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: # Choose a brick that doesn't cleanly divide into the bounding box message-block-shape: [192,64,64] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true masksegmentation: mask-roi: {roi_name} batch-size: 5 block-statistics-file: erased-block-statistics.h5 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, roi_mask_s5, input_segmentation_name, output_segmentation_name
def setup_sparseblockstats(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo _ = 0 volume_layout = [[_, _, _, _, _, _, _, _], [_, _, _, _, _, 4, _, _], [_, 1, 1, 1, _, _, 1, 1], [_, 1, _, _, _, _, _, _], [_, 1, _, _, _, _, _, _], [_, 1, _, 2, 2, 2, 2, _], [_, _, _, _, _, _, _, _], [_, 3, _, _, _, 3, _, 1]] lowres_volume = np.zeros((4, 8, 8), np.uint64) lowres_volume[:] = volume_layout volume = upsample(lowres_volume, 16) assert volume.shape == (64, 128, 128) input_segmentation_name = 'sparseblockstats-input' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), volume) # Mask is same as input, but times 10 mask_volume = volume * 10 mask_segmentation_name = 'sparseblockstats-mask' create_labelmap_instance(dvid_address, repo_uuid, mask_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, mask_segmentation_name, (0, 0, 0), mask_volume) config_text = textwrap.dedent(f"""\ workflow-name: sparseblockstats cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,64] bounding-box: [[0,0,0], [128,128,64]] mask-input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {mask_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,64] bounding-box: [[0,0,0], [128,128,64]] sparseblockstats: mask-labels: [20,40] # Avoids the top-left block """) template_dir = tempfile.mkdtemp(suffix="sparseblockstats-template") with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) yaml = YAML() yaml.default_flow_style = False with open(f"{template_dir}/workflow.yaml", 'w') as f: yaml.dump(config, f) return template_dir, config, volume, mask_volume, dvid_address, repo_uuid
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'labelmapcopy-segmentation-input' output_segmentation_name = 'labelmapcopy-segmentation-output' partial_output_segmentation_name = 'labelmapcopy-segmentation-partial-output' max_scale = 3 already_exists = False try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=max_scale) create_labelmap_instance(dvid_address, repo_uuid, partial_output_segmentation_name, max_scale=max_scale) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): already_exists = True expected_vols = {} for scale in range(1 + max_scale): if scale == 0: scaled_vol = random_segmentation else: scaled_vol = downsample(scaled_vol, 2, 'labels-numba') expected_vols[scale] = scaled_vol if not already_exists: scaled_box = round_box([(0, 0, 0), scaled_vol.shape], 64, 'out') aligned_vol = np.zeros(scaled_box[1], np.uint64) overwrite_subvol(aligned_vol, [(0, 0, 0), scaled_vol.shape], scaled_vol) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), aligned_vol, scale=scale) if not already_exists: # Create a 'partial' output volume that is the same (bitwise) as the input except for some blocks. scaled_box = np.array([(0, 0, 0), random_segmentation.shape]) scaled_box[1, -1] = 192 for scale in range(1 + max_scale): scaled_box = round_box(scaled_box // (2**scale), 64, 'out') raw_blocks = fetch_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, scaled_box, scale, supervoxels=True, format='raw-response') post_labelmap_blocks(dvid_address, repo_uuid, partial_output_segmentation_name, [(0, 0, 0)], raw_blocks, scale, is_raw=True) block = np.random.randint(1_000_000, 1_000_010, size=(64, 64, 64), dtype=np.uint64) post_labelmap_voxels(dvid_address, repo_uuid, partial_output_segmentation_name, (0, 128, 64), block, 0, downres=True) partial_vol = fetch_labelmap_voxels(dvid_address, repo_uuid, partial_output_segmentation_name, [(0, 0, 0), random_segmentation.shape], supervoxels=True) template_dir = tempfile.mkdtemp(suffix="labelmapcopy-template") config_text = textwrap.dedent(f"""\ workflow-name: labelmapcopy cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [512,64,64] available-scales: [0,1,2,3] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true labelmapcopy: slab-shape: [512,128,64] dont-overwrite-identical-blocks: true """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, expected_vols, partial_vol, dvid_address, repo_uuid, output_segmentation_name, partial_output_segmentation_name
def setup_dvid_segmentation_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-sparsemeshes-input' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3) label_vol = create_test_object().astype(np.uint64) label_vol *= 100 corners = np.array(list(ndrange((0, 0, 0), label_vol.shape, (64, 64, 64)))) # Drop it away from (0,0,0) to make sure the workflow handles arbitrary locations corners += 256 blockwise_vol = view_as_blocks(label_vol, (64, 64, 64)) blocks = blockwise_vol.reshape(-1, 64, 64, 64) # post to dvid post_labelarray_blocks(dvid_address, repo_uuid, input_segmentation_name, corners, blocks, downres=True, noindexing=False) template_dir = tempfile.mkdtemp(suffix="sparsemeshes-template") config_text = textwrap.dedent(f"""\ workflow-name: sparsemeshes cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} geometry: bounding-box: [[0,0,0], [128,128,128]] block-width: 64 available-scales: [0,1,2,3] sparsemeshes: min-scale: 0 max-analysis-volume: 1e6 # Too small for scale 0, will use scale 1 bodies: [100, 200] # 200 doesn't exist, will fail (see below) smoothing-iterations: 3 decimation-fraction: 0.1 format: obj output-directory: meshes """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, dvid_address, repo_uuid
def init_labelmap_nodes(): # Five supervoxels are each 1x3x3, arranged in a single row like this: # [[[1 1 1 2 2 2 3 3 3 4 4 4 5 5 5] # [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5] # [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]]] supervoxel_vol = np.zeros((1,3,15), np.uint64) supervoxel_vol[:] = (np.arange(15, dtype=np.uint64) // 3).reshape(1,1,15) supervoxel_vol += 1 np.set_printoptions(linewidth=100) #print(supervoxel_vol) # Merge table: Merge them all together id_a = np.array([1, 2, 3, 4], np.uint64) id_b = np.array([2, 3, 4, 5], np.uint64) xa = np.array([2, 5, 8, 11], np.uint32) ya = np.array([1, 1, 1, 1], np.uint32) za = np.array([0, 0, 0, 0], np.uint32) xb = np.array([3, 6, 9, 12], np.uint32) yb = np.array([1, 1, 1, 1], np.uint32) zb = np.array([0, 0, 0, 0], np.uint32) # Weak edge between 3 and 4 score = np.array([0.4, 0.4, 0.8, 0.4], np.float32) merge_table = pd.DataFrame({'id_a': id_a, 'id_b': id_b, 'xa': xa, 'ya': ya, 'za': za, 'xb': xb, 'yb': yb, 'zb': zb, 'score': score}) merge_table = merge_table[['id_a', 'id_b', 'xa', 'ya', 'za', 'xb', 'yb', 'zb', 'score']] merge_table_path = f'{TEST_DATA_DIR}/merge-table.npy' np.save(merge_table_path, merge_table.to_records(index=False)) create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation', max_scale=2) create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation-scratch', max_scale=2) # Expand to 64**3 supervoxel_block = np.zeros((64,64,64), np.uint64) supervoxel_block[:1,:3,:15] = supervoxel_vol post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation', (0,0,0), supervoxel_block) post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation-scratch', (0,0,0), supervoxel_block) post_commit(TEST_SERVER, TEST_REPO, 'supervoxels') # # Create a child node for agglo mappings # r = requests.post(f'http://{TEST_SERVER}/api/node/{TEST_REPO}/newversion', json={'note': 'agglo'}) # r.raise_for_status() # agglo_uuid = r.json["child"] # Merge everything agglo_uuid = TEST_REPO post_merge(TEST_SERVER, agglo_uuid, 'segmentation', 1, [2, 3, 4, 5]) mapping = np.array([[1,1],[2,1],[3,1],[4,1],[5,1]], np.uint64) #mapping = pd.DataFrame(mapping, columns=['sv', 'body']).set_index('sv')['body'] mapping_path = f'{TEST_DATA_DIR}/mapping.npy' np.save(mapping_path, mapping) return merge_table_path, mapping_path, supervoxel_vol
def main(): # Early exit if we're dumping the config # (Parse it ourselves to allow omission of otherwise required parameters.) if ({'--dump-config-template', '-d'} & {*sys.argv}): dump_default_config(ConfigSchema, sys.stdout, "yaml-with-comments") sys.exit(0) parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--dump-config-template', '-d', action='store_true', help='Dump out a template yaml config file and exit.') parser.add_argument('--count', '-c', type=int, help='How many points to generate.') parser.add_argument('--roi', '-r', help='Limit points to the given ROI.') parser.add_argument('--body', '-b', type=int, help='Limit points to the given body.') parser.add_argument( '--tbars', '-t', action='store_true', help= 'If given, limit points to the tbars of the given body, from the "synapses" instance in the input UUID.' ) parser.add_argument( '--skeleton', '-s', action='store_true', help= 'If given, choose the points from the nodes of the skeleton for the given body.' ) parser.add_argument( '--generate-points-only', '-g', action='store_true', help= "If given, generate the points list, but don't write neighborhood segmentations" ) parser.add_argument( '--points', '-p', help= 'A CSV file containing the points to use instead of automatically generating them.' ) parser.add_argument( '--ng-links', '-n', action='store_true', help='If given, include neuroglancer links in the output CSV.' 'Your config should specify the basic neuroglancer view settings; only the "position" will be overwritten in each link.' ) parser.add_argument('config') args = parser.parse_args() configure_default_logging() config = load_config(args.config, ConfigSchema) update_ng_settings(config) input_seg = [*config["input"].values()] output_seg = [*config["output"].values()] radius = config["radius"] random_seed = config["random-seed"] if config["enforce-minimum-distance"]: minimum_distance = 2 * radius else: minimum_distance = 0 if args.points and any( [args.count, args.roi, args.body, args.tbars, args.skeleton]): msg = ("If you're providing your own list of points, you shouldn't" " specify any of the auto-generation arguments, such as" " --count --roi --body --tbars") sys.exit(msg) if not args.points and not any( [args.count, args.roi, args.body, args.tbars, args.skeleton]): msg = "You must provide a list of points or specify how to auto-generate them." sys.exit(msg) if args.points: assert args.points.endswith('.csv') name, _ = os.path.splitext(args.points) output_path = name + '-neighborhoods.csv' points = pd.read_csv(args.points) else: points = autogen_points(input_seg, args.count, args.roi, args.body, args.tbars, args.skeleton, random_seed, minimum_distance) uuid = input_seg[1] output_path = f'neighborhoods-from-{uuid[:6]}' if not any([args.roi, args.body, args.tbars, args.skeleton]): output_path += input_seg[2] else: if args.roi: output_path += f'-{args.roi}' if args.body: output_path += f'-{args.body}' if args.tbars: output_path += '-tbars' if args.skeleton: output_path += '-skeleton' assignment_path = output_path + '.json' csv_path = output_path + '.csv' kd = scipy.spatial.cKDTree(points[[*'zyx']].values) if len(kd.query_pairs(2 * radius)) > 0: msg = ( "Some of the chosen points are closer to each other than 2x the " f"configured radius ({radius}). Their neighborhood segments may " "be mangled in the output.") logger.warning(msg) cols = [*'xyz'] + list({*points.columns} - {*'xyz'}) points = points[cols] if args.generate_points_only: add_link_col(points, config) export_as_html(points, csv_path) if not args.ng_links: del points['link'] points.to_csv(csv_path, index=False, header=True, quoting=csv.QUOTE_NONE) sys.exit(0) try: input_info = fetch_instance_info(*input_seg) except Exception: sys.exit( f"Couldn't find input segmentation instance: {' / '.join(input_seg)}" ) try: fetch_instance_info(*output_seg) except Exception: logger.info( f"Output labelmap not found. Creating new label instance: {' / '.join(output_seg)}" ) # Copy details from input instance. # But only provide a single value for each, even though the info provides three. # Otherwise, DVID kicks back errors like this: # Setting for 'VoxelUnits' was not a string: [nanometers nanometers nanometers] settings = { 'block_size': input_info['Extended']['BlockSize'][0], 'voxel_size': input_info['Extended']['VoxelSize'][0], 'voxel_units': input_info['Extended']['VoxelUnits'][0], 'max_scale': input_info['Extended']['MaxDownresLevel'] } create_labelmap_instance(*output_seg, **settings) # Also create keyvalue for meshes create_instance(*output_seg[:2], output_seg[2] + '_meshes', 'keyvalue') results_df = write_point_neighborhoods(input_seg, output_seg, points, radius, args.body) add_link_col(results_df, config) export_as_html(results_df, csv_path) write_assignment_file(output_seg, results_df, assignment_path, config) if not args.ng_links: del results_df['link'] results_df.to_csv(csv_path, index=False, header=True, quoting=csv.QUOTE_NONE)
def load_roi_label_volume(server, uuid, rois_or_neuprint, box_s5=[None, None], export_path=None, export_labelmap=None): """ Fetch several ROIs from DVID and combine them into a single label volume or mask. The label values in the returned volume correspond to the order in which the ROI names were passed in, starting at label 1. This function is essentially a convenience function around fetch_combined_roi_volume(), but in this case it will optionally auto-fetch the ROI list, and auto-export the volume. Args: server: DVID server uuid: DVID uuid rois_or_neuprint: Either a list of ROIs or a neuprint server from which to obtain the roi list. box_s5: If you want to restrict the ROIs to a particular subregion, you may pass your own bounding box (at scale 5). Alternatively, you may pass the name of a segmentation instance from DVID whose bounding box will be used. export_path: If you want the ROI volume to be exported to disk, provide a path name ending with .npy or .h5. export_labelmap: If you want the ROI volume to be exported to a DVID labelmap instance, Provide the instance name, or a tuple of (server, uuid, instance). Returns: (roi_vol, roi_box), containing the fetched label volume and the bounding box it corresponds to, in DVID scale-5 coordinates. Note: If you have a list of (full-res) points to extract from the returned volume, pass a DataFrame with columns ['z','y','x'] to the following function. If you already downloaded the roi_vol (above), provide it. Otherwise, leave out those args and it will be fetched first. Adds columns to the input DF (in-place) for 'roi' (str) and 'roi_label' (int). >>> from neuclease.dvid import determine_point_rois >>> determine_point_rois(*master, rois, point_df, roi_vol, roi_box) """ if isinstance(box_s5, str): # Assume that this is a segmentation instance whose dimensions should be used # Fetch the maximum extents of the segmentation, # and rescale it for scale-5. seg_box = fetch_volume_box(server, uuid, box_s5) box_s5 = round_box(seg_box, (2**5), 'out') // 2**5 box_s5[0] = (0, 0, 0) if export_labelmap: assert isinstance(box_s5, np.ndarray) assert not (box_s5 % 64).any(), \ ("If exporting to a labelmap instance, please supply " "an explicit box and make sure it is block-aligned.") if isinstance(rois_or_neuprint, (str, neuprint.Client)): if isinstance(rois_or_neuprint, str): npclient = neuprint.Client(rois_or_neuprint) else: npclient = rois_or_neuprint # Fetch ROI names from neuprint q = "MATCH (m: Meta) RETURN m.superLevelRois as rois" rois = npclient.fetch_custom(q)['rois'].iloc[0] rois = sorted(rois) # # Remove '.*ACA' ROIs. Apparently there is some # # problem with them. (They overlap with other ROIs.) # rois = [*filter(lambda r: 'ACA' not in r, rois)] else: assert isinstance(rois_or_neuprint, collections.abc.Iterable) rois = rois_or_neuprint # Fetch each ROI and write it into a volume with Timer(f"Fetching combined ROI volume for {len(rois)} ROIs", logger): roi_vol, roi_box, overlap_stats = fetch_combined_roi_volume( server, uuid, rois, box_zyx=box_s5) if len(overlap_stats) > 0: logger.warn( f"Some ROIs overlap! Here's an incomplete list of overlapping pairs:\n{overlap_stats}" ) # Export to npy/h5py for external use if export_path: with Timer(f"Exporting to {export_path}", logger): if export_path.endswith('.npy'): np.save(export_path, roi_vol) elif export_path.endswith('.h5'): with h5py.File(export_path, 'w') as f: f.create_dataset('rois_scale_5', data=roi_vol, chunks=True) if export_labelmap: if isinstance(export_labelmap, str): export_labelmap = (server, uuid, export_labelmap) assert len(export_labelmap) == 3 with Timer(f"Exporting to {export_labelmap[2]}", logger): if export_labelmap[2] not in fetch_repo_instances( server, uuid, 'labelmap'): create_labelmap_instance( *export_labelmap, voxel_size=8 * (2**5), max_scale=6) # FIXME: hard-coded voxel size # It's really important to use this block shape. # See https://github.com/janelia-flyem/dvid/issues/342 boxes = boxes_from_grid(roi_box, (256, 256, 256), clipped=True) for box in tqdm_proxy(boxes): block = extract_subvol(roi_vol, box - roi_box[0]) post_labelmap_voxels(*export_labelmap, box[0], block, scale=0, downres=True) return roi_vol, roi_box, rois
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-input' output_segmentation_name = 'segmentation-output-from-dvid' try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): pass post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), random_segmentation) # Make sure the output is empty (if it exists) if output_segmentation_name in fetch_repo_instances( dvid_address, repo_uuid): z = np.zeros((256, 256, 256), np.uint64) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), z, 0, True) template_dir = tempfile.mkdtemp( suffix="copysegmentation-from-dvid-template") config_text = textwrap.dedent(f"""\ workflow-name: copysegmentation cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,512] bounding-box: [[0,0,100], [256,200,256]] adapters: {{}} output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true geometry: {{}} # Auto-set from input copysegmentation: pyramid-depth: 1 slab-depth: 128 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_segmentation_name
def setup_dvid_to_zarr(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-input' try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): pass post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), random_segmentation) template_dir = tempfile.mkdtemp( suffix="copysegmentation-from-dvid-template") output_path = 'output.zarr' config_text = textwrap.dedent(f"""\ workflow-name: copysegmentation cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,512] bounding-box: [[0,0,100], [256,200,256]] adapters: {{}} output: zarr: path: {output_path} dataset: s0 create-if-necessary: true creation-settings: dtype: uint64 chunk-shape: [64,64,64] geometry: {{}} # Auto-set from input copysegmentation: pyramid-depth: 1 slab-depth: 128 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_path