def setup_dvid_segmentation_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo # Start with a low-res map of the test data # and scale it up 16x to achieve a 128-cube _ = 0 # 0 1 2 3 4 5 6 7 volume_layout = [[ [_, _, _, _, _, 6, 6, 6], # 0 [_, 1, 1, 2, 2, _, 6, _], # 1 [_, 1, 1, 2, 2, _, _, _], # 2 [_, 1, 1, 2, 8, _, 7, 7], # 3 [_, 1, 1, 2, 8, _, 7, 7], # 4 [_, _, _, _, _, _, _, _], # 5 [_, 4, 4, 4, _, _, _, _], # 6 [_, 3, 3, 3, _, 5, 9, _] ]] # 7 # 0 1 2 3 4 5 6 7 lowres_volume = np.zeros((4, 8, 8), np.uint64) lowres_volume[:] = volume_layout volume = upsample(lowres_volume, 16) assert volume.shape == (64, 128, 128) input_segmentation_name = 'findadjacencies-input' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), volume) template_dir = tempfile.mkdtemp(suffix="findadjacencies-from-dvid") config_text = textwrap.dedent(f"""\ workflow-name: findadjacencies cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [128,64,64] findadjacencies: output-table: output.csv find-closest-using-scale: 0 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, volume, dvid_address, repo_uuid
def setup_tsv_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-decimatemeshes-input' test_volume, object_boxes, object_sizes = create_test_segmentation() create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0,0,0), test_volume, downres=True, noindexing=False) tsv_name = 'segmentation-decimatemeshes-tsv' create_tarsupervoxel_instance(dvid_address, repo_uuid, tsv_name, input_segmentation_name, '.drc') # Post supervoxel meshes meshes = Mesh.from_label_volume(test_volume, progress=False) meshes_data = {f"{label}.drc": mesh.serialize(fmt='drc') for label, mesh in meshes.items()} post_load(dvid_address, repo_uuid, tsv_name, meshes_data) # Merge two of the objects (100 and 300) post_merge(dvid_address, repo_uuid, input_segmentation_name, 100, [300]) object_boxes[100] = box_union(object_boxes[100], object_boxes[300]) del object_boxes[300] object_sizes[100] += object_sizes[300] del object_sizes[300] meshes[100] = Mesh.concatenate_meshes((meshes[100], meshes[300])) del meshes[300] return dvid_address, repo_uuid, tsv_name, object_boxes, object_sizes, meshes
def copy_vnc_subvolume(box_zyx, copy_grayscale=True, copy_segmentation=True, chunk_shape=(64, 64, 2048)): assert not (box_zyx % 64).any(), \ "Only 64px block-aligned volumes can be copied." import numpy as np from neuclease.util import boxes_from_grid, tqdm_proxy, round_box from neuclease.dvid import find_master, fetch_raw, post_raw, fetch_subvol, post_labelmap_voxels vnc_master = ('emdata4:8200', find_master('emdata4:8200')) NUM_SCALES = 8 num_voxels = np.prod(box_zyx[1] - box_zyx[0]) if copy_grayscale: logger.info( f"Copying grayscale from box {box_zyx[:,::-1].tolist()} ({num_voxels/1e6:.1f} Mvox) for {NUM_SCALES} scales" ) for scale in tqdm_proxy(range(NUM_SCALES)): if scale == 0: input_name = 'grayscalejpeg' output_name = 'local-grayscalejpeg' else: input_name = f'grayscalejpeg_{scale}' output_name = f'local-grayscalejpeg_{scale}' scaled_box_zyx = np.maximum(box_zyx // 2**scale, 1) scaled_box_zyx = round_box(scaled_box_zyx, 64, 'out') for chunk_box in tqdm_proxy(boxes_from_grid(scaled_box_zyx, chunk_shape, clipped=True), leave=False): chunk = fetch_subvol(*vnc_master, input_name, chunk_box, progress=False) post_raw(*vnc_master, output_name, chunk_box[0], chunk) if copy_segmentation: logger.info( f"Copying segmentation from box {box_zyx[:,::-1].tolist()} ({num_voxels/1e6:.2f} Mvox)" ) for chunk_box in tqdm_proxy( boxes_from_grid(box_zyx, chunk_shape, clipped=True)): chunk = fetch_raw(*vnc_master, 'segmentation', chunk_box, dtype=np.uint64) post_labelmap_voxels(*vnc_master, 'local-segmentation', chunk_box[0], chunk, downres=True) # TODO: Update label indexes? logger.info("DONE")
def test_post_hierarchical_cleaves(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup uuid = post_branch(dvid_server, dvid_repo, 'segmentation-post_hierarchical_cleaves', '') instance_info = dvid_server, uuid, 'segmentation-post_hierarchical_cleaves' create_labelmap_instance(*instance_info) svs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] groups = [1, 1, 2, 2, 3, 3, 3, 3, 3, 4] svs = np.asarray(svs, np.uint64) # Post some supervoxels in multiple blocks, just to prove that post_hierarchical_cleaves() # doesn't assume that the labelindex has the same length as the mapping. sv_vol = np.zeros((128, 64, 64), np.uint64) sv_vol[0, 0, :len(svs)] = svs sv_vol[64, 0, 0:len(svs):2] = svs[::2] post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol) post_merge(*instance_info, 1, svs[1:]) group_mapping = pd.Series(index=svs, data=groups) final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping) assert (fetch_mapping(*instance_info, svs) == final_table['body'].values).all() assert (final_table.drop_duplicates( ['group']) == final_table.drop_duplicates(['group', 'body'])).all().all() assert (final_table.drop_duplicates( ['body']) == final_table.drop_duplicates(['group', 'body'])).all().all() # Since the mapping included all supervoxels in the body, # the last group is left with the original label. assert final_table.iloc[-1]['body'] == 1 # Now merge them all together and try again, but leave # two supevoxels out of the groups this time. merges = set(pd.unique(final_table['body'].values)) - set([1]) post_merge(*instance_info, 1, list(merges)) group_mapping = pd.Series(index=svs[:-2], data=groups[:-2]) final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping) assert len( final_table.query('body == 1') ) == 0, "Did not expect any of the groups to retain the original body ID!" assert (fetch_mapping(*instance_info, svs[:-2]) == final_table['body'].values).all() assert (final_table.drop_duplicates( ['group']) == final_table.drop_duplicates(['group', 'body'])).all().all() assert (final_table.drop_duplicates( ['body']) == final_table.drop_duplicates(['group', 'body'])).all().all() assert (fetch_mapping(*instance_info, svs[-2:]) == 1).all()
def test_labelindex(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup # Need an unlocked node to test these posts uuid = post_branch(dvid_server, dvid_repo, 'test_labelindex', 'test_labelindex') instance_info = (dvid_server, uuid, 'segmentation-scratch') # Write some random data sv = 99 vol = sv * np.random.randint(2, size=(128, 128, 128), dtype=np.uint64) offset = np.array((64, 64, 64)) # DVID will generate the index. post_labelmap_voxels(*instance_info, offset, vol) # Compute labelindex table from scratch rows = [] for block_coord in ndrange(offset, offset + vol.shape, (64, 64, 64)): block_coord = np.array(block_coord) block_box = np.array((block_coord, block_coord + 64)) block = extract_subvol(vol, block_box - offset) count = (block == sv).sum() rows.append([*block_coord, sv, count]) index_df = pd.DataFrame(rows, columns=['z', 'y', 'x', 'sv', 'count']) # Check DVID's generated labelindex table against expected labelindex_tuple = fetch_labelindex(*instance_info, sv, format='pandas') assert labelindex_tuple.label == sv labelindex_tuple.blocks.sort_values(['z', 'y', 'x', 'sv'], inplace=True) labelindex_tuple.blocks.reset_index(drop=True, inplace=True) assert (labelindex_tuple.blocks == index_df).all().all() # Check our protobuf against DVID's index_tuple = PandasLabelIndex(index_df, sv, 1, datetime.datetime.now().isoformat(), 'someuser') labelindex = create_labelindex(index_tuple) # Since labelindex block entries are not required to be sorted, # dvid might return them in a different order. # Hence this comparison function which sorts them first. def compare_proto_blocks(left, right): left_blocks = sorted(left.blocks.items()) right_blocks = sorted(right.blocks.items()) return left_blocks == right_blocks dvid_labelindex = fetch_labelindex(*instance_info, sv, format='protobuf') assert compare_proto_blocks(labelindex, dvid_labelindex) # Check post/get roundtrip post_labelindex(*instance_info, sv, labelindex) dvid_labelindex = fetch_labelindex(*instance_info, sv, format='protobuf') assert compare_proto_blocks(labelindex, dvid_labelindex)
def test_fetch_sparsevol_coarse_via_labelindex(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup # Create a labelmap volume with 3 blocks. # # Supervoxels are arranged like this: # # | 1 2 | 3 4 | 5 6 | # # After merging [2,3,4,5], bodies will be: # # | 1 2 | 2 4 | 5 6 | # vol_shape = (64, 64, 256) sv_vol = np.zeros(vol_shape, np.uint64) sv_vol[:, :, 0:32] = 1 sv_vol[:, :, 32:64] = 2 sv_vol[:, :, 64:96] = 3 sv_vol[:, :, 96:128] = 4 sv_vol[:, :, 128:160] = 5 sv_vol[:, :, 160:192] = 6 instance_info = dvid_server, dvid_repo, 'segmentation-test-sparsevol-coarse' create_labelmap_instance(*instance_info) post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol) post_merge(*instance_info, 2, [3, 4, 5]) body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 2, method='protobuf') expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2) assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist()) body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 2, method='pandas') expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2) assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist()) sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 3, supervoxels=True, method='protobuf') expected_sv_svc = fetch_sparsevol_coarse(*instance_info, 3, supervoxels=True) assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist()) sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info, 3, supervoxels=True, method='pandas') expected_sv_svc = fetch_sparsevol_coarse(*instance_info, 3, supervoxels=True) assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist())
def setup_hdf5_segmentation_input(setup_dvid_repo, write_hdf5_volume): volume_path, random_segmentation = write_hdf5_volume dvid_address, repo_uuid = setup_dvid_repo template_dir = tempfile.mkdtemp( suffix="copysegmentation-from-hdf5-template") output_segmentation_name = 'segmentation-output-from-hdf5' # Make sure the output is empty (if it exists) if output_segmentation_name in fetch_repo_instances( dvid_address, repo_uuid): z = np.zeros((256, 256, 256), np.uint64) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), z, 0, True) config_text = textwrap.dedent(f"""\ workflow-name: copysegmentation cluster-type: {CLUSTER_TYPE} input: hdf5: path: {volume_path} dataset: volume geometry: message-block-shape: [64,64,256] # note: this is weird because normally we stripe in the X direction... bounding-box: [[0,0,100], [256,200,256]] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true geometry: {{}} # Auto-set from input copysegmentation: pyramid-depth: 1 slab-depth: 128 download-pre-downsampled: false """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_segmentation_name
def test_fetch_labelindices(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup # Need an unlocked node to test these posts uuid = post_branch(dvid_server, dvid_repo, 'test_labelindices', 'test_labelindices') instance_info = (dvid_server, uuid, 'segmentation-scratch') # Write some random data vol = np.random.randint(1, 10, size=(128, 128, 128), dtype=np.uint64) offset = np.array((64, 64, 64)) # DVID will generate the index. post_labelmap_voxels(*instance_info, offset, vol) labelindices = fetch_labelindices(*instance_info, list(range(1, 10))) for sv, li in zip(range(1, 10), labelindices.indices): # This function is already tested elsewhere, so we'll use it as a reference li2 = fetch_labelindex(*instance_info, sv) assert li == li2 labelindices = fetch_labelindices(*instance_info, list(range(1, 10)), format='list-of-protobuf') for sv, li in zip(range(1, 10), labelindices): # This function is already tested elsewhere, so we'll use it as a reference li2 = fetch_labelindex(*instance_info, sv) assert li == li2 labelindices = fetch_labelindices(*instance_info, list(range(1, 10)), format='pandas') for sv, li in zip(range(1, 10), labelindices): # This function is already tested elsewhere, so we'll use it as a reference li2 = fetch_labelindex(*instance_info, sv, format='pandas') li_df = li.blocks.sort_values(['z', 'y', 'x']).reset_index(drop=True) li2_df = li2.blocks.sort_values(['z', 'y', 'x']).reset_index(drop=True) assert (li_df == li2_df).all().all() # Test the copy function (just do a round-trip -- hopefully I didn't swap src and dest anywhere...) copy_labelindices(instance_info, instance_info, list(range(1, 10)), batch_size=2) copy_labelindices(instance_info, instance_info, list(range(1, 10)), batch_size=2, processes=2)
def test_post_labelmap_voxels(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup instance_info = DvidInstanceInfo(dvid_server, dvid_repo, 'segmentation-scratch') # Write some random data and read it back. vol = np.random.randint(10, size=(128, 128, 128), dtype=np.uint64) offset = (64, 64, 64) post_labelmap_voxels(dvid_server, dvid_repo, 'segmentation-scratch', offset, vol, 0) complete_voxels = fetch_labelmap_voxels(*instance_info, [(0, 0, 0), (256, 256, 256)], supervoxels=True) assert (complete_voxels[64:192, 64:192, 64:192] == vol).all()
def setup_segmentation_input(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-createmeshes-input' test_volume, object_boxes, object_sizes = create_test_segmentation() create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), test_volume, downres=True, noindexing=False) return dvid_address, repo_uuid, input_segmentation_name, object_boxes, object_sizes
def process_point(seg_src, seg_dst, point, radius, src_body, dst_body): """ Generate a neighborhood segment around a particular point. Upload the voxels for the segment and the corresponding mesh. """ r = radius src_box = np.asarray((point - r, point + r + 1)) src_vol = fetch_labelmap_voxels(*seg_src, src_box) if src_body is None: src_body = src_vol[r, r, r] if dst_body is None: # Generate a neighborhood segment ID from the coordinate. # Divide by 4 to ensure the coordinates fit within 2^53. # (The segment ID will not retain the full resolution of # the coordinate, but that's usually OK for our purposes.) dst_body = encode_point_to_uint64(point // 4, 17) mask = (src_vol == src_body) & sphere_mask(r) dst_box = round_box(src_box, 64, 'out') dst_vol = fetch_labelmap_voxels(*seg_dst, dst_box) dst_view = dst_vol[b2s(*(src_box - dst_box[0]))] dst_view[mask] = dst_body post_labelmap_voxels(*seg_dst, dst_box[0], dst_vol, downres=True) # Mesh needs to be written in nm, hence 8x mesh = Mesh.from_binary_vol(mask, 8 * src_box, smoothing_rounds=2) mesh.simplify(0.05, in_memory=True) post_key(*seg_dst[:2], f'{seg_dst[2]}_meshes', f'{dst_body}.ngmesh', mesh.serialize(fmt='ngmesh')) centroid = src_box[0] + mask_centroid(mask, True) top_z = mask.sum(axis=(1, 2)).nonzero()[0][0] top_coords = np.transpose(mask[top_z].nonzero()) top_point = src_box[0] + (top_z, *top_coords[len(top_coords) // 2]) return point, centroid, top_point, src_body, dst_body, mask.sum()
def test_fetch_mutations(labelmap_setup): dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup uuid = post_branch(dvid_server, dvid_repo, 'segmentation-fetch_mutations', '') instance = 'segmentation-fetch_mutations' create_labelmap_instance(dvid_server, uuid, instance) voxels = np.zeros((64, 64, 64), dtype=np.uint64) voxels[0, 0, :10] = [*range(1, 11)] post_labelmap_voxels(dvid_server, uuid, instance, (0, 0, 0), voxels) post_merge(dvid_server, uuid, instance, 1, [2, 3, 4]) post_merge(dvid_server, uuid, instance, 5, [6, 7, 8]) post_commit(dvid_server, uuid, '') uuid2 = post_newversion(dvid_server, uuid, '') post_merge(dvid_server, uuid2, instance, 9, [10]) post_merge(dvid_server, uuid2, instance, 1, [5, 10]) mut_df = fetch_mutations(dvid_server, uuid2, instance, dag_filter='leaf-only') assert len(mut_df) == 2 assert (mut_df['uuid'] == uuid2).all() assert (mut_df['action'] == 'merge').all() assert (mut_df['target_body'] == [9, 1]).all() mut_df = fetch_mutations(dvid_server, uuid2, instance, dag_filter='leaf-and-parents') assert len(mut_df) == 4 assert (mut_df['uuid'] == [uuid, uuid, uuid2, uuid2]).all() assert (mut_df['action'] == 'merge').all() assert (mut_df['target_body'] == [1, 5, 9, 1]).all()
def test_copysegmentation_from_hdf5_to_dvid_output_mask( setup_hdf5_segmentation_input, disable_auto_retry): template_dir, config, input_volume, dvid_address, repo_uuid, _output_segmentation_name = setup_hdf5_segmentation_input # make sure we get a fresh output output_segmentation_name = 'copyseg-with-output-mask' config["output"]["dvid"]["segmentation-name"] = output_segmentation_name output_volume = np.zeros((256, 256, 256), np.uint64) mask = np.zeros((256, 256, 256), dtype=bool) masked_labels = [5, 10, 15, 20] # Start with an output that is striped (but along on block boundaries) for label, (z_start, z_stop) in enumerate(zip(range(0, 250, 10), range(10, 260, 10))): output_volume[z_start:z_stop] = label if label in masked_labels: mask[z_start:z_stop] = True # We expect the output to remain unchanged except in the masked voxels. expected_vol = np.where(mask, input_volume, output_volume) # make sure we get a fresh output output_segmentation_name = 'copyseg-with-output-mask' config["output"]["dvid"]["segmentation-name"] = output_segmentation_name config["copysegmentation"]["output-mask-labels"] = masked_labels max_scale = config["copysegmentation"]["pyramid-depth"] create_labelmap_instance(dvid_address, repo_uuid, output_segmentation_name, max_scale=max_scale) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), output_volume) setup = template_dir, config, expected_vol, dvid_address, repo_uuid, output_segmentation_name _box_zyx, _expected_vol, _output_vol = _run_to_dvid(setup)
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-input' output_segmentation_name = 'segmentation-output-from-dvid' try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): pass post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), random_segmentation) # Make sure the output is empty (if it exists) if output_segmentation_name in fetch_repo_instances( dvid_address, repo_uuid): z = np.zeros((256, 256, 256), np.uint64) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), z, 0, True) template_dir = tempfile.mkdtemp( suffix="copysegmentation-from-dvid-template") config_text = textwrap.dedent(f"""\ workflow-name: copysegmentation cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,512] bounding-box: [[0,0,100], [256,200,256]] adapters: {{}} output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true geometry: {{}} # Auto-set from input copysegmentation: pyramid-depth: 1 slab-depth: 128 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_segmentation_name
def setup_dvid_to_zarr(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'segmentation-input' try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): pass post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), random_segmentation) template_dir = tempfile.mkdtemp( suffix="copysegmentation-from-dvid-template") output_path = 'output.zarr' config_text = textwrap.dedent(f"""\ workflow-name: copysegmentation cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,512] bounding-box: [[0,0,100], [256,200,256]] adapters: {{}} output: zarr: path: {output_path} dataset: s0 create-if-necessary: true creation-settings: dtype: uint64 chunk-shape: [64,64,64] geometry: {{}} # Auto-set from input copysegmentation: pyramid-depth: 1 slab-depth: 128 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_path
def load_roi_label_volume(server, uuid, rois_or_neuprint, box_s5=[None, None], export_path=None, export_labelmap=None): """ Fetch several ROIs from DVID and combine them into a single label volume or mask. The label values in the returned volume correspond to the order in which the ROI names were passed in, starting at label 1. This function is essentially a convenience function around fetch_combined_roi_volume(), but in this case it will optionally auto-fetch the ROI list, and auto-export the volume. Args: server: DVID server uuid: DVID uuid rois_or_neuprint: Either a list of ROIs or a neuprint server from which to obtain the roi list. box_s5: If you want to restrict the ROIs to a particular subregion, you may pass your own bounding box (at scale 5). Alternatively, you may pass the name of a segmentation instance from DVID whose bounding box will be used. export_path: If you want the ROI volume to be exported to disk, provide a path name ending with .npy or .h5. export_labelmap: If you want the ROI volume to be exported to a DVID labelmap instance, Provide the instance name, or a tuple of (server, uuid, instance). Returns: (roi_vol, roi_box), containing the fetched label volume and the bounding box it corresponds to, in DVID scale-5 coordinates. Note: If you have a list of (full-res) points to extract from the returned volume, pass a DataFrame with columns ['z','y','x'] to the following function. If you already downloaded the roi_vol (above), provide it. Otherwise, leave out those args and it will be fetched first. Adds columns to the input DF (in-place) for 'roi' (str) and 'roi_label' (int). >>> from neuclease.dvid import determine_point_rois >>> determine_point_rois(*master, rois, point_df, roi_vol, roi_box) """ if isinstance(box_s5, str): # Assume that this is a segmentation instance whose dimensions should be used # Fetch the maximum extents of the segmentation, # and rescale it for scale-5. seg_box = fetch_volume_box(server, uuid, box_s5) box_s5 = round_box(seg_box, (2**5), 'out') // 2**5 box_s5[0] = (0, 0, 0) if export_labelmap: assert isinstance(box_s5, np.ndarray) assert not (box_s5 % 64).any(), \ ("If exporting to a labelmap instance, please supply " "an explicit box and make sure it is block-aligned.") if isinstance(rois_or_neuprint, (str, neuprint.Client)): if isinstance(rois_or_neuprint, str): npclient = neuprint.Client(rois_or_neuprint) else: npclient = rois_or_neuprint # Fetch ROI names from neuprint q = "MATCH (m: Meta) RETURN m.superLevelRois as rois" rois = npclient.fetch_custom(q)['rois'].iloc[0] rois = sorted(rois) # # Remove '.*ACA' ROIs. Apparently there is some # # problem with them. (They overlap with other ROIs.) # rois = [*filter(lambda r: 'ACA' not in r, rois)] else: assert isinstance(rois_or_neuprint, collections.abc.Iterable) rois = rois_or_neuprint # Fetch each ROI and write it into a volume with Timer(f"Fetching combined ROI volume for {len(rois)} ROIs", logger): roi_vol, roi_box, overlap_stats = fetch_combined_roi_volume( server, uuid, rois, box_zyx=box_s5) if len(overlap_stats) > 0: logger.warn( f"Some ROIs overlap! Here's an incomplete list of overlapping pairs:\n{overlap_stats}" ) # Export to npy/h5py for external use if export_path: with Timer(f"Exporting to {export_path}", logger): if export_path.endswith('.npy'): np.save(export_path, roi_vol) elif export_path.endswith('.h5'): with h5py.File(export_path, 'w') as f: f.create_dataset('rois_scale_5', data=roi_vol, chunks=True) if export_labelmap: if isinstance(export_labelmap, str): export_labelmap = (server, uuid, export_labelmap) assert len(export_labelmap) == 3 with Timer(f"Exporting to {export_labelmap[2]}", logger): if export_labelmap[2] not in fetch_repo_instances( server, uuid, 'labelmap'): create_labelmap_instance( *export_labelmap, voxel_size=8 * (2**5), max_scale=6) # FIXME: hard-coded voxel size # It's really important to use this block shape. # See https://github.com/janelia-flyem/dvid/issues/342 boxes = boxes_from_grid(roi_box, (256, 256, 256), clipped=True) for box in tqdm_proxy(boxes): block = extract_subvol(roi_vol, box - roi_box[0]) post_labelmap_voxels(*export_labelmap, box[0], block, scale=0, downres=True) return roi_vol, roi_box, rois
def setup_connectedcomponents_dvid(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo _ = 0 volume_layout = [[_, _, _, _, _, _, _, _], [_, _, _, _, _, 4, _, _], [_, 1, 1, 1, _, _, 1, 1], [_, 1, _, _, _, _, _, _], [_, 1, _, _, _, _, _, _], [_, 1, _, 2, 2, 2, 2, _], [_, _, _, _, _, _, _, _], [_, 3, _, _, _, 3, _, 1]] lowres_volume = np.zeros((4, 8, 8), np.uint64) lowres_volume[:] = volume_layout volume = upsample(lowres_volume, 16) assert volume.shape == (64, 128, 128) input_segmentation_name = 'cc-input' output_segmentation_name = 'cc-output' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), volume) # Post data to the output -- don't leave it empty, # or we run into 'maxlabel' issues related to dvid issue #284 # https://github.com/janelia-flyem/dvid/issues/284 create_labelmap_instance(dvid_address, repo_uuid, output_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name, (0, 0, 0), volume) config_text = textwrap.dedent(f"""\ workflow-name: connectedcomponents cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,64] bounding-box: [[0,0,0], [128,128,64]] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true geometry: {{}} # Auto-set from input connectedcomponents: orig-max-label: 4 halo: 1 subset-labels: [1,2,4] # Not 3 compute-block-statistics: true log-relabeled-objects: true """) template_dir = tempfile.mkdtemp(suffix="connectedcomponents-template") with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) yaml = YAML() yaml.default_flow_style = False with open(f"{template_dir}/workflow.yaml", 'w') as f: yaml.dump(config, f) return template_dir, config, volume, dvid_address, repo_uuid, output_segmentation_name
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo # Since the same UUID is re-used for each test case, # this counter is a little hack used to make sure the segmentation # has a unique name each time, so that previous test cases don't # affect subsequent test casess. global test_case_counter test_case_counter += 1 # Normally the MaskSegmentation workflow is used to update # a segmentation instance from a parent uuid to a child uuid. # But for this test, we'll simulate that by writing to two # different instances in the same uuid. input_segmentation_name = f'masksegmentation-input-{test_case_counter}' output_segmentation_name = f'masksegmentation-output-from-dvid-{test_case_counter}' # Agglomerate some supervoxels into bodies # Choose supervoxels that intersect three Z-planes at 64, 128, 192 svs_1 = np.unique(random_segmentation[64]) svs_2 = np.unique(random_segmentation[128]) svs_3 = np.unique(random_segmentation[192]) for instance in (input_segmentation_name, output_segmentation_name): create_labelmap_instance(dvid_address, repo_uuid, instance, max_scale=MAX_SCALE) # Start with an empty mapping (the repo/instance are re-used for each test case) post_labelmap_voxels(dvid_address, repo_uuid, instance, (0, 0, 0), random_segmentation, downres=True) post_merge(dvid_address, repo_uuid, instance, svs_1[0], svs_1[1:]) post_merge(dvid_address, repo_uuid, instance, svs_2[0], svs_2[1:]) post_merge(dvid_address, repo_uuid, instance, svs_3[0], svs_3[1:]) # Create an ROI to test with -- a sphere with scale-5 resolution shape_s5 = np.array(random_segmentation.shape) // 2**5 midpoint_s5 = shape_s5 / 2 radius = midpoint_s5.min() coords_s5 = ndindex_array(*shape_s5) distances = np.sqrt(np.sum((coords_s5 - midpoint_s5)**2, axis=1)) keep = (distances < radius) coords_s5 = coords_s5[keep, :] roi_ranges = runlength_encode_to_ranges(coords_s5) roi_name = 'masksegmentation-test-roi' try: create_instance(dvid_address, repo_uuid, roi_name, 'roi') except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): pass post_roi(dvid_address, repo_uuid, roi_name, roi_ranges) roi_mask_s5 = np.zeros(shape_s5, dtype=bool) roi_mask_s5[(*coords_s5.transpose(), )] = True template_dir = tempfile.mkdtemp( suffix="masksegmentation-from-dvid-template") config_text = textwrap.dedent(f"""\ workflow-name: masksegmentation cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: # Choose a brick that doesn't cleanly divide into the bounding box message-block-shape: [192,64,64] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true masksegmentation: mask-roi: {roi_name} batch-size: 5 block-statistics-file: erased-block-statistics.h5 """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, random_segmentation, dvid_address, repo_uuid, roi_mask_s5, input_segmentation_name, output_segmentation_name
def setup_sparseblockstats(setup_dvid_repo): dvid_address, repo_uuid = setup_dvid_repo _ = 0 volume_layout = [[_, _, _, _, _, _, _, _], [_, _, _, _, _, 4, _, _], [_, 1, 1, 1, _, _, 1, 1], [_, 1, _, _, _, _, _, _], [_, 1, _, _, _, _, _, _], [_, 1, _, 2, 2, 2, 2, _], [_, _, _, _, _, _, _, _], [_, 3, _, _, _, 3, _, 1]] lowres_volume = np.zeros((4, 8, 8), np.uint64) lowres_volume[:] = volume_layout volume = upsample(lowres_volume, 16) assert volume.shape == (64, 128, 128) input_segmentation_name = 'sparseblockstats-input' create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), volume) # Mask is same as input, but times 10 mask_volume = volume * 10 mask_segmentation_name = 'sparseblockstats-mask' create_labelmap_instance(dvid_address, repo_uuid, mask_segmentation_name) post_labelmap_voxels(dvid_address, repo_uuid, mask_segmentation_name, (0, 0, 0), mask_volume) config_text = textwrap.dedent(f"""\ workflow-name: sparseblockstats cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,64] bounding-box: [[0,0,0], [128,128,64]] mask-input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {mask_segmentation_name} supervoxels: true geometry: message-block-shape: [64,64,64] bounding-box: [[0,0,0], [128,128,64]] sparseblockstats: mask-labels: [20,40] # Avoids the top-left block """) template_dir = tempfile.mkdtemp(suffix="sparseblockstats-template") with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) yaml = YAML() yaml.default_flow_style = False with open(f"{template_dir}/workflow.yaml", 'w') as f: yaml.dump(config, f) return template_dir, config, volume, mask_volume, dvid_address, repo_uuid
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'labelmapcopy-segmentation-input' output_segmentation_name = 'labelmapcopy-segmentation-output' partial_output_segmentation_name = 'labelmapcopy-segmentation-partial-output' max_scale = 3 already_exists = False try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=max_scale) create_labelmap_instance(dvid_address, repo_uuid, partial_output_segmentation_name, max_scale=max_scale) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): already_exists = True expected_vols = {} for scale in range(1 + max_scale): if scale == 0: scaled_vol = random_segmentation else: scaled_vol = downsample(scaled_vol, 2, 'labels-numba') expected_vols[scale] = scaled_vol if not already_exists: scaled_box = round_box([(0, 0, 0), scaled_vol.shape], 64, 'out') aligned_vol = np.zeros(scaled_box[1], np.uint64) overwrite_subvol(aligned_vol, [(0, 0, 0), scaled_vol.shape], scaled_vol) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), aligned_vol, scale=scale) if not already_exists: # Create a 'partial' output volume that is the same (bitwise) as the input except for some blocks. scaled_box = np.array([(0, 0, 0), random_segmentation.shape]) scaled_box[1, -1] = 192 for scale in range(1 + max_scale): scaled_box = round_box(scaled_box // (2**scale), 64, 'out') raw_blocks = fetch_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, scaled_box, scale, supervoxels=True, format='raw-response') post_labelmap_blocks(dvid_address, repo_uuid, partial_output_segmentation_name, [(0, 0, 0)], raw_blocks, scale, is_raw=True) block = np.random.randint(1_000_000, 1_000_010, size=(64, 64, 64), dtype=np.uint64) post_labelmap_voxels(dvid_address, repo_uuid, partial_output_segmentation_name, (0, 128, 64), block, 0, downres=True) partial_vol = fetch_labelmap_voxels(dvid_address, repo_uuid, partial_output_segmentation_name, [(0, 0, 0), random_segmentation.shape], supervoxels=True) template_dir = tempfile.mkdtemp(suffix="labelmapcopy-template") config_text = textwrap.dedent(f"""\ workflow-name: labelmapcopy cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [512,64,64] available-scales: [0,1,2,3] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true labelmapcopy: slab-shape: [512,128,64] dont-overwrite-identical-blocks: true """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, expected_vols, partial_vol, dvid_address, repo_uuid, output_segmentation_name, partial_output_segmentation_name
def init_labelmap_nodes(): # Five supervoxels are each 1x3x3, arranged in a single row like this: # [[[1 1 1 2 2 2 3 3 3 4 4 4 5 5 5] # [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5] # [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]]] supervoxel_vol = np.zeros((1,3,15), np.uint64) supervoxel_vol[:] = (np.arange(15, dtype=np.uint64) // 3).reshape(1,1,15) supervoxel_vol += 1 np.set_printoptions(linewidth=100) #print(supervoxel_vol) # Merge table: Merge them all together id_a = np.array([1, 2, 3, 4], np.uint64) id_b = np.array([2, 3, 4, 5], np.uint64) xa = np.array([2, 5, 8, 11], np.uint32) ya = np.array([1, 1, 1, 1], np.uint32) za = np.array([0, 0, 0, 0], np.uint32) xb = np.array([3, 6, 9, 12], np.uint32) yb = np.array([1, 1, 1, 1], np.uint32) zb = np.array([0, 0, 0, 0], np.uint32) # Weak edge between 3 and 4 score = np.array([0.4, 0.4, 0.8, 0.4], np.float32) merge_table = pd.DataFrame({'id_a': id_a, 'id_b': id_b, 'xa': xa, 'ya': ya, 'za': za, 'xb': xb, 'yb': yb, 'zb': zb, 'score': score}) merge_table = merge_table[['id_a', 'id_b', 'xa', 'ya', 'za', 'xb', 'yb', 'zb', 'score']] merge_table_path = f'{TEST_DATA_DIR}/merge-table.npy' np.save(merge_table_path, merge_table.to_records(index=False)) create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation', max_scale=2) create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation-scratch', max_scale=2) # Expand to 64**3 supervoxel_block = np.zeros((64,64,64), np.uint64) supervoxel_block[:1,:3,:15] = supervoxel_vol post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation', (0,0,0), supervoxel_block) post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation-scratch', (0,0,0), supervoxel_block) post_commit(TEST_SERVER, TEST_REPO, 'supervoxels') # # Create a child node for agglo mappings # r = requests.post(f'http://{TEST_SERVER}/api/node/{TEST_REPO}/newversion', json={'note': 'agglo'}) # r.raise_for_status() # agglo_uuid = r.json["child"] # Merge everything agglo_uuid = TEST_REPO post_merge(TEST_SERVER, agglo_uuid, 'segmentation', 1, [2, 3, 4, 5]) mapping = np.array([[1,1],[2,1],[3,1],[4,1],[5,1]], np.uint64) #mapping = pd.DataFrame(mapping, columns=['sv', 'body']).set_index('sv')['body'] mapping_path = f'{TEST_DATA_DIR}/mapping.npy' np.save(mapping_path, mapping) return merge_table_path, mapping_path, supervoxel_vol