コード例 #1
0
def setup_tsv_input(setup_dvid_repo):
    dvid_address, repo_uuid = setup_dvid_repo

    input_segmentation_name = 'segmentation-decimatemeshes-input'
    test_volume, object_boxes, object_sizes = create_test_segmentation()

    create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3)
    post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0,0,0), test_volume, downres=True, noindexing=False)

    tsv_name = 'segmentation-decimatemeshes-tsv'
    create_tarsupervoxel_instance(dvid_address, repo_uuid, tsv_name, input_segmentation_name, '.drc')
 
    # Post supervoxel meshes
    meshes = Mesh.from_label_volume(test_volume, progress=False)
    meshes_data = {f"{label}.drc": mesh.serialize(fmt='drc') for label, mesh in meshes.items()}
    post_load(dvid_address, repo_uuid, tsv_name, meshes_data)
    
    # Merge two of the objects (100 and 300)
    post_merge(dvid_address, repo_uuid, input_segmentation_name, 100, [300])
    object_boxes[100] = box_union(object_boxes[100], object_boxes[300])
    del object_boxes[300]
    
    object_sizes[100] += object_sizes[300]
    del object_sizes[300]
    
    meshes[100] = Mesh.concatenate_meshes((meshes[100], meshes[300]))
    del meshes[300]
    
    return dvid_address, repo_uuid, tsv_name, object_boxes, object_sizes, meshes
コード例 #2
0
ファイル: test_dvid.py プロジェクト: y2mk1ng/neuclease
def test_post_hierarchical_cleaves(labelmap_setup):
    dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup

    uuid = post_branch(dvid_server, dvid_repo,
                       'segmentation-post_hierarchical_cleaves', '')
    instance_info = dvid_server, uuid, 'segmentation-post_hierarchical_cleaves'
    create_labelmap_instance(*instance_info)

    svs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    groups = [1, 1, 2, 2, 3, 3, 3, 3, 3, 4]

    svs = np.asarray(svs, np.uint64)

    # Post some supervoxels in multiple blocks, just to prove that post_hierarchical_cleaves()
    # doesn't assume that the labelindex has the same length as the mapping.
    sv_vol = np.zeros((128, 64, 64), np.uint64)
    sv_vol[0, 0, :len(svs)] = svs
    sv_vol[64, 0, 0:len(svs):2] = svs[::2]

    post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol)

    post_merge(*instance_info, 1, svs[1:])

    group_mapping = pd.Series(index=svs, data=groups)
    final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping)

    assert (fetch_mapping(*instance_info,
                          svs) == final_table['body'].values).all()
    assert (final_table.drop_duplicates(
        ['group']) == final_table.drop_duplicates(['group',
                                                   'body'])).all().all()
    assert (final_table.drop_duplicates(
        ['body']) == final_table.drop_duplicates(['group',
                                                  'body'])).all().all()

    # Since the mapping included all supervoxels in the body,
    # the last group is left with the original label.
    assert final_table.iloc[-1]['body'] == 1

    # Now merge them all together and try again, but leave
    # two supevoxels out of the groups this time.
    merges = set(pd.unique(final_table['body'].values)) - set([1])
    post_merge(*instance_info, 1, list(merges))

    group_mapping = pd.Series(index=svs[:-2], data=groups[:-2])
    final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping)

    assert len(
        final_table.query('body == 1')
    ) == 0, "Did not expect any of the groups to retain the original body ID!"
    assert (fetch_mapping(*instance_info,
                          svs[:-2]) == final_table['body'].values).all()
    assert (final_table.drop_duplicates(
        ['group']) == final_table.drop_duplicates(['group',
                                                   'body'])).all().all()
    assert (final_table.drop_duplicates(
        ['body']) == final_table.drop_duplicates(['group',
                                                  'body'])).all().all()
    assert (fetch_mapping(*instance_info, svs[-2:]) == 1).all()
コード例 #3
0
ファイル: test_dvid.py プロジェクト: y2mk1ng/neuclease
def test_fetch_sparsevol_coarse_via_labelindex(labelmap_setup):
    dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup

    # Create a labelmap volume with 3 blocks.
    #
    # Supervoxels are arranged like this:
    #
    #   | 1 2 | 3 4 | 5 6 |
    #
    # After merging [2,3,4,5], bodies will be:
    #
    #   | 1 2 | 2 4 | 5 6 |
    #
    vol_shape = (64, 64, 256)
    sv_vol = np.zeros(vol_shape, np.uint64)
    sv_vol[:, :, 0:32] = 1
    sv_vol[:, :, 32:64] = 2
    sv_vol[:, :, 64:96] = 3
    sv_vol[:, :, 96:128] = 4
    sv_vol[:, :, 128:160] = 5
    sv_vol[:, :, 160:192] = 6

    instance_info = dvid_server, dvid_repo, 'segmentation-test-sparsevol-coarse'
    create_labelmap_instance(*instance_info)
    post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol)

    post_merge(*instance_info, 2, [3, 4, 5])

    body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
                                                     2,
                                                     method='protobuf')
    expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2)
    assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist())

    body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
                                                     2,
                                                     method='pandas')
    expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2)
    assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist())

    sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
                                                   3,
                                                   supervoxels=True,
                                                   method='protobuf')
    expected_sv_svc = fetch_sparsevol_coarse(*instance_info,
                                             3,
                                             supervoxels=True)
    assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist())

    sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
                                                   3,
                                                   supervoxels=True,
                                                   method='pandas')
    expected_sv_svc = fetch_sparsevol_coarse(*instance_info,
                                             3,
                                             supervoxels=True)
    assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist())
コード例 #4
0
def test_extract_edges_with_large_gap(labelmap_setup):
    """
    If a large gap exists between a supervoxel and the rest of the body,
    we won't find an edge for it, but there should be no crash.
    """
    dvid_server, dvid_repo, merge_table_path, mapping_path, _supervoxel_vol = labelmap_setup
    orig_merge_table = load_merge_table(merge_table_path,
                                        mapping_path,
                                        normalize=True)

    merge_graph = LabelmapMergeGraph(merge_table_path)
    merge_graph.apply_mapping(mapping_path)

    uuid = post_branch(dvid_server, dvid_repo, f'test_extract_edges_large_gap',
                       '')

    # Exercise a corner case:
    # Add a new supervoxel to the body, far away from the others.
    # (No edge will be added for that supervoxel.)
    block_99 = 99 * np.ones((64, 64, 64), np.uint64)
    DVIDNodeService(dvid_server, uuid).put_labels3D('segmentation', block_99,
                                                    (128, 0, 0))
    post_merge(dvid_server, uuid, 'segmentation', 1, [99])

    root_logger = logging.getLogger()
    oldlevel = root_logger.level
    try:
        # Hide warnings for this call; they're intentional.
        logging.getLogger().setLevel(logging.ERROR)
        _mutid, dvid_supervoxels, edges, _scores = merge_graph.extract_edges(
            dvid_server, uuid, 'segmentation', 1)
    finally:
        root_logger.setLevel(oldlevel)

    assert (dvid_supervoxels == [1, 2, 3, 4, 5, 99]).all()
    assert (orig_merge_table[['id_a', 'id_b']].values == edges).all().all(), \
        f"Original merge table doesn't match fetched:\n{orig_merge_table}\n\n{edges}\n"
コード例 #5
0
ファイル: test_dvid.py プロジェクト: janelia-flyem/neuclease
def test_fetch_mutations(labelmap_setup):
    dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup

    uuid = post_branch(dvid_server, dvid_repo, 'segmentation-fetch_mutations',
                       '')

    instance = 'segmentation-fetch_mutations'
    create_labelmap_instance(dvid_server, uuid, instance)

    voxels = np.zeros((64, 64, 64), dtype=np.uint64)
    voxels[0, 0, :10] = [*range(1, 11)]

    post_labelmap_voxels(dvid_server, uuid, instance, (0, 0, 0), voxels)

    post_merge(dvid_server, uuid, instance, 1, [2, 3, 4])
    post_merge(dvid_server, uuid, instance, 5, [6, 7, 8])

    post_commit(dvid_server, uuid, '')
    uuid2 = post_newversion(dvid_server, uuid, '')

    post_merge(dvid_server, uuid2, instance, 9, [10])
    post_merge(dvid_server, uuid2, instance, 1, [5, 10])

    mut_df = fetch_mutations(dvid_server,
                             uuid2,
                             instance,
                             dag_filter='leaf-only')
    assert len(mut_df) == 2
    assert (mut_df['uuid'] == uuid2).all()
    assert (mut_df['action'] == 'merge').all()
    assert (mut_df['target_body'] == [9, 1]).all()

    mut_df = fetch_mutations(dvid_server,
                             uuid2,
                             instance,
                             dag_filter='leaf-and-parents')
    assert len(mut_df) == 4
    assert (mut_df['uuid'] == [uuid, uuid, uuid2, uuid2]).all()
    assert (mut_df['action'] == 'merge').all()
    assert (mut_df['target_body'] == [1, 5, 9, 1]).all()
コード例 #6
0
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation):
    dvid_address, repo_uuid = setup_dvid_repo

    # Since the same UUID is re-used for each test case,
    # this counter is a little hack used to make sure the segmentation
    # has a unique name each time, so that previous test cases don't
    # affect subsequent test casess.
    global test_case_counter
    test_case_counter += 1

    # Normally the MaskSegmentation workflow is used to update
    # a segmentation instance from a parent uuid to a child uuid.
    # But for this test, we'll simulate that by writing to two
    # different instances in the same uuid.
    input_segmentation_name = f'masksegmentation-input-{test_case_counter}'
    output_segmentation_name = f'masksegmentation-output-from-dvid-{test_case_counter}'

    # Agglomerate some supervoxels into bodies
    # Choose supervoxels that intersect three Z-planes at 64, 128, 192
    svs_1 = np.unique(random_segmentation[64])
    svs_2 = np.unique(random_segmentation[128])
    svs_3 = np.unique(random_segmentation[192])

    for instance in (input_segmentation_name, output_segmentation_name):
        create_labelmap_instance(dvid_address,
                                 repo_uuid,
                                 instance,
                                 max_scale=MAX_SCALE)

        # Start with an empty mapping (the repo/instance are re-used for each test case)
        post_labelmap_voxels(dvid_address,
                             repo_uuid,
                             instance, (0, 0, 0),
                             random_segmentation,
                             downres=True)
        post_merge(dvid_address, repo_uuid, instance, svs_1[0], svs_1[1:])
        post_merge(dvid_address, repo_uuid, instance, svs_2[0], svs_2[1:])
        post_merge(dvid_address, repo_uuid, instance, svs_3[0], svs_3[1:])

    # Create an ROI to test with -- a sphere with scale-5 resolution
    shape_s5 = np.array(random_segmentation.shape) // 2**5
    midpoint_s5 = shape_s5 / 2
    radius = midpoint_s5.min()

    coords_s5 = ndindex_array(*shape_s5)
    distances = np.sqrt(np.sum((coords_s5 - midpoint_s5)**2, axis=1))
    keep = (distances < radius)
    coords_s5 = coords_s5[keep, :]

    roi_ranges = runlength_encode_to_ranges(coords_s5)
    roi_name = 'masksegmentation-test-roi'

    try:
        create_instance(dvid_address, repo_uuid, roi_name, 'roi')
    except HTTPError as ex:
        if ex.response is not None and 'already exists' in ex.response.content.decode(
                'utf-8'):
            pass

    post_roi(dvid_address, repo_uuid, roi_name, roi_ranges)

    roi_mask_s5 = np.zeros(shape_s5, dtype=bool)
    roi_mask_s5[(*coords_s5.transpose(), )] = True

    template_dir = tempfile.mkdtemp(
        suffix="masksegmentation-from-dvid-template")

    config_text = textwrap.dedent(f"""\
        workflow-name: masksegmentation
        cluster-type: {CLUSTER_TYPE}
         
        input:
          dvid:
            server: {dvid_address}
            uuid: {repo_uuid}
            segmentation-name: {input_segmentation_name}
            supervoxels: true
           
          geometry:
            # Choose a brick that doesn't cleanly divide into the bounding box
            message-block-shape: [192,64,64]
 
        output:
          dvid:
            server: {dvid_address}
            uuid: {repo_uuid}
            segmentation-name: {output_segmentation_name}
            supervoxels: true
            disable-indexing: true
 
        masksegmentation:
          mask-roi: {roi_name}
          batch-size: 5
          block-statistics-file: erased-block-statistics.h5
    """)

    with open(f"{template_dir}/workflow.yaml", 'w') as f:
        f.write(config_text)

    yaml = YAML()
    with StringIO(config_text) as f:
        config = yaml.load(f)

    return template_dir, config, random_segmentation, dvid_address, repo_uuid, roi_mask_s5, input_segmentation_name, output_segmentation_name
コード例 #7
0
ファイル: conftest.py プロジェクト: stuarteberg/pydvid
def init_labelmap_nodes():
    # Five supervoxels are each 1x3x3, arranged in a single row like this:
    # [[[1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]
    #   [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]
    #   [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]]]
    supervoxel_vol = np.zeros((1,3,15), np.uint64)
    supervoxel_vol[:] = (np.arange(15, dtype=np.uint64) // 3).reshape(1,1,15)
    supervoxel_vol += 1
    np.set_printoptions(linewidth=100)
    #print(supervoxel_vol)

    # Merge table: Merge them all together
    id_a = np.array([1, 2, 3, 4], np.uint64)
    id_b = np.array([2, 3, 4, 5], np.uint64)

    xa = np.array([2, 5, 8, 11], np.uint32)
    ya = np.array([1, 1, 1, 1], np.uint32)
    za = np.array([0, 0, 0, 0], np.uint32)

    xb = np.array([3, 6, 9, 12], np.uint32)
    yb = np.array([1, 1, 1, 1], np.uint32)
    zb = np.array([0, 0, 0, 0], np.uint32)

    # Weak edge between 3 and 4
    score = np.array([0.4, 0.4, 0.8, 0.4], np.float32)

    merge_table = pd.DataFrame({'id_a': id_a, 'id_b': id_b,
                                'xa': xa, 'ya': ya, 'za': za,
                                'xb': xb, 'yb': yb, 'zb': zb,
                                'score': score})
    merge_table = merge_table[['id_a', 'id_b', 'xa', 'ya', 'za', 'xb', 'yb', 'zb', 'score']]

    merge_table_path = f'{TEST_DATA_DIR}/merge-table.npy'
    np.save(merge_table_path, merge_table.to_records(index=False))
    
    create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation', max_scale=2)
    create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation-scratch', max_scale=2)

    # Expand to 64**3
    supervoxel_block = np.zeros((64,64,64), np.uint64)
    supervoxel_block[:1,:3,:15] = supervoxel_vol
    post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation', (0,0,0), supervoxel_block)
    post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation-scratch', (0,0,0), supervoxel_block)

    post_commit(TEST_SERVER, TEST_REPO, 'supervoxels')

#     # Create a child node for agglo mappings
#     r = requests.post(f'http://{TEST_SERVER}/api/node/{TEST_REPO}/newversion', json={'note': 'agglo'})
#     r.raise_for_status()
#     agglo_uuid = r.json["child"]

    # Merge everything
    agglo_uuid = TEST_REPO
    post_merge(TEST_SERVER, agglo_uuid, 'segmentation', 1, [2, 3, 4, 5])

    mapping = np.array([[1,1],[2,1],[3,1],[4,1],[5,1]], np.uint64)
    #mapping = pd.DataFrame(mapping, columns=['sv', 'body']).set_index('sv')['body']
    
    mapping_path = f'{TEST_DATA_DIR}/mapping.npy'
    np.save(mapping_path, mapping)

    return merge_table_path, mapping_path, supervoxel_vol