def test_maxlabel_and_friends(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
# Need an unlocked node to test these posts
uuid = post_branch(dvid_server, dvid_repo, 'test_maxlabel_and_friends',
'test_maxlabel_and_friends')
instance_info = (dvid_server, uuid, 'segmentation-scratch')
max_label = fetch_maxlabel(*instance_info)
next_label = fetch_nextlabel(*instance_info)
assert max_label + 1 == next_label
start, end = post_nextlabel(*instance_info, 5)
assert start == max_label + 1
assert end == start + 5 - 1
max_label = fetch_maxlabel(*instance_info)
next_label = fetch_nextlabel(*instance_info)
assert next_label == max_label + 1 == end + 1
new_max = next_label + 10
post_maxlabel(*instance_info, new_max)
max_label = fetch_maxlabel(*instance_info)
assert max_label == new_max
next_label = fetch_nextlabel(*instance_info)
assert max_label + 1 == next_label
def _setup_test_append_edges_for_split(labelmap_setup, branch_name):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, supervoxel_vol = labelmap_setup
uuid = post_branch(dvid_server, dvid_repo, branch_name, '')
# Split supervoxel 3 (see conftest.init_labelmap_nodes)
# Remove the first column of pixels from it.
# supervoxel 3 starts in column 6
assert (supervoxel_vol == 3).nonzero()[2][0] == 6
rle = [
[6, 0, 0, 1], # x,y,z,runlength
[6, 1, 0, 1],
[6, 2, 0, 1]
]
rle = np.array(rle, np.uint32)
header = np.array([0, 3, 0, 0], np.uint8)
voxels = np.array([0], np.uint32)
num_spans = np.array([len(rle)], np.uint32)
payload = bytes(header) + bytes(voxels) + bytes(num_spans) + bytes(rle)
split_sv, remain_sv = post_split_supervoxel(dvid_server, uuid,
'segmentation', 3, payload)
return uuid, split_sv, remain_sv
def test_post_hierarchical_cleaves(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
uuid = post_branch(dvid_server, dvid_repo,
'segmentation-post_hierarchical_cleaves', '')
instance_info = dvid_server, uuid, 'segmentation-post_hierarchical_cleaves'
create_labelmap_instance(*instance_info)
svs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
groups = [1, 1, 2, 2, 3, 3, 3, 3, 3, 4]
svs = np.asarray(svs, np.uint64)
# Post some supervoxels in multiple blocks, just to prove that post_hierarchical_cleaves()
# doesn't assume that the labelindex has the same length as the mapping.
sv_vol = np.zeros((128, 64, 64), np.uint64)
sv_vol[0, 0, :len(svs)] = svs
sv_vol[64, 0, 0:len(svs):2] = svs[::2]
post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol)
post_merge(*instance_info, 1, svs[1:])
group_mapping = pd.Series(index=svs, data=groups)
final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping)
assert (fetch_mapping(*instance_info,
svs) == final_table['body'].values).all()
assert (final_table.drop_duplicates(
['group']) == final_table.drop_duplicates(['group',
'body'])).all().all()
assert (final_table.drop_duplicates(
['body']) == final_table.drop_duplicates(['group',
'body'])).all().all()
# Since the mapping included all supervoxels in the body,
# the last group is left with the original label.
assert final_table.iloc[-1]['body'] == 1
# Now merge them all together and try again, but leave
# two supevoxels out of the groups this time.
merges = set(pd.unique(final_table['body'].values)) - set([1])
post_merge(*instance_info, 1, list(merges))
group_mapping = pd.Series(index=svs[:-2], data=groups[:-2])
final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping)
assert len(
final_table.query('body == 1')
) == 0, "Did not expect any of the groups to retain the original body ID!"
assert (fetch_mapping(*instance_info,
svs[:-2]) == final_table['body'].values).all()
assert (final_table.drop_duplicates(
['group']) == final_table.drop_duplicates(['group',
'body'])).all().all()
assert (final_table.drop_duplicates(
['body']) == final_table.drop_duplicates(['group',
'body'])).all().all()
assert (fetch_mapping(*instance_info, svs[-2:]) == 1).all()
def test_labelindex(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
# Need an unlocked node to test these posts
uuid = post_branch(dvid_server, dvid_repo, 'test_labelindex',
'test_labelindex')
instance_info = (dvid_server, uuid, 'segmentation-scratch')
# Write some random data
sv = 99
vol = sv * np.random.randint(2, size=(128, 128, 128), dtype=np.uint64)
offset = np.array((64, 64, 64))
# DVID will generate the index.
post_labelmap_voxels(*instance_info, offset, vol)
# Compute labelindex table from scratch
rows = []
for block_coord in ndrange(offset, offset + vol.shape, (64, 64, 64)):
block_coord = np.array(block_coord)
block_box = np.array((block_coord, block_coord + 64))
block = extract_subvol(vol, block_box - offset)
count = (block == sv).sum()
rows.append([*block_coord, sv, count])
index_df = pd.DataFrame(rows, columns=['z', 'y', 'x', 'sv', 'count'])
# Check DVID's generated labelindex table against expected
labelindex_tuple = fetch_labelindex(*instance_info, sv, format='pandas')
assert labelindex_tuple.label == sv
labelindex_tuple.blocks.sort_values(['z', 'y', 'x', 'sv'], inplace=True)
labelindex_tuple.blocks.reset_index(drop=True, inplace=True)
assert (labelindex_tuple.blocks == index_df).all().all()
# Check our protobuf against DVID's
index_tuple = PandasLabelIndex(index_df, sv, 1,
datetime.datetime.now().isoformat(),
'someuser')
labelindex = create_labelindex(index_tuple)
# Since labelindex block entries are not required to be sorted,
# dvid might return them in a different order.
# Hence this comparison function which sorts them first.
def compare_proto_blocks(left, right):
left_blocks = sorted(left.blocks.items())
right_blocks = sorted(right.blocks.items())
return left_blocks == right_blocks
dvid_labelindex = fetch_labelindex(*instance_info, sv, format='protobuf')
assert compare_proto_blocks(labelindex, dvid_labelindex)
# Check post/get roundtrip
post_labelindex(*instance_info, sv, labelindex)
dvid_labelindex = fetch_labelindex(*instance_info, sv, format='protobuf')
assert compare_proto_blocks(labelindex, dvid_labelindex)
def test_fetch_labelindices(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
# Need an unlocked node to test these posts
uuid = post_branch(dvid_server, dvid_repo, 'test_labelindices',
'test_labelindices')
instance_info = (dvid_server, uuid, 'segmentation-scratch')
# Write some random data
vol = np.random.randint(1, 10, size=(128, 128, 128), dtype=np.uint64)
offset = np.array((64, 64, 64))
# DVID will generate the index.
post_labelmap_voxels(*instance_info, offset, vol)
labelindices = fetch_labelindices(*instance_info, list(range(1, 10)))
for sv, li in zip(range(1, 10), labelindices.indices):
# This function is already tested elsewhere, so we'll use it as a reference
li2 = fetch_labelindex(*instance_info, sv)
assert li == li2
labelindices = fetch_labelindices(*instance_info,
list(range(1, 10)),
format='list-of-protobuf')
for sv, li in zip(range(1, 10), labelindices):
# This function is already tested elsewhere, so we'll use it as a reference
li2 = fetch_labelindex(*instance_info, sv)
assert li == li2
labelindices = fetch_labelindices(*instance_info,
list(range(1, 10)),
format='pandas')
for sv, li in zip(range(1, 10), labelindices):
# This function is already tested elsewhere, so we'll use it as a reference
li2 = fetch_labelindex(*instance_info, sv, format='pandas')
li_df = li.blocks.sort_values(['z', 'y', 'x']).reset_index(drop=True)
li2_df = li2.blocks.sort_values(['z', 'y', 'x']).reset_index(drop=True)
assert (li_df == li2_df).all().all()
# Test the copy function (just do a round-trip -- hopefully I didn't swap src and dest anywhere...)
copy_labelindices(instance_info,
instance_info,
list(range(1, 10)),
batch_size=2)
copy_labelindices(instance_info,
instance_info,
list(range(1, 10)),
batch_size=2,
processes=2)
def test_fetch_mutations(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
uuid = post_branch(dvid_server, dvid_repo, 'segmentation-fetch_mutations',
'')
instance = 'segmentation-fetch_mutations'
create_labelmap_instance(dvid_server, uuid, instance)
voxels = np.zeros((64, 64, 64), dtype=np.uint64)
voxels[0, 0, :10] = [*range(1, 11)]
post_labelmap_voxels(dvid_server, uuid, instance, (0, 0, 0), voxels)
post_merge(dvid_server, uuid, instance, 1, [2, 3, 4])
post_merge(dvid_server, uuid, instance, 5, [6, 7, 8])
post_commit(dvid_server, uuid, '')
uuid2 = post_newversion(dvid_server, uuid, '')
post_merge(dvid_server, uuid2, instance, 9, [10])
post_merge(dvid_server, uuid2, instance, 1, [5, 10])
mut_df = fetch_mutations(dvid_server,
uuid2,
instance,
dag_filter='leaf-only')
assert len(mut_df) == 2
assert (mut_df['uuid'] == uuid2).all()
assert (mut_df['action'] == 'merge').all()
assert (mut_df['target_body'] == [9, 1]).all()
mut_df = fetch_mutations(dvid_server,
uuid2,
instance,
dag_filter='leaf-and-parents')
assert len(mut_df) == 4
assert (mut_df['uuid'] == [uuid, uuid, uuid2, uuid2]).all()
assert (mut_df['action'] == 'merge').all()
assert (mut_df['target_body'] == [1, 5, 9, 1]).all()
def test_extract_edges_with_large_gap(labelmap_setup):
"""
If a large gap exists between a supervoxel and the rest of the body,
we won't find an edge for it, but there should be no crash.
"""
dvid_server, dvid_repo, merge_table_path, mapping_path, _supervoxel_vol = labelmap_setup
orig_merge_table = load_merge_table(merge_table_path,
mapping_path,
normalize=True)
merge_graph = LabelmapMergeGraph(merge_table_path)
merge_graph.apply_mapping(mapping_path)
uuid = post_branch(dvid_server, dvid_repo, f'test_extract_edges_large_gap',
'')
# Exercise a corner case:
# Add a new supervoxel to the body, far away from the others.
# (No edge will be added for that supervoxel.)
block_99 = 99 * np.ones((64, 64, 64), np.uint64)
DVIDNodeService(dvid_server, uuid).put_labels3D('segmentation', block_99,
(128, 0, 0))
post_merge(dvid_server, uuid, 'segmentation', 1, [99])
root_logger = logging.getLogger()
oldlevel = root_logger.level
try:
# Hide warnings for this call; they're intentional.
logging.getLogger().setLevel(logging.ERROR)
_mutid, dvid_supervoxels, edges, _scores = merge_graph.extract_edges(
dvid_server, uuid, 'segmentation', 1)
finally:
root_logger.setLevel(oldlevel)
assert (dvid_supervoxels == [1, 2, 3, 4, 5, 99]).all()
assert (orig_merge_table[['id_a', 'id_b']].values == edges).all().all(), \
f"Original merge table doesn't match fetched:\n{orig_merge_table}\n\n{edges}\n"
def _test_extract_edges(labelmap_setup, force_dirty_mapping):
"""
Implementation for testing extract_edges(), starting either with a "clean" mapping
(in which the body column is already correct beforehand),
or a "dirty" mapping (in which the body column is not correct beforehand).
"""
dvid_server, dvid_repo, merge_table_path, mapping_path, _supervoxel_vol = labelmap_setup
instance_info = DvidInstanceInfo(dvid_server, dvid_repo, 'segmentation')
orig_merge_table = load_merge_table(merge_table_path,
mapping_path,
normalize=True)
merge_graph = LabelmapMergeGraph(merge_table_path)
merge_graph.apply_mapping(mapping_path)
if force_dirty_mapping:
# A little white-box manipulation here to ensure that the mapping is dirty
merge_graph.merge_table_df['body'] = np.uint64(0)
merge_graph.mapping[:] = np.uint64(0)
# First test: If nothing has changed in DVID, we get all rows.
# We should be able to repeat this with the same results
# (Make sure the cache is repopulated correctly.)
_mutid, dvid_supervoxels, edges, _scores = merge_graph.extract_edges(
*instance_info, 1)
assert (dvid_supervoxels == [1, 2, 3, 4, 5]).all()
assert (orig_merge_table[['id_a', 'id_b']].values == edges).all().all(), \
f"Original merge table doesn't match fetched:\n{orig_merge_table}\n\n{edges}\n"
# Now change the mapping in DVID and verify it is reflected in the extracted rows.
# For this test, we'll cleave supervoxels [4,5] from the rest of the body.
uuid = post_branch(dvid_server, dvid_repo,
f'extract-rows-test-{force_dirty_mapping}', '')
cleaved_body = post_cleave(dvid_server, uuid, 'segmentation', 1, [4, 5])
cleaved_mutid = fetch_mutation_id(dvid_server, uuid, 'segmentation', 1)
if force_dirty_mapping:
# A little white-box manipulation here to ensure that the mapping is dirty
merge_graph.mapping.loc[2] = 0
merge_graph.merge_table_df['body'].values[0:2] = np.uint64(0)
mutid, dvid_supervoxels, edges, _scores = merge_graph.extract_edges(
dvid_server, uuid, 'segmentation', 1)
assert (dvid_supervoxels == [1, 2, 3]).all()
_cleaved_svs = set([4, 5])
assert (edges == orig_merge_table[[
'id_a', 'id_b'
]].query('id_a not in @_cleaved_svs and id_b not in @_cleaved_svs')
).all().all()
assert mutid == cleaved_mutid, "Expected cached mutation ID to match DVID"
cleaved_mutid = fetch_mutation_id(dvid_server, uuid, 'segmentation',
cleaved_body)
# Check the other body
mutid, dvid_supervoxels, edges, _scores = merge_graph.extract_edges(
dvid_server, uuid, 'segmentation', cleaved_body)
assert (edges == orig_merge_table[[
'id_a', 'id_b'
]].query('id_a in @_cleaved_svs and id_b in @_cleaved_svs')).all().all()
assert mutid == cleaved_mutid, "Expected cached mutation ID to match DVID"
