def test_ndrange():
r = ndrange((1, 2, 3), (3, 5, 6), (1, 2, 2))
expected = [(1, 2, 3), (1, 2, 5), (1, 4, 3), (1, 4, 5), (2, 2, 3),
(2, 2, 5), (2, 4, 3), (2, 4, 5)]
assert len(r) == len(expected)
assert list(r) == expected
def test_labelindex(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
# Need an unlocked node to test these posts
uuid = post_branch(dvid_server, dvid_repo, 'test_labelindex',
'test_labelindex')
instance_info = (dvid_server, uuid, 'segmentation-scratch')
# Write some random data
sv = 99
vol = sv * np.random.randint(2, size=(128, 128, 128), dtype=np.uint64)
offset = np.array((64, 64, 64))
# DVID will generate the index.
post_labelmap_voxels(*instance_info, offset, vol)
# Compute labelindex table from scratch
rows = []
for block_coord in ndrange(offset, offset + vol.shape, (64, 64, 64)):
block_coord = np.array(block_coord)
block_box = np.array((block_coord, block_coord + 64))
block = extract_subvol(vol, block_box - offset)
count = (block == sv).sum()
rows.append([*block_coord, sv, count])
index_df = pd.DataFrame(rows, columns=['z', 'y', 'x', 'sv', 'count'])
# Check DVID's generated labelindex table against expected
labelindex_tuple = fetch_labelindex(*instance_info, sv, format='pandas')
assert labelindex_tuple.label == sv
labelindex_tuple.blocks.sort_values(['z', 'y', 'x', 'sv'], inplace=True)
labelindex_tuple.blocks.reset_index(drop=True, inplace=True)
assert (labelindex_tuple.blocks == index_df).all().all()
# Check our protobuf against DVID's
index_tuple = PandasLabelIndex(index_df, sv, 1,
datetime.datetime.now().isoformat(),
'someuser')
labelindex = create_labelindex(index_tuple)
# Since labelindex block entries are not required to be sorted,
# dvid might return them in a different order.
# Hence this comparison function which sorts them first.
def compare_proto_blocks(left, right):
left_blocks = sorted(left.blocks.items())
right_blocks = sorted(right.blocks.items())
return left_blocks == right_blocks
dvid_labelindex = fetch_labelindex(*instance_info, sv, format='protobuf')
assert compare_proto_blocks(labelindex, dvid_labelindex)
# Check post/get roundtrip
post_labelindex(*instance_info, sv, labelindex)
dvid_labelindex = fetch_labelindex(*instance_info, sv, format='protobuf')
assert compare_proto_blocks(labelindex, dvid_labelindex)
def place_test_object(label, corner):
corner = np.array(corner)
label_vol = create_test_object().astype(np.uint64)
label_vol *= label
corners = np.array(list(ndrange((0,0,0), label_vol.shape, (64,64,64))))
# Drop it away from (0,0,0) to make sure the workflow handles arbitrary locations
corners += corner
blockwise_vol = view_as_blocks(label_vol, (64,64,64))
blocks = blockwise_vol.reshape(-1,64,64,64)
# post to dvid
post_labelarray_blocks(dvid_address, repo_uuid, input_segmentation_name, corners, blocks, downres=True, noindexing=False)
return np.array([corner, corner + label_vol.shape])
def test_is_box_coverage_complete():
full_box = [[0, 50, 100], [100, 150, 200]]
assert is_box_coverage_complete([full_box], full_box)
full_box = np.array(full_box)
boxes = [[[0, 50, 100], [50, 150, 200]], [[50, 50, 100], [100, 150, 200]]]
assert is_box_coverage_complete(boxes, full_box)
boxes = [[[0, 50, 100], [50, 150, 200]], [[50, 50, 100], [75, 150, 200]]]
assert not is_box_coverage_complete(boxes, full_box)
box_shape = np.array((50, 50, 50))
boxes = [(box, box + box_shape)
for box in ndrange(full_box[0], full_box[1], box_shape)]
assert is_box_coverage_complete(boxes, full_box)
assert not is_box_coverage_complete(boxes[:-1], full_box)
assert not is_box_coverage_complete(boxes[1:], full_box)
# Try boxes that exceed the bounds of full_box
boxes = [[[0, 0, 100], [50, 150, 200]], [[50, 50, 100], [100, 150, 300]]]
assert is_box_coverage_complete(boxes, full_box)
boxes = [[[0, 0, 100], [50, 150, 200]], [[50, 50, 100], [75, 150, 300]]]
assert not is_box_coverage_complete(boxes, full_box)
def setup_dvid_segmentation_input(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-sparsemeshes-input'
create_labelmap_instance(dvid_address,
repo_uuid,
input_segmentation_name,
max_scale=3)
label_vol = create_test_object().astype(np.uint64)
label_vol *= 100
corners = np.array(list(ndrange((0, 0, 0), label_vol.shape, (64, 64, 64))))
# Drop it away from (0,0,0) to make sure the workflow handles arbitrary locations
corners += 256
blockwise_vol = view_as_blocks(label_vol, (64, 64, 64))
blocks = blockwise_vol.reshape(-1, 64, 64, 64)
# post to dvid
post_labelarray_blocks(dvid_address,
repo_uuid,
input_segmentation_name,
corners,
blocks,
downres=True,
noindexing=False)
template_dir = tempfile.mkdtemp(suffix="sparsemeshes-template")
config_text = textwrap.dedent(f"""\
workflow-name: sparsemeshes
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
geometry:
bounding-box: [[0,0,0], [128,128,128]]
block-width: 64
available-scales: [0,1,2,3]
sparsemeshes:
min-scale: 0
max-analysis-volume: 1e6 # Too small for scale 0, will use scale 1
bodies: [100, 200] # 200 doesn't exist, will fail (see below)
smoothing-iterations: 3
decimation-fraction: 0.1
format: obj
output-directory: meshes
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, dvid_address, repo_uuid
def test_connectedcomponents(setup_connectedcomponents_hdf5_zarr,
disable_auto_retry):
template_dir, _config, input_vol = setup_connectedcomponents_hdf5_zarr
execution_dir, workflow = launch_flow(template_dir, 1)
final_config = workflow.config
output_path = final_config["output"]["zarr"]["path"]
dset_name = final_config["output"]["zarr"]["dataset"]
store = zarr.NestedDirectoryStore(output_path)
f = zarr.open(store=store, mode='r')
output_vol = f[dset_name][:]
assert output_vol.shape == input_vol.shape
final_labels = pd.unique(output_vol.reshape(-1))
# Never change label 0
assert 0 in final_labels
assert ((input_vol == 0) == (output_vol == 0)).all()
# Single-component objects
assert 2 in final_labels
assert 4 in final_labels
assert ((input_vol == 2) == (output_vol == 2)).all()
assert ((input_vol == 4) == (output_vol == 4)).all()
# Split objects
assert 1 not in final_labels
assert 3 not in final_labels
for corner in map(np.array, ndrange((0, 0, 0), (1, 8, 8), (1, 4, 4))):
box = (corner, corner + (1, 4, 4))
input_block = extract_subvol(input_vol, box)
output_block = extract_subvol(output_vol, box)
for orig_label in [1, 3]:
if orig_label in input_block:
positions = (input_block == orig_label)
assert (input_block[positions] != output_block[positions]).all(), \
f"original label {orig_label} was not split!"
assert (output_block[positions] > input_vol.max()).all(), \
f"original label {orig_label} was not split!"
# This block-based assertion is not generally true for all possible input,
# but our test data blocks are set up so that this is a valid check.
# (No block happens to contain more than one final CC that came from the same original label.)
assert (output_block[positions] == output_block[positions][0]).all(), \
f"original label {orig_label} ended up over-segmentated"
# Check CSV output
df = pd.read_csv(f'{execution_dir}/relabeled-objects.csv')
assert len(df.query('orig_label == 0')) == 0
assert len(df.query('orig_label == 1')) == 3
assert len(df.query('orig_label == 2')) == 0
assert len(df.query('orig_label == 3')) == 2
assert len(df.query('orig_label == 4')) == 0
assert not df['final_label'].duplicated().any()
assert (df['final_label'] > input_vol.max()).all()
def test_connectedcomponents_dvid_subset_labels(setup_connectedcomponents_dvid,
disable_auto_retry):
template_dir, _config, input_vol, dvid_address, repo_uuid, output_segmentation_name = setup_connectedcomponents_dvid
execution_dir, workflow = launch_flow(template_dir, 1)
_final_config = workflow.config
output_vol = fetch_labelmap_voxels(dvid_address,
repo_uuid,
output_segmentation_name,
[(0, 0, 0), input_vol.shape],
supervoxels=True)
assert output_vol.shape == input_vol.shape
final_labels = pd.unique(output_vol.reshape(-1))
# Never change label 0
assert 0 in final_labels
assert ((input_vol == 0) == (output_vol == 0)).all()
# Single-component objects
assert 2 in final_labels
assert 4 in final_labels
assert ((input_vol == 2) == (output_vol == 2)).all()
assert ((input_vol == 4) == (output_vol == 4)).all()
# Omitted from analysis; left unsplit
assert 3 in final_labels
assert ((input_vol == 3) == (output_vol == 3)).all()
# Split objects
assert 1 not in final_labels
for corner in map(np.array, ndrange((0, 0, 0), (1, 8, 8), (1, 4, 4))):
box = (corner, corner + 4)
input_block = extract_subvol(input_vol, box)
output_block = extract_subvol(output_vol, box)
for orig_label in [1]:
if orig_label in input_block:
positions = (input_block == orig_label)
assert (input_block[positions] != output_block[positions]).all(), \
f"original label {orig_label} was not split!"
assert (output_block[positions] > input_vol.max()).all(), \
f"original label {orig_label} was not split!"
# This block-based assertion is not generally true for all possible input,
# but our test data blocks are set up so that this is a valid check.
# (No block happens to contain more than one final CC that came from the same original label.)
assert (output_block[positions] == output_block[positions][0]).all(), \
f"original label {orig_label} ended up over-segmentated"
#
# Check CSV output
#
df = pd.read_csv(f'{execution_dir}/relabeled-objects.csv')
assert len(df.query('orig_label == 0')) == 0
assert len(df.query('orig_label == 1')) == 3
assert len(df.query('orig_label == 2')) == 0
assert len(df.query('orig_label == 3')) == 0 # 3 was not touched.
assert len(df.query('orig_label == 4')) == 0
assert not df['final_label'].duplicated().any()
assert (df['final_label'] > input_vol.max()).all()
#
# Check block stats
#
with h5py.File(f'{execution_dir}/block-statistics.h5', 'r') as f:
stats_df = pd.DataFrame(f['stats'][:])
for row in stats_df.itertuples():
corner = np.array((row.z, row.y, row.x))
block_box = np.array([corner, corner + 64])
block = extract_subvol(output_vol, block_box)
assert (block == row.segment_id).sum() == row.count