def setup_dvid_segmentation_input(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
# Start with a low-res map of the test data
# and scale it up 16x to achieve a 128-cube
_ = 0
# 0 1 2 3 4 5 6 7
volume_layout = [[
[_, _, _, _, _, 6, 6, 6], # 0
[_, 1, 1, 2, 2, _, 6, _], # 1
[_, 1, 1, 2, 2, _, _, _], # 2
[_, 1, 1, 2, 8, _, 7, 7], # 3
[_, 1, 1, 2, 8, _, 7, 7], # 4
[_, _, _, _, _, _, _, _], # 5
[_, 4, 4, 4, _, _, _, _], # 6
[_, 3, 3, 3, _, 5, 9, _]
]] # 7
# 0 1 2 3 4 5 6 7
lowres_volume = np.zeros((4, 8, 8), np.uint64)
lowres_volume[:] = volume_layout
volume = upsample(lowres_volume, 16)
assert volume.shape == (64, 128, 128)
input_segmentation_name = 'findadjacencies-input'
create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name)
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name,
(0, 0, 0), volume)
template_dir = tempfile.mkdtemp(suffix="findadjacencies-from-dvid")
config_text = textwrap.dedent(f"""\
workflow-name: findadjacencies
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [128,64,64]
findadjacencies:
output-table: output.csv
find-closest-using-scale: 0
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, volume, dvid_address, repo_uuid
def setup_tsv_input(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-decimatemeshes-input'
test_volume, object_boxes, object_sizes = create_test_segmentation()
create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3)
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0,0,0), test_volume, downres=True, noindexing=False)
tsv_name = 'segmentation-decimatemeshes-tsv'
create_tarsupervoxel_instance(dvid_address, repo_uuid, tsv_name, input_segmentation_name, '.drc')
# Post supervoxel meshes
meshes = Mesh.from_label_volume(test_volume, progress=False)
meshes_data = {f"{label}.drc": mesh.serialize(fmt='drc') for label, mesh in meshes.items()}
post_load(dvid_address, repo_uuid, tsv_name, meshes_data)
# Merge two of the objects (100 and 300)
post_merge(dvid_address, repo_uuid, input_segmentation_name, 100, [300])
object_boxes[100] = box_union(object_boxes[100], object_boxes[300])
del object_boxes[300]
object_sizes[100] += object_sizes[300]
del object_sizes[300]
meshes[100] = Mesh.concatenate_meshes((meshes[100], meshes[300]))
del meshes[300]
return dvid_address, repo_uuid, tsv_name, object_boxes, object_sizes, meshes
def setup_segmentation_input(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-stitchedmesh-input'
create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=3)
def place_test_object(label, corner):
corner = np.array(corner)
label_vol = create_test_object().astype(np.uint64)
label_vol *= label
corners = np.array(list(ndrange((0,0,0), label_vol.shape, (64,64,64))))
# Drop it away from (0,0,0) to make sure the workflow handles arbitrary locations
corners += corner
blockwise_vol = view_as_blocks(label_vol, (64,64,64))
blocks = blockwise_vol.reshape(-1,64,64,64)
# post to dvid
post_labelarray_blocks(dvid_address, repo_uuid, input_segmentation_name, corners, blocks, downres=True, noindexing=False)
return np.array([corner, corner + label_vol.shape])
object_boxes = []
for label, corner in zip([100,200,300], [(256,256,256), (512,512,512), (1024,1024,1024)]):
box = place_test_object(label, corner)
object_boxes.append( box )
return dvid_address, repo_uuid, input_segmentation_name, object_boxes
def test_post_hierarchical_cleaves(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
uuid = post_branch(dvid_server, dvid_repo,
'segmentation-post_hierarchical_cleaves', '')
instance_info = dvid_server, uuid, 'segmentation-post_hierarchical_cleaves'
create_labelmap_instance(*instance_info)
svs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
groups = [1, 1, 2, 2, 3, 3, 3, 3, 3, 4]
svs = np.asarray(svs, np.uint64)
# Post some supervoxels in multiple blocks, just to prove that post_hierarchical_cleaves()
# doesn't assume that the labelindex has the same length as the mapping.
sv_vol = np.zeros((128, 64, 64), np.uint64)
sv_vol[0, 0, :len(svs)] = svs
sv_vol[64, 0, 0:len(svs):2] = svs[::2]
post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol)
post_merge(*instance_info, 1, svs[1:])
group_mapping = pd.Series(index=svs, data=groups)
final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping)
assert (fetch_mapping(*instance_info,
svs) == final_table['body'].values).all()
assert (final_table.drop_duplicates(
['group']) == final_table.drop_duplicates(['group',
'body'])).all().all()
assert (final_table.drop_duplicates(
['body']) == final_table.drop_duplicates(['group',
'body'])).all().all()
# Since the mapping included all supervoxels in the body,
# the last group is left with the original label.
assert final_table.iloc[-1]['body'] == 1
# Now merge them all together and try again, but leave
# two supevoxels out of the groups this time.
merges = set(pd.unique(final_table['body'].values)) - set([1])
post_merge(*instance_info, 1, list(merges))
group_mapping = pd.Series(index=svs[:-2], data=groups[:-2])
final_table = post_hierarchical_cleaves(*instance_info, 1, group_mapping)
assert len(
final_table.query('body == 1')
) == 0, "Did not expect any of the groups to retain the original body ID!"
assert (fetch_mapping(*instance_info,
svs[:-2]) == final_table['body'].values).all()
assert (final_table.drop_duplicates(
['group']) == final_table.drop_duplicates(['group',
'body'])).all().all()
assert (final_table.drop_duplicates(
['body']) == final_table.drop_duplicates(['group',
'body'])).all().all()
assert (fetch_mapping(*instance_info, svs[-2:]) == 1).all()
def test_fetch_sparsevol_coarse_via_labelindex(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
# Create a labelmap volume with 3 blocks.
#
# Supervoxels are arranged like this:
#
# | 1 2 | 3 4 | 5 6 |
#
# After merging [2,3,4,5], bodies will be:
#
# | 1 2 | 2 4 | 5 6 |
#
vol_shape = (64, 64, 256)
sv_vol = np.zeros(vol_shape, np.uint64)
sv_vol[:, :, 0:32] = 1
sv_vol[:, :, 32:64] = 2
sv_vol[:, :, 64:96] = 3
sv_vol[:, :, 96:128] = 4
sv_vol[:, :, 128:160] = 5
sv_vol[:, :, 160:192] = 6
instance_info = dvid_server, dvid_repo, 'segmentation-test-sparsevol-coarse'
create_labelmap_instance(*instance_info)
post_labelmap_voxels(*instance_info, (0, 0, 0), sv_vol)
post_merge(*instance_info, 2, [3, 4, 5])
body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
2,
method='protobuf')
expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2)
assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist())
body_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
2,
method='pandas')
expected_body_svc = fetch_sparsevol_coarse(*instance_info, 2)
assert sorted(body_svc.tolist()) == sorted(expected_body_svc.tolist())
sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
3,
supervoxels=True,
method='protobuf')
expected_sv_svc = fetch_sparsevol_coarse(*instance_info,
3,
supervoxels=True)
assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist())
sv_svc = fetch_sparsevol_coarse_via_labelindex(*instance_info,
3,
supervoxels=True,
method='pandas')
expected_sv_svc = fetch_sparsevol_coarse(*instance_info,
3,
supervoxels=True)
assert sorted(sv_svc.tolist()) == sorted(expected_sv_svc.tolist())
def setup_segmentation_input(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-createmeshes-input'
test_volume, object_boxes, object_sizes = create_test_segmentation()
create_labelmap_instance(dvid_address,
repo_uuid,
input_segmentation_name,
max_scale=3)
post_labelmap_voxels(dvid_address,
repo_uuid,
input_segmentation_name, (0, 0, 0),
test_volume,
downres=True,
noindexing=False)
return dvid_address, repo_uuid, input_segmentation_name, object_boxes, object_sizes
def _create_segmentation_instance(self, volume_config):
"""
Create a segmentation volume in DVID according to the given configuration.
In DVID, segmentation instances are stored via a special instance type, 'labelmap',
which has several features, including built-in multiscale support, supervoxel-to-body
mapping, and sparse retrieval of body locations.
"""
if self.instance_name in fetch_repo_instances(self.server, self.uuid):
logger.info(
f"'{self.instance_name}' already exists, skipping creation")
return
settings = volume_config["dvid"]["creation-settings"]
block_width = volume_config["geometry"]["block-width"]
pyramid_depth = settings["max-scale"]
if pyramid_depth == -1:
pyramid_depth = choose_pyramid_depth(self.bounding_box_zyx, 512)
if settings["compression"] != DvidInstanceCreationSettingsSchema[
"properties"]["compression"]["default"]:
raise RuntimeError(
"Alternative compression methods are not permitted on labelmap instances. "
"Please remove the 'compression' setting from your config.")
if settings["background"] != 0:
raise RuntimeError(
"Labelmap instances do not support custom background values. "
"Please remove 'background' from your config.")
create_labelmap_instance(self.server, self.uuid, self.instance_name,
settings["versioned"], settings["tags"],
block_width, settings["voxel-size"],
settings["voxel-units"],
settings["enable-index"], pyramid_depth)
# Workaround for https://github.com/janelia-flyem/dvid/issues/344
# Write an empty block to the first and last blocks to set the
# bounding-box in DVID now, without any concurrency issues.
empty_block = np.zeros((64, 64, 64), np.uint64)
first_block_start, last_block_stop = round_box(self.bounding_box_zyx,
64, 'out')
last_block_start = last_block_stop - 64
self.write_subvolume(empty_block, first_block_start)
self.write_subvolume(empty_block, last_block_start)
def test_fetch_mutations(labelmap_setup):
dvid_server, dvid_repo, _merge_table_path, _mapping_path, _supervoxel_vol = labelmap_setup
uuid = post_branch(dvid_server, dvid_repo, 'segmentation-fetch_mutations',
'')
instance = 'segmentation-fetch_mutations'
create_labelmap_instance(dvid_server, uuid, instance)
voxels = np.zeros((64, 64, 64), dtype=np.uint64)
voxels[0, 0, :10] = [*range(1, 11)]
post_labelmap_voxels(dvid_server, uuid, instance, (0, 0, 0), voxels)
post_merge(dvid_server, uuid, instance, 1, [2, 3, 4])
post_merge(dvid_server, uuid, instance, 5, [6, 7, 8])
post_commit(dvid_server, uuid, '')
uuid2 = post_newversion(dvid_server, uuid, '')
post_merge(dvid_server, uuid2, instance, 9, [10])
post_merge(dvid_server, uuid2, instance, 1, [5, 10])
mut_df = fetch_mutations(dvid_server,
uuid2,
instance,
dag_filter='leaf-only')
assert len(mut_df) == 2
assert (mut_df['uuid'] == uuid2).all()
assert (mut_df['action'] == 'merge').all()
assert (mut_df['target_body'] == [9, 1]).all()
mut_df = fetch_mutations(dvid_server,
uuid2,
instance,
dag_filter='leaf-and-parents')
assert len(mut_df) == 4
assert (mut_df['uuid'] == [uuid, uuid, uuid2, uuid2]).all()
assert (mut_df['action'] == 'merge').all()
assert (mut_df['target_body'] == [1, 5, 9, 1]).all()
def test_copysegmentation_from_hdf5_to_dvid_output_mask(
setup_hdf5_segmentation_input, disable_auto_retry):
template_dir, config, input_volume, dvid_address, repo_uuid, _output_segmentation_name = setup_hdf5_segmentation_input
# make sure we get a fresh output
output_segmentation_name = 'copyseg-with-output-mask'
config["output"]["dvid"]["segmentation-name"] = output_segmentation_name
output_volume = np.zeros((256, 256, 256), np.uint64)
mask = np.zeros((256, 256, 256), dtype=bool)
masked_labels = [5, 10, 15, 20]
# Start with an output that is striped (but along on block boundaries)
for label, (z_start,
z_stop) in enumerate(zip(range(0, 250, 10), range(10, 260,
10))):
output_volume[z_start:z_stop] = label
if label in masked_labels:
mask[z_start:z_stop] = True
# We expect the output to remain unchanged except in the masked voxels.
expected_vol = np.where(mask, input_volume, output_volume)
# make sure we get a fresh output
output_segmentation_name = 'copyseg-with-output-mask'
config["output"]["dvid"]["segmentation-name"] = output_segmentation_name
config["copysegmentation"]["output-mask-labels"] = masked_labels
max_scale = config["copysegmentation"]["pyramid-depth"]
create_labelmap_instance(dvid_address,
repo_uuid,
output_segmentation_name,
max_scale=max_scale)
post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name,
(0, 0, 0), output_volume)
setup = template_dir, config, expected_vol, dvid_address, repo_uuid, output_segmentation_name
_box_zyx, _expected_vol, _output_vol = _run_to_dvid(setup)
def setup_connectedcomponents_dvid(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
_ = 0
volume_layout = [[_, _, _, _, _, _, _, _], [_, _, _, _, _, 4, _, _],
[_, 1, 1, 1, _, _, 1, 1], [_, 1, _, _, _, _, _, _],
[_, 1, _, _, _, _, _, _], [_, 1, _, 2, 2, 2, 2, _],
[_, _, _, _, _, _, _, _], [_, 3, _, _, _, 3, _, 1]]
lowres_volume = np.zeros((4, 8, 8), np.uint64)
lowres_volume[:] = volume_layout
volume = upsample(lowres_volume, 16)
assert volume.shape == (64, 128, 128)
input_segmentation_name = 'cc-input'
output_segmentation_name = 'cc-output'
create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name)
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name,
(0, 0, 0), volume)
# Post data to the output -- don't leave it empty,
# or we run into 'maxlabel' issues related to dvid issue #284
# https://github.com/janelia-flyem/dvid/issues/284
create_labelmap_instance(dvid_address, repo_uuid, output_segmentation_name)
post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name,
(0, 0, 0), volume)
config_text = textwrap.dedent(f"""\
workflow-name: connectedcomponents
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [64,64,64]
bounding-box: [[0,0,0], [128,128,64]]
output:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {output_segmentation_name}
supervoxels: true
disable-indexing: true
create-if-necessary: true
geometry: {{}} # Auto-set from input
connectedcomponents:
orig-max-label: 4
halo: 1
subset-labels: [1,2,4] # Not 3
compute-block-statistics: true
log-relabeled-objects: true
""")
template_dir = tempfile.mkdtemp(suffix="connectedcomponents-template")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
yaml = YAML()
yaml.default_flow_style = False
with open(f"{template_dir}/workflow.yaml", 'w') as f:
yaml.dump(config, f)
return template_dir, config, volume, dvid_address, repo_uuid, output_segmentation_name
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation):
dvid_address, repo_uuid = setup_dvid_repo
# Since the same UUID is re-used for each test case,
# this counter is a little hack used to make sure the segmentation
# has a unique name each time, so that previous test cases don't
# affect subsequent test casess.
global test_case_counter
test_case_counter += 1
# Normally the MaskSegmentation workflow is used to update
# a segmentation instance from a parent uuid to a child uuid.
# But for this test, we'll simulate that by writing to two
# different instances in the same uuid.
input_segmentation_name = f'masksegmentation-input-{test_case_counter}'
output_segmentation_name = f'masksegmentation-output-from-dvid-{test_case_counter}'
# Agglomerate some supervoxels into bodies
# Choose supervoxels that intersect three Z-planes at 64, 128, 192
svs_1 = np.unique(random_segmentation[64])
svs_2 = np.unique(random_segmentation[128])
svs_3 = np.unique(random_segmentation[192])
for instance in (input_segmentation_name, output_segmentation_name):
create_labelmap_instance(dvid_address,
repo_uuid,
instance,
max_scale=MAX_SCALE)
# Start with an empty mapping (the repo/instance are re-used for each test case)
post_labelmap_voxels(dvid_address,
repo_uuid,
instance, (0, 0, 0),
random_segmentation,
downres=True)
post_merge(dvid_address, repo_uuid, instance, svs_1[0], svs_1[1:])
post_merge(dvid_address, repo_uuid, instance, svs_2[0], svs_2[1:])
post_merge(dvid_address, repo_uuid, instance, svs_3[0], svs_3[1:])
# Create an ROI to test with -- a sphere with scale-5 resolution
shape_s5 = np.array(random_segmentation.shape) // 2**5
midpoint_s5 = shape_s5 / 2
radius = midpoint_s5.min()
coords_s5 = ndindex_array(*shape_s5)
distances = np.sqrt(np.sum((coords_s5 - midpoint_s5)**2, axis=1))
keep = (distances < radius)
coords_s5 = coords_s5[keep, :]
roi_ranges = runlength_encode_to_ranges(coords_s5)
roi_name = 'masksegmentation-test-roi'
try:
create_instance(dvid_address, repo_uuid, roi_name, 'roi')
except HTTPError as ex:
if ex.response is not None and 'already exists' in ex.response.content.decode(
'utf-8'):
pass
post_roi(dvid_address, repo_uuid, roi_name, roi_ranges)
roi_mask_s5 = np.zeros(shape_s5, dtype=bool)
roi_mask_s5[(*coords_s5.transpose(), )] = True
template_dir = tempfile.mkdtemp(
suffix="masksegmentation-from-dvid-template")
config_text = textwrap.dedent(f"""\
workflow-name: masksegmentation
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
# Choose a brick that doesn't cleanly divide into the bounding box
message-block-shape: [192,64,64]
output:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {output_segmentation_name}
supervoxels: true
disable-indexing: true
masksegmentation:
mask-roi: {roi_name}
batch-size: 5
block-statistics-file: erased-block-statistics.h5
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, random_segmentation, dvid_address, repo_uuid, roi_mask_s5, input_segmentation_name, output_segmentation_name
def setup_sparseblockstats(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
_ = 0
volume_layout = [[_, _, _, _, _, _, _, _], [_, _, _, _, _, 4, _, _],
[_, 1, 1, 1, _, _, 1, 1], [_, 1, _, _, _, _, _, _],
[_, 1, _, _, _, _, _, _], [_, 1, _, 2, 2, 2, 2, _],
[_, _, _, _, _, _, _, _], [_, 3, _, _, _, 3, _, 1]]
lowres_volume = np.zeros((4, 8, 8), np.uint64)
lowres_volume[:] = volume_layout
volume = upsample(lowres_volume, 16)
assert volume.shape == (64, 128, 128)
input_segmentation_name = 'sparseblockstats-input'
create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name)
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name,
(0, 0, 0), volume)
# Mask is same as input, but times 10
mask_volume = volume * 10
mask_segmentation_name = 'sparseblockstats-mask'
create_labelmap_instance(dvid_address, repo_uuid, mask_segmentation_name)
post_labelmap_voxels(dvid_address, repo_uuid, mask_segmentation_name,
(0, 0, 0), mask_volume)
config_text = textwrap.dedent(f"""\
workflow-name: sparseblockstats
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [64,64,64]
bounding-box: [[0,0,0], [128,128,64]]
mask-input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {mask_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [64,64,64]
bounding-box: [[0,0,0], [128,128,64]]
sparseblockstats:
mask-labels: [20,40] # Avoids the top-left block
""")
template_dir = tempfile.mkdtemp(suffix="sparseblockstats-template")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
yaml = YAML()
yaml.default_flow_style = False
with open(f"{template_dir}/workflow.yaml", 'w') as f:
yaml.dump(config, f)
return template_dir, config, volume, mask_volume, dvid_address, repo_uuid
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'labelmapcopy-segmentation-input'
output_segmentation_name = 'labelmapcopy-segmentation-output'
partial_output_segmentation_name = 'labelmapcopy-segmentation-partial-output'
max_scale = 3
already_exists = False
try:
create_labelmap_instance(dvid_address,
repo_uuid,
input_segmentation_name,
max_scale=max_scale)
create_labelmap_instance(dvid_address,
repo_uuid,
partial_output_segmentation_name,
max_scale=max_scale)
except HTTPError as ex:
if ex.response is not None and 'already exists' in ex.response.content.decode(
'utf-8'):
already_exists = True
expected_vols = {}
for scale in range(1 + max_scale):
if scale == 0:
scaled_vol = random_segmentation
else:
scaled_vol = downsample(scaled_vol, 2, 'labels-numba')
expected_vols[scale] = scaled_vol
if not already_exists:
scaled_box = round_box([(0, 0, 0), scaled_vol.shape], 64, 'out')
aligned_vol = np.zeros(scaled_box[1], np.uint64)
overwrite_subvol(aligned_vol, [(0, 0, 0), scaled_vol.shape],
scaled_vol)
post_labelmap_voxels(dvid_address,
repo_uuid,
input_segmentation_name, (0, 0, 0),
aligned_vol,
scale=scale)
if not already_exists:
# Create a 'partial' output volume that is the same (bitwise) as the input except for some blocks.
scaled_box = np.array([(0, 0, 0), random_segmentation.shape])
scaled_box[1, -1] = 192
for scale in range(1 + max_scale):
scaled_box = round_box(scaled_box // (2**scale), 64, 'out')
raw_blocks = fetch_labelmap_voxels(dvid_address,
repo_uuid,
input_segmentation_name,
scaled_box,
scale,
supervoxels=True,
format='raw-response')
post_labelmap_blocks(dvid_address,
repo_uuid,
partial_output_segmentation_name, [(0, 0, 0)],
raw_blocks,
scale,
is_raw=True)
block = np.random.randint(1_000_000,
1_000_010,
size=(64, 64, 64),
dtype=np.uint64)
post_labelmap_voxels(dvid_address,
repo_uuid,
partial_output_segmentation_name, (0, 128, 64),
block,
0,
downres=True)
partial_vol = fetch_labelmap_voxels(dvid_address,
repo_uuid,
partial_output_segmentation_name,
[(0, 0, 0), random_segmentation.shape],
supervoxels=True)
template_dir = tempfile.mkdtemp(suffix="labelmapcopy-template")
config_text = textwrap.dedent(f"""\
workflow-name: labelmapcopy
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [512,64,64]
available-scales: [0,1,2,3]
output:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {output_segmentation_name}
supervoxels: true
disable-indexing: true
create-if-necessary: true
labelmapcopy:
slab-shape: [512,128,64]
dont-overwrite-identical-blocks: true
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, expected_vols, partial_vol, dvid_address, repo_uuid, output_segmentation_name, partial_output_segmentation_name
def setup_dvid_segmentation_input(setup_dvid_repo):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-sparsemeshes-input'
create_labelmap_instance(dvid_address,
repo_uuid,
input_segmentation_name,
max_scale=3)
label_vol = create_test_object().astype(np.uint64)
label_vol *= 100
corners = np.array(list(ndrange((0, 0, 0), label_vol.shape, (64, 64, 64))))
# Drop it away from (0,0,0) to make sure the workflow handles arbitrary locations
corners += 256
blockwise_vol = view_as_blocks(label_vol, (64, 64, 64))
blocks = blockwise_vol.reshape(-1, 64, 64, 64)
# post to dvid
post_labelarray_blocks(dvid_address,
repo_uuid,
input_segmentation_name,
corners,
blocks,
downres=True,
noindexing=False)
template_dir = tempfile.mkdtemp(suffix="sparsemeshes-template")
config_text = textwrap.dedent(f"""\
workflow-name: sparsemeshes
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
geometry:
bounding-box: [[0,0,0], [128,128,128]]
block-width: 64
available-scales: [0,1,2,3]
sparsemeshes:
min-scale: 0
max-analysis-volume: 1e6 # Too small for scale 0, will use scale 1
bodies: [100, 200] # 200 doesn't exist, will fail (see below)
smoothing-iterations: 3
decimation-fraction: 0.1
format: obj
output-directory: meshes
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, dvid_address, repo_uuid
def init_labelmap_nodes():
# Five supervoxels are each 1x3x3, arranged in a single row like this:
# [[[1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]
# [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]
# [1 1 1 2 2 2 3 3 3 4 4 4 5 5 5]]]
supervoxel_vol = np.zeros((1,3,15), np.uint64)
supervoxel_vol[:] = (np.arange(15, dtype=np.uint64) // 3).reshape(1,1,15)
supervoxel_vol += 1
np.set_printoptions(linewidth=100)
#print(supervoxel_vol)
# Merge table: Merge them all together
id_a = np.array([1, 2, 3, 4], np.uint64)
id_b = np.array([2, 3, 4, 5], np.uint64)
xa = np.array([2, 5, 8, 11], np.uint32)
ya = np.array([1, 1, 1, 1], np.uint32)
za = np.array([0, 0, 0, 0], np.uint32)
xb = np.array([3, 6, 9, 12], np.uint32)
yb = np.array([1, 1, 1, 1], np.uint32)
zb = np.array([0, 0, 0, 0], np.uint32)
# Weak edge between 3 and 4
score = np.array([0.4, 0.4, 0.8, 0.4], np.float32)
merge_table = pd.DataFrame({'id_a': id_a, 'id_b': id_b,
'xa': xa, 'ya': ya, 'za': za,
'xb': xb, 'yb': yb, 'zb': zb,
'score': score})
merge_table = merge_table[['id_a', 'id_b', 'xa', 'ya', 'za', 'xb', 'yb', 'zb', 'score']]
merge_table_path = f'{TEST_DATA_DIR}/merge-table.npy'
np.save(merge_table_path, merge_table.to_records(index=False))
create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation', max_scale=2)
create_labelmap_instance(TEST_SERVER, TEST_REPO, 'segmentation-scratch', max_scale=2)
# Expand to 64**3
supervoxel_block = np.zeros((64,64,64), np.uint64)
supervoxel_block[:1,:3,:15] = supervoxel_vol
post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation', (0,0,0), supervoxel_block)
post_labelmap_voxels(TEST_SERVER, TEST_REPO, 'segmentation-scratch', (0,0,0), supervoxel_block)
post_commit(TEST_SERVER, TEST_REPO, 'supervoxels')
# # Create a child node for agglo mappings
# r = requests.post(f'http://{TEST_SERVER}/api/node/{TEST_REPO}/newversion', json={'note': 'agglo'})
# r.raise_for_status()
# agglo_uuid = r.json["child"]
# Merge everything
agglo_uuid = TEST_REPO
post_merge(TEST_SERVER, agglo_uuid, 'segmentation', 1, [2, 3, 4, 5])
mapping = np.array([[1,1],[2,1],[3,1],[4,1],[5,1]], np.uint64)
#mapping = pd.DataFrame(mapping, columns=['sv', 'body']).set_index('sv')['body']
mapping_path = f'{TEST_DATA_DIR}/mapping.npy'
np.save(mapping_path, mapping)
return merge_table_path, mapping_path, supervoxel_vol
def main():
# Early exit if we're dumping the config
# (Parse it ourselves to allow omission of otherwise required parameters.)
if ({'--dump-config-template', '-d'} & {*sys.argv}):
dump_default_config(ConfigSchema, sys.stdout, "yaml-with-comments")
sys.exit(0)
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--dump-config-template',
'-d',
action='store_true',
help='Dump out a template yaml config file and exit.')
parser.add_argument('--count',
'-c',
type=int,
help='How many points to generate.')
parser.add_argument('--roi', '-r', help='Limit points to the given ROI.')
parser.add_argument('--body',
'-b',
type=int,
help='Limit points to the given body.')
parser.add_argument(
'--tbars',
'-t',
action='store_true',
help=
'If given, limit points to the tbars of the given body, from the "synapses" instance in the input UUID.'
)
parser.add_argument(
'--skeleton',
'-s',
action='store_true',
help=
'If given, choose the points from the nodes of the skeleton for the given body.'
)
parser.add_argument(
'--generate-points-only',
'-g',
action='store_true',
help=
"If given, generate the points list, but don't write neighborhood segmentations"
)
parser.add_argument(
'--points',
'-p',
help=
'A CSV file containing the points to use instead of automatically generating them.'
)
parser.add_argument(
'--ng-links',
'-n',
action='store_true',
help='If given, include neuroglancer links in the output CSV.'
'Your config should specify the basic neuroglancer view settings; only the "position" will be overwritten in each link.'
)
parser.add_argument('config')
args = parser.parse_args()
configure_default_logging()
config = load_config(args.config, ConfigSchema)
update_ng_settings(config)
input_seg = [*config["input"].values()]
output_seg = [*config["output"].values()]
radius = config["radius"]
random_seed = config["random-seed"]
if config["enforce-minimum-distance"]:
minimum_distance = 2 * radius
else:
minimum_distance = 0
if args.points and any(
[args.count, args.roi, args.body, args.tbars, args.skeleton]):
msg = ("If you're providing your own list of points, you shouldn't"
" specify any of the auto-generation arguments, such as"
" --count --roi --body --tbars")
sys.exit(msg)
if not args.points and not any(
[args.count, args.roi, args.body, args.tbars, args.skeleton]):
msg = "You must provide a list of points or specify how to auto-generate them."
sys.exit(msg)
if args.points:
assert args.points.endswith('.csv')
name, _ = os.path.splitext(args.points)
output_path = name + '-neighborhoods.csv'
points = pd.read_csv(args.points)
else:
points = autogen_points(input_seg, args.count, args.roi, args.body,
args.tbars, args.skeleton, random_seed,
minimum_distance)
uuid = input_seg[1]
output_path = f'neighborhoods-from-{uuid[:6]}'
if not any([args.roi, args.body, args.tbars, args.skeleton]):
output_path += input_seg[2]
else:
if args.roi:
output_path += f'-{args.roi}'
if args.body:
output_path += f'-{args.body}'
if args.tbars:
output_path += '-tbars'
if args.skeleton:
output_path += '-skeleton'
assignment_path = output_path + '.json'
csv_path = output_path + '.csv'
kd = scipy.spatial.cKDTree(points[[*'zyx']].values)
if len(kd.query_pairs(2 * radius)) > 0:
msg = (
"Some of the chosen points are closer to each other than 2x the "
f"configured radius ({radius}). Their neighborhood segments may "
"be mangled in the output.")
logger.warning(msg)
cols = [*'xyz'] + list({*points.columns} - {*'xyz'})
points = points[cols]
if args.generate_points_only:
add_link_col(points, config)
export_as_html(points, csv_path)
if not args.ng_links:
del points['link']
points.to_csv(csv_path,
index=False,
header=True,
quoting=csv.QUOTE_NONE)
sys.exit(0)
try:
input_info = fetch_instance_info(*input_seg)
except Exception:
sys.exit(
f"Couldn't find input segmentation instance: {' / '.join(input_seg)}"
)
try:
fetch_instance_info(*output_seg)
except Exception:
logger.info(
f"Output labelmap not found. Creating new label instance: {' / '.join(output_seg)}"
)
# Copy details from input instance.
# But only provide a single value for each, even though the info provides three.
# Otherwise, DVID kicks back errors like this:
# Setting for 'VoxelUnits' was not a string: [nanometers nanometers nanometers]
settings = {
'block_size': input_info['Extended']['BlockSize'][0],
'voxel_size': input_info['Extended']['VoxelSize'][0],
'voxel_units': input_info['Extended']['VoxelUnits'][0],
'max_scale': input_info['Extended']['MaxDownresLevel']
}
create_labelmap_instance(*output_seg, **settings)
# Also create keyvalue for meshes
create_instance(*output_seg[:2], output_seg[2] + '_meshes', 'keyvalue')
results_df = write_point_neighborhoods(input_seg, output_seg, points,
radius, args.body)
add_link_col(results_df, config)
export_as_html(results_df, csv_path)
write_assignment_file(output_seg, results_df, assignment_path, config)
if not args.ng_links:
del results_df['link']
results_df.to_csv(csv_path,
index=False,
header=True,
quoting=csv.QUOTE_NONE)
def load_roi_label_volume(server,
uuid,
rois_or_neuprint,
box_s5=[None, None],
export_path=None,
export_labelmap=None):
"""
Fetch several ROIs from DVID and combine them into a single label volume or mask.
The label values in the returned volume correspond to the order in which the ROI
names were passed in, starting at label 1.
This function is essentially a convenience function around fetch_combined_roi_volume(),
but in this case it will optionally auto-fetch the ROI list, and auto-export the volume.
Args:
server:
DVID server
uuid:
DVID uuid
rois_or_neuprint:
Either a list of ROIs or a neuprint server from which to obtain the roi list.
box_s5:
If you want to restrict the ROIs to a particular subregion,
you may pass your own bounding box (at scale 5).
Alternatively, you may pass the name of a segmentation
instance from DVID whose bounding box will be used.
export_path:
If you want the ROI volume to be exported to disk,
provide a path name ending with .npy or .h5.
export_labelmap:
If you want the ROI volume to be exported to a DVID labelmap instance,
Provide the instance name, or a tuple of (server, uuid, instance).
Returns:
(roi_vol, roi_box), containing the fetched label volume and the
bounding box it corresponds to, in DVID scale-5 coordinates.
Note:
If you have a list of (full-res) points to extract from the returned volume,
pass a DataFrame with columns ['z','y','x'] to the following function.
If you already downloaded the roi_vol (above), provide it.
Otherwise, leave out those args and it will be fetched first.
Adds columns to the input DF (in-place) for 'roi' (str) and 'roi_label' (int).
>>> from neuclease.dvid import determine_point_rois
>>> determine_point_rois(*master, rois, point_df, roi_vol, roi_box)
"""
if isinstance(box_s5, str):
# Assume that this is a segmentation instance whose dimensions should be used
# Fetch the maximum extents of the segmentation,
# and rescale it for scale-5.
seg_box = fetch_volume_box(server, uuid, box_s5)
box_s5 = round_box(seg_box, (2**5), 'out') // 2**5
box_s5[0] = (0, 0, 0)
if export_labelmap:
assert isinstance(box_s5, np.ndarray)
assert not (box_s5 % 64).any(), \
("If exporting to a labelmap instance, please supply "
"an explicit box and make sure it is block-aligned.")
if isinstance(rois_or_neuprint, (str, neuprint.Client)):
if isinstance(rois_or_neuprint, str):
npclient = neuprint.Client(rois_or_neuprint)
else:
npclient = rois_or_neuprint
# Fetch ROI names from neuprint
q = "MATCH (m: Meta) RETURN m.superLevelRois as rois"
rois = npclient.fetch_custom(q)['rois'].iloc[0]
rois = sorted(rois)
# # Remove '.*ACA' ROIs. Apparently there is some
# # problem with them. (They overlap with other ROIs.)
# rois = [*filter(lambda r: 'ACA' not in r, rois)]
else:
assert isinstance(rois_or_neuprint, collections.abc.Iterable)
rois = rois_or_neuprint
# Fetch each ROI and write it into a volume
with Timer(f"Fetching combined ROI volume for {len(rois)} ROIs", logger):
roi_vol, roi_box, overlap_stats = fetch_combined_roi_volume(
server, uuid, rois, box_zyx=box_s5)
if len(overlap_stats) > 0:
logger.warn(
f"Some ROIs overlap! Here's an incomplete list of overlapping pairs:\n{overlap_stats}"
)
# Export to npy/h5py for external use
if export_path:
with Timer(f"Exporting to {export_path}", logger):
if export_path.endswith('.npy'):
np.save(export_path, roi_vol)
elif export_path.endswith('.h5'):
with h5py.File(export_path, 'w') as f:
f.create_dataset('rois_scale_5', data=roi_vol, chunks=True)
if export_labelmap:
if isinstance(export_labelmap, str):
export_labelmap = (server, uuid, export_labelmap)
assert len(export_labelmap) == 3
with Timer(f"Exporting to {export_labelmap[2]}", logger):
if export_labelmap[2] not in fetch_repo_instances(
server, uuid, 'labelmap'):
create_labelmap_instance(
*export_labelmap, voxel_size=8 * (2**5),
max_scale=6) # FIXME: hard-coded voxel size
# It's really important to use this block shape.
# See https://github.com/janelia-flyem/dvid/issues/342
boxes = boxes_from_grid(roi_box, (256, 256, 256), clipped=True)
for box in tqdm_proxy(boxes):
block = extract_subvol(roi_vol, box - roi_box[0])
post_labelmap_voxels(*export_labelmap,
box[0],
block,
scale=0,
downres=True)
return roi_vol, roi_box, rois
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-input'
output_segmentation_name = 'segmentation-output-from-dvid'
try:
create_labelmap_instance(dvid_address, repo_uuid,
input_segmentation_name)
except HTTPError as ex:
if ex.response is not None and 'already exists' in ex.response.content.decode(
'utf-8'):
pass
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name,
(0, 0, 0), random_segmentation)
# Make sure the output is empty (if it exists)
if output_segmentation_name in fetch_repo_instances(
dvid_address, repo_uuid):
z = np.zeros((256, 256, 256), np.uint64)
post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name,
(0, 0, 0), z, 0, True)
template_dir = tempfile.mkdtemp(
suffix="copysegmentation-from-dvid-template")
config_text = textwrap.dedent(f"""\
workflow-name: copysegmentation
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [64,64,512]
bounding-box: [[0,0,100], [256,200,256]]
adapters: {{}}
output:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {output_segmentation_name}
supervoxels: true
disable-indexing: true
create-if-necessary: true
geometry: {{}} # Auto-set from input
copysegmentation:
pyramid-depth: 1
slab-depth: 128
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_segmentation_name
def setup_dvid_to_zarr(setup_dvid_repo, random_segmentation):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-input'
try:
create_labelmap_instance(dvid_address, repo_uuid,
input_segmentation_name)
except HTTPError as ex:
if ex.response is not None and 'already exists' in ex.response.content.decode(
'utf-8'):
pass
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name,
(0, 0, 0), random_segmentation)
template_dir = tempfile.mkdtemp(
suffix="copysegmentation-from-dvid-template")
output_path = 'output.zarr'
config_text = textwrap.dedent(f"""\
workflow-name: copysegmentation
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [64,64,512]
bounding-box: [[0,0,100], [256,200,256]]
adapters: {{}}
output:
zarr:
path: {output_path}
dataset: s0
create-if-necessary: true
creation-settings:
dtype: uint64
chunk-shape: [64,64,64]
geometry: {{}} # Auto-set from input
copysegmentation:
pyramid-depth: 1
slab-depth: 128
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_path
