def test_dvid_volume_service_grayscale(setup_dvid_repo, disable_auto_retry):
server, uuid = setup_dvid_repo
instance_name = 'test-dvs-grayscale'
volume = np.random.randint(100, size=(256, 192, 128), dtype=np.uint8)
max_scale = 2
voxel_dimensions = [4.0, 4.0, 32.0]
config_text = textwrap.dedent(f"""\
dvid:
server: {server}
uuid: {uuid}
grayscale-name: {instance_name}
create-if-necessary: true
creation-settings:
max-scale: {max_scale}
voxel-size: {voxel_dimensions}
geometry:
bounding-box: [[0,0,0], {list(volume.shape[::-1])}]
""")
yaml = YAML()
with StringIO(config_text) as f:
volume_config = yaml.load(f)
assert instance_name not in fetch_repo_instances(server, uuid)
service = VolumeService.create_from_config(volume_config)
repo_instances = fetch_repo_instances(server, uuid)
info = fetch_instance_info(server, uuid, instance_name)
assert info["Extended"]["VoxelSize"] == voxel_dimensions
scaled_volumes = {}
for scale in range(max_scale + 1):
if scale == 0:
assert instance_name in repo_instances
assert repo_instances[instance_name] == 'uint8blk'
else:
assert f"{instance_name}_{scale}" in repo_instances
assert repo_instances[f"{instance_name}_{scale}"] == 'uint8blk'
vol = downsample(volume, 2**scale,
'label') # label downsampling is easier to test with
aligned_shape = (np.ceil(np.array(vol.shape) / 64) * 64).astype(int)
aligned_vol = np.zeros(aligned_shape, np.uint8)
overwrite_subvol(aligned_vol, [(0, 0, 0), aligned_shape], aligned_vol)
service.write_subvolume(aligned_vol, (0, 0, 0), scale)
scaled_volumes[scale] = aligned_vol
box = np.array([[40, 80, 40], [240, 160, 100]])
for scale in range(max_scale + 1):
scaled_box = box // 2**scale
vol = service.get_subvolume(scaled_box, scale)
assert (vol == extract_subvol(scaled_volumes[scale], scaled_box)).all()
def test_dvid_volume_service_branch(setup_dvid_repo, disable_auto_retry):
server, uuid = setup_dvid_repo
instance_name = 'test-dvs-branch'
volume = np.random.randint(100, size=(256, 192, 128), dtype=np.uint8)
max_scale = 2
voxel_dimensions = [4.0, 4.0, 32.0]
config_text = textwrap.dedent(f"""\
dvid:
server: {server}
uuid: master
grayscale-name: {instance_name}
create-if-necessary: true
creation-settings:
max-scale: {max_scale}
voxel-size: {voxel_dimensions}
geometry:
bounding-box: [[0,0,0], {list(volume.shape[::-1])}]
""")
yaml = YAML()
with StringIO(config_text) as f:
volume_config = yaml.load(f)
assert instance_name not in fetch_repo_instances(server, uuid)
service = VolumeService.create_from_config(volume_config)
assert service.uuid == uuid
def main():
# Create the destination instance if necessary.
dst_instances = fetch_repo_instances(*dst_node, 'annotation')
if dst_syn not in dst_instances:
logger.info(f"Creating instance '{dst_syn}'")
create_instance(*dst_node, dst_syn, 'annotation')
# Check to see if the sync already exists; add it if necessary
syn_info = fetch_instance_info(*dst_node, dst_syn)
if len(syn_info["Base"]["Syncs"]) == 0:
logger.info(f"Adding a sync to '{dst_syn}' from '{dst_seg}'")
post_sync(*dst_node, dst_syn, [dst_seg])
elif syn_info["Base"]["Syncs"][0] != dst_seg:
other_seg = syn_info["Base"]["Syncs"][0]
raise RuntimeError(
f"Can't create a sync to '{dst_seg}'. "
f"Your instance is already sync'd to a different segmentation: {other_seg}"
)
# Fetch segmentation extents
bounding_box_zyx = fetch_volume_box(*src_node, src_seg).tolist()
# Break into block-aligned chunks (boxes) that are long in the X direction
# (optimal access pattern for dvid read/write)
boxes = boxes_from_grid(bounding_box_zyx, (256, 256, 6400), clipped=True)
# Use a process pool to copy the chunks in parallel.
compute_parallel(copy_syn_blocks,
boxes,
processes=PROCESSES,
ordered=False)
def _create_grayscale_instances(self, volume_config):
"""
Create the grayscale instance(s) in DVID for the given volume configuration.
In DVID, grayscale data is stored in instances of type 'uint8blk',
which has no concept of scale.
Instead, multi-scale volumes are represented by creating multiple instances,
with the scale indicated by a suffix (except for scale 0).
For example:
- grayscale # scale 0
- grayscale_1
- grayscale_2
- grayscale_3
- ...
"""
settings = volume_config["dvid"]["creation-settings"]
block_width = volume_config["geometry"]["block-width"]
pyramid_depth = settings["max-scale"]
if pyramid_depth == -1:
pyramid_depth = choose_pyramid_depth(self.bounding_box_zyx, 512)
repo_instances = fetch_repo_instances(self.server, self.uuid)
# Bottom level of pyramid is listed as neuroglancer-compatible
extend_list_value(self.server, self.uuid, '.meta', 'neuroglancer',
[self.instance_name])
for scale in range(pyramid_depth + 1):
scaled_output_box_zyx = round_box(self.bounding_box_zyx, 2**scale,
'out') // 2**scale
if scale == 0:
scaled_instance_name = self.instance_name
else:
scaled_instance_name = f"{self.instance_name}_{scale}"
if scaled_instance_name in repo_instances:
logger.info(
f"'{scaled_instance_name}' already exists, skipping creation"
)
else:
create_voxel_instance(
self.server, self.uuid, scaled_instance_name, 'uint8blk',
settings["versioned"], settings["compression"],
settings["tags"], block_width, settings["voxel-size"],
settings["voxel-units"], settings["background"])
update_extents(self.server, self.uuid, scaled_instance_name,
scaled_output_box_zyx)
# Higher-levels of the pyramid should not appear in the DVID console.
extend_list_value(self.server, self.uuid, '.meta', 'restrictions',
[scaled_instance_name])
def _check_instance(server, uuid, instance):
"""
Verify that the instance is a valid destination for the LabelIndices we're about to ingest.
"""
if fetch_repo_instances(server, uuid)[instance] != 'labelmap':
raise RuntimeError(f"DVID instance is not a labelmap: {instance}")
info = fetch_instance_info(server, uuid, instance)
bz, by, bx = info["Extended"]["BlockSize"]
assert bz == by == bx == 64, \
"The code below makes the hard-coded assumption that the instance block width is 64."
def setup_hdf5_segmentation_input(setup_dvid_repo, write_hdf5_volume):
volume_path, random_segmentation = write_hdf5_volume
dvid_address, repo_uuid = setup_dvid_repo
template_dir = tempfile.mkdtemp(
suffix="copysegmentation-from-hdf5-template")
output_segmentation_name = 'segmentation-output-from-hdf5'
# Make sure the output is empty (if it exists)
if output_segmentation_name in fetch_repo_instances(
dvid_address, repo_uuid):
z = np.zeros((256, 256, 256), np.uint64)
post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name,
(0, 0, 0), z, 0, True)
config_text = textwrap.dedent(f"""\
workflow-name: copysegmentation
cluster-type: {CLUSTER_TYPE}
input:
hdf5:
path: {volume_path}
dataset: volume
geometry:
message-block-shape: [64,64,256] # note: this is weird because normally we stripe in the X direction...
bounding-box: [[0,0,100], [256,200,256]]
output:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {output_segmentation_name}
supervoxels: true
disable-indexing: true
create-if-necessary: true
geometry: {{}} # Auto-set from input
copysegmentation:
pyramid-depth: 1
slab-depth: 128
download-pre-downsampled: false
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_segmentation_name
def _create_segmentation_instance(self, volume_config):
"""
Create a segmentation volume in DVID according to the given configuration.
In DVID, segmentation instances are stored via a special instance type, 'labelmap',
which has several features, including built-in multiscale support, supervoxel-to-body
mapping, and sparse retrieval of body locations.
"""
if self.instance_name in fetch_repo_instances(self.server, self.uuid):
logger.info(
f"'{self.instance_name}' already exists, skipping creation")
return
settings = volume_config["dvid"]["creation-settings"]
block_width = volume_config["geometry"]["block-width"]
pyramid_depth = settings["max-scale"]
if pyramid_depth == -1:
pyramid_depth = choose_pyramid_depth(self.bounding_box_zyx, 512)
if settings["compression"] != DvidInstanceCreationSettingsSchema[
"properties"]["compression"]["default"]:
raise RuntimeError(
"Alternative compression methods are not permitted on labelmap instances. "
"Please remove the 'compression' setting from your config.")
if settings["background"] != 0:
raise RuntimeError(
"Labelmap instances do not support custom background values. "
"Please remove 'background' from your config.")
create_labelmap_instance(self.server, self.uuid, self.instance_name,
settings["versioned"], settings["tags"],
block_width, settings["voxel-size"],
settings["voxel-units"],
settings["enable-index"], pyramid_depth)
# Workaround for https://github.com/janelia-flyem/dvid/issues/344
# Write an empty block to the first and last blocks to set the
# bounding-box in DVID now, without any concurrency issues.
empty_block = np.zeros((64, 64, 64), np.uint64)
first_block_start, last_block_stop = round_box(self.bounding_box_zyx,
64, 'out')
last_block_start = last_block_stop - 64
self.write_subvolume(empty_block, first_block_start)
self.write_subvolume(empty_block, last_block_start)
def ingest_mapping(server,
uuid,
instance,
mutid,
segment_to_body_df,
subset_labels=None,
batch_size=100_000):
"""
Ingest the forward-map (supervoxel-to-body) into DVID via the .../mappings endpoint
Args:
server, uuid, instance_name:
DVID instance info
mutid:
The mutation ID to use for all mappings
segment_to_body_df:
DataFrame. Must have columns ['segment_id', 'body_id']
batch_size:
Approximately how many mapping pairs to pack into a single REST call.
"""
assert list(segment_to_body_df.columns) == AGGLO_MAP_COLUMNS
if fetch_repo_instances(server, uuid)[instance] != 'labelmap':
raise RuntimeError(f"DVID instance is not a labelmap: {instance}")
segment_to_body_df.sort_values(['body_id', 'segment_id'], inplace=True)
if subset_labels is not None:
segment_to_body_df = segment_to_body_df.query(
'body_id in @subset_labels')
if len(segment_to_body_df) == 0:
raise RuntimeError(
"None of the selected bodies have any mappings to post!")
mappings = segment_to_body_df.set_index('segment_id')['body_id']
post_mappings(server,
uuid,
instance,
mappings,
mutid,
batch_size=batch_size)
def copy_synapses(src_loc, dst_loc, processes):
"""
See caveats in the module docstring above.
"""
src_loc = Location(*src_loc)
dst_loc = Location(*dst_loc)
# Create the destination instance if necessary.
dst_instances = fetch_repo_instances(*dst_loc[:2], 'annotation')
if dst_loc.syn_instance not in dst_instances:
logger.info(f"Creating instance '{dst_loc.syn_instance}'")
create_instance(*dst_loc, 'annotation')
# Check to see if the sync already exists; add it if necessary
syn_info = fetch_instance_info(*dst_loc[:3])
if len(syn_info["Base"]["Syncs"]) == 0:
logger.info(
f"Adding a sync to '{dst_loc.syn_instance}' from '{dst_loc.seg_instance}'"
)
post_sync(*dst_loc[:3], [dst_loc.seg_instance])
elif syn_info["Base"]["Syncs"][0] != dst_loc.seg_instance:
other_seg = syn_info["Base"]["Syncs"][0]
raise RuntimeError(
f"Can't create a sync to '{dst_loc.seg_instance}'. "
f"Your instance is already sync'd to a different segmentation: {other_seg}"
)
# Fetch segmentation extents
bounding_box_zyx = fetch_volume_box(*src_loc[:2],
src_loc.seg_instance).tolist()
# Break into block-aligned chunks (boxes) that are long in the X direction
# (optimal access pattern for dvid read/write)
boxes = boxes_from_grid(bounding_box_zyx, (256, 256, 6400), clipped=True)
# Use a process pool to copy the chunks in parallel.
fn = partial(copy_syn_blocks, src_loc, dst_loc)
compute_parallel(fn, boxes, processes=processes, ordered=False)
def _init_services(self):
"""
Initialize the input and output services,
and fill in 'auto' config values as needed.
Also check the service configurations for errors.
"""
input_config = self.config["input"]
output_config = self.config["output"]
mgr_options = self.config["resource-manager"]
options = self.config["copysegmentation"]
slab_depth = options["slab-depth"]
pyramid_depth = options["pyramid-depth"]
permit_inconsistent_pyramids = options["permit-inconsistent-pyramid"]
self.mgr_client = ResourceManagerClient(mgr_options["server"],
mgr_options["port"])
self.input_service = VolumeService.create_from_config(
input_config, self.mgr_client)
brick_shape = self.input_service.preferred_message_shape
if slab_depth % brick_shape[0] != 0:
self.input_service.preferred_message_shape[0]
logger.warning(
f"Your slab-depth {slab_depth} is not a multiple of the input's brick width {brick_shape[0]}"
)
if isinstance(self.input_service.base_service, DvidVolumeService):
assert input_config["dvid"]["supervoxels"], \
'DVID input service config must use "supervoxels: true"'
# Special handling for creation of multi-scale outputs:
# auto-configure the pyramid depths
multiscale_output_type = None
for t in ["dvid", "n5", "zarr"]:
if t in output_config and not hasattr(output_config[t],
'from_default'):
multiscale_output_type = t
if multiscale_output_type:
out_fmt = multiscale_output_type
if output_config[out_fmt]["create-if-necessary"]:
if self.config["copysegmentation"][
"skip-scale-0-write"] and pyramid_depth == 0:
# Nothing to write. Maybe the user is just computing block statistics.
msg = (
"Since your config specifies no pyramid levels to write, no output instance will be created. "
"Avoid this warning by removing 'create-if-necessary' from your config"
)
logger.warning(msg)
output_config[out_fmt]["create-if-necessary"] = False
else:
max_scale = output_config[out_fmt]["creation-settings"][
"max-scale"]
if max_scale not in (-1, pyramid_depth):
msg = (
f"Inconsistent max-scale ({max_scale}) and pyramid-depth ({pyramid_depth}). "
"Omit max-scale from your creation-settings.")
raise RuntimeError(msg)
output_config[out_fmt]["creation-settings"][
"max-scale"] = pyramid_depth
# Replace 'auto' dimensions with input bounding box
replace_default_entries(output_config["geometry"]["bounding-box"],
self.input_service.bounding_box_zyx[:, ::-1])
self.output_service = VolumeService.create_from_config(
output_config, self.mgr_client)
output_service = self.output_service
assert isinstance(output_service, VolumeServiceWriter)
if "dvid" in output_config:
assert output_config["dvid"]["supervoxels"], \
'DVID output service config must use "supervoxels: true"'
if output_service.instance_name in fetch_repo_instances(
output_service.server, output_service.uuid):
existing_depth = self._read_pyramid_depth()
if pyramid_depth not in (
-1,
existing_depth) and not permit_inconsistent_pyramids:
raise Exception(
f"Can't set pyramid-depth to {pyramid_depth}: "
f"Data instance '{output_service.instance_name}' already existed, with depth {existing_depth}"
)
# These services aren't supported because we copied some geometry (bounding-box)
# directly from the input service.
assert not isinstance(output_service, TransposedVolumeService)
assert not isinstance(
output_service,
ScaledVolumeService) or output_service.scale_delta == 0
if isinstance(self.output_service.base_service, DvidVolumeService):
assert output_service.base_service.disable_indexing, \
"During ingestion, dvid labelmap indexing should be disabled.\n" \
"Please add 'disable-indexing: true' to your output dvid config."
logger.info(
f"Output bounding box (xyz) is: {output_service.bounding_box_zyx[:,::-1].tolist()}"
)
input_shape = -np.subtract(*self.input_service.bounding_box_zyx)
output_shape = -np.subtract(*output_service.bounding_box_zyx)
assert not any(np.array(output_service.preferred_message_shape) % output_service.block_width), \
"Output message-block-shape should be a multiple of the block size in all dimensions."
assert (input_shape == output_shape).all(), \
"Input bounding box and output bounding box do not have the same dimensions"
if ("apply-labelmap" in output_config["adapters"]) and (
output_config["adapters"]["apply-labelmap"]["file-type"] !=
"__invalid__"):
assert output_config["adapters"]["apply-labelmap"]["apply-when"] == "reading-and-writing", \
"Labelmap will be applied to voxels during pre-write and post-read (due to block padding).\n"\
"You cannot use this workflow with non-idempotent labelmaps, unless your data is already perfectly block aligned."
def _prepare_output(self):
"""
If necessary, create the output directory or
DVID instance so that meshes can be written to it.
"""
input_cfg = self.config["input"]
output_cfg = self.config["output"]
options = self.config["svdecimate"]
## directory output
if 'directory' in output_cfg:
# Convert to absolute so we can chdir with impunity later.
output_cfg['directory'] = os.path.abspath(output_cfg['directory'])
os.makedirs(output_cfg['directory'], exist_ok=True)
return
##
## DVID output (either keyvalue or tarsupervoxels)
##
(instance_type,) = output_cfg.keys()
server = output_cfg[instance_type]['server']
uuid = output_cfg[instance_type]['uuid']
instance = output_cfg[instance_type]['instance']
# If the output server or uuid is left blank,
# we assume it should be auto-filled from the input settings.
if server == "" or uuid == "":
assert "dvid" in input_cfg
if server == "":
output_cfg[instance_type]['server'] = input_cfg["dvid"]["server"]
if uuid == "":
output_cfg[instance_type]['uuid'] = input_cfg["dvid"]["uuid"]
# Resolve in case a branch was given instead of a specific uuid
server = output_cfg[instance_type]['server']
uuid = output_cfg[instance_type]['uuid']
uuid = resolve_ref(server, uuid)
if is_locked(server, uuid):
info = fetch_server_info(server)
if "Mode" in info and info["Mode"] == "allow writes on committed nodes":
logger.warn(f"Output is a locked node ({uuid}), but server is in full-write mode. Proceeding.")
elif os.environ.get("DVID_ADMIN_TOKEN", ""):
logger.warn(f"Output is a locked node ({uuid}), but you defined DVID_ADMIN_TOKEN. Proceeding.")
else:
raise RuntimeError(f"Can't write to node {uuid} because it is locked.")
if instance_type == 'tarsupervoxels' and not self.input_is_labelmap_supervoxels():
msg = ("You shouldn't write to a tarsupervoxels instance unless "
"you're reading supervoxels from a labelmap input.\n"
"Use a labelmap input source, and set supervoxels: true")
raise RuntimeError(msg)
existing_instances = fetch_repo_instances(server, uuid)
if instance in existing_instances:
# Instance exists -- nothing to do.
return
if not output_cfg[instance_type]['create-if-necessary']:
msg = (f"Output instance '{instance}' does not exist, "
"and your config did not specify create-if-necessary")
raise RuntimeError(msg)
assert instance_type in ('tarsupervoxels', 'keyvalue')
## keyvalue output
if instance_type == "keyvalue":
create_instance(server, uuid, instance, "keyvalue", tags=["type=meshes"])
return
## tarsupervoxels output
sync_instance = output_cfg["tarsupervoxels"]["sync-to"]
if not sync_instance:
# Auto-fill a default 'sync-to' instance using the input segmentation, if possible.
info = fetch_instance_info(*[input_cfg["dvid"][k] for k in ("server", "uuid", "tarsupervoxels-instance")])
syncs = info['Base']['Syncs']
if syncs:
sync_instance = syncs[0]
if not sync_instance:
msg = ("Can't create a tarsupervoxels instance unless "
"you specify a 'sync-to' labelmap instance name.")
raise RuntimeError(msg)
if sync_instance not in existing_instances:
msg = ("Can't sync to labelmap instance '{sync_instance}': "
"it doesn't exist on the output server.")
raise RuntimeError(msg)
create_tarsupervoxel_instance(server, uuid, instance, sync_instance, options["format"])
def __init__(self, volume_config, resource_manager_client=None):
validate(volume_config, DvidGenericVolumeSchema, inject_defaults=True)
assert 'apply-labelmap' not in volume_config["dvid"].keys(), \
("The apply-labelmap section should be in the 'adapters' section, (parallel to 'dvid' and 'geometry'), "
"not nested within the 'dvid' section!")
##
## server, uuid
##
## Note:
## self.uuid will be resolved, but volume_config["dvid"]["uuid"]
## will not be overwritten. It will remain unresolved.
##
self._server = volume_config["dvid"]["server"]
self._uuid = resolve_ref(volume_config["dvid"]["server"],
volume_config["dvid"]["uuid"])
self._throttle = volume_config["dvid"]["accept-throttling"]
##
## instance, dtype, etc.
##
config_block_width = volume_config["geometry"]["block-width"]
assert ('segmentation-name' in volume_config["dvid"]) ^ ('grayscale-name' in volume_config["dvid"]), \
"Config error: Specify either segmentation-name or grayscale-name (not both)"
if "segmentation-name" in volume_config["dvid"]:
self._instance_name = volume_config["dvid"]["segmentation-name"]
self._dtype = np.uint64
elif "grayscale-name" in volume_config["dvid"]:
self._instance_name = volume_config["dvid"]["grayscale-name"]
self._dtype = np.uint8
self._dtype_nbytes = np.dtype(self._dtype).type().nbytes
try:
instance_info = fetch_instance_info(self._server, self._uuid,
self._instance_name)
except HTTPError as ex:
if ex.response.status_code != 400:
raise
if not volume_config["dvid"]["create-if-necessary"]:
existing_instances = fetch_repo_instances(
self._server, self._uuid)
if self._instance_name not in existing_instances:
raise RuntimeError(
f"Instance '{self._instance_name}' does not exist in {self._server} / {self._uuid}."
"Add 'create-if-necessary: true' to your config if you want it to be created.'"
)
raise
# Instance doesn't exist yet -- we are going to create it.
if "segmentation-name" in volume_config["dvid"]:
self._instance_type = 'labelmap' # get_voxels doesn't really care if it's labelarray or labelmap...
self._is_labels = True
else:
self._instance_type = 'uint8blk'
self._is_labels = False
block_width = config_block_width
else:
self._instance_type = instance_info["Base"]["TypeName"]
self._is_labels = self._instance_type in ('labelblk', 'labelarray',
'labelmap')
if self._instance_type == "googlevoxels" and instance_info[
"Extended"]["Scales"][0]["channelType"] == "UINT64":
self._is_labels = True
bs_x, bs_y, bs_z = instance_info["Extended"]["BlockSize"]
assert (bs_x == bs_y == bs_z), "Expected blocks to be cubes."
block_width = bs_x
if "disable-indexing" in volume_config["dvid"]:
self.disable_indexing = volume_config["dvid"]["disable-indexing"]
else:
self.disable_indexing = DvidSegmentationServiceSchema[
"properties"]["disable-indexing"]["default"]
if "enable-downres" in volume_config["dvid"]:
self.enable_downres = volume_config["dvid"]["enable-downres"]
else:
self.enable_downres = DvidSegmentationServiceSchema["properties"][
"enable-downres"]["default"]
if "gzip-level" in volume_config["dvid"]:
self.gzip_level = volume_config["dvid"]["gzip-level"]
else:
self.gzip_level = DvidSegmentationServiceSchema["properties"][
"gzip-level"]["default"]
# Whether or not to read the supervoxels from the labelmap instance instead of agglomerated labels.
self.supervoxels = ("supervoxels" in volume_config["dvid"]) and (
volume_config["dvid"]["supervoxels"])
##
## default block width
##
assert config_block_width in (-1, block_width), \
f"DVID volume block-width ({config_block_width}) from config does not match server metadata ({block_width})"
if block_width == -1:
# No block-width specified; choose default
block_width = 64
##
## bounding-box
##
bounding_box_zyx = np.array(
volume_config["geometry"]["bounding-box"])[:, ::-1]
try:
stored_extents = fetch_volume_box(self._server, self.uuid,
self._instance_name)
except HTTPError:
assert -1 not in bounding_box_zyx.flat[:], \
f"Instance '{self._instance_name}' does not yet exist on the server, "\
"so your volume_config must specify explicit values for bounding-box"
else:
if stored_extents is not None and stored_extents.any():
replace_default_entries(bounding_box_zyx, stored_extents)
##
## message-block-shape
##
preferred_message_shape_zyx = np.array(
volume_config["geometry"]["message-block-shape"][::-1])
replace_default_entries(preferred_message_shape_zyx,
[block_width, block_width, 100 * block_width])
##
## available-scales
##
available_scales = list(volume_config["geometry"]["available-scales"])
##
## resource_manager_client
##
if resource_manager_client is None:
# Dummy client
resource_manager_client = ResourceManagerClient("", 0)
##
## Special setting to override resource manager for sparse coords
##
try:
use_resource_manager_for_sparse_coords = volume_config["dvid"][
"use-resource-manager-for-sparse-coords"]
except KeyError:
# Grayscale doesn't have this setting
use_resource_manager_for_sparse_coords = False
##
## Store members
##
self._resource_manager_client = resource_manager_client
self._block_width = block_width
self._bounding_box_zyx = bounding_box_zyx
self._preferred_message_shape_zyx = preferred_message_shape_zyx
self._available_scales = available_scales
self._use_resource_manager_for_sparse_coords = use_resource_manager_for_sparse_coords
self.write_empty_blocks = volume_config["dvid"]["write-empty-blocks"]
##
## Overwrite config entries that we might have modified
##
volume_config["geometry"]["block-width"] = self._block_width
volume_config["geometry"][
"bounding-box"] = self._bounding_box_zyx[:, ::-1].tolist()
volume_config["geometry"][
"message-block-shape"] = self._preferred_message_shape_zyx[::
-1].tolist(
)
# TODO: Check the server for available scales and overwrite in the config?
#volume_config["geometry"]["available-scales"] = [0]
if volume_config["dvid"]["create-if-necessary"]:
self._create_instance(volume_config)
def _init_services(self):
"""
Initialize the input and output services,
and fill in 'auto' config values as needed.
Also check the service configurations for errors.
"""
input_config = self.config["input"]
output_config = self.config["output"]
mgr_options = self.config["resource-manager"]
options = self.config["labelmapcopy"]
self.mgr_client = ResourceManagerClient(mgr_options["server"],
mgr_options["port"])
self.input_service = VolumeService.create_from_config(
input_config, self.mgr_client)
assert input_config["dvid"]["supervoxels"], \
'DVID input service config must use "supervoxels: true"'
assert output_config["dvid"]["supervoxels"], \
'DVID output service config must use "supervoxels: true"'
input_service = self.input_service
max_scale = options["max-scale"]
if max_scale == -1:
info = fetch_instance_info(*input_service.instance_triple)
max_scale = int(info["Extended"]["MaxDownresLevel"])
options["max-scale"] = max_scale
assert not (set(range(1+max_scale)) - set(input_service.available_scales)), \
"Your input config's 'available-scales' must include all levels you wish to copy."
assert len(options["slab-shape"]) == 3
slab_shape_zyx = np.array(options["slab-shape"][::-1])
# FIXME: Should be a whole slab (per the docs above), not just the brick shape!
replace_default_entries(slab_shape_zyx,
input_service.preferred_message_shape)
options["slab-shape"] = slab_shape_zyx[::-1].tolist()
assert (slab_shape_zyx % input_service.preferred_message_shape[0] == 0).all(), \
"slab-shape must be divisible by the brick shape"
# Transposed/remapped services aren't supported because we're not going to inflate the downloaded blocks.
assert all(not isinstance(svc, TransposedVolumeService)
for svc in input_service.service_chain)
assert all(not isinstance(svc, LabelmappedVolumeService)
for svc in input_service.service_chain)
assert not (input_service.bounding_box_zyx % input_service.block_width).any(), \
"Input bounding-box should be a multiple of the block size in all dimensions."
assert not (input_service.preferred_message_shape % input_service.block_width).any(), \
"Input message-block-shape should be a multiple of the block size in all dimensions."
assert all(not isinstance( svc, ScaledVolumeService ) or svc.scale_delta == 0 for svc in input_service.service_chain), \
"For now, we don't support rescaled input, though it would be possible in theory."
if options["record-only"]:
# Don't need to check output setting if we're not writing
self.output_service = None
assert options[
"record-label-sets"], "If using 'record-only', you must set 'record-label-sets', too."
assert not options["dont-overwrite-identical-blocks"], \
"In record only mode, the output service can't be accessed, and you can't use dont-overwrite-identical-blocks"
return
if output_config["dvid"]["create-if-necessary"]:
creation_depth = output_config["dvid"]["creation-settings"][
"max-scale"]
if creation_depth not in (-1, max_scale):
msg = (
f"Inconsistent max-scale options in the labelmapcopy config options ({max_scale}) and creation-settings options ({creation_depth}). "
"Omit max-scale from your creation-settings.")
raise RuntimeError(msg)
output_config["dvid"]["creation-settings"]["max-scale"] = max_scale
# Replace 'auto' dimensions with input bounding box
replace_default_entries(output_config["geometry"]["bounding-box"],
input_service.bounding_box_zyx[:, ::-1])
self.output_service = VolumeService.create_from_config(
output_config, self.mgr_client)
output_service = self.output_service
assert isinstance(output_service, VolumeServiceWriter)
if output_service.instance_name in fetch_repo_instances(
output_service.server, output_service.uuid):
info = fetch_instance_info(*output_service.instance_triple)
existing_depth = int(info["Extended"]["MaxDownresLevel"])
if max_scale not in (-1, existing_depth):
raise Exception(
f"Can't set pyramid-depth to {max_scale}: \n"
f"Data instance '{output_service.instance_name}' already existed, with depth {existing_depth}.\n"
f"For now, you are required to populate ALL scales of the output, or create a new output instance from scratch."
)
assert all(not isinstance(svc, TransposedVolumeService)
for svc in output_service.service_chain)
assert all(not isinstance(svc, LabelmappedVolumeService)
for svc in output_service.service_chain)
assert all(
not isinstance(svc, ScaledVolumeService) or svc.scale_delta == 0
for svc in output_service.service_chain)
# Output can't be a scaled service because we copied some geometry (bounding-box)
# directly from the input service.
assert not isinstance(
output_service,
ScaledVolumeService) or output_service.scale_delta == 0
assert output_service.base_service.disable_indexing, \
"During ingestion, indexing should be disabled.\n" \
"Please add 'disable-indexing':true to your output dvid config."
logger.info(
f"Output bounding box (xyz) is: {output_service.bounding_box_zyx[:,::-1].tolist()}"
)
assert (input_service.bounding_box_zyx == output_service.bounding_box_zyx).all(), \
"Input and output service bounding boxes must match exactly."
assert input_service.block_width == output_service.block_width, \
"Input and output must use the same block-width"
assert not (output_service.bounding_box_zyx % output_service.block_width).any(), \
"Output bounding-box should be a multiple of the block size in all dimensions."
assert not (output_service.preferred_message_shape % output_service.block_width).any(), \
"Output message-block-shape should be a multiple of the block size in all dimensions."
def test_dvid_volume_service_labelmap(setup_dvid_repo, random_segmentation,
disable_auto_retry):
server, uuid = setup_dvid_repo
instance_name = 'test-dvs-labelmap'
volume = random_segmentation[:256, :192, :128]
max_scale = 2
voxel_dimensions = [4.0, 4.0, 32.0]
config_text = textwrap.dedent(f"""\
dvid:
server: {server}
uuid: {uuid}
segmentation-name: {instance_name}
supervoxels: true
create-if-necessary: true
creation-settings:
max-scale: {max_scale}
voxel-size: {voxel_dimensions}
geometry:
bounding-box: [[0,0,0], {list(volume.shape[::-1])}]
message-block-shape: [64,64,64]
""")
yaml = YAML()
with StringIO(config_text) as f:
volume_config = yaml.load(f)
assert instance_name not in fetch_repo_instances(server, uuid)
service = VolumeService.create_from_config(volume_config)
repo_instances = fetch_repo_instances(server, uuid)
assert instance_name in repo_instances
assert repo_instances[instance_name] == 'labelmap'
info = fetch_instance_info(server, uuid, instance_name)
assert info["Extended"]["VoxelSize"] == voxel_dimensions
scaled_volumes = {}
for scale in range(max_scale + 1):
vol = downsample(volume, 2**scale, 'label')
aligned_shape = (np.ceil(np.array(vol.shape) / 64) * 64).astype(int)
aligned_vol = np.zeros(aligned_shape, np.uint64)
overwrite_subvol(aligned_vol, [(0, 0, 0), vol.shape], vol)
service.write_subvolume(aligned_vol, (0, 0, 0), scale)
scaled_volumes[scale] = aligned_vol
box = np.array([[40, 80, 40], [240, 160, 100]])
for scale in range(max_scale + 1):
scaled_box = box // 2**scale
vol = service.get_subvolume(scaled_box, scale)
assert (vol == extract_subvol(scaled_volumes[scale], scaled_box)).all()
#
# Check sparse coords function
#
labels = list({*pd.unique(volume.reshape(-1))} - {0})
brick_coords_df = service.sparse_brick_coords_for_labels(labels)
assert brick_coords_df.columns.tolist() == ['z', 'y', 'x', 'label']
assert set(brick_coords_df['label'].values) == set(labels), \
"Some labels were missing from the sparse brick coords!"
def ndi(shape):
return np.indices(shape).reshape(len(shape), -1).transpose()
expected_df = pd.DataFrame(ndi(volume.shape), columns=[*'zyx'])
expected_df['label'] = volume.reshape(-1)
expected_df['z'] //= 64
expected_df['y'] //= 64
expected_df['x'] //= 64
expected_df = expected_df.drop_duplicates()
expected_df['z'] *= 64
expected_df['y'] *= 64
expected_df['x'] *= 64
expected_df = expected_df.query('label != 0')
expected_df.sort_values(['z', 'y', 'x', 'label'], inplace=True)
brick_coords_df.sort_values(['z', 'y', 'x', 'label'], inplace=True)
expected_df.reset_index(drop=True, inplace=True)
brick_coords_df.reset_index(drop=True, inplace=True)
assert expected_df.shape == brick_coords_df.shape
assert (brick_coords_df == expected_df).all().all()
#
# Check sample_labels()
#
points = [np.random.randint(d, size=(10, )) for d in vol.shape]
points = np.transpose(points)
labels = service.sample_labels(points)
assert (labels == volume[(*points.transpose(), )]).all()
def load_roi_label_volume(server,
uuid,
rois_or_neuprint,
box_s5=[None, None],
export_path=None,
export_labelmap=None):
"""
Fetch several ROIs from DVID and combine them into a single label volume or mask.
The label values in the returned volume correspond to the order in which the ROI
names were passed in, starting at label 1.
This function is essentially a convenience function around fetch_combined_roi_volume(),
but in this case it will optionally auto-fetch the ROI list, and auto-export the volume.
Args:
server:
DVID server
uuid:
DVID uuid
rois_or_neuprint:
Either a list of ROIs or a neuprint server from which to obtain the roi list.
box_s5:
If you want to restrict the ROIs to a particular subregion,
you may pass your own bounding box (at scale 5).
Alternatively, you may pass the name of a segmentation
instance from DVID whose bounding box will be used.
export_path:
If you want the ROI volume to be exported to disk,
provide a path name ending with .npy or .h5.
export_labelmap:
If you want the ROI volume to be exported to a DVID labelmap instance,
Provide the instance name, or a tuple of (server, uuid, instance).
Returns:
(roi_vol, roi_box), containing the fetched label volume and the
bounding box it corresponds to, in DVID scale-5 coordinates.
Note:
If you have a list of (full-res) points to extract from the returned volume,
pass a DataFrame with columns ['z','y','x'] to the following function.
If you already downloaded the roi_vol (above), provide it.
Otherwise, leave out those args and it will be fetched first.
Adds columns to the input DF (in-place) for 'roi' (str) and 'roi_label' (int).
>>> from neuclease.dvid import determine_point_rois
>>> determine_point_rois(*master, rois, point_df, roi_vol, roi_box)
"""
if isinstance(box_s5, str):
# Assume that this is a segmentation instance whose dimensions should be used
# Fetch the maximum extents of the segmentation,
# and rescale it for scale-5.
seg_box = fetch_volume_box(server, uuid, box_s5)
box_s5 = round_box(seg_box, (2**5), 'out') // 2**5
box_s5[0] = (0, 0, 0)
if export_labelmap:
assert isinstance(box_s5, np.ndarray)
assert not (box_s5 % 64).any(), \
("If exporting to a labelmap instance, please supply "
"an explicit box and make sure it is block-aligned.")
if isinstance(rois_or_neuprint, (str, neuprint.Client)):
if isinstance(rois_or_neuprint, str):
npclient = neuprint.Client(rois_or_neuprint)
else:
npclient = rois_or_neuprint
# Fetch ROI names from neuprint
q = "MATCH (m: Meta) RETURN m.superLevelRois as rois"
rois = npclient.fetch_custom(q)['rois'].iloc[0]
rois = sorted(rois)
# # Remove '.*ACA' ROIs. Apparently there is some
# # problem with them. (They overlap with other ROIs.)
# rois = [*filter(lambda r: 'ACA' not in r, rois)]
else:
assert isinstance(rois_or_neuprint, collections.abc.Iterable)
rois = rois_or_neuprint
# Fetch each ROI and write it into a volume
with Timer(f"Fetching combined ROI volume for {len(rois)} ROIs", logger):
roi_vol, roi_box, overlap_stats = fetch_combined_roi_volume(
server, uuid, rois, box_zyx=box_s5)
if len(overlap_stats) > 0:
logger.warn(
f"Some ROIs overlap! Here's an incomplete list of overlapping pairs:\n{overlap_stats}"
)
# Export to npy/h5py for external use
if export_path:
with Timer(f"Exporting to {export_path}", logger):
if export_path.endswith('.npy'):
np.save(export_path, roi_vol)
elif export_path.endswith('.h5'):
with h5py.File(export_path, 'w') as f:
f.create_dataset('rois_scale_5', data=roi_vol, chunks=True)
if export_labelmap:
if isinstance(export_labelmap, str):
export_labelmap = (server, uuid, export_labelmap)
assert len(export_labelmap) == 3
with Timer(f"Exporting to {export_labelmap[2]}", logger):
if export_labelmap[2] not in fetch_repo_instances(
server, uuid, 'labelmap'):
create_labelmap_instance(
*export_labelmap, voxel_size=8 * (2**5),
max_scale=6) # FIXME: hard-coded voxel size
# It's really important to use this block shape.
# See https://github.com/janelia-flyem/dvid/issues/342
boxes = boxes_from_grid(roi_box, (256, 256, 256), clipped=True)
for box in tqdm_proxy(boxes):
block = extract_subvol(roi_vol, box - roi_box[0])
post_labelmap_voxels(*export_labelmap,
box[0],
block,
scale=0,
downres=True)
return roi_vol, roi_box, rois
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation):
dvid_address, repo_uuid = setup_dvid_repo
input_segmentation_name = 'segmentation-input'
output_segmentation_name = 'segmentation-output-from-dvid'
try:
create_labelmap_instance(dvid_address, repo_uuid,
input_segmentation_name)
except HTTPError as ex:
if ex.response is not None and 'already exists' in ex.response.content.decode(
'utf-8'):
pass
post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name,
(0, 0, 0), random_segmentation)
# Make sure the output is empty (if it exists)
if output_segmentation_name in fetch_repo_instances(
dvid_address, repo_uuid):
z = np.zeros((256, 256, 256), np.uint64)
post_labelmap_voxels(dvid_address, repo_uuid, output_segmentation_name,
(0, 0, 0), z, 0, True)
template_dir = tempfile.mkdtemp(
suffix="copysegmentation-from-dvid-template")
config_text = textwrap.dedent(f"""\
workflow-name: copysegmentation
cluster-type: {CLUSTER_TYPE}
input:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {input_segmentation_name}
supervoxels: true
geometry:
message-block-shape: [64,64,512]
bounding-box: [[0,0,100], [256,200,256]]
adapters: {{}}
output:
dvid:
server: {dvid_address}
uuid: {repo_uuid}
segmentation-name: {output_segmentation_name}
supervoxels: true
disable-indexing: true
create-if-necessary: true
geometry: {{}} # Auto-set from input
copysegmentation:
pyramid-depth: 1
slab-depth: 128
""")
with open(f"{template_dir}/workflow.yaml", 'w') as f:
f.write(config_text)
yaml = YAML()
with StringIO(config_text) as f:
config = yaml.load(f)
return template_dir, config, random_segmentation, dvid_address, repo_uuid, output_segmentation_name