def test_copysegmentation_from_hdf5_to_dvid_input_mask( setup_hdf5_segmentation_input, disable_auto_retry): template_dir, config, volume, dvid_address, repo_uuid, _output_segmentation_name = setup_hdf5_segmentation_input # make sure we get a fresh output output_segmentation_name = 'copyseg-with-input-mask' config["output"]["dvid"]["segmentation-name"] = output_segmentation_name # Select only even IDs all_labels = pd.unique(volume.reshape(-1)) even_labels = all_labels[all_labels % 2 == 0] config["copysegmentation"]["input-mask-labels"] = even_labels.tolist() # Add an offset, which is added to both the input volume AND the mask labels offset = 2000 config["copysegmentation"]["add-offset-to-ids"] = offset input_box = np.array(config["input"]["geometry"]["bounding-box"])[:, ::-1] volume = np.where((volume % 2) == 0, volume + offset, 0) expected_vol = np.zeros_like(volume) overwrite_subvol(expected_vol, input_box, extract_subvol(volume, input_box)) setup = template_dir, config, expected_vol, dvid_address, repo_uuid, output_segmentation_name _box_zyx, _expected_vol, _output_vol = _run_to_dvid(setup)
def test_copysegmentation_from_dvid_to_dvid_input_mask( setup_dvid_segmentation_input, disable_auto_retry): template_dir, config, volume, dvid_address, repo_uuid, _output_segmentation_name = setup_dvid_segmentation_input # make sure we get a fresh output output_segmentation_name = 'copyseg-with-input-mask-from-dvid' config["output"]["dvid"]["segmentation-name"] = output_segmentation_name # Add an offset, which is added to both the input volume AND the mask labels offset = 2000 config["copysegmentation"]["add-offset-to-ids"] = offset # Select some labels that don't extend throughout the whole volume selected_labels = pd.unique(volume[150, 64:128, 64:128].reshape(-1)) assert 0 not in selected_labels selected_coords = np.array( mask_for_labels(volume, selected_labels).nonzero()).transpose() selected_box = np.array( [selected_coords.min(axis=0), 1 + selected_coords.max(axis=0)]) input_box = np.array(config["input"]["geometry"]["bounding-box"])[:, ::-1] subvol_box = box_intersection(input_box, selected_box) selected_subvol = extract_subvol(volume, subvol_box).copy() selected_subvol = apply_mask_for_labels(selected_subvol, selected_labels) config["copysegmentation"]["input-mask-labels"] = selected_labels.tolist() selected_subvol = np.where(selected_subvol, selected_subvol + offset, 0) expected_vol = np.zeros(volume.shape, np.uint64) overwrite_subvol(expected_vol, subvol_box, selected_subvol) setup = template_dir, config, expected_vol, dvid_address, repo_uuid, output_segmentation_name _box_zyx, _expected_vol, _output_vol = _run_to_dvid(setup)
def test_dvid_volume_service_grayscale(setup_dvid_repo, disable_auto_retry): server, uuid = setup_dvid_repo instance_name = 'test-dvs-grayscale' volume = np.random.randint(100, size=(256, 192, 128), dtype=np.uint8) max_scale = 2 voxel_dimensions = [4.0, 4.0, 32.0] config_text = textwrap.dedent(f"""\ dvid: server: {server} uuid: {uuid} grayscale-name: {instance_name} create-if-necessary: true creation-settings: max-scale: {max_scale} voxel-size: {voxel_dimensions} geometry: bounding-box: [[0,0,0], {list(volume.shape[::-1])}] """) yaml = YAML() with StringIO(config_text) as f: volume_config = yaml.load(f) assert instance_name not in fetch_repo_instances(server, uuid) service = VolumeService.create_from_config(volume_config) repo_instances = fetch_repo_instances(server, uuid) info = fetch_instance_info(server, uuid, instance_name) assert info["Extended"]["VoxelSize"] == voxel_dimensions scaled_volumes = {} for scale in range(max_scale + 1): if scale == 0: assert instance_name in repo_instances assert repo_instances[instance_name] == 'uint8blk' else: assert f"{instance_name}_{scale}" in repo_instances assert repo_instances[f"{instance_name}_{scale}"] == 'uint8blk' vol = downsample(volume, 2**scale, 'label') # label downsampling is easier to test with aligned_shape = (np.ceil(np.array(vol.shape) / 64) * 64).astype(int) aligned_vol = np.zeros(aligned_shape, np.uint8) overwrite_subvol(aligned_vol, [(0, 0, 0), aligned_shape], aligned_vol) service.write_subvolume(aligned_vol, (0, 0, 0), scale) scaled_volumes[scale] = aligned_vol box = np.array([[40, 80, 40], [240, 160, 100]]) for scale in range(max_scale + 1): scaled_box = box // 2**scale vol = service.get_subvolume(scaled_box, scale) assert (vol == extract_subvol(scaled_volumes[scale], scaled_box)).all()
def assemble_brick_fragments( fragments ): """ Given a list of Bricks with identical logical_boxes, splice their volumes together into a final Brick that contains a full volume containing all of the fragments. Note: Brick 'fragments' are also just Bricks, whose physical_box does not cover the entire logical_box for the brick. Each fragment's physical_box indicates where that fragment's data should be located within the final returned Brick. Returns: A Brick containing the data from all fragments, UNLESS the fully assembled fragments would not intersect with the Brick's own logical_box (i.e. all fragments fall within the halo), in which case None is returned. Note: If the fragment physical_boxes are not disjoint, the results are undefined. """ fragments = list(fragments) # All logical boxes must be the same logical_boxes = np.asarray([frag.logical_box for frag in fragments]) assert (logical_boxes == logical_boxes[0]).all(), \ "Cannot assemble brick fragments from different logical boxes. "\ "They belong to different bricks!" final_logical_box = logical_boxes[0] # The final physical box is the min/max of all fragment physical extents. physical_boxes = np.array([frag.physical_box for frag in fragments]) assert physical_boxes.ndim == 3 # (N, 2, Dim) assert physical_boxes.shape == ( len(fragments), 2, final_logical_box.shape[1] ) final_physical_box = np.asarray( ( np.min( physical_boxes[:,0,:], axis=0 ), np.max( physical_boxes[:,1,:], axis=0 ) ) ) interior_box = box_intersection(final_physical_box, final_logical_box) if (interior_box[1] - interior_box[0] < 1).any(): # All fragments lie completely within the halo return None final_volume_shape = final_physical_box[1] - final_physical_box[0] dtype = fragments[0].volume.dtype final_volume = np.zeros(final_volume_shape, dtype) for frag in fragments: internal_box = frag.physical_box - final_physical_box[0] overwrite_subvol(final_volume, internal_box, frag.volume) # Destroy original to save RAM frag.destroy() brick = Brick( final_logical_box, final_physical_box, final_volume ) brick.compress() return brick
def zero_fill(vol, box, full_box): """ Given a volume, it's corresponding box, and a 'full box' that encompasses it, Return a volume that fills the full box, padding with zeros if necessary. """ if (box == full_box).all(): return vol else: full_vol = np.zeros(full_box[1] - full_box[0], vol.dtype) overwrite_subvol(full_vol, box - full_box[0], vol) return full_vol
def download(bounding_box_zyx, output_path): shape = bounding_box_zyx[1] - bounding_box_zyx[0] with h5py.File(output_path, 'w') as f: gray_dset = f.create_dataset('grayscale', shape=shape, dtype=np.uint8, chunks=True) seg_dset = f.create_dataset('segmentation', shape=shape, dtype=np.uint64, chunks=True, compression='gzip') print("Downloading grayscale...") block_shape = (256, 256, 256) block_boxes = boxes_from_grid(bounding_box_zyx, block_shape, clipped=True) for block_box in tqdm(block_boxes): relative_box = block_box - bounding_box_zyx[0] block_gray = fetch_raw(*GRAYSCALE, block_box) overwrite_subvol(gray_dset, relative_box, block_gray) print("") print("Downloading segmentation...") block_boxes = boxes_from_grid(bounding_box_zyx, block_shape, clipped=True) for block_box in tqdm(block_boxes): relative_box = block_box - bounding_box_zyx[0] block_seg = fetch_labelmap_voxels(*SEGMENTATION, block_box) overwrite_subvol(seg_dset, relative_box, block_seg) print("") print("DONE")
def setup_dvid_segmentation_input(setup_dvid_repo, random_segmentation): dvid_address, repo_uuid = setup_dvid_repo input_segmentation_name = 'labelmapcopy-segmentation-input' output_segmentation_name = 'labelmapcopy-segmentation-output' partial_output_segmentation_name = 'labelmapcopy-segmentation-partial-output' max_scale = 3 already_exists = False try: create_labelmap_instance(dvid_address, repo_uuid, input_segmentation_name, max_scale=max_scale) create_labelmap_instance(dvid_address, repo_uuid, partial_output_segmentation_name, max_scale=max_scale) except HTTPError as ex: if ex.response is not None and 'already exists' in ex.response.content.decode( 'utf-8'): already_exists = True expected_vols = {} for scale in range(1 + max_scale): if scale == 0: scaled_vol = random_segmentation else: scaled_vol = downsample(scaled_vol, 2, 'labels-numba') expected_vols[scale] = scaled_vol if not already_exists: scaled_box = round_box([(0, 0, 0), scaled_vol.shape], 64, 'out') aligned_vol = np.zeros(scaled_box[1], np.uint64) overwrite_subvol(aligned_vol, [(0, 0, 0), scaled_vol.shape], scaled_vol) post_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, (0, 0, 0), aligned_vol, scale=scale) if not already_exists: # Create a 'partial' output volume that is the same (bitwise) as the input except for some blocks. scaled_box = np.array([(0, 0, 0), random_segmentation.shape]) scaled_box[1, -1] = 192 for scale in range(1 + max_scale): scaled_box = round_box(scaled_box // (2**scale), 64, 'out') raw_blocks = fetch_labelmap_voxels(dvid_address, repo_uuid, input_segmentation_name, scaled_box, scale, supervoxels=True, format='raw-response') post_labelmap_blocks(dvid_address, repo_uuid, partial_output_segmentation_name, [(0, 0, 0)], raw_blocks, scale, is_raw=True) block = np.random.randint(1_000_000, 1_000_010, size=(64, 64, 64), dtype=np.uint64) post_labelmap_voxels(dvid_address, repo_uuid, partial_output_segmentation_name, (0, 128, 64), block, 0, downres=True) partial_vol = fetch_labelmap_voxels(dvid_address, repo_uuid, partial_output_segmentation_name, [(0, 0, 0), random_segmentation.shape], supervoxels=True) template_dir = tempfile.mkdtemp(suffix="labelmapcopy-template") config_text = textwrap.dedent(f"""\ workflow-name: labelmapcopy cluster-type: {CLUSTER_TYPE} input: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {input_segmentation_name} supervoxels: true geometry: message-block-shape: [512,64,64] available-scales: [0,1,2,3] output: dvid: server: {dvid_address} uuid: {repo_uuid} segmentation-name: {output_segmentation_name} supervoxels: true disable-indexing: true create-if-necessary: true labelmapcopy: slab-shape: [512,128,64] dont-overwrite-identical-blocks: true """) with open(f"{template_dir}/workflow.yaml", 'w') as f: f.write(config_text) yaml = YAML() with StringIO(config_text) as f: config = yaml.load(f) return template_dir, config, expected_vols, partial_vol, dvid_address, repo_uuid, output_segmentation_name, partial_output_segmentation_name
def pad_brick_data_from_volume_source( padding_grid, volume_accessor_func, brick ): """ Expand the given Brick's data until its physical_box is aligned with the given padding_grid. The data in the expanded region will be sourced from the given volume_accessor_func. Note: padding_grid need not be identical to the grid the Brick was created with, but it must divide evenly into that grid. For instance, if padding_grid happens to be the same as the brick's own native grid, then the phyiscal_box is expanded to align perfectly with the logical_box on all sides: +-------------+ +-------------+ | physical | | | same | |__________| | | physical | | | --> | and | | logical | | logical | |_____________| |_____________| Args: brick: Brick padding_grid: Grid volume_accessor_func: Callable with signature: f(box) -> ndarray Returns: Brick Note: It is not legal to call this function unless the Brick's physical_box lies completely within the logical_box (i.e. no halos allowed). Furthremore, the padding_grid is not permitted to use a halo, either. (These restrictions could be fixed, but the current version of this function has these requirements.) Note: If no padding is necessary, then the original Brick is returned (no copy is made). """ assert isinstance(padding_grid, Grid) assert not padding_grid.halo_shape.any() block_shape = padding_grid.block_shape assert ((brick.logical_box - padding_grid.offset) % block_shape == 0).all(), \ f"Padding grid {padding_grid.offset} must be aligned with brick logical_box: {brick.logical_box}" # Subtract offset to calculate the needed padding offset_physical_box = brick.physical_box - padding_grid.offset if (offset_physical_box % block_shape == 0).all(): # Internal data is already aligned to the padding_grid. return brick offset_padded_box = np.array([offset_physical_box[0] // block_shape * block_shape, (offset_physical_box[1] + block_shape - 1) // block_shape * block_shape]) # Re-add offset padded_box = offset_padded_box + padding_grid.offset assert (padded_box[0] >= brick.logical_box[0]).all() assert (padded_box[1] <= brick.logical_box[1]).all() # Initialize a new volume of the fully-padded shape padded_volume_shape = padded_box[1] - padded_box[0] padded_volume = np.zeros(padded_volume_shape, dtype=brick.volume.dtype) # Overwrite the previously existing data in the new padded volume orig_box = brick.physical_box orig_box_within_padded = orig_box - padded_box[0] overwrite_subvol(padded_volume, orig_box_within_padded, brick.volume) # Check for a non-zero-volume halo on all six sides. halo_boxes = [] for axis in range(padded_volume.ndim): if orig_box[0,axis] != padded_box[0,axis]: leading_halo_box = padded_box.copy() leading_halo_box[1, axis] = orig_box[0,axis] halo_boxes.append(leading_halo_box) if orig_box[1,axis] != padded_box[1,axis]: trailing_halo_box = padded_box.copy() trailing_halo_box[0, axis] = orig_box[1,axis] halo_boxes.append(trailing_halo_box) assert halo_boxes, \ "How could halo_boxes be empty if there was padding needed?" for halo_box in halo_boxes: # Retrieve padding data for one halo side halo_volume = volume_accessor_func(halo_box) # Overwrite in the final padded volume halo_box_within_padded = halo_box - padded_box[0] overwrite_subvol(padded_volume, halo_box_within_padded, halo_volume) return Brick( brick.logical_box, padded_box, padded_volume )
def _init_mask(self): """ - read the mask ROI as a volume - dilate/erode it if necessary - invert it if necessary - save to .h5 (just for offline debug) - return the scale-5 mask and its scale-5 bounding-box """ options = self.config["masksegmentation"] roi = options["mask-roi"] invert_mask = options["invert-mask"] max_scale = options["max-pyramid-scale"] roi_dilation = options["dilate-roi"] roi_erosion = options["erode-roi"] seg_dilation = options["dilate-segmentation"] block_width = self.output_service.block_width # Select a mask_box that's large enough to divide evenly into the # block width even when reduced to the highest scale we'll be processing. seg_box = round_box(self.input_service.bounding_box_zyx, block_width * 2**max_scale) seg_box_s5 = round_box(seg_box, 2**5) // (2**5) with Timer(f"Loading ROI '{roi}'", logger): roi_mask, _ = fetch_roi(self.input_service.server, self.input_service.uuid, roi, format='mask', mask_box=seg_box_s5) with h5py.File('roi-mask.h5', 'w') as f: f.create_dataset('mask', data=roi_mask.view(np.uint8), chunks=(128, 128, 128)) assert not (roi_dilation and roi_erosion) if roi_dilation > 0: with Timer(f"Dilating ROI by {roi_dilation}", logger): roi_mask = vigra.filters.multiBinaryDilation( roi_mask, roi_dilation) with h5py.File('dilated-roi-mask.h5', 'w') as f: f.create_dataset('mask', data=roi_mask.view(np.uint8), chunks=(128, 128, 128)) if roi_erosion > 0: with Timer(f"Eroding ROI by {roi_erosion}", logger): roi_mask = vigra.filters.multiBinaryErosion( roi_mask, roi_erosion) with h5py.File('eroded-roi-mask.h5', 'w') as f: f.create_dataset('mask', data=roi_mask.view(np.uint8), chunks=(128, 128, 128)) assert not seg_dilation or invert_mask, \ "Can't use 'dilate-segmentation'. The segmentation isn't downloaded unless 'invert-mask' is used." if invert_mask: with Timer("Inverting mask", logger): # Initialize the mask with entire segmentation at scale 5, # then subtract the roi from it. boxes = [ *boxes_from_grid(seg_box_s5, (64, 64, 2048), clipped=True) ] input_service = self.input_service def fetch_seg_mask_s5(box_s5): seg_s5 = input_service.get_subvolume(box_s5, scale=5) return box_s5, (seg_s5 != 0) boxes_and_mask = dask.bag.from_sequence( boxes, 1).map(fetch_seg_mask_s5).compute() seg_mask = np.zeros(box_shape(seg_box_s5), bool) for box_s5, box_mask in boxes_and_mask: overwrite_subvol(seg_mask, box_s5, box_mask) if seg_dilation == 0: with h5py.File('segmentation-mask.h5', 'w') as f: f.create_dataset('mask', data=seg_mask.view(np.uint8), chunks=(128, 128, 128)) else: with Timer(f"Dilating segmentation by {seg_dilation}", logger): seg_mask = vigra.filters.multiBinaryDilation( seg_mask, seg_dilation) with h5py.File('dilated-segmentation-mask.h5', 'w') as f: f.create_dataset('mask', data=seg_mask.view(np.uint8), chunks=(128, 128, 128)) seg_mask[roi_mask] = False roi_mask = seg_mask with h5py.File('final-mask.h5', 'w') as f: f.create_dataset('mask', data=roi_mask.view(np.uint8), chunks=(128, 128, 128)) # Downsample the roi_mask to dvid-block resolution, just to see how many blocks it touches. block_mask = view_as_blocks(roi_mask, (2, 2, 2)).any(axis=(3, 4, 5)) blocks_touched = block_mask.sum() voxel_total = blocks_touched * (block_width**3) logger.info( f"Mask touches {blocks_touched} blocks ({voxel_total / 1e9:.1f} Gigavoxels)" ) return roi_mask, seg_box_s5
def assemble_brick_fragments(fragments, output_accessor_fn=None): """ Given a list of Bricks with identical logical_boxes, splice their volumes together into a final Brick that contains a full volume containing all of the fragments. Note: Brick 'fragments' are also just Bricks, whose physical_box does not cover the entire logical_box for the brick. Each fragment's physical_box indicates where that fragment's data should be located within the final returned Brick. Args: fragments: TODO: docs. output_accessor_fn: Callable with signature: f(box) -> ndarray TODO: docs. Returns: A Brick containing the data from all fragments, UNLESS the fully assembled fragments would not intersect with the Brick's own logical_box (i.e. all fragments fall within the halo), in which case None is returned. Note: If the fragment physical_boxes are not disjoint, the results are undefined. That is, if two fragments overlap, there's no guarantee about which one "wins" for the overlapping region. """ fragments = list(fragments) # All logical boxes must be the same logical_boxes = np.asarray([frag.logical_box for frag in fragments]) assert (logical_boxes == logical_boxes[0]).all(), \ "Cannot assemble brick fragments from different logical boxes. "\ "They belong to different bricks!" final_logical_box = fragments[0].logical_box final_location_id = fragments[0].location_id # The final physical box is the min/max of all fragment physical extents. physical_boxes = np.array([frag.physical_box for frag in fragments]) assert physical_boxes.ndim == 3 # (N, 2, Dim) assert physical_boxes.shape == (len(fragments), 2, final_logical_box.shape[1]) final_physical_box = np.asarray( (np.min(physical_boxes[:, 0, :], axis=0), np.max(physical_boxes[:, 1, :], axis=0))) intersects_interior = False for frag_pbox in physical_boxes: interior_box = box_intersection(frag_pbox, final_logical_box) if (interior_box[1] - interior_box[0] > 0).all(): intersects_interior = True if not intersects_interior: # All fragments lie completely within the halo; # none intersect with the interior logical_box, # so we don't bother keeping this brick. return None final_volume_shape = final_physical_box[1] - final_physical_box[0] dtype = fragments[0].volume.dtype # If the physical boxes don't completely fill the final_box, # then we will need to use the output_accessor_fn (if given). if output_accessor_fn is None or is_box_coverage_complete( physical_boxes, final_physical_box): final_volume = np.zeros(final_volume_shape, dtype) else: final_volume = output_accessor_fn(final_physical_box) for frag in fragments: internal_box = frag.physical_box - final_physical_box[0] overwrite_subvol(final_volume, internal_box, frag.volume) # Recompress fragment now that we're done with it. frag.compress() ## It's tempting to destroy the fragment to save RAM, ## but the fragment might be needed by more than one final brick. ## (Also, it might be needed twice if a Worker gets restarted.) # frag.destroy() compression = fragments[0].compression brick = Brick(final_logical_box, final_physical_box, final_volume, location_id=final_location_id, compression=compression) brick.compress() return brick
def pad_brick_data_from_volume_source(padding_grid, volume_accessor_func, brick): """ Expand the given Brick's data until its physical_box is aligned with the given padding_grid. The data in the expanded region will be sourced from the given volume_accessor_func. Note: padding_grid need not be identical to the grid the Brick was created with, but it must divide evenly into that grid. For instance, if padding_grid happens to be the same as the brick's own native grid, then the phyiscal_box is expanded to align perfectly with the logical_box on all sides: +-------------+ +-------------+ | physical | | | same | |__________| | | physical | | | --> | and | | logical | | logical | |_____________| |_____________| Args: brick: Brick padding_grid: Grid volume_accessor_func: Callable with signature: f(box) -> ndarray Returns: Brick Note: It is not legal to call this function unless the Brick's physical_box lies completely within the logical_box (i.e. no halos allowed). Furthremore, the padding_grid is not permitted to use a halo, either. (These restrictions could be fixed, but the current version of this function has these requirements.) Note: If no padding is necessary, then the original Brick is returned (no copy is made). """ assert isinstance(padding_grid, Grid) assert not padding_grid.halo_shape.any() block_shape = padding_grid.block_shape assert ((brick.logical_box - padding_grid.offset) % block_shape == 0).all(), \ f"Padding grid {padding_grid.offset} must be aligned with brick logical_box: {brick.logical_box}" # Subtract offset to calculate the needed padding offset_physical_box = brick.physical_box - padding_grid.offset if (offset_physical_box % block_shape == 0).all(): # Internal data is already aligned to the padding_grid. return brick offset_padded_box = np.array([ offset_physical_box[0] // block_shape * block_shape, (offset_physical_box[1] + block_shape - 1) // block_shape * block_shape ]) # Re-add offset padded_box = offset_padded_box + padding_grid.offset assert (padded_box[0] >= brick.logical_box[0]).all() assert (padded_box[1] <= brick.logical_box[1]).all() # Initialize a new volume of the fully-padded shape padded_volume_shape = padded_box[1] - padded_box[0] padded_volume = np.zeros(padded_volume_shape, dtype=brick.volume.dtype) # Overwrite the previously existing data in the new padded volume orig_box = brick.physical_box orig_box_within_padded = orig_box - padded_box[0] overwrite_subvol(padded_volume, orig_box_within_padded, brick.volume) # Check for a non-zero-volume halo on all six sides. halo_boxes = [] for axis in range(padded_volume.ndim): if orig_box[0, axis] != padded_box[0, axis]: leading_halo_box = padded_box.copy() leading_halo_box[1, axis] = orig_box[0, axis] halo_boxes.append(leading_halo_box) if orig_box[1, axis] != padded_box[1, axis]: trailing_halo_box = padded_box.copy() trailing_halo_box[0, axis] = orig_box[1, axis] halo_boxes.append(trailing_halo_box) assert halo_boxes, \ "How could halo_boxes be empty if there was padding needed?" for halo_box in halo_boxes: # Retrieve padding data for one halo side halo_volume = volume_accessor_func(halo_box) # Overwrite in the final padded volume halo_box_within_padded = halo_box - padded_box[0] overwrite_subvol(padded_volume, halo_box_within_padded, halo_volume) new_brick = Brick(brick.logical_box, padded_box, padded_volume, location_id=brick.location_id, compression=brick.compression) brick.compress() return new_brick
def test_dvid_volume_service_labelmap(setup_dvid_repo, random_segmentation, disable_auto_retry): server, uuid = setup_dvid_repo instance_name = 'test-dvs-labelmap' volume = random_segmentation[:256, :192, :128] max_scale = 2 voxel_dimensions = [4.0, 4.0, 32.0] config_text = textwrap.dedent(f"""\ dvid: server: {server} uuid: {uuid} segmentation-name: {instance_name} supervoxels: true create-if-necessary: true creation-settings: max-scale: {max_scale} voxel-size: {voxel_dimensions} geometry: bounding-box: [[0,0,0], {list(volume.shape[::-1])}] message-block-shape: [64,64,64] """) yaml = YAML() with StringIO(config_text) as f: volume_config = yaml.load(f) assert instance_name not in fetch_repo_instances(server, uuid) service = VolumeService.create_from_config(volume_config) repo_instances = fetch_repo_instances(server, uuid) assert instance_name in repo_instances assert repo_instances[instance_name] == 'labelmap' info = fetch_instance_info(server, uuid, instance_name) assert info["Extended"]["VoxelSize"] == voxel_dimensions scaled_volumes = {} for scale in range(max_scale + 1): vol = downsample(volume, 2**scale, 'label') aligned_shape = (np.ceil(np.array(vol.shape) / 64) * 64).astype(int) aligned_vol = np.zeros(aligned_shape, np.uint64) overwrite_subvol(aligned_vol, [(0, 0, 0), vol.shape], vol) service.write_subvolume(aligned_vol, (0, 0, 0), scale) scaled_volumes[scale] = aligned_vol box = np.array([[40, 80, 40], [240, 160, 100]]) for scale in range(max_scale + 1): scaled_box = box // 2**scale vol = service.get_subvolume(scaled_box, scale) assert (vol == extract_subvol(scaled_volumes[scale], scaled_box)).all() # # Check sparse coords function # labels = list({*pd.unique(volume.reshape(-1))} - {0}) brick_coords_df = service.sparse_brick_coords_for_labels(labels) assert brick_coords_df.columns.tolist() == ['z', 'y', 'x', 'label'] assert set(brick_coords_df['label'].values) == set(labels), \ "Some labels were missing from the sparse brick coords!" def ndi(shape): return np.indices(shape).reshape(len(shape), -1).transpose() expected_df = pd.DataFrame(ndi(volume.shape), columns=[*'zyx']) expected_df['label'] = volume.reshape(-1) expected_df['z'] //= 64 expected_df['y'] //= 64 expected_df['x'] //= 64 expected_df = expected_df.drop_duplicates() expected_df['z'] *= 64 expected_df['y'] *= 64 expected_df['x'] *= 64 expected_df = expected_df.query('label != 0') expected_df.sort_values(['z', 'y', 'x', 'label'], inplace=True) brick_coords_df.sort_values(['z', 'y', 'x', 'label'], inplace=True) expected_df.reset_index(drop=True, inplace=True) brick_coords_df.reset_index(drop=True, inplace=True) assert expected_df.shape == brick_coords_df.shape assert (brick_coords_df == expected_df).all().all() # # Check sample_labels() # points = [np.random.randint(d, size=(10, )) for d in vol.shape] points = np.transpose(points) labels = service.sample_labels(points) assert (labels == volume[(*points.transpose(), )]).all()