def __call__(self, chunk):
assert 3 == chunk.ndim
voxel_offset = chunk.voxel_offset
num_mips = self.stop_mip - self.chunk_mip
# tinybrain use F order and require 4D array!
chunk2 = np.transpose(chunk)
# chunk2 = np.reshape(chunk2, (*chunk2.shape, 1))
chunk2 = np.expand_dims(chunk2, 3)
if np.issubdtype(chunk.dtype, np.floating) or chunk.dtype == np.uint8:
pyramid = tinybrain.downsample_with_averaging(
chunk2, factor=self.factor[::-1], num_mips=num_mips)
else:
pyramid = tinybrain.downsample_segmentation(
chunk2, factor=self.factor[::-1], num_mips=num_mips)
for mip in range(self.start_mip, self.stop_mip):
# the first chunk in pyramid is already downsampled!
downsampled_chunk = pyramid[mip - self.chunk_mip - 1]
# compute new offset, only downsample the y,x dimensions
offset = np.divide(
voxel_offset,
np.asarray([
self.factor[0]**(mip - self.chunk_mip),
self.factor[1]**(mip - self.chunk_mip),
self.factor[2]**(mip - self.chunk_mip)
]))
bbox = Bbox.from_delta(offset, downsampled_chunk.shape[0:3][::-1])
# upload downsampled chunk, note that we should use F order in the indexing
self.vols[mip][bbox.to_slices()[::-1]] = downsampled_chunk
def create_bounding_boxes(chunk_size: tuple,
chunk_overlap: tuple = (0, 0, 0),
roi_start: tuple = None,
roi_stop: tuple = None,
layer_path: str = None,
mip: int = 0,
grid_size: tuple = None,
verbose: bool = True):
if layer_path:
vol = CloudVolume(layer_path, mip=mip)
# dataset shape as z,y,x
dataset_size = vol.mip_shape(mip)[:3][::-1]
dataset_offset = vol.mip_voxel_offset(mip)[::-1]
if roi_stop is None:
roi_stop = Vec(
*[o + s for o, s in zip(dataset_offset, dataset_size)])
if roi_start is None:
# note that we normally start from -overlap to keep the chunks aligned!
roi_start = dataset_offset - chunk_overlap
chunk_size = Vec(*chunk_size)
chunk_overlap = Vec(*chunk_overlap)
stride = chunk_size - chunk_overlap
if isinstance(grid_size, tuple):
grid_size = Vec(*grid_size)
assert roi_start is not None
if isinstance(roi_start, tuple):
roi_start = Vec(*roi_start)
if roi_stop is None:
roi_stop = roi_start + stride * grid_size + chunk_overlap
elif isinstance(roi_stop, tuple):
roi_stop = Vec(*roi_stop)
roi_size = roi_stop - roi_start
if grid_size is None:
grid_size = (roi_size - chunk_overlap) // stride + 1
# the stride should not be zero if there is more than one chunks
for g, s in zip(grid_size, stride):
if g > 1:
assert s > 0
final_output_stop = roi_start + (grid_size - 1) * stride + chunk_size
if verbose:
print('\nroi start: ', roi_start)
print('stride: ', stride)
print('grid size: ', grid_size)
print('final output stop: ', final_output_stop)
bboxes = []
for (z, y, x) in product(range(grid_size[0]), range(grid_size[1]),
range(grid_size[2])):
chunk_start = roi_start + Vec(z, y, x) * stride
bbox = Bbox.from_delta(chunk_start, chunk_size)
bboxes.append(bbox)
return bboxes
def coordinates2bbox(start: tuple, size: tuple = None, stop: tuple = None):
# use bounding box of volume
start = Vec(*start)
if size is None:
assert stop is not None
size = stop - start
else:
size = Vec(*size)
return Bbox.from_delta(start, size)
def test_cutout(self):
print('test volume cutout...')
operator = CutoutOperator(self.volume_path, mip=self.mip)
offset = (4, 64, 64)
shape = (28, 320, 320)
output_bbox = Bbox.from_delta(offset, shape)
chunk = operator(output_bbox)
self.assertEqual(offset, chunk.global_offset)
self.assertTrue(chunk == self.img[4:-4, 64:-64, 64:-64])
shutil.rmtree('/tmp/test')
def test_bounding_box():
bbox = Bbox.from_delta((1, 3, 2), (64, 32, 8))
bbox = BoundingBox.from_bbox(bbox)
assert bbox.start == Cartesian(1, 3, 2)
assert bbox.stop == Cartesian(65, 35, 10)
bbox = bbox.clone()
assert isinstance(bbox, BoundingBox)
minpt = Cartesian(1, 2, 3)
maxpt = Cartesian(2, 3, 4)
bbox = BoundingBox(minpt, maxpt)
bbox = BoundingBox.from_center(Cartesian(1, 2, 3), 3)
assert bbox == BoundingBox.from_list([-2, -1, 0, 4, 5, 6])
bbox = BoundingBox.from_center(Cartesian(1, 2, 3), 3, even_size=False)
assert bbox == BoundingBox.from_list([-2, -1, 0, 5, 6, 7])
def test_blackout_sections(self):
print('test blackout sections...')
operator = CutoutOperator(self.volume_path,
mip=self.mip,
blackout_sections=True)
offset = (4, 64, 64)
shape = (28, 320, 320)
output_bbox = Bbox.from_delta(offset, shape)
chunk = operator(output_bbox)
img = np.copy(self.img)
for z in self.blackout_section_ids:
img[z, :, :] = 0
img = img[4:-4, 64:-64, 64:-64]
self.assertTrue(img == chunk)
shutil.rmtree('/tmp/test')
def __call__(self, chunk):
"""
TODO: implement using precomputed lookup tables.
The intensity value map could be precomputed as a list,
and we only need to replace the voxel intensity according to the list.
"""
# this is a image, not affinitymap
assert chunk.ndim == 3
image = np.transpose(chunk).astype(np.float32)
# translate to xyz order since cloudvolume is working this F order
offset = Vec(*chunk.global_offset[::-1])
shape = Vec(*image.shape)
bounds = Bbox.from_delta(offset, shape)
zlevels = self.fetch_z_levels(bounds)
# number of bits per voxel
nbits = np.dtype(chunk.dtype).itemsize * 8
maxval = float(2**nbits - 1)
for z in range(bounds.minpt.z, bounds.maxpt.z):
imagez = z - bounds.minpt.z
zlevel = zlevels[imagez]
(lower, upper) = self.find_section_clamping_values(
zlevel, self.clip_fraction, 1 - self.clip_fraction)
if lower == upper:
continue
img = image[:, :, imagez]
img = (img - float(lower)) * (maxval /
(float(upper) - float(lower)))
image[:, :, imagez] = img
image = np.round(image)
minval = self.minval if self.minval is not None else 0.0
maxval = self.maxval if self.maxval is not None else maxval
image = np.clip(image, minval, maxval)
chunk = np.transpose(image).astype(chunk.dtype)
chunk = Chunk(chunk, global_offset=(*offset, )[::-1])
return chunk
def create_bounding_boxes(chunk_size:tuple, overlap: tuple=(0,0,0),
start:tuple=None, layer_path: str=None, mip:int=0,
grid_size: tuple=None, verbose: bool=True):
if layer_path:
vol = CloudVolume(layer_path, mip=mip)
# dataset shape as z,y,x
dataset_shape = vol.mip_shape(mip)[:3][::-1]
dataset_offset = vol.mip_voxel_offset(mip)[::-1]
chunk_size = Vec(*chunk_size)
overlap = Vec(*overlap)
stride = chunk_size - overlap
if start is None:
# note that we normally start from -overlap to keep the chunks aligned!
start = dataset_offset - overlap
volume_size = dataset_shape
else:
start = Vec(*start)
if grid_size is None:
volume_size = dataset_shape - (start - dataset_offset)
grid_size = (volume_size-overlap) // stride + 1
# the stride should not be zero if there is more than one chunks
for g, s in zip(grid_size, stride):
if g > 1:
assert s > 0
if verbose:
print('\nstart: ', start)
print('stride: ', stride)
print('grid size: ', grid_size)
print('chunk_size: ', chunk_size, '\n')
bboxes = []
for (z, y, x) in tqdm(product(range(grid_size[0]), range(grid_size[1]),
range(grid_size[2]))):
chunk_start = start + Vec(z, y, x) * stride
bbox = Bbox.from_delta(chunk_start, chunk_size)
bboxes.append( bbox )
return bboxes
def __call__(self, chunk, log={'timer': {}}, output_bbox=None):
start = time.time()
chunk = self._auto_convert_dtype(chunk)
chunk_slices = chunk.slices
# transpose czyx to xyzc order
arr = np.transpose(chunk)
self.volume[chunk_slices[::-1]] = arr
if self.create_thumbnail:
self._create_thumbnail(chunk)
# add timer for save operation itself
log['timer'][self.name] = time.time() - start
if self.upload_log:
if output_bbox is None:
output_bbox = Bbox.from_delta((0, 0, 0), chunk.shape)
self._upload_log(log, output_bbox)
def from_manual_setup(cls,
chunk_size: Union[Vec, tuple],
chunk_overlap: Union[Vec, tuple] = Vec(0, 0, 0),
roi_start: Union[Vec, tuple] = None,
roi_stop: Union[Vec, tuple] = None,
roi_size: Union[Vec, tuple] = None,
grid_size: Union[Vec, tuple] = None,
respect_chunk_size: bool = True,
aligned_block_size: Union[Vec, tuple] = None,
layer_path: str = None,
mip: int = 0):
if layer_path:
if layer_path.endswith('.h5'):
assert os.path.exists(layer_path)
with h5py.File(layer_path, mode='r') as file:
for key in file.keys():
if 'offset' in key:
roi_start = Vec(*(file[key]))
elif 'voxel_size' not in key:
if roi_size is None:
roi_size = Vec(*file[key].shape[-3:])
if roi_start is None:
roi_start = Vec(0, 0, 0)
roi_stop = roi_start + roi_size
else:
vol = CloudVolume(layer_path, mip=mip)
# dataset shape as z,y,x
dataset_size = vol.mip_shape(mip)[:3][::-1]
dataset_offset = vol.mip_voxel_offset(mip)[::-1]
if roi_size is None:
roi_size = Vec(*dataset_size)
if roi_stop is None:
roi_stop = Vec(
*[o + s for o, s in zip(dataset_offset, dataset_size)])
if roi_start is None:
# note that we normally start from -overlap to keep the chunks aligned!
roi_start = dataset_offset - chunk_overlap
assert roi_start is not None
if roi_size is None and roi_stop is None and grid_size is None:
grid_size = Vec(1, 1, 1)
if isinstance(chunk_size, tuple):
chunk_size = Vec(*chunk_size)
if isinstance(chunk_overlap, tuple):
chunk_overlap = Vec(*chunk_overlap)
if isinstance(roi_start, tuple):
roi_start = Vec(*roi_start)
if isinstance(roi_size, tuple):
roi_size = Vec(*roi_size)
if isinstance(grid_size, tuple):
grid_size = Vec(*grid_size)
if isinstance(roi_stop, tuple):
roi_stop = Vec(*roi_stop)
stride = chunk_size - chunk_overlap
if roi_stop is None:
roi_stop = roi_start + stride * grid_size + chunk_overlap
if aligned_block_size is not None:
if not isinstance(aligned_block_size, Vec):
aligned_block_size = Vec(*aligned_block_size)
assert np.all(aligned_block_size <= chunk_size)
assert np.alltrue(chunk_size % aligned_block_size == 0)
roi_start -= roi_start % aligned_block_size
assert len(aligned_block_size) == 3
assert len(roi_stop) == 3
for idx in range(3):
if roi_stop[idx] % aligned_block_size[idx] > 0:
roi_stop[idx] += aligned_block_size[
idx] - roi_stop[idx] % aligned_block_size[idx]
if roi_size is None:
roi_size = roi_stop - roi_start
if grid_size is None:
grid_size = (roi_size - chunk_overlap) / stride
grid_size = tuple(ceil(x) for x in grid_size)
grid_size = Vec(*grid_size)
# the stride should not be zero if there is more than one chunks
for g, s in zip(grid_size, stride):
if g > 1:
assert s > 0
final_output_stop = roi_start + (grid_size - 1) * stride + chunk_size
logging.info(f'\nroi start: {roi_start}')
logging.info(f'stride: {stride}')
logging.info(f'grid size: {grid_size}')
logging.info(f'final output stop: {final_output_stop}')
print('grid size: ', grid_size)
bboxes = []
for (gz, gy, gx) in product(range(grid_size[0]), range(grid_size[1]),
range(grid_size[2])):
chunk_start = roi_start + Vec(gz, gy, gx) * stride
bbox = Bbox.from_delta(chunk_start, chunk_size)
if not respect_chunk_size:
bbox.maxpt = np.minimum(bbox.maxpt, roi_stop)
bboxes.append(bbox)
return cls(bboxes)
def bbox(self) -> Bbox:
"""
:getter: the cloudvolume bounding box in the big volume
"""
return Bbox.from_delta(self.global_offset, self.array.shape)
def test_bbox(self):
self.assertEqual(self.chunk.bbox,
Bbox.from_delta(self.voxel_offset, self.size))
def test_create_from_bounding_box(self):
bbox = Bbox.from_delta(self.voxel_offset, self.size)
Chunk.from_bbox( bbox )
