def __call__(self, chunk): assert 3 == chunk.ndim voxel_offset = chunk.voxel_offset num_mips = self.stop_mip - self.chunk_mip # tinybrain use F order and require 4D array! chunk2 = np.transpose(chunk) # chunk2 = np.reshape(chunk2, (*chunk2.shape, 1)) chunk2 = np.expand_dims(chunk2, 3) if np.issubdtype(chunk.dtype, np.floating) or chunk.dtype == np.uint8: pyramid = tinybrain.downsample_with_averaging( chunk2, factor=self.factor[::-1], num_mips=num_mips) else: pyramid = tinybrain.downsample_segmentation( chunk2, factor=self.factor[::-1], num_mips=num_mips) for mip in range(self.start_mip, self.stop_mip): # the first chunk in pyramid is already downsampled! downsampled_chunk = pyramid[mip - self.chunk_mip - 1] # compute new offset, only downsample the y,x dimensions offset = np.divide( voxel_offset, np.asarray([ self.factor[0]**(mip - self.chunk_mip), self.factor[1]**(mip - self.chunk_mip), self.factor[2]**(mip - self.chunk_mip) ])) bbox = Bbox.from_delta(offset, downsampled_chunk.shape[0:3][::-1]) # upload downsampled chunk, note that we should use F order in the indexing self.vols[mip][bbox.to_slices()[::-1]] = downsampled_chunk
def create_bounding_boxes(chunk_size: tuple, chunk_overlap: tuple = (0, 0, 0), roi_start: tuple = None, roi_stop: tuple = None, layer_path: str = None, mip: int = 0, grid_size: tuple = None, verbose: bool = True): if layer_path: vol = CloudVolume(layer_path, mip=mip) # dataset shape as z,y,x dataset_size = vol.mip_shape(mip)[:3][::-1] dataset_offset = vol.mip_voxel_offset(mip)[::-1] if roi_stop is None: roi_stop = Vec( *[o + s for o, s in zip(dataset_offset, dataset_size)]) if roi_start is None: # note that we normally start from -overlap to keep the chunks aligned! roi_start = dataset_offset - chunk_overlap chunk_size = Vec(*chunk_size) chunk_overlap = Vec(*chunk_overlap) stride = chunk_size - chunk_overlap if isinstance(grid_size, tuple): grid_size = Vec(*grid_size) assert roi_start is not None if isinstance(roi_start, tuple): roi_start = Vec(*roi_start) if roi_stop is None: roi_stop = roi_start + stride * grid_size + chunk_overlap elif isinstance(roi_stop, tuple): roi_stop = Vec(*roi_stop) roi_size = roi_stop - roi_start if grid_size is None: grid_size = (roi_size - chunk_overlap) // stride + 1 # the stride should not be zero if there is more than one chunks for g, s in zip(grid_size, stride): if g > 1: assert s > 0 final_output_stop = roi_start + (grid_size - 1) * stride + chunk_size if verbose: print('\nroi start: ', roi_start) print('stride: ', stride) print('grid size: ', grid_size) print('final output stop: ', final_output_stop) bboxes = [] for (z, y, x) in product(range(grid_size[0]), range(grid_size[1]), range(grid_size[2])): chunk_start = roi_start + Vec(z, y, x) * stride bbox = Bbox.from_delta(chunk_start, chunk_size) bboxes.append(bbox) return bboxes
def coordinates2bbox(start: tuple, size: tuple = None, stop: tuple = None): # use bounding box of volume start = Vec(*start) if size is None: assert stop is not None size = stop - start else: size = Vec(*size) return Bbox.from_delta(start, size)
def test_cutout(self): print('test volume cutout...') operator = CutoutOperator(self.volume_path, mip=self.mip) offset = (4, 64, 64) shape = (28, 320, 320) output_bbox = Bbox.from_delta(offset, shape) chunk = operator(output_bbox) self.assertEqual(offset, chunk.global_offset) self.assertTrue(chunk == self.img[4:-4, 64:-64, 64:-64]) shutil.rmtree('/tmp/test')
def test_bounding_box(): bbox = Bbox.from_delta((1, 3, 2), (64, 32, 8)) bbox = BoundingBox.from_bbox(bbox) assert bbox.start == Cartesian(1, 3, 2) assert bbox.stop == Cartesian(65, 35, 10) bbox = bbox.clone() assert isinstance(bbox, BoundingBox) minpt = Cartesian(1, 2, 3) maxpt = Cartesian(2, 3, 4) bbox = BoundingBox(minpt, maxpt) bbox = BoundingBox.from_center(Cartesian(1, 2, 3), 3) assert bbox == BoundingBox.from_list([-2, -1, 0, 4, 5, 6]) bbox = BoundingBox.from_center(Cartesian(1, 2, 3), 3, even_size=False) assert bbox == BoundingBox.from_list([-2, -1, 0, 5, 6, 7])
def test_blackout_sections(self): print('test blackout sections...') operator = CutoutOperator(self.volume_path, mip=self.mip, blackout_sections=True) offset = (4, 64, 64) shape = (28, 320, 320) output_bbox = Bbox.from_delta(offset, shape) chunk = operator(output_bbox) img = np.copy(self.img) for z in self.blackout_section_ids: img[z, :, :] = 0 img = img[4:-4, 64:-64, 64:-64] self.assertTrue(img == chunk) shutil.rmtree('/tmp/test')
def __call__(self, chunk): """ TODO: implement using precomputed lookup tables. The intensity value map could be precomputed as a list, and we only need to replace the voxel intensity according to the list. """ # this is a image, not affinitymap assert chunk.ndim == 3 image = np.transpose(chunk).astype(np.float32) # translate to xyz order since cloudvolume is working this F order offset = Vec(*chunk.global_offset[::-1]) shape = Vec(*image.shape) bounds = Bbox.from_delta(offset, shape) zlevels = self.fetch_z_levels(bounds) # number of bits per voxel nbits = np.dtype(chunk.dtype).itemsize * 8 maxval = float(2**nbits - 1) for z in range(bounds.minpt.z, bounds.maxpt.z): imagez = z - bounds.minpt.z zlevel = zlevels[imagez] (lower, upper) = self.find_section_clamping_values( zlevel, self.clip_fraction, 1 - self.clip_fraction) if lower == upper: continue img = image[:, :, imagez] img = (img - float(lower)) * (maxval / (float(upper) - float(lower))) image[:, :, imagez] = img image = np.round(image) minval = self.minval if self.minval is not None else 0.0 maxval = self.maxval if self.maxval is not None else maxval image = np.clip(image, minval, maxval) chunk = np.transpose(image).astype(chunk.dtype) chunk = Chunk(chunk, global_offset=(*offset, )[::-1]) return chunk
def create_bounding_boxes(chunk_size:tuple, overlap: tuple=(0,0,0), start:tuple=None, layer_path: str=None, mip:int=0, grid_size: tuple=None, verbose: bool=True): if layer_path: vol = CloudVolume(layer_path, mip=mip) # dataset shape as z,y,x dataset_shape = vol.mip_shape(mip)[:3][::-1] dataset_offset = vol.mip_voxel_offset(mip)[::-1] chunk_size = Vec(*chunk_size) overlap = Vec(*overlap) stride = chunk_size - overlap if start is None: # note that we normally start from -overlap to keep the chunks aligned! start = dataset_offset - overlap volume_size = dataset_shape else: start = Vec(*start) if grid_size is None: volume_size = dataset_shape - (start - dataset_offset) grid_size = (volume_size-overlap) // stride + 1 # the stride should not be zero if there is more than one chunks for g, s in zip(grid_size, stride): if g > 1: assert s > 0 if verbose: print('\nstart: ', start) print('stride: ', stride) print('grid size: ', grid_size) print('chunk_size: ', chunk_size, '\n') bboxes = [] for (z, y, x) in tqdm(product(range(grid_size[0]), range(grid_size[1]), range(grid_size[2]))): chunk_start = start + Vec(z, y, x) * stride bbox = Bbox.from_delta(chunk_start, chunk_size) bboxes.append( bbox ) return bboxes
def __call__(self, chunk, log={'timer': {}}, output_bbox=None): start = time.time() chunk = self._auto_convert_dtype(chunk) chunk_slices = chunk.slices # transpose czyx to xyzc order arr = np.transpose(chunk) self.volume[chunk_slices[::-1]] = arr if self.create_thumbnail: self._create_thumbnail(chunk) # add timer for save operation itself log['timer'][self.name] = time.time() - start if self.upload_log: if output_bbox is None: output_bbox = Bbox.from_delta((0, 0, 0), chunk.shape) self._upload_log(log, output_bbox)
def from_manual_setup(cls, chunk_size: Union[Vec, tuple], chunk_overlap: Union[Vec, tuple] = Vec(0, 0, 0), roi_start: Union[Vec, tuple] = None, roi_stop: Union[Vec, tuple] = None, roi_size: Union[Vec, tuple] = None, grid_size: Union[Vec, tuple] = None, respect_chunk_size: bool = True, aligned_block_size: Union[Vec, tuple] = None, layer_path: str = None, mip: int = 0): if layer_path: if layer_path.endswith('.h5'): assert os.path.exists(layer_path) with h5py.File(layer_path, mode='r') as file: for key in file.keys(): if 'offset' in key: roi_start = Vec(*(file[key])) elif 'voxel_size' not in key: if roi_size is None: roi_size = Vec(*file[key].shape[-3:]) if roi_start is None: roi_start = Vec(0, 0, 0) roi_stop = roi_start + roi_size else: vol = CloudVolume(layer_path, mip=mip) # dataset shape as z,y,x dataset_size = vol.mip_shape(mip)[:3][::-1] dataset_offset = vol.mip_voxel_offset(mip)[::-1] if roi_size is None: roi_size = Vec(*dataset_size) if roi_stop is None: roi_stop = Vec( *[o + s for o, s in zip(dataset_offset, dataset_size)]) if roi_start is None: # note that we normally start from -overlap to keep the chunks aligned! roi_start = dataset_offset - chunk_overlap assert roi_start is not None if roi_size is None and roi_stop is None and grid_size is None: grid_size = Vec(1, 1, 1) if isinstance(chunk_size, tuple): chunk_size = Vec(*chunk_size) if isinstance(chunk_overlap, tuple): chunk_overlap = Vec(*chunk_overlap) if isinstance(roi_start, tuple): roi_start = Vec(*roi_start) if isinstance(roi_size, tuple): roi_size = Vec(*roi_size) if isinstance(grid_size, tuple): grid_size = Vec(*grid_size) if isinstance(roi_stop, tuple): roi_stop = Vec(*roi_stop) stride = chunk_size - chunk_overlap if roi_stop is None: roi_stop = roi_start + stride * grid_size + chunk_overlap if aligned_block_size is not None: if not isinstance(aligned_block_size, Vec): aligned_block_size = Vec(*aligned_block_size) assert np.all(aligned_block_size <= chunk_size) assert np.alltrue(chunk_size % aligned_block_size == 0) roi_start -= roi_start % aligned_block_size assert len(aligned_block_size) == 3 assert len(roi_stop) == 3 for idx in range(3): if roi_stop[idx] % aligned_block_size[idx] > 0: roi_stop[idx] += aligned_block_size[ idx] - roi_stop[idx] % aligned_block_size[idx] if roi_size is None: roi_size = roi_stop - roi_start if grid_size is None: grid_size = (roi_size - chunk_overlap) / stride grid_size = tuple(ceil(x) for x in grid_size) grid_size = Vec(*grid_size) # the stride should not be zero if there is more than one chunks for g, s in zip(grid_size, stride): if g > 1: assert s > 0 final_output_stop = roi_start + (grid_size - 1) * stride + chunk_size logging.info(f'\nroi start: {roi_start}') logging.info(f'stride: {stride}') logging.info(f'grid size: {grid_size}') logging.info(f'final output stop: {final_output_stop}') print('grid size: ', grid_size) bboxes = [] for (gz, gy, gx) in product(range(grid_size[0]), range(grid_size[1]), range(grid_size[2])): chunk_start = roi_start + Vec(gz, gy, gx) * stride bbox = Bbox.from_delta(chunk_start, chunk_size) if not respect_chunk_size: bbox.maxpt = np.minimum(bbox.maxpt, roi_stop) bboxes.append(bbox) return cls(bboxes)
def bbox(self) -> Bbox: """ :getter: the cloudvolume bounding box in the big volume """ return Bbox.from_delta(self.global_offset, self.array.shape)
def test_bbox(self): self.assertEqual(self.chunk.bbox, Bbox.from_delta(self.voxel_offset, self.size))
def test_create_from_bounding_box(self): bbox = Bbox.from_delta(self.voxel_offset, self.size) Chunk.from_bbox( bbox )