def test_bbox_to_mip(): info = { 'data_type': 'uint8', 'mesh': '', 'num_channels': 1, 'scales': [ { 'chunk_sizes': [[64, 64, 1]], 'encoding': 'raw', 'key': '4_4_40', 'resolution': [4, 4, 40], 'size': [1024, 1024, 32], 'voxel_offset': [35, 0, 1], }, { 'chunk_sizes': [[64, 64, 1]], 'encoding': 'raw', 'key': '8_8_40', 'resolution': [8, 8, 40], 'size': [512, 512, 32], 'voxel_offset': [17, 0, 1], }, { 'chunk_sizes': [[64, 64, 1]], 'encoding': 'raw', 'key': '16_16_40', 'resolution': [16, 16, 40], 'size': [256, 256, 32], 'voxel_offset': [8, 0, 1], }, { 'chunk_sizes': [[64, 64, 1]], 'encoding': 'raw', 'key': '32_32_40', 'resolution': [32, 32, 40], 'size': [128, 128, 32], 'voxel_offset': [4, 0, 1], }, ], 'type': 'image' } cv = CloudVolume('file:///tmp/removeme/bbox_to_mip', info=info) bbox = Bbox( (35,0,1), (1024, 1024, 32)) res = cv.bbox_to_mip(bbox, 0, 3) assert res.minpt.x == 4 assert res.minpt.y == 0 assert res.minpt.z == 1 bbox = Bbox( (4, 0, 1), (128, 128, 32) ) res = cv.bbox_to_mip(bbox, 3, 0) assert res.minpt.x == 32 assert res.minpt.y == 0 assert res.minpt.z == 1 res = cv.bbox_to_mip(bbox, 0, 0) assert res == bbox
def create_quantize_tasks(src_layer, dest_layer, shape, mip=0, fill_missing=False, chunk_size=(128, 128, 64), encoding='raw', bounds=None): shape = Vec(*shape) info = create_quantized_affinity_info(src_layer, dest_layer, shape, mip, chunk_size, encoding) destvol = CloudVolume(dest_layer, info=info, mip=mip) destvol.commit_info() downsample_scales.create_downsample_scales(dest_layer, mip=mip, ds_shape=shape, chunk_size=chunk_size, encoding=encoding) if bounds is None: bounds = destvol.mip_bounds(mip) else: bounds = destvol.bbox_to_mip(bounds, mip=0, to_mip=mip) bounds = bounds.expand_to_chunk_size(destvol.mip_chunk_size(mip), destvol.mip_voxel_offset(mip)) class QuantizeTasksIterator(FinelyDividedTaskIterator): def task(self, shape, offset): return partial( QuantizeTask, source_layer_path=src_layer, dest_layer_path=dest_layer, shape=shape.tolist(), offset=offset.tolist(), fill_missing=fill_missing, mip=mip, ) def on_finish(self): destvol.provenance.sources = [src_layer] destvol.provenance.processing.append({ 'method': { 'task': 'QuantizeTask', 'source_layer_path': src_layer, 'dest_layer_path': dest_layer, 'shape': shape.tolist(), 'fill_missing': fill_missing, 'mip': mip, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) destvol.commit_provenance() return QuantizeTasksIterator(bounds, shape)
def _draw_bounding_cube(cv_path: str, bbox, mip: int, pad: Tuple[int, int, int]): from cloudvolume import CloudVolume from cloudvolume.lib import Vec from cloudvolume.lib import Bbox from .utils import draw_bounding_cube cv = CloudVolume(cv_path, mip=mip, fill_missing=True) pad = Vec(*pad) mip0_bbox = Bbox.from_list(bbox) vol_start = mip0_bbox.minpt vol_stop = mip0_bbox.maxpt vol_bbox = cv.bbox_to_mip( # pylint: disable=no-member Bbox(vol_start - pad, vol_stop + pad), mip, mip ) draw_bbox = cv.bbox_to_mip(mip0_bbox, mip, mip) # pylint: disable=no-member arr = cv[vol_bbox.to_slices()][:, :, :, 0] # pylint: disable=unsubscriptable-object local_draw_bbox = draw_bbox - vol_bbox.minpt if any(x != 0 for x in pad): draw_bounding_cube(arr, local_draw_bbox, val=255) return arr, draw_bbox
def create_blackout_tasks(cloudpath: str, bounds: Bbox, mip: int = 0, shape: ShapeType = (2048, 2048, 64), value: int = 0, non_aligned_writes: bool = False): vol = CloudVolume(cloudpath, mip=mip) shape = Vec(*shape) bounds = Bbox.create(bounds) bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) if not non_aligned_writes: bounds = bounds.expand_to_chunk_size(vol.chunk_size, vol.voxel_offset) bounds = Bbox.clamp(bounds, vol.mip_bounds(mip)) class BlackoutTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): bounded_shape = min2(shape, vol.bounds.maxpt - offset) return partial( igneous.tasks.BlackoutTask, cloudpath=cloudpath, mip=mip, shape=shape.clone(), offset=offset.clone(), value=value, non_aligned_writes=non_aligned_writes, ) def on_finish(self): vol.provenance.processing.append({ 'method': { 'task': 'BlackoutTask', 'cloudpath': cloudpath, 'mip': mip, 'non_aligned_writes': non_aligned_writes, 'value': value, 'shape': shape.tolist(), 'bounds': [ bounds.minpt.tolist(), bounds.maxpt.tolist(), ], }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) return BlackoutTaskIterator(bounds, shape)
def create_blackout_tasks(cloudpath, bounds, mip=0, shape=(2048, 2048, 64), value=0, non_aligned_writes=False): vol = CloudVolume(cloudpath, mip=mip) shape = Vec(*shape) bounds = Bbox.create(bounds) bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) bounds = Bbox.clamp(bounds, vol.mip_bounds(mip)) class BlackoutTaskIterator(): def __len__(self): return num_tasks(bounds, shape) def __iter__(self): for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape): bounded_shape = min2(shape, vol.bounds.maxpt - startpt) yield igneous.tasks.BlackoutTask( cloudpath=cloudpath, mip=mip, shape=shape.clone(), offset=startpt.clone(), value=value, non_aligned_writes=non_aligned_writes, ) vol.provenance.processing.append({ 'method': { 'task': 'BlackoutTask', 'cloudpath': cloudpath, 'mip': mip, 'non_aligned_writes': non_aligned_writes, 'value': value, 'shape': shape.tolist(), 'bounds': [ bounds.minpt.tolist(), bounds.maxpt.tolist(), ], }, 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) return BlackoutTaskIterator()
def create_touch_tasks(self, cloudpath, mip=0, shape=(2048, 2048, 64), bounds=None): vol = CloudVolume(cloudpath, mip=mip) shape = Vec(*shape) if bounds is None: bounds = vol.bounds.clone() bounds = Bbox.create(bounds) bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) bounds = Bbox.clamp(bounds, vol.mip_bounds(mip)) class TouchTaskIterator(): def __len__(self): return num_tasks(bounds, shape) def __iter__(self): for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape): bounded_shape = min2(shape, vol.bounds.maxpt - startpt) yield igneous.tasks.TouchTask( cloudpath=cloudpath, shape=bounded_shape.clone(), offset=startpt.clone(), mip=mip, ) vol.provenance.processing.append({ 'method': { 'task': 'TouchTask', 'mip': mip, 'shape': shape.tolist(), 'bounds': [ bounds.minpt.tolist(), bounds.maxpt.tolist(), ], }, 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return TouchTaskIterator()
def execute(self): vol = CloudVolume(self.layer_path, mip=self.mip) highres_bbox = Bbox( self.offset, self.offset + self.shape ) top_mip = min(vol.available_mips[-1], self.mip + self.num_mips) for mip in range(self.mip, top_mip + 1): vol.mip = mip bbox = vol.bbox_to_mip(highres_bbox, self.mip, mip) bbox = bbox.round_to_chunk_size(vol.underlying, offset=vol.bounds.minpt) bbox = Bbox.clamp(bbox, vol.bounds) if bbox.volume() == 0: continue vol.delete(bbox)
def create_touch_tasks( self, cloudpath, mip=0, shape=(2048, 2048, 64), bounds=None ): vol = CloudVolume(cloudpath, mip=mip) shape = Vec(*shape) if bounds is None: bounds = vol.bounds.clone() bounds = Bbox.create(bounds) bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) bounds = Bbox.clamp(bounds, vol.mip_bounds(mip)) class TouchTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): bounded_shape = min2(shape, vol.bounds.maxpt - offset) return igneous.tasks.TouchTask( cloudpath=cloudpath, shape=bounded_shape.clone(), offset=offset.clone(), mip=mip, ) def on_finish(self): vol.provenance.processing.append({ 'method': { 'task': 'TouchTask', 'mip': mip, 'shape': shape.tolist(), 'bounds': [ bounds.minpt.tolist(), bounds.maxpt.tolist(), ], }, 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return TouchTaskIterator(bounds, shape)
def DeleteTask(layer_path:str, shape, offset, mip=0, num_mips=5): """Delete a block of images inside a layer on all mip levels.""" shape = Vec(*shape) offset = Vec(*offset) vol = CloudVolume(layer_path, mip=mip, max_redirects=0) highres_bbox = Bbox( offset, offset + shape ) top_mip = min(vol.available_mips[-1], mip + num_mips) for mip_i in range(mip, top_mip + 1): vol.mip = mip_i bbox = vol.bbox_to_mip(highres_bbox, mip, mip_i) bbox = bbox.round_to_chunk_size(vol.chunk_size, offset=vol.bounds.minpt) bbox = Bbox.clamp(bbox, vol.bounds) if bbox.volume() == 0: continue vol.delete(bbox)
def create_transfer_tasks(src_layer_path, dest_layer_path, chunk_size=None, shape=Vec(2048, 2048, 64), fill_missing=False, translate=(0, 0, 0), bounds=None, mip=0, preserve_chunk_size=True, encoding=None): """ Transfer data from one data layer to another. It's possible to transfer from a lower resolution mip level within a given bounding box. The bounding box should be specified in terms of the highest resolution. """ shape = Vec(*shape) vol = CloudVolume(src_layer_path, mip=mip) translate = Vec(*translate) // vol.downsample_ratio if not chunk_size: chunk_size = vol.info['scales'][mip]['chunk_sizes'][0] chunk_size = Vec(*chunk_size) try: dvol = CloudVolume(dest_layer_path, mip=mip) except Exception: # no info file info = copy.deepcopy(vol.info) dvol = CloudVolume(dest_layer_path, info=info) dvol.commit_info() if encoding is not None: dvol.info['scales'][mip]['encoding'] = encoding dvol.info['scales'] = dvol.info['scales'][:mip + 1] dvol.info['scales'][mip]['chunk_sizes'] = [chunk_size.tolist()] dvol.commit_info() create_downsample_scales(dest_layer_path, mip=mip, ds_shape=shape, preserve_chunk_size=preserve_chunk_size, encoding=encoding) if bounds is None: bounds = vol.bounds.clone() else: bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) bounds = Bbox.clamp(bounds, dvol.bounds) dvol_bounds = dvol.mip_bounds(mip).clone() class TransferTaskIterator(object): def __len__(self): return int(reduce(operator.mul, np.ceil(bounds.size3() / shape))) def __iter__(self): for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape): task_shape = min2(shape.clone(), dvol_bounds.maxpt - startpt) yield TransferTask( src_path=src_layer_path, dest_path=dest_layer_path, shape=task_shape, offset=startpt.clone(), fill_missing=fill_missing, translate=translate, mip=mip, ) job_details = { 'method': { 'task': 'TransferTask', 'src': src_layer_path, 'dest': dest_layer_path, 'shape': list(map(int, shape)), 'fill_missing': fill_missing, 'translate': list(map(int, translate)), 'bounds': [bounds.minpt.tolist(), bounds.maxpt.tolist()], 'mip': mip, }, 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), } dvol = CloudVolume(dest_layer_path) dvol.provenance.sources = [src_layer_path] dvol.provenance.processing.append(job_details) dvol.commit_provenance() if vol.path.protocol != 'boss': vol.provenance.processing.append(job_details) vol.commit_provenance() return TransferTaskIterator()
def create_graphene_meshing_tasks( cloudpath, timestamp, mip, simplification=True, max_simplification_error=40, mesh_dir=None, cdn_cache=False, object_ids=None, progress=False, fill_missing=False, sharding=None, draco_compression_level=1, bounds=None ): cv = CloudVolume(cloudpath, mip=mip) if mip < cv.meta.watershed_mip: raise ValueError("Must mesh at or above the watershed mip level. Watershed MIP: {} Got: {}".format( cv.meta.watershed_mip, mip )) if mesh_dir is None: mesh_dir = 'meshes' cv.info['mesh'] = mesh_dir # necessary to set the mesh.commit_info() dir right if not 'mesh' in cv.info: cv.commit_info() watershed_downsample_ratio = cv.resolution // cv.meta.resolution(cv.meta.watershed_mip) shape = Vec(*cv.meta.graph_chunk_size) // watershed_downsample_ratio cv.mesh.meta.info['@type'] = 'neuroglancer_legacy_mesh' cv.mesh.meta.info['mip'] = cv.mip cv.mesh.meta.info['chunk_size'] = list(shape) if sharding: cv.mesh.meta.info['sharding'] = sharding cv.mesh.meta.commit_info() simplification = (0 if not simplification else 100) class GrapheneMeshTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): return GrapheneMeshTask( cloudpath=cloudpath, shape=shape.clone(), offset=offset.clone(), mip=int(mip), simplification_factor=simplification, max_simplification_error=max_simplification_error, draco_compression_level=draco_compression_level, mesh_dir=mesh_dir, cache_control=('' if cdn_cache else 'no-cache'), progress=progress, fill_missing=fill_missing, timestamp=timestamp, ) def on_finish(self): cv.provenance.processing.append({ 'method': { 'task': 'GrapheneMeshTask', 'cloudpath': cv.cloudpath, 'shape': cv.meta.graph_chunk_size, 'mip': int(mip), 'simplification': simplification, 'max_simplification_error': max_simplification_error, 'mesh_dir': mesh_dir, 'fill_missing': fill_missing, 'cdn_cache': cdn_cache, 'timestamp': timestamp, 'draco_compression_level': draco_compression_level, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) cv.commit_provenance() if bounds is None: bounds = cv.meta.bounds(mip).clone() else: bounds = cv.bbox_to_mip(bounds, mip=0, to_mip=mip) bounds = Bbox.clamp(bounds, cv.bounds) bounds = bounds.expand_to_chunk_size(shape, cv.voxel_offset) return GrapheneMeshTaskIterator(bounds, shape)
class ReadPrecomputedOperator(OperatorBase): def __init__(self, volume_path: str, mip: int = 0, expand_margin_size=(0, 0, 0), fill_missing: bool = False, validate_mip: int = None, blackout_sections: bool = None, dry_run: bool = False, name: str = 'cutout'): super().__init__(name=name) self.volume_path = volume_path self.mip = mip self.expand_margin_size = expand_margin_size self.fill_missing = fill_missing self.validate_mip = validate_mip self.blackout_sections = blackout_sections self.dry_run = dry_run if blackout_sections: with Storage(volume_path) as stor: self.blackout_section_ids = stor.get_json( 'blackout_section_ids.json')['section_ids'] verbose = (logging.getLogger().getEffectiveLevel() <= 30) self.vol = CloudVolume(self.volume_path, bounded=False, fill_missing=self.fill_missing, progress=verbose, mip=self.mip, cache=False, green_threads=True) def __call__(self, output_bbox): chunk_slices = tuple( slice(s.start - m, s.stop + m) for s, m in zip(output_bbox.to_slices(), self.expand_margin_size)) if self.dry_run: input_bbox = Bbox.from_slices(chunk_slices) return Chunk.from_bbox(input_bbox) logging.info('cutout {} from {}'.format(chunk_slices[::-1], self.volume_path)) # always reverse the indexes since cloudvolume use x,y,z indexing chunk = self.vol[chunk_slices[::-1]] chunk = np.asarray(chunk) # the cutout is fortran ordered, so need to transpose and make it C order chunk = chunk.transpose() # we can delay this transpose later # actually we do not need to make it contiguous # chunk = np.ascontiguousarray(chunk) # if the channel number is 1, squeeze it as 3d array # this should not be neccessary # TODO: remove this step and use 4D array all over this package. # always use 4D array will simplify some operations voxel_offset = tuple(s.start for s in chunk_slices) if chunk.shape[0] == 1: chunk = np.squeeze(chunk, axis=0) else: voxel_offset = (0, ) + voxel_offset chunk = Chunk(chunk, voxel_offset=voxel_offset, voxel_size=tuple(self.vol.resolution[::-1])) if self.blackout_sections: chunk = self._blackout_sections(chunk) if self.validate_mip: self._validate_chunk(chunk) return chunk def _blackout_sections(self, chunk): """ make some sections black. this was normally used for the section with bad alignment. The ConvNet was supposed to handle them better with black image. TODO: make this function as a separate operator """ # current code only works with 3d image assert chunk.ndim == 3, "current code assumes that the chunk is 3D image." for z in self.blackout_section_ids: z0 = z - chunk.voxel_offset[0] if z0 >= 0 and z0 < chunk.shape[0]: chunk[z0, :, :] = 0 return chunk def _validate_chunk(self, chunk): """ check that all the input voxels was downloaded without black region We have found some black regions in previous inference run, so hopefully this will solve the problem. """ if chunk.ndim == 4 and chunk.shape[0] > 1: chunk = chunk[0, :, :, :] validate_vol = CloudVolume(self.volume_path, bounded=False, fill_missing=self.fill_missing, progress=False, mip=self.validate_mip, cache=False, green_threads=True) chunk_mip = self.mip logging.info('validate chunk in mip {}'.format(self.validate_mip)) assert self.validate_mip >= chunk_mip # only use the region corresponds to higher mip level # clamp the surrounding regions in XY plane # this assumes that the input dataset was downsampled starting from the # beginning offset in the info file voxel_offset = chunk.voxel_offset # factor3 follows xyz order in CloudVolume factor3 = np.array([ 2**(self.validate_mip - chunk_mip), 2 **(self.validate_mip - chunk_mip), 1 ], dtype=np.int32) clamped_offset = tuple(go + f - (go - vo) % f for go, vo, f in zip( voxel_offset[::-1], self.vol.voxel_offset, factor3)) clamped_stop = tuple( go + s - (go + s - vo) % f for go, s, vo, f in zip(voxel_offset[::-1], chunk.shape[::-1], vol.voxel_offset, factor3)) clamped_slices = tuple( slice(o, s) for o, s in zip(clamped_offset, clamped_stop)) clamped_bbox = Bbox.from_slices(clamped_slices) clamped_input = chunk.cutout(clamped_slices[::-1]) # transform to xyz order clamped_input = np.transpose(clamped_input) # get the corresponding bounding box for validation validate_bbox = self.vol.bbox_to_mip(clamped_bbox, mip=chunk_mip, to_mip=self.validate_mip) #validate_bbox = clamped_bbox // factor3 # downsample the input using avaraging # keep the z as it is since the mip only applies to xy plane # recursivly downsample the input # if we do it directly, the downsampled input will not be the same with the recursive one # because of the rounding error of integer division for _ in range(self.validate_mip - chunk_mip): clamped_input = downsample_with_averaging(clamped_input, (2, 2, 1)) # validation by template matching assert validate_by_template_matching(clamped_input) validate_input = validate_vol[validate_bbox.to_slices()] if validate_input.shape[3] == 1: validate_input = np.squeeze(validate_input, axis=3) # use the validate input to check the downloaded input assert np.alltrue(validate_input == clamped_input)