def test_bbox_to_mip():
info = {
'data_type': 'uint8',
'mesh': '',
'num_channels': 1,
'scales': [
{
'chunk_sizes': [[64, 64, 1]],
'encoding': 'raw',
'key': '4_4_40',
'resolution': [4, 4, 40],
'size': [1024, 1024, 32],
'voxel_offset': [35, 0, 1],
},
{
'chunk_sizes': [[64, 64, 1]],
'encoding': 'raw',
'key': '8_8_40',
'resolution': [8, 8, 40],
'size': [512, 512, 32],
'voxel_offset': [17, 0, 1],
},
{
'chunk_sizes': [[64, 64, 1]],
'encoding': 'raw',
'key': '16_16_40',
'resolution': [16, 16, 40],
'size': [256, 256, 32],
'voxel_offset': [8, 0, 1],
},
{
'chunk_sizes': [[64, 64, 1]],
'encoding': 'raw',
'key': '32_32_40',
'resolution': [32, 32, 40],
'size': [128, 128, 32],
'voxel_offset': [4, 0, 1],
},
],
'type': 'image'
}
cv = CloudVolume('file:///tmp/removeme/bbox_to_mip', info=info)
bbox = Bbox( (35,0,1), (1024, 1024, 32))
res = cv.bbox_to_mip(bbox, 0, 3)
assert res.minpt.x == 4
assert res.minpt.y == 0
assert res.minpt.z == 1
bbox = Bbox( (4, 0, 1), (128, 128, 32) )
res = cv.bbox_to_mip(bbox, 3, 0)
assert res.minpt.x == 32
assert res.minpt.y == 0
assert res.minpt.z == 1
res = cv.bbox_to_mip(bbox, 0, 0)
assert res == bbox
def create_quantize_tasks(src_layer,
dest_layer,
shape,
mip=0,
fill_missing=False,
chunk_size=(128, 128, 64),
encoding='raw',
bounds=None):
shape = Vec(*shape)
info = create_quantized_affinity_info(src_layer, dest_layer, shape, mip,
chunk_size, encoding)
destvol = CloudVolume(dest_layer, info=info, mip=mip)
destvol.commit_info()
downsample_scales.create_downsample_scales(dest_layer,
mip=mip,
ds_shape=shape,
chunk_size=chunk_size,
encoding=encoding)
if bounds is None:
bounds = destvol.mip_bounds(mip)
else:
bounds = destvol.bbox_to_mip(bounds, mip=0, to_mip=mip)
bounds = bounds.expand_to_chunk_size(destvol.mip_chunk_size(mip),
destvol.mip_voxel_offset(mip))
class QuantizeTasksIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return partial(
QuantizeTask,
source_layer_path=src_layer,
dest_layer_path=dest_layer,
shape=shape.tolist(),
offset=offset.tolist(),
fill_missing=fill_missing,
mip=mip,
)
def on_finish(self):
destvol.provenance.sources = [src_layer]
destvol.provenance.processing.append({
'method': {
'task': 'QuantizeTask',
'source_layer_path': src_layer,
'dest_layer_path': dest_layer,
'shape': shape.tolist(),
'fill_missing': fill_missing,
'mip': mip,
},
'by':
operator_contact(),
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
destvol.commit_provenance()
return QuantizeTasksIterator(bounds, shape)
def _draw_bounding_cube(cv_path: str, bbox, mip: int, pad: Tuple[int, int, int]):
from cloudvolume import CloudVolume
from cloudvolume.lib import Vec
from cloudvolume.lib import Bbox
from .utils import draw_bounding_cube
cv = CloudVolume(cv_path, mip=mip, fill_missing=True)
pad = Vec(*pad)
mip0_bbox = Bbox.from_list(bbox)
vol_start = mip0_bbox.minpt
vol_stop = mip0_bbox.maxpt
vol_bbox = cv.bbox_to_mip( # pylint: disable=no-member
Bbox(vol_start - pad, vol_stop + pad), mip, mip
)
draw_bbox = cv.bbox_to_mip(mip0_bbox, mip, mip) # pylint: disable=no-member
arr = cv[vol_bbox.to_slices()][:, :, :, 0] # pylint: disable=unsubscriptable-object
local_draw_bbox = draw_bbox - vol_bbox.minpt
if any(x != 0 for x in pad):
draw_bounding_cube(arr, local_draw_bbox, val=255)
return arr, draw_bbox
def create_blackout_tasks(cloudpath: str,
bounds: Bbox,
mip: int = 0,
shape: ShapeType = (2048, 2048, 64),
value: int = 0,
non_aligned_writes: bool = False):
vol = CloudVolume(cloudpath, mip=mip)
shape = Vec(*shape)
bounds = Bbox.create(bounds)
bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip)
if not non_aligned_writes:
bounds = bounds.expand_to_chunk_size(vol.chunk_size, vol.voxel_offset)
bounds = Bbox.clamp(bounds, vol.mip_bounds(mip))
class BlackoutTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
bounded_shape = min2(shape, vol.bounds.maxpt - offset)
return partial(
igneous.tasks.BlackoutTask,
cloudpath=cloudpath,
mip=mip,
shape=shape.clone(),
offset=offset.clone(),
value=value,
non_aligned_writes=non_aligned_writes,
)
def on_finish(self):
vol.provenance.processing.append({
'method': {
'task': 'BlackoutTask',
'cloudpath': cloudpath,
'mip': mip,
'non_aligned_writes': non_aligned_writes,
'value': value,
'shape': shape.tolist(),
'bounds': [
bounds.minpt.tolist(),
bounds.maxpt.tolist(),
],
},
'by':
operator_contact(),
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
return BlackoutTaskIterator(bounds, shape)
def create_blackout_tasks(cloudpath,
bounds,
mip=0,
shape=(2048, 2048, 64),
value=0,
non_aligned_writes=False):
vol = CloudVolume(cloudpath, mip=mip)
shape = Vec(*shape)
bounds = Bbox.create(bounds)
bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip)
bounds = Bbox.clamp(bounds, vol.mip_bounds(mip))
class BlackoutTaskIterator():
def __len__(self):
return num_tasks(bounds, shape)
def __iter__(self):
for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape):
bounded_shape = min2(shape, vol.bounds.maxpt - startpt)
yield igneous.tasks.BlackoutTask(
cloudpath=cloudpath,
mip=mip,
shape=shape.clone(),
offset=startpt.clone(),
value=value,
non_aligned_writes=non_aligned_writes,
)
vol.provenance.processing.append({
'method': {
'task': 'BlackoutTask',
'cloudpath': cloudpath,
'mip': mip,
'non_aligned_writes': non_aligned_writes,
'value': value,
'shape': shape.tolist(),
'bounds': [
bounds.minpt.tolist(),
bounds.maxpt.tolist(),
],
},
'by':
OPERATOR_CONTACT,
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
return BlackoutTaskIterator()
def create_touch_tasks(self,
cloudpath,
mip=0,
shape=(2048, 2048, 64),
bounds=None):
vol = CloudVolume(cloudpath, mip=mip)
shape = Vec(*shape)
if bounds is None:
bounds = vol.bounds.clone()
bounds = Bbox.create(bounds)
bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip)
bounds = Bbox.clamp(bounds, vol.mip_bounds(mip))
class TouchTaskIterator():
def __len__(self):
return num_tasks(bounds, shape)
def __iter__(self):
for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape):
bounded_shape = min2(shape, vol.bounds.maxpt - startpt)
yield igneous.tasks.TouchTask(
cloudpath=cloudpath,
shape=bounded_shape.clone(),
offset=startpt.clone(),
mip=mip,
)
vol.provenance.processing.append({
'method': {
'task': 'TouchTask',
'mip': mip,
'shape': shape.tolist(),
'bounds': [
bounds.minpt.tolist(),
bounds.maxpt.tolist(),
],
},
'by':
OPERATOR_CONTACT,
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return TouchTaskIterator()
def execute(self):
vol = CloudVolume(self.layer_path, mip=self.mip)
highres_bbox = Bbox( self.offset, self.offset + self.shape )
top_mip = min(vol.available_mips[-1], self.mip + self.num_mips)
for mip in range(self.mip, top_mip + 1):
vol.mip = mip
bbox = vol.bbox_to_mip(highres_bbox, self.mip, mip)
bbox = bbox.round_to_chunk_size(vol.underlying, offset=vol.bounds.minpt)
bbox = Bbox.clamp(bbox, vol.bounds)
if bbox.volume() == 0:
continue
vol.delete(bbox)
def create_touch_tasks(
self, cloudpath,
mip=0, shape=(2048, 2048, 64),
bounds=None
):
vol = CloudVolume(cloudpath, mip=mip)
shape = Vec(*shape)
if bounds is None:
bounds = vol.bounds.clone()
bounds = Bbox.create(bounds)
bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip)
bounds = Bbox.clamp(bounds, vol.mip_bounds(mip))
class TouchTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
bounded_shape = min2(shape, vol.bounds.maxpt - offset)
return igneous.tasks.TouchTask(
cloudpath=cloudpath,
shape=bounded_shape.clone(),
offset=offset.clone(),
mip=mip,
)
def on_finish(self):
vol.provenance.processing.append({
'method': {
'task': 'TouchTask',
'mip': mip,
'shape': shape.tolist(),
'bounds': [
bounds.minpt.tolist(),
bounds.maxpt.tolist(),
],
},
'by': OPERATOR_CONTACT,
'date': strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return TouchTaskIterator(bounds, shape)
def DeleteTask(layer_path:str, shape, offset, mip=0, num_mips=5):
"""Delete a block of images inside a layer on all mip levels."""
shape = Vec(*shape)
offset = Vec(*offset)
vol = CloudVolume(layer_path, mip=mip, max_redirects=0)
highres_bbox = Bbox( offset, offset + shape )
top_mip = min(vol.available_mips[-1], mip + num_mips)
for mip_i in range(mip, top_mip + 1):
vol.mip = mip_i
bbox = vol.bbox_to_mip(highres_bbox, mip, mip_i)
bbox = bbox.round_to_chunk_size(vol.chunk_size, offset=vol.bounds.minpt)
bbox = Bbox.clamp(bbox, vol.bounds)
if bbox.volume() == 0:
continue
vol.delete(bbox)
def create_transfer_tasks(src_layer_path,
dest_layer_path,
chunk_size=None,
shape=Vec(2048, 2048, 64),
fill_missing=False,
translate=(0, 0, 0),
bounds=None,
mip=0,
preserve_chunk_size=True,
encoding=None):
"""
Transfer data from one data layer to another. It's possible
to transfer from a lower resolution mip level within a given
bounding box. The bounding box should be specified in terms of
the highest resolution.
"""
shape = Vec(*shape)
vol = CloudVolume(src_layer_path, mip=mip)
translate = Vec(*translate) // vol.downsample_ratio
if not chunk_size:
chunk_size = vol.info['scales'][mip]['chunk_sizes'][0]
chunk_size = Vec(*chunk_size)
try:
dvol = CloudVolume(dest_layer_path, mip=mip)
except Exception: # no info file
info = copy.deepcopy(vol.info)
dvol = CloudVolume(dest_layer_path, info=info)
dvol.commit_info()
if encoding is not None:
dvol.info['scales'][mip]['encoding'] = encoding
dvol.info['scales'] = dvol.info['scales'][:mip + 1]
dvol.info['scales'][mip]['chunk_sizes'] = [chunk_size.tolist()]
dvol.commit_info()
create_downsample_scales(dest_layer_path,
mip=mip,
ds_shape=shape,
preserve_chunk_size=preserve_chunk_size,
encoding=encoding)
if bounds is None:
bounds = vol.bounds.clone()
else:
bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip)
bounds = Bbox.clamp(bounds, dvol.bounds)
dvol_bounds = dvol.mip_bounds(mip).clone()
class TransferTaskIterator(object):
def __len__(self):
return int(reduce(operator.mul, np.ceil(bounds.size3() / shape)))
def __iter__(self):
for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape):
task_shape = min2(shape.clone(), dvol_bounds.maxpt - startpt)
yield TransferTask(
src_path=src_layer_path,
dest_path=dest_layer_path,
shape=task_shape,
offset=startpt.clone(),
fill_missing=fill_missing,
translate=translate,
mip=mip,
)
job_details = {
'method': {
'task': 'TransferTask',
'src': src_layer_path,
'dest': dest_layer_path,
'shape': list(map(int, shape)),
'fill_missing': fill_missing,
'translate': list(map(int, translate)),
'bounds': [bounds.minpt.tolist(),
bounds.maxpt.tolist()],
'mip': mip,
},
'by': OPERATOR_CONTACT,
'date': strftime('%Y-%m-%d %H:%M %Z'),
}
dvol = CloudVolume(dest_layer_path)
dvol.provenance.sources = [src_layer_path]
dvol.provenance.processing.append(job_details)
dvol.commit_provenance()
if vol.path.protocol != 'boss':
vol.provenance.processing.append(job_details)
vol.commit_provenance()
return TransferTaskIterator()
def create_graphene_meshing_tasks(
cloudpath, timestamp, mip,
simplification=True, max_simplification_error=40,
mesh_dir=None, cdn_cache=False, object_ids=None,
progress=False, fill_missing=False, sharding=None,
draco_compression_level=1, bounds=None
):
cv = CloudVolume(cloudpath, mip=mip)
if mip < cv.meta.watershed_mip:
raise ValueError("Must mesh at or above the watershed mip level. Watershed MIP: {} Got: {}".format(
cv.meta.watershed_mip, mip
))
if mesh_dir is None:
mesh_dir = 'meshes'
cv.info['mesh'] = mesh_dir # necessary to set the mesh.commit_info() dir right
if not 'mesh' in cv.info:
cv.commit_info()
watershed_downsample_ratio = cv.resolution // cv.meta.resolution(cv.meta.watershed_mip)
shape = Vec(*cv.meta.graph_chunk_size) // watershed_downsample_ratio
cv.mesh.meta.info['@type'] = 'neuroglancer_legacy_mesh'
cv.mesh.meta.info['mip'] = cv.mip
cv.mesh.meta.info['chunk_size'] = list(shape)
if sharding:
cv.mesh.meta.info['sharding'] = sharding
cv.mesh.meta.commit_info()
simplification = (0 if not simplification else 100)
class GrapheneMeshTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return GrapheneMeshTask(
cloudpath=cloudpath,
shape=shape.clone(),
offset=offset.clone(),
mip=int(mip),
simplification_factor=simplification,
max_simplification_error=max_simplification_error,
draco_compression_level=draco_compression_level,
mesh_dir=mesh_dir,
cache_control=('' if cdn_cache else 'no-cache'),
progress=progress,
fill_missing=fill_missing,
timestamp=timestamp,
)
def on_finish(self):
cv.provenance.processing.append({
'method': {
'task': 'GrapheneMeshTask',
'cloudpath': cv.cloudpath,
'shape': cv.meta.graph_chunk_size,
'mip': int(mip),
'simplification': simplification,
'max_simplification_error': max_simplification_error,
'mesh_dir': mesh_dir,
'fill_missing': fill_missing,
'cdn_cache': cdn_cache,
'timestamp': timestamp,
'draco_compression_level': draco_compression_level,
},
'by': operator_contact(),
'date': strftime('%Y-%m-%d %H:%M %Z'),
})
cv.commit_provenance()
if bounds is None:
bounds = cv.meta.bounds(mip).clone()
else:
bounds = cv.bbox_to_mip(bounds, mip=0, to_mip=mip)
bounds = Bbox.clamp(bounds, cv.bounds)
bounds = bounds.expand_to_chunk_size(shape, cv.voxel_offset)
return GrapheneMeshTaskIterator(bounds, shape)
class ReadPrecomputedOperator(OperatorBase):
def __init__(self,
volume_path: str,
mip: int = 0,
expand_margin_size=(0, 0, 0),
fill_missing: bool = False,
validate_mip: int = None,
blackout_sections: bool = None,
dry_run: bool = False,
name: str = 'cutout'):
super().__init__(name=name)
self.volume_path = volume_path
self.mip = mip
self.expand_margin_size = expand_margin_size
self.fill_missing = fill_missing
self.validate_mip = validate_mip
self.blackout_sections = blackout_sections
self.dry_run = dry_run
if blackout_sections:
with Storage(volume_path) as stor:
self.blackout_section_ids = stor.get_json(
'blackout_section_ids.json')['section_ids']
verbose = (logging.getLogger().getEffectiveLevel() <= 30)
self.vol = CloudVolume(self.volume_path,
bounded=False,
fill_missing=self.fill_missing,
progress=verbose,
mip=self.mip,
cache=False,
green_threads=True)
def __call__(self, output_bbox):
chunk_slices = tuple(
slice(s.start - m, s.stop + m)
for s, m in zip(output_bbox.to_slices(), self.expand_margin_size))
if self.dry_run:
input_bbox = Bbox.from_slices(chunk_slices)
return Chunk.from_bbox(input_bbox)
logging.info('cutout {} from {}'.format(chunk_slices[::-1],
self.volume_path))
# always reverse the indexes since cloudvolume use x,y,z indexing
chunk = self.vol[chunk_slices[::-1]]
chunk = np.asarray(chunk)
# the cutout is fortran ordered, so need to transpose and make it C order
chunk = chunk.transpose()
# we can delay this transpose later
# actually we do not need to make it contiguous
# chunk = np.ascontiguousarray(chunk)
# if the channel number is 1, squeeze it as 3d array
# this should not be neccessary
# TODO: remove this step and use 4D array all over this package.
# always use 4D array will simplify some operations
voxel_offset = tuple(s.start for s in chunk_slices)
if chunk.shape[0] == 1:
chunk = np.squeeze(chunk, axis=0)
else:
voxel_offset = (0, ) + voxel_offset
chunk = Chunk(chunk,
voxel_offset=voxel_offset,
voxel_size=tuple(self.vol.resolution[::-1]))
if self.blackout_sections:
chunk = self._blackout_sections(chunk)
if self.validate_mip:
self._validate_chunk(chunk)
return chunk
def _blackout_sections(self, chunk):
"""
make some sections black.
this was normally used for the section with bad alignment.
The ConvNet was supposed to handle them better with black image.
TODO: make this function as a separate operator
"""
# current code only works with 3d image
assert chunk.ndim == 3, "current code assumes that the chunk is 3D image."
for z in self.blackout_section_ids:
z0 = z - chunk.voxel_offset[0]
if z0 >= 0 and z0 < chunk.shape[0]:
chunk[z0, :, :] = 0
return chunk
def _validate_chunk(self, chunk):
"""
check that all the input voxels was downloaded without black region
We have found some black regions in previous inference run,
so hopefully this will solve the problem.
"""
if chunk.ndim == 4 and chunk.shape[0] > 1:
chunk = chunk[0, :, :, :]
validate_vol = CloudVolume(self.volume_path,
bounded=False,
fill_missing=self.fill_missing,
progress=False,
mip=self.validate_mip,
cache=False,
green_threads=True)
chunk_mip = self.mip
logging.info('validate chunk in mip {}'.format(self.validate_mip))
assert self.validate_mip >= chunk_mip
# only use the region corresponds to higher mip level
# clamp the surrounding regions in XY plane
# this assumes that the input dataset was downsampled starting from the
# beginning offset in the info file
voxel_offset = chunk.voxel_offset
# factor3 follows xyz order in CloudVolume
factor3 = np.array([
2**(self.validate_mip - chunk_mip), 2
**(self.validate_mip - chunk_mip), 1
],
dtype=np.int32)
clamped_offset = tuple(go + f - (go - vo) % f for go, vo, f in zip(
voxel_offset[::-1], self.vol.voxel_offset, factor3))
clamped_stop = tuple(
go + s - (go + s - vo) % f
for go, s, vo, f in zip(voxel_offset[::-1], chunk.shape[::-1],
vol.voxel_offset, factor3))
clamped_slices = tuple(
slice(o, s) for o, s in zip(clamped_offset, clamped_stop))
clamped_bbox = Bbox.from_slices(clamped_slices)
clamped_input = chunk.cutout(clamped_slices[::-1])
# transform to xyz order
clamped_input = np.transpose(clamped_input)
# get the corresponding bounding box for validation
validate_bbox = self.vol.bbox_to_mip(clamped_bbox,
mip=chunk_mip,
to_mip=self.validate_mip)
#validate_bbox = clamped_bbox // factor3
# downsample the input using avaraging
# keep the z as it is since the mip only applies to xy plane
# recursivly downsample the input
# if we do it directly, the downsampled input will not be the same with the recursive one
# because of the rounding error of integer division
for _ in range(self.validate_mip - chunk_mip):
clamped_input = downsample_with_averaging(clamped_input, (2, 2, 1))
# validation by template matching
assert validate_by_template_matching(clamped_input)
validate_input = validate_vol[validate_bbox.to_slices()]
if validate_input.shape[3] == 1:
validate_input = np.squeeze(validate_input, axis=3)
# use the validate input to check the downloaded input
assert np.alltrue(validate_input == clamped_input)
