def _upload_spatial_index(self, bbox, mesh_bboxes):
cf = CloudFiles(self.layer_path, progress=self.options['progress'])
cf.put_json(
f"{self._mesh_dir}/{bbox.to_filename()}.spatial",
mesh_bboxes,
compress=self.options['compress'],
cache_control=False,
)
def MeshSpatialIndex(
cloudpath:str,
shape:Tuple[int,int,int],
offset:Tuple[int,int,int],
mip:int = 0,
fill_missing:bool=False,
compress:Optional[Union[str,bool]] = 'gzip',
mesh_dir:Optional[str] = None
) -> None:
"""
The main way to add a spatial index is to use the MeshTask,
but old datasets or broken datasets may need it to be
reconstituted. An alternative use is create the spatial index
over a different area size than the mesh task.
"""
cv = CloudVolume(
cloudpath, mip=mip,
bounded=False, fill_missing=fill_missing
)
cf = CloudFiles(cloudpath)
bounds = Bbox(Vec(*offset), Vec(*shape) + Vec(*offset))
bounds = Bbox.clamp(bounds, cv.bounds)
data_bounds = bounds.clone()
data_bounds.maxpt += 1 # match typical Marching Cubes overlap
precision = cv.mesh.spatial_index.precision
resolution = cv.resolution
if not mesh_dir:
mesh_dir = cv.info["mesh"]
# remap: old img -> img
img, remap = cv.download(data_bounds, renumber=True)
img = img[...,0]
slcs = find_objects(img)
del img
reverse_map = { v:k for k,v in remap.items() } # img -> old img
bboxes = {}
for label, slc in enumerate(slcs):
if slc is None:
continue
mesh_bounds = Bbox.from_slices(slc)
mesh_bounds += Vec(*offset)
mesh_bounds *= Vec(*resolution, dtype=np.float32)
bboxes[str(reverse_map[label+1])] = \
mesh_bounds.astype(resolution.dtype).to_list()
bounds = bounds.astype(resolution.dtype) * resolution
cf.put_json(
f"{mesh_dir}/{bounds.to_filename(precision)}.spatial",
bboxes,
compress=compress,
cache_control=False,
)
class MeshManifestOperator(OperatorBase):
"""Create mesh manifest files for Neuroglancer visualization."""
def __init__(self,
volume_path: str,
lod: int = 0,
name: str = 'mesh-manifest'):
"""
Parameters
------------
volume_path:
path to store mesh manifest files
lod:
level of detail. we always use 0!
"""
super().__init__(name=name)
self.lod = lod
vol = CloudVolume(volume_path)
info = vol.info
assert 'mesh' in info
self.mesh_path = os.path.join(volume_path, info['mesh'])
self.storage = CloudFiles(self.mesh_path)
def __call__(self, prefix: Union[int, str], digits: int) -> None:
assert int(prefix) < 10**digits
prefix = str(prefix).zfill(digits)
id2filenames = defaultdict(list)
for filename in tqdm(self.storage.list(prefix=prefix),
desc='list mesh files'):
filename = os.path.basename(filename)
# `match` implies the beginning (^). `search` matches whole string
matches = re.search(r'(\d+):(\d+):', filename)
if not matches:
continue
seg_id, lod = matches.groups()
seg_id, lod = int(seg_id), int(lod)
# currently we are not using `level of detail`, it is always 0
# will need to adjust code if we start using variants
assert lod == self.lod
id2filenames[seg_id].append(filename)
for seg_id, frags in tqdm(id2filenames.items(),
desc='upload aggregated manifest file'):
logging.info(f'segment id: {seg_id}')
logging.info(f'fragments: {frags}')
self.storage.put_json(
path=f'{seg_id}:{self.lod}',
content={"fragments": frags},
)
# the last few hundred files will not be uploaded without sleeping!
sleep(0.01)
def _upload_spatial_index(self, bbox, mesh_bboxes):
cf = CloudFiles(self.layer_path, progress=self.options['progress'])
precision = self._volume.mesh.spatial_index.precision
resolution = self._volume.resolution
bbox = bbox.astype(resolution.dtype) * resolution
cf.put_json(
f"{self._mesh_dir}/{bbox.to_filename(precision)}.spatial",
mesh_bboxes,
compress=self.options['compress'],
cache_control=False,
)
def upload_spatial_index(self, vol, path, bbox, skeletons):
spatial_index = {}
for segid, skel in tqdm(skeletons.items(),
disable=(not vol.progress),
desc="Extracting Bounding Boxes"):
segid_bbx = Bbox.from_points(skel.vertices)
spatial_index[segid] = segid_bbx.to_list()
bbox = bbox * vol.resolution
cf = CloudFiles(path, progress=vol.progress)
cf.put_json(
path=f"{bbox.to_filename()}.spatial",
content=spatial_index,
compress='gzip',
cache_control=False,
)
def test_read_write(s3, protocol, num_threads, green):
from cloudfiles import CloudFiles, exceptions
url = compute_url(protocol, "rw")
cf = CloudFiles(url, num_threads=num_threads, green=green)
content = b'some_string'
cf.put('info', content, compress=None, cache_control='no-cache')
cf['info2'] = content
assert cf.get('info') == content
assert cf['info2'] == content
assert cf['info2', 0:3] == content[0:3]
assert cf['info2', :] == content[:]
assert cf.get('nonexistentfile') is None
assert cf.get('info', return_dict=True) == {"info": content}
assert cf.get(['info', 'info2'], return_dict=True) == {
"info": content,
"info2": content
}
del cf['info2']
assert cf.exists('info2') == False
num_infos = max(num_threads, 1)
results = cf.get(['info' for i in range(num_infos)])
assert len(results) == num_infos
assert results[0]['path'] == 'info'
assert results[0]['content'] == content
assert all(map(lambda x: x['error'] is None, results))
assert cf.get(['nonexistentfile'])[0]['content'] is None
cf.delete('info')
cf.put_json('info', {'omg': 'wow'}, cache_control='no-cache')
results = cf.get_json('info')
assert results == {'omg': 'wow'}
cf.delete('info')
if protocol == 'file':
rmtree(url)
def setUp(self):
print('test volume cutout...')
# compute parameters
self.mip = 0
self.size = (36, 448, 448)
# create image dataset using cloud-volume
img = np.random.randint(0, 256, size=self.size)
self.img = img.astype(np.uint8)
# save the input to disk
self.volume_path = 'file:///tmp/test/cutout/' + generate_random_string(
)
CloudVolume.from_numpy(np.transpose(self.img),
vol_path=self.volume_path)
# prepare blackout section ids
self.blackout_section_ids = [17, 20]
ids = {'section_ids': self.blackout_section_ids}
stor = CloudFiles(self.volume_path)
stor.put_json('blackout_section_ids.json', ids)
def create_xfer_meshes_tasks(
src:str,
dest:str,
mesh_dir:Optional[str] = None,
magnitude=2,
):
cv_src = CloudVolume(src)
cf_dest = CloudFiles(dest)
if not mesh_dir:
info = cf_dest.get_json("info")
if info.get("mesh", None):
mesh_dir = info.get("mesh")
cf_dest.put_json(f"{mesh_dir}/info", cv_src.mesh.meta.info)
alphabet = [ str(i) for i in range(10) ]
if cv_src.mesh.meta.is_sharded():
alphabet += [ 'a', 'b', 'c', 'd', 'e', 'f' ]
prefixes = itertools.product(*([ alphabet ] * magnitude))
prefixes = [ "".join(x) for x in prefixes ]
# explicitly enumerate all prefixes smaller than the magnitude.
for i in range(1, magnitude):
explicit_prefix = itertools.product(*([ alphabet ] * i))
explicit_prefix = [ "".join(x) for x in explicit_prefix ]
if cv_src.mesh.meta.is_sharded():
prefixes += [ f"{x}." for x in explicit_prefix ]
else:
prefixes += [ f"{x}:0" for x in explicit_prefix ]
return [
partial(TransferMeshFilesTask,
src=src,
dest=dest,
prefix=prefix,
mesh_dir=mesh_dir,
)
for prefix in prefixes
]
def configure_multires_info(
cloudpath:str,
vertex_quantization_bits:int,
mesh_dir:str
):
"""
Computes properties and uploads a multires
mesh info file
"""
assert vertex_quantization_bits in (10, 16)
vol = CloudVolume(cloudpath)
mesh_dir = mesh_dir or vol.info.get("mesh", None)
if not "mesh" in vol.info:
vol.info['mesh'] = mesh_dir
vol.commit_info()
res = vol.meta.resolution(vol.mesh.meta.mip)
cf = CloudFiles(cloudpath)
info_filename = f'{mesh_dir}/info'
mesh_info = cf.get_json(info_filename) or {}
new_mesh_info = copy.deepcopy(mesh_info)
new_mesh_info['@type'] = "neuroglancer_multilod_draco"
new_mesh_info['vertex_quantization_bits'] = vertex_quantization_bits
new_mesh_info['transform'] = [
res[0], 0, 0, 0,
0, res[1], 0, 0,
0, 0, res[2], 0,
]
new_mesh_info['lod_scale_multiplier'] = 1.0
if new_mesh_info != mesh_info:
cf.put_json(
info_filename, new_mesh_info,
cache_control="no-cache"
)
def execute(self):
srccv = CloudVolume(self.src_path, mip=self.mip, fill_missing=True)
# Accumulate a histogram of the luminance levels
nbits = np.dtype(srccv.dtype).itemsize * 8
levels = np.zeros(shape=(2 ** nbits,), dtype=np.uint64)
bounds = Bbox(self.offset, self.shape[:3] + self.offset)
bounds = Bbox.clamp(bounds, srccv.bounds)
bboxes = self.select_bounding_boxes(bounds)
for bbox in bboxes:
img2d = srccv[bbox.to_slices()].reshape((bbox.volume()))
cts = np.bincount(img2d)
levels[0:len(cts)] += cts.astype(np.uint64)
covered_area = sum([bbx.volume() for bbx in bboxes])
bboxes = [(bbox.volume(), bbox.size3()) for bbox in bboxes]
bboxes.sort(key=lambda x: x[0])
biggest = bboxes[-1][1]
output = {
"levels": levels.tolist(),
"patch_size": biggest.tolist(),
"num_patches": len(bboxes),
"coverage_ratio": covered_area / self.shape.rectVolume(),
}
path = self.levels_path if self.levels_path else self.src_path
path = os.path.join(path, 'levels')
cf = CloudFiles(path)
cf.put_json(
path="{}/{}".format(self.mip, self.offset.z),
content=output,
cache_control='no-cache'
)
class MeshOperator(OperatorBase):
"""Create mesh files from segmentation."""
def __init__(self,
output_path: str,
output_format: str,
mip: int = None,
voxel_size: tuple = (1, 1, 1),
simplification_factor: int = 100,
max_simplification_error: int = 8,
manifest: bool = False,
shard: bool = False,
name: str = 'mesh'):
"""
Parameters
------------
output_path:
path to store mesh files
output_format:
format of output {'ply', 'obj', 'precomputed'}
voxel_size:
size of voxels
simplification_factor:
mesh simplification factor.
max_simplification_error:
maximum tolerance error of meshing.
manifest:
create manifest files or not. This should
not be True if you are only doing meshing for a segmentation chunk.
name:
operator name.
Note that some functions are adopted from igneous.
"""
super().__init__(name=name)
self.simplification_factor = simplification_factor
self.max_simplification_error = max_simplification_error
# zmesh use fortran order, translate zyx to xyz
self.output_path = output_path
self.output_format = output_format
self.manifest = manifest
self.shard = shard
if manifest:
assert output_format == 'precomputed'
if output_format == 'precomputed':
# adjust the mesh path according to info
vol = CloudVolume(self.output_path, mip)
info = vol.info
if 'mesh' not in info:
# add mesh to info and update it
info['mesh'] = 'mesh_err_{}'.format(max_simplification_error)
vol.info = info
vol.commit_info()
self.mesh_path = os.path.join(output_path, info['mesh'])
self.voxel_size = vol.resolution[::-1]
self.mesher = Mesher( vol.resolution )
else:
self.mesh_path = output_path
self.mesher = Mesher(voxel_size[::-1])
self.storage = CloudFiles(self.mesh_path)
def _get_mesh_data(self, obj_id, offset):
mesh = self.mesher.get_mesh(
obj_id,
normals=False,
simplification_factor=self.simplification_factor,
max_simplification_error=self.max_simplification_error)
# delete high resolution mesh
self.mesher.erase(obj_id)
if self.output_format == 'precomputed':
mesh.vertices[:] += offset[::-1] * self.voxel_size[::-1]
data = mesh.to_precomputed()
elif self.output_format == 'ply':
data = mesh.to_ply()
elif self.output_format == 'obj':
data = mesh.to_obj()
else:
raise NotImplementedError
mesh_bounds = Bbox(
np.amin(mesh.vertices, axis=0),
np.amax(mesh.vertices, axis=0)
)
return data, mesh_bounds
def _get_file_name(self, bbox, obj_id):
if self.output_format == 'precomputed':
# bbox is in z,y,x order, should transform to x,y,z order
bbox2 = Bbox.from_slices(bbox.to_slices()[::-1])
return '{}:0:{}'.format(obj_id, bbox2.to_filename())
elif self.output_format == 'ply':
return '{}.ply'.format(obj_id)
elif self.output_format == 'obj':
return '{}.obj'.format(obj_id)
else:
raise ValueError('unsupported format!')
def __call__(self, seg: Chunk):
"""Meshing the segmentation.
Parameters
------------
seg:
3D segmentation chunk.
"""
if seg is None:
return
assert isinstance(seg, Chunk)
assert seg.ndim == 3
assert np.issubdtype(seg.dtype, np.integer)
bbox = seg.bbox
# use ndarray after getting the bounding box
seg = seg.array
logging.info('computing meshes from segmentation...')
self.mesher.mesh(seg)
logging.info('write mesh to storage...')
if self.shard:
assert 'precomputed' in self.output_format
meshes = []
mesh_bboxes = {}
for obj_id in self.mesher.ids():
data, mesh_bbox = self._get_mesh_data(obj_id, bbox.minpt)
meshes.append(data)
mesh_bboxes[obj_id] = mesh_bbox.to_list()
# use shared format in default!
self.storage.put(
f"{self.mesh_path}/{bbox.to_filename()}.frags",
content=pickle.dumps(meshes),
compress='gzip',
content_type="application/python-pickle",
cache_control=False,
)
self.storage.put_json(
f"{self.mesh_path}/{bbox.to_filename()}.spatial",
mesh_bboxes,
compress='gzip',
cache_control=False,
)
else:
if 'precomputed' in self.output_format:
compress = 'gzip'
else:
compress = None
for obj_id in tqdm(self.mesher.ids(), desc='writing out meshes'):
# print('object id: ', obj_id)
data, _ = self._get_mesh_data(obj_id, bbox.minpt)
file_name = self._get_file_name(bbox, obj_id)
self.storage.put(
file_name, data,
cache_control=None,
compress=compress
)
# create manifest file
if self.manifest:
self.storage.put_json(
f'{obj_id}:0',
{'fragments': [file_name]}
)
self.storage.put_json(
'info',
{"@type": "neuroglancer_legacy_mesh"}
)
# release memory
self.mesher.clear()
class WritePrecomputedOperator(OperatorBase):
def __init__(self,
volume_path: str,
mip: int,
upload_log: bool = True,
create_thumbnail: bool = False,
intensity_threshold: int = None,
name: str = 'write-precomputed'):
super().__init__(name=name)
self.upload_log = upload_log
self.create_thumbnail = create_thumbnail
self.mip = mip
self.intensity_threshold = intensity_threshold
# if not volume_path.startswith('precomputed://'):
# volume_path += 'precomputed://'
self.volume_path = volume_path
# gevent.monkey.patch_all(thread=False)
self.volume = CloudVolume(self.volume_path,
fill_missing=True,
bounded=False,
autocrop=True,
mip=self.mip,
cache=False,
green_threads=True,
delete_black_uploads=True,
progress=True)
#parallel=True,
if upload_log:
log_path = os.path.join(volume_path, 'log')
self.log_storage = CloudFiles(log_path)
def create_chunk_with_zeros(self, bbox, num_channels, dtype):
"""Create a fake all zero chunk.
this is used in skip some operation based on mask."""
shape = (num_channels, *bbox.size3())
arr = np.zeros(shape, dtype=dtype)
chunk = Chunk(arr, voxel_offset=(0, *bbox.minpt))
return chunk
def __call__(self, chunk, log=None):
assert isinstance(chunk, Chunk)
logging.info('save chunk.')
start = time.time()
if self.intensity_threshold is not None and np.all(
chunk.array < self.intensity_threshold):
print(
'the voxel intensity in this chunk are all below intensity threshold, return directly without saving anything.'
)
return
chunk = self._auto_convert_dtype(chunk, self.volume)
# transpose czyx to xyzc order
arr = np.transpose(chunk.array)
self.volume[chunk.slices[::-1]] = arr
if self.create_thumbnail:
self._create_thumbnail(chunk)
# add timer for save operation itself
if log:
log['timer'][self.name] = time.time() - start
if self.upload_log:
self._upload_log(log, chunk.bbox)
def _auto_convert_dtype(self, chunk, volume):
"""convert the data type to fit volume datatype"""
if np.issubdtype(volume.dtype, np.floating) and np.issubdtype(
chunk.dtype, np.uint8):
chunk = chunk.astype(volume.dtype)
chunk /= 255.
# chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max
elif np.issubdtype(volume.dtype, np.uint8) and np.issubdtype(
chunk.dtype, np.floating):
chunk.max() <= 1.
chunk *= 255
if volume.dtype != chunk.dtype:
print(
yellow(f'converting chunk data type {chunk.dtype} ' +
f'to volume data type: {volume.dtype}'))
# float_chunk = chunk.astype(np.float64)
# chunk = float_chunk / np.iinfo(chunk.dtype).max * np.iinfo(self.volume.dtype).max
# chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max
return chunk.astype(volume.dtype)
else:
return chunk
def _create_thumbnail(self, chunk):
logging.info('creating thumbnail...')
thumbnail_layer_path = os.path.join(self.volume_path, 'thumbnail')
thumbnail_volume = CloudVolume(thumbnail_layer_path,
compress='gzip',
fill_missing=True,
bounded=False,
autocrop=True,
mip=self.mip,
cache=False,
green_threads=True,
delete_black_uploads=True,
progress=False)
# only use the last channel, it is the Z affinity
# if this is affinitymap
image = chunk[-1, :, :, :]
if np.issubdtype(image.dtype, np.floating):
image = (image * 255).astype(np.uint8)
#self.thumbnail_operator(image)
# transpose to xyzc
image = np.transpose(image)
image_bbox = BoundingBox.from_slices(chunk.slices[::-1][:3])
downsample_and_upload(image,
image_bbox,
thumbnail_volume,
Vec(*(image.shape)),
mip=self.mip,
max_mip=6,
axis='z',
skip_first=True,
only_last_mip=True)
def _upload_log(self, log, output_bbox):
assert log
assert isinstance(output_bbox, BoundingBox)
logging.info(f'uploaded log: {log}')
# write to google cloud storage
self.log_storage.put_json(output_bbox.to_filename() + '.json',
content=json.dumps(log))
def create_meshing_tasks(
layer_path, mip, shape=(448, 448, 448),
simplification=True, max_simplification_error=40,
mesh_dir=None, cdn_cache=False, dust_threshold=None,
object_ids=None, progress=False, fill_missing=False,
encoding='precomputed', spatial_index=True, sharded=False,
compress='gzip'
):
shape = Vec(*shape)
vol = CloudVolume(layer_path, mip)
if mesh_dir is None:
mesh_dir = 'mesh_mip_{}_err_{}'.format(mip, max_simplification_error)
if not 'mesh' in vol.info:
vol.info['mesh'] = mesh_dir
vol.commit_info()
cf = CloudFiles(layer_path)
info_filename = '{}/info'.format(mesh_dir)
mesh_info = cf.get_json(info_filename) or {}
mesh_info['@type'] = 'neuroglancer_legacy_mesh'
mesh_info['mip'] = int(vol.mip)
mesh_info['chunk_size'] = shape.tolist()
if spatial_index:
mesh_info['spatial_index'] = {
'resolution': vol.resolution.tolist(),
'chunk_size': (shape*vol.resolution).tolist(),
}
cf.put_json(info_filename, mesh_info)
class MeshTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return MeshTask(
shape=shape.clone(),
offset=offset.clone(),
layer_path=layer_path,
mip=vol.mip,
simplification_factor=(0 if not simplification else 100),
max_simplification_error=max_simplification_error,
mesh_dir=mesh_dir,
cache_control=('' if cdn_cache else 'no-cache'),
dust_threshold=dust_threshold,
progress=progress,
object_ids=object_ids,
fill_missing=fill_missing,
encoding=encoding,
spatial_index=spatial_index,
sharded=sharded,
compress=compress,
)
def on_finish(self):
vol.provenance.processing.append({
'method': {
'task': 'MeshTask',
'layer_path': layer_path,
'mip': vol.mip,
'shape': shape.tolist(),
'simplification': simplification,
'max_simplification_error': max_simplification_error,
'mesh_dir': mesh_dir,
'fill_missing': fill_missing,
'cdn_cache': cdn_cache,
'dust_threshold': dust_threshold,
'encoding': encoding,
'object_ids': object_ids,
'spatial_index': spatial_index,
'sharded': sharded,
'compress': compress,
},
'by': operator_contact(),
'date': strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return MeshTaskIterator(vol.mip_bounds(mip), shape)
def create_spatial_index_mesh_tasks(
cloudpath:str,
shape:Tuple[int,int,int] = (448,448,448),
mip:int = 0,
fill_missing:bool = False,
compress:Optional[Union[str,bool]] = 'gzip',
mesh_dir:Optional[str] = None
):
"""
The main way to add a spatial index is to use the MeshTask,
but old datasets or broken datasets may need it to be
reconstituted. An alternative use is create the spatial index
over a different area size than the mesh task.
"""
shape = Vec(*shape)
vol = CloudVolume(cloudpath, mip=mip)
if mesh_dir is None:
mesh_dir = f"mesh_mip_{mip}_err_40"
if not "mesh" in vol.info:
vol.info['mesh'] = mesh_dir
vol.commit_info()
cf = CloudFiles(cloudpath)
info_filename = '{}/info'.format(mesh_dir)
mesh_info = cf.get_json(info_filename) or {}
new_mesh_info = copy.deepcopy(mesh_info)
new_mesh_info['@type'] = new_mesh_info.get('@type', 'neuroglancer_legacy_mesh')
new_mesh_info['mip'] = new_mesh_info.get("mip", int(vol.mip))
new_mesh_info['chunk_size'] = shape.tolist()
new_mesh_info['spatial_index'] = {
'resolution': vol.resolution.tolist(),
'chunk_size': (shape * vol.resolution).tolist(),
}
if new_mesh_info != mesh_info:
cf.put_json(info_filename, new_mesh_info)
class SpatialIndexMeshTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return partial(MeshSpatialIndex,
cloudpath=cloudpath,
shape=shape,
offset=offset,
mip=int(mip),
fill_missing=bool(fill_missing),
compress=compress,
mesh_dir=mesh_dir,
)
def on_finish(self):
vol.provenance.processing.append({
'method': {
'task': 'MeshSpatialIndex',
'cloudpath': vol.cloudpath,
'shape': shape.tolist(),
'mip': int(mip),
'mesh_dir': mesh_dir,
'fill_missing': fill_missing,
'compress': compress,
},
'by': operator_contact(),
'date': strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return SpatialIndexMeshTaskIterator(vol.bounds, shape)
def maybe_cache_info(self):
if self.enabled:
cf = CloudFiles('file://' + self.path)
cf.put_json('info', self.meta.info)
def create_spatial_index_skeleton_tasks(
cloudpath: str,
shape: Tuple[int, int, int] = (448, 448, 448),
mip: int = 0,
fill_missing: bool = False,
compress: Optional[Union[str, bool]] = 'gzip',
skel_dir: Optional[str] = None):
"""
The main way to add a spatial index is to use the SkeletonTask,
but old datasets or broken datasets may need it to be
reconstituted. An alternative use is create the spatial index
over a different area size than the skeleton task.
"""
shape = Vec(*shape)
vol = CloudVolume(cloudpath, mip=mip)
if skel_dir is None and not vol.info.get("skeletons", None):
skel_dir = f"skeletons_mip_{mip}"
elif skel_dir is None and vol.info.get("skeletons", None):
skel_dir = vol.info["skeletons"]
if not "skeletons" in vol.info:
vol.info['skeletons'] = skel_dir
vol.commit_info()
cf = CloudFiles(cloudpath)
info_filename = cf.join(skel_dir, 'info')
skel_info = cf.get_json(info_filename) or {}
new_skel_info = copy.deepcopy(skel_info)
new_skel_info['@type'] = new_skel_info.get('@type',
'neuroglancer_skeletons')
new_skel_info['mip'] = new_skel_info.get("mip", int(vol.mip))
new_skel_info['chunk_size'] = shape.tolist()
new_skel_info['spatial_index'] = {
'resolution': vol.resolution.tolist(),
'chunk_size': (shape * vol.resolution).tolist(),
}
if new_skel_info != skel_info:
cf.put_json(info_filename, new_skel_info)
vol = CloudVolume(cloudpath, mip=mip) # reload spatial_index
class SpatialIndexSkeletonTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return partial(
SpatialIndexTask,
cloudpath=cloudpath,
shape=shape,
offset=offset,
subdir=skel_dir,
precision=vol.skeleton.spatial_index.precision,
mip=int(mip),
fill_missing=bool(fill_missing),
compress=compress,
)
def on_finish(self):
vol.provenance.processing.append({
'method': {
'task': 'SpatialIndexTask',
'cloudpath': vol.cloudpath,
'shape': shape.tolist(),
'mip': int(mip),
'subdir': skel_dir,
'fill_missing': fill_missing,
'compress': compress,
},
'by':
operator_contact(),
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return SpatialIndexSkeletonTaskIterator(vol.bounds, shape)
