def fetch_z_levels(self, bounds):
cf = CloudFiles(self.levels_path)
levelfilenames = [
cf.join('levels', f"{self.mip}", f"{z}")
for z in range(bounds.minpt.z, bounds.maxpt.z)
]
levels = cf.get(levelfilenames)
errors = [
level['path'] \
for level in levels if level['content'] == None
]
if len(errors):
raise Exception(", ".join(
errors) + " were not defined. Did you run a LuminanceLevelsTask for these slices?")
levels = [(
int(os.path.basename(item['path'])),
json.loads(item['content'].decode('utf-8'))
) for item in levels ]
levels.sort(key=lambda x: x[0])
levels = [x[1] for x in levels]
return [ np.array(x['levels'], dtype=np.uint64) for x in levels ]
def SpatialIndexTask(
cloudpath: str,
shape: Tuple[int, int, int],
offset: Tuple[int, int, int],
subdir: str,
precision: int,
mip: int = 0,
fill_missing: bool = False,
compress: Optional[Union[str, bool]] = 'gzip',
) -> None:
"""
The main way to add a spatial index is to use the MeshTask or SkeletonTask,
but old datasets or broken datasets may need it to be
reconstituted. An alternative use is create the spatial index
over a different area size than the mesh or skeleton task.
"""
cv = CloudVolume(cloudpath,
mip=mip,
bounded=False,
fill_missing=fill_missing)
cf = CloudFiles(cloudpath)
bounds = Bbox(Vec(*offset), Vec(*shape) + Vec(*offset))
bounds = Bbox.clamp(bounds, cv.bounds)
data_bounds = bounds.clone()
data_bounds.maxpt += 1 # match typical Marching Cubes overlap
resolution = cv.resolution
# remap: old img -> img
img, remap = cv.download(data_bounds, renumber=True)
img = img[..., 0]
slcs = find_objects(img)
del img
reverse_map = {v: k for k, v in remap.items()} # img -> old img
bboxes = {}
for label, slc in enumerate(slcs):
if slc is None:
continue
obj_bounds = Bbox.from_slices(slc)
obj_bounds += Vec(*offset)
obj_bounds *= Vec(*resolution, dtype=np.float32)
bboxes[str(reverse_map[label+1])] = \
obj_bounds.astype(resolution.dtype).to_list()
bounds = bounds.astype(resolution.dtype) * resolution
cf.put_json(
cf.join(subdir, f"{bounds.to_filename(precision)}.spatial"),
bboxes,
compress=compress,
cache_control=False,
)
def download(self, bbox, mip, parallel=1, renumber=False):
if parallel != 1:
raise ValueError("Only parallel=1 is supported for n5.")
elif renumber != False:
raise ValueError("Only renumber=False is supported for n5.")
bounds = Bbox.clamp(bbox, self.meta.bounds(mip))
if self.autocrop:
image, bounds = autocropfn(self.meta, image, bounds, mip)
if bounds.subvoxel():
raise exceptions.EmptyRequestException(
f'Requested less than one pixel of volume. {bounds}')
cf = CloudFiles(self.meta.cloudpath, progress=self.config.progress)
realized_bbox = bbox.expand_to_chunk_size(self.meta.chunk_size(mip))
grid_bbox = realized_bbox // self.meta.chunk_size(mip)
urls = [
cf.join(f"s{mip}", str(x), str(y), str(z))
for x, y, z in xyzrange(grid_bbox.minpt, grid_bbox.maxpt)
]
all_chunks = cf.get(urls, parallel=parallel, return_dict=True)
shape = list(bbox.size3()) + [self.meta.num_channels]
renderbuffer = np.zeros(shape=shape, dtype=self.meta.dtype, order='F')
sep = '/'
if cf._path.protocol == "file":
sep = os.path.sep
if sep == '\\':
sep = '\\\\' # compensate for regexp escaping
regexp = re.compile(
rf"s(?P<mip>\d+){sep}(?P<x>\d+){sep}(?P<y>\d+){sep}(?P<z>\d+)")
for fname, binary in all_chunks.items():
m = re.search(regexp, fname).groupdict()
assert mip == int(m["mip"])
gridpoint = Vec(*[int(i) for i in [m["x"], m["y"], m["z"]]])
chunk_bbox = Bbox(gridpoint,
gridpoint + 1) * self.meta.chunk_size(mip)
chunk_bbox = Bbox.clamp(chunk_bbox, self.meta.bounds(mip))
default_shape = list(chunk_bbox.size3()) + [self.meta.num_channels]
chunk, chunk_shape = self.parse_chunk(binary, mip, fname,
default_shape)
chunk_bbox = Bbox(chunk_bbox.minpt,
chunk_bbox.minpt + Vec(*chunk_shape[:3]))
chunk_bbox = Bbox.clamp(chunk_bbox, self.meta.bounds(mip))
shade(renderbuffer, bbox, chunk, chunk_bbox)
return VolumeCutout.from_volume(self.meta, mip, renderbuffer, bbox)
def MeshManifestPrefixTask(layer_path: str,
prefix: str,
lod: int = 0,
mesh_dir: Optional[str] = None):
"""
Finalize mesh generation by post-processing chunk fragment
lists into mesh fragment manifests.
These are necessary for neuroglancer to know which mesh
fragments to download for a given segid.
If we parallelize using prefixes single digit prefixes ['0','1',..'9'] all meshes will
be correctly processed. But if we do ['10','11',..'99'] meshes from [0,9] won't get
processed and need to be handle specifically by creating tasks that will process
a single mesh ['0:','1:',..'9:']
"""
cf = CloudFiles(layer_path)
info = cf.get_json('info')
if mesh_dir is None and 'mesh' in info:
mesh_dir = info['mesh']
prefix = cf.join(mesh_dir, prefix)
segids = defaultdict(list)
regexp = re.compile(r'(\d+):(\d+):')
for filename in cf.list(prefix=prefix):
filename = os.path.basename(filename)
# `match` implies the beginning (^). `search` matches whole string
matches = re.search(regexp, filename)
if not matches:
continue
segid, mlod = matches.groups()
segid, mlod = int(segid), int(mlod)
if mlod != lod:
continue
segids[segid].append(filename)
items = ((f"{mesh_dir}/{segid}:{lod}", {
"fragments": frags
}) for segid, frags in segids.items())
cf.put_jsons(items)
def MeshManifestFilesystemTask(
layer_path: str,
lod: int = 0,
mesh_dir: Optional[str] = None,
):
cf = CloudFiles(layer_path)
info = cf.get_json('info')
if mesh_dir is None and 'mesh' in info:
mesh_dir = info['mesh']
filepath = cloudfiles.paths.asfilepath(cf.join(layer_path, mesh_dir))
segids = defaultdict(list)
regexp = re.compile(r'(\d+):(\d+):')
for entry in os.scandir(filepath):
if not entry.is_file():
continue
filename = os.path.basename(entry.name)
# `match` implies the beginning (^). `search` matches whole string
matches = re.search(regexp, filename)
if not matches:
continue
segid, mlod = matches.groups()
segid, mlod = int(segid), int(mlod)
if mlod != lod:
continue
filename, ext = os.path.splitext(filename)
segids[segid].append(filename)
items = ((f"{mesh_dir}/{segid}:{lod}", {
"fragments": frags
}) for segid, frags in segids.items())
cf.put_jsons(items)
def fetch_info(self):
cf = CloudFiles(self.cloudpath, secrets=self.config.secrets)
self.attributes["root"] = cf.get_json("attributes.json")
if 'pixelResolution' in self.attributes["root"]:
resolution = self.attributes["root"]["pixelResolution"][
"dimensions"]
else:
resolution = self.attributes["root"]["resolution"]
scale_dirs = [
cf.join(f"s{i}", "attributes.json")
for i in range(len(self.attributes["root"]["scales"]))
]
scale_attrs = cf.get_json(scale_dirs)
self.attributes["scales"] = scale_attrs
# glossing over that each scale can have
# a different data type, but usually it
# should all be the same
data_type = scale_attrs[0]["dataType"]
info = PrecomputedMetadata.create_info(
num_channels=1,
layer_type="image",
data_type=data_type,
encoding=scale_attrs[0]["compression"]["type"],
resolution=resolution,
voxel_offset=[0, 0, 0],
volume_size=scale_attrs[0]["dimensions"][:3],
chunk_size=scale_attrs[0]["blockSize"],
)
for scale in scale_attrs[1:]:
self.add_scale(scale["downsamplingFactors"],
chunk_size=scale["blockSize"],
encoding=scale["compression"]["type"],
info=info)
return info
def create_spatial_index_skeleton_tasks(
cloudpath: str,
shape: Tuple[int, int, int] = (448, 448, 448),
mip: int = 0,
fill_missing: bool = False,
compress: Optional[Union[str, bool]] = 'gzip',
skel_dir: Optional[str] = None):
"""
The main way to add a spatial index is to use the SkeletonTask,
but old datasets or broken datasets may need it to be
reconstituted. An alternative use is create the spatial index
over a different area size than the skeleton task.
"""
shape = Vec(*shape)
vol = CloudVolume(cloudpath, mip=mip)
if skel_dir is None and not vol.info.get("skeletons", None):
skel_dir = f"skeletons_mip_{mip}"
elif skel_dir is None and vol.info.get("skeletons", None):
skel_dir = vol.info["skeletons"]
if not "skeletons" in vol.info:
vol.info['skeletons'] = skel_dir
vol.commit_info()
cf = CloudFiles(cloudpath)
info_filename = cf.join(skel_dir, 'info')
skel_info = cf.get_json(info_filename) or {}
new_skel_info = copy.deepcopy(skel_info)
new_skel_info['@type'] = new_skel_info.get('@type',
'neuroglancer_skeletons')
new_skel_info['mip'] = new_skel_info.get("mip", int(vol.mip))
new_skel_info['chunk_size'] = shape.tolist()
new_skel_info['spatial_index'] = {
'resolution': vol.resolution.tolist(),
'chunk_size': (shape * vol.resolution).tolist(),
}
if new_skel_info != skel_info:
cf.put_json(info_filename, new_skel_info)
vol = CloudVolume(cloudpath, mip=mip) # reload spatial_index
class SpatialIndexSkeletonTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return partial(
SpatialIndexTask,
cloudpath=cloudpath,
shape=shape,
offset=offset,
subdir=skel_dir,
precision=vol.skeleton.spatial_index.precision,
mip=int(mip),
fill_missing=bool(fill_missing),
compress=compress,
)
def on_finish(self):
vol.provenance.processing.append({
'method': {
'task': 'SpatialIndexTask',
'cloudpath': vol.cloudpath,
'shape': shape.tolist(),
'mip': int(mip),
'subdir': skel_dir,
'fill_missing': fill_missing,
'compress': compress,
},
'by':
operator_contact(),
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return SpatialIndexSkeletonTaskIterator(vol.bounds, shape)
