def fetch_z_levels(self, bounds): cf = CloudFiles(self.levels_path) levelfilenames = [ cf.join('levels', f"{self.mip}", f"{z}") for z in range(bounds.minpt.z, bounds.maxpt.z) ] levels = cf.get(levelfilenames) errors = [ level['path'] \ for level in levels if level['content'] == None ] if len(errors): raise Exception(", ".join( errors) + " were not defined. Did you run a LuminanceLevelsTask for these slices?") levels = [( int(os.path.basename(item['path'])), json.loads(item['content'].decode('utf-8')) ) for item in levels ] levels.sort(key=lambda x: x[0]) levels = [x[1] for x in levels] return [ np.array(x['levels'], dtype=np.uint64) for x in levels ]
def SpatialIndexTask( cloudpath: str, shape: Tuple[int, int, int], offset: Tuple[int, int, int], subdir: str, precision: int, mip: int = 0, fill_missing: bool = False, compress: Optional[Union[str, bool]] = 'gzip', ) -> None: """ The main way to add a spatial index is to use the MeshTask or SkeletonTask, but old datasets or broken datasets may need it to be reconstituted. An alternative use is create the spatial index over a different area size than the mesh or skeleton task. """ cv = CloudVolume(cloudpath, mip=mip, bounded=False, fill_missing=fill_missing) cf = CloudFiles(cloudpath) bounds = Bbox(Vec(*offset), Vec(*shape) + Vec(*offset)) bounds = Bbox.clamp(bounds, cv.bounds) data_bounds = bounds.clone() data_bounds.maxpt += 1 # match typical Marching Cubes overlap resolution = cv.resolution # remap: old img -> img img, remap = cv.download(data_bounds, renumber=True) img = img[..., 0] slcs = find_objects(img) del img reverse_map = {v: k for k, v in remap.items()} # img -> old img bboxes = {} for label, slc in enumerate(slcs): if slc is None: continue obj_bounds = Bbox.from_slices(slc) obj_bounds += Vec(*offset) obj_bounds *= Vec(*resolution, dtype=np.float32) bboxes[str(reverse_map[label+1])] = \ obj_bounds.astype(resolution.dtype).to_list() bounds = bounds.astype(resolution.dtype) * resolution cf.put_json( cf.join(subdir, f"{bounds.to_filename(precision)}.spatial"), bboxes, compress=compress, cache_control=False, )
def download(self, bbox, mip, parallel=1, renumber=False): if parallel != 1: raise ValueError("Only parallel=1 is supported for n5.") elif renumber != False: raise ValueError("Only renumber=False is supported for n5.") bounds = Bbox.clamp(bbox, self.meta.bounds(mip)) if self.autocrop: image, bounds = autocropfn(self.meta, image, bounds, mip) if bounds.subvoxel(): raise exceptions.EmptyRequestException( f'Requested less than one pixel of volume. {bounds}') cf = CloudFiles(self.meta.cloudpath, progress=self.config.progress) realized_bbox = bbox.expand_to_chunk_size(self.meta.chunk_size(mip)) grid_bbox = realized_bbox // self.meta.chunk_size(mip) urls = [ cf.join(f"s{mip}", str(x), str(y), str(z)) for x, y, z in xyzrange(grid_bbox.minpt, grid_bbox.maxpt) ] all_chunks = cf.get(urls, parallel=parallel, return_dict=True) shape = list(bbox.size3()) + [self.meta.num_channels] renderbuffer = np.zeros(shape=shape, dtype=self.meta.dtype, order='F') sep = '/' if cf._path.protocol == "file": sep = os.path.sep if sep == '\\': sep = '\\\\' # compensate for regexp escaping regexp = re.compile( rf"s(?P<mip>\d+){sep}(?P<x>\d+){sep}(?P<y>\d+){sep}(?P<z>\d+)") for fname, binary in all_chunks.items(): m = re.search(regexp, fname).groupdict() assert mip == int(m["mip"]) gridpoint = Vec(*[int(i) for i in [m["x"], m["y"], m["z"]]]) chunk_bbox = Bbox(gridpoint, gridpoint + 1) * self.meta.chunk_size(mip) chunk_bbox = Bbox.clamp(chunk_bbox, self.meta.bounds(mip)) default_shape = list(chunk_bbox.size3()) + [self.meta.num_channels] chunk, chunk_shape = self.parse_chunk(binary, mip, fname, default_shape) chunk_bbox = Bbox(chunk_bbox.minpt, chunk_bbox.minpt + Vec(*chunk_shape[:3])) chunk_bbox = Bbox.clamp(chunk_bbox, self.meta.bounds(mip)) shade(renderbuffer, bbox, chunk, chunk_bbox) return VolumeCutout.from_volume(self.meta, mip, renderbuffer, bbox)
def MeshManifestPrefixTask(layer_path: str, prefix: str, lod: int = 0, mesh_dir: Optional[str] = None): """ Finalize mesh generation by post-processing chunk fragment lists into mesh fragment manifests. These are necessary for neuroglancer to know which mesh fragments to download for a given segid. If we parallelize using prefixes single digit prefixes ['0','1',..'9'] all meshes will be correctly processed. But if we do ['10','11',..'99'] meshes from [0,9] won't get processed and need to be handle specifically by creating tasks that will process a single mesh ['0:','1:',..'9:'] """ cf = CloudFiles(layer_path) info = cf.get_json('info') if mesh_dir is None and 'mesh' in info: mesh_dir = info['mesh'] prefix = cf.join(mesh_dir, prefix) segids = defaultdict(list) regexp = re.compile(r'(\d+):(\d+):') for filename in cf.list(prefix=prefix): filename = os.path.basename(filename) # `match` implies the beginning (^). `search` matches whole string matches = re.search(regexp, filename) if not matches: continue segid, mlod = matches.groups() segid, mlod = int(segid), int(mlod) if mlod != lod: continue segids[segid].append(filename) items = ((f"{mesh_dir}/{segid}:{lod}", { "fragments": frags }) for segid, frags in segids.items()) cf.put_jsons(items)
def MeshManifestFilesystemTask( layer_path: str, lod: int = 0, mesh_dir: Optional[str] = None, ): cf = CloudFiles(layer_path) info = cf.get_json('info') if mesh_dir is None and 'mesh' in info: mesh_dir = info['mesh'] filepath = cloudfiles.paths.asfilepath(cf.join(layer_path, mesh_dir)) segids = defaultdict(list) regexp = re.compile(r'(\d+):(\d+):') for entry in os.scandir(filepath): if not entry.is_file(): continue filename = os.path.basename(entry.name) # `match` implies the beginning (^). `search` matches whole string matches = re.search(regexp, filename) if not matches: continue segid, mlod = matches.groups() segid, mlod = int(segid), int(mlod) if mlod != lod: continue filename, ext = os.path.splitext(filename) segids[segid].append(filename) items = ((f"{mesh_dir}/{segid}:{lod}", { "fragments": frags }) for segid, frags in segids.items()) cf.put_jsons(items)
def fetch_info(self): cf = CloudFiles(self.cloudpath, secrets=self.config.secrets) self.attributes["root"] = cf.get_json("attributes.json") if 'pixelResolution' in self.attributes["root"]: resolution = self.attributes["root"]["pixelResolution"][ "dimensions"] else: resolution = self.attributes["root"]["resolution"] scale_dirs = [ cf.join(f"s{i}", "attributes.json") for i in range(len(self.attributes["root"]["scales"])) ] scale_attrs = cf.get_json(scale_dirs) self.attributes["scales"] = scale_attrs # glossing over that each scale can have # a different data type, but usually it # should all be the same data_type = scale_attrs[0]["dataType"] info = PrecomputedMetadata.create_info( num_channels=1, layer_type="image", data_type=data_type, encoding=scale_attrs[0]["compression"]["type"], resolution=resolution, voxel_offset=[0, 0, 0], volume_size=scale_attrs[0]["dimensions"][:3], chunk_size=scale_attrs[0]["blockSize"], ) for scale in scale_attrs[1:]: self.add_scale(scale["downsamplingFactors"], chunk_size=scale["blockSize"], encoding=scale["compression"]["type"], info=info) return info
def create_spatial_index_skeleton_tasks( cloudpath: str, shape: Tuple[int, int, int] = (448, 448, 448), mip: int = 0, fill_missing: bool = False, compress: Optional[Union[str, bool]] = 'gzip', skel_dir: Optional[str] = None): """ The main way to add a spatial index is to use the SkeletonTask, but old datasets or broken datasets may need it to be reconstituted. An alternative use is create the spatial index over a different area size than the skeleton task. """ shape = Vec(*shape) vol = CloudVolume(cloudpath, mip=mip) if skel_dir is None and not vol.info.get("skeletons", None): skel_dir = f"skeletons_mip_{mip}" elif skel_dir is None and vol.info.get("skeletons", None): skel_dir = vol.info["skeletons"] if not "skeletons" in vol.info: vol.info['skeletons'] = skel_dir vol.commit_info() cf = CloudFiles(cloudpath) info_filename = cf.join(skel_dir, 'info') skel_info = cf.get_json(info_filename) or {} new_skel_info = copy.deepcopy(skel_info) new_skel_info['@type'] = new_skel_info.get('@type', 'neuroglancer_skeletons') new_skel_info['mip'] = new_skel_info.get("mip", int(vol.mip)) new_skel_info['chunk_size'] = shape.tolist() new_skel_info['spatial_index'] = { 'resolution': vol.resolution.tolist(), 'chunk_size': (shape * vol.resolution).tolist(), } if new_skel_info != skel_info: cf.put_json(info_filename, new_skel_info) vol = CloudVolume(cloudpath, mip=mip) # reload spatial_index class SpatialIndexSkeletonTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): return partial( SpatialIndexTask, cloudpath=cloudpath, shape=shape, offset=offset, subdir=skel_dir, precision=vol.skeleton.spatial_index.precision, mip=int(mip), fill_missing=bool(fill_missing), compress=compress, ) def on_finish(self): vol.provenance.processing.append({ 'method': { 'task': 'SpatialIndexTask', 'cloudpath': vol.cloudpath, 'shape': shape.tolist(), 'mip': int(mip), 'subdir': skel_dir, 'fill_missing': fill_missing, 'compress': compress, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return SpatialIndexSkeletonTaskIterator(vol.bounds, shape)