def test_ingest_segmentation(): delete_layer() storage, data = create_layer(size=(256,256,128,1), offset=(0,0,0), layer_type='segmentation') cv = CloudVolume(storage.layer_path) assert len(cv.scales) == 3 assert len(cv.available_mips) == 3 slice64 = np.s_[0:64, 0:64, 0:64] cv.mip = 0 assert np.all(cv[slice64] == data[slice64]) assert len(cv.available_mips) == 3 assert np.array_equal(cv.mip_volume_size(0), [ 256, 256, 128 ]) assert np.array_equal(cv.mip_volume_size(1), [ 128, 128, 128 ]) assert np.array_equal(cv.mip_volume_size(2), [ 64, 64, 128 ]) slice64 = np.s_[0:64, 0:64, 0:64] cv.mip = 0 assert np.all(cv[slice64] == data[slice64]) data_ds1 = downsample.downsample_segmentation(data, factor=[2, 2, 1, 1]) cv.mip = 1 assert np.all(cv[slice64] == data_ds1[slice64]) data_ds2 = downsample.downsample_segmentation(data_ds1, factor=[2, 2, 1, 1]) cv.mip = 2 assert np.all(cv[slice64] == data_ds2[slice64])
def test_downsample_with_offset(): delete_layer() storage, data = create_layer(size=(512,512,128,1), offset=(3,7,11)) cv = CloudVolume(storage.layer_path) assert len(cv.scales) == 4 assert len(cv.available_mips) == 4 cv.commit_info() create_downsampling_tasks(MockTaskQueue(), storage.layer_path, mip=0, num_mips=3) cv.refresh_info() assert len(cv.available_mips) == 4 assert np.array_equal(cv.mip_volume_size(0), [ 512, 512, 128 ]) assert np.array_equal(cv.mip_volume_size(1), [ 256, 256, 128 ]) assert np.array_equal(cv.mip_volume_size(2), [ 128, 128, 128 ]) assert np.array_equal(cv.mip_volume_size(3), [ 64, 64, 128 ]) assert np.all(cv.mip_voxel_offset(3) == (0,0,11)) cv.mip = 0 assert np.all(cv[3:67, 7:71, 11:75] == data[0:64, 0:64, 0:64]) data_ds1 = downsample.downsample_with_averaging(data, factor=[2, 2, 1, 1]) cv.mip = 1 assert np.all(cv[1:33, 3:35, 11:75] == data_ds1[0:32, 0:32, 0:64]) data_ds2 = downsample.downsample_with_averaging(data_ds1, factor=[2, 2, 1, 1]) cv.mip = 2 assert np.all(cv[0:16, 1:17, 11:75] == data_ds2[0:16, 0:16, 0:64]) data_ds3 = downsample.downsample_with_averaging(data_ds2, factor=[2, 2, 1, 1]) cv.mip = 3 assert np.all(cv[0:8, 0:8, 11:75] == data_ds3[0:8,0:8,0:64])
def test_downsample_w_missing(): delete_layer() storage, data = create_layer(size=(512,512,128,1), offset=(3,7,11)) cv = CloudVolume(storage.layer_path) assert len(cv.scales) == 4 assert len(cv.available_mips) == 4 delete_layer() cv.commit_info() try: create_downsampling_tasks(MockTaskQueue(), storage.layer_path, mip=0, num_mips=3, fill_missing=False) except EmptyVolumeException: pass create_downsampling_tasks(MockTaskQueue(), storage.layer_path, mip=0, num_mips=3, fill_missing=True) cv.refresh_info() assert len(cv.available_mips) == 4 assert np.array_equal(cv.mip_volume_size(0), [ 512, 512, 128 ]) assert np.array_equal(cv.mip_volume_size(1), [ 256, 256, 128 ]) assert np.array_equal(cv.mip_volume_size(2), [ 128, 128, 128 ]) assert np.array_equal(cv.mip_volume_size(3), [ 64, 64, 128 ]) assert np.all(cv.mip_voxel_offset(3) == (0,0,11)) cv.mip = 0 cv.fill_missing = True assert np.count_nonzero(cv[3:67, 7:71, 11:75]) == 0
def test_boss_download(): vol = CloudVolume('gs://seunglab-test/test_v0/image') bossvol = CloudVolume('boss://automated_testing/test_v0/image') vimg = vol[:, :, :5] bimg = bossvol[:, :, :5] assert np.all(bimg == vimg) assert bimg.dtype == vimg.dtype vol.bounded = False vol.fill_missing = True bossvol.bounded = False bossvol.fill_missing = True assert np.all(vol[-100:100, -100:100, -10:10] == bossvol[-100:100, -100:100, -10:10]) # BOSS using a different algorithm for creating downsamples # so hard to compare 1:1 w/ pixels. bossvol.bounded = True bossvol.fill_missing = False bossvol.mip = 1 bimg = bossvol[:, :, 5:6] assert np.any(bimg > 0)
def test_downsample_no_offset(compression_method): delete_layer() storage, data = create_layer(size=(1024, 1024, 128, 1), offset=(0, 0, 0)) cv = CloudVolume(storage.layer_path) assert len(cv.scales) == 1 assert len(cv.available_mips) == 1 cv.commit_info() tq = MockTaskQueue() tasks = create_downsampling_tasks(storage.layer_path, mip=0, num_mips=4, compress=compression_method) tq.insert_all(tasks) cv.refresh_info() assert len(cv.available_mips) == 5 assert np.array_equal(cv.mip_volume_size(0), [1024, 1024, 128]) assert np.array_equal(cv.mip_volume_size(1), [512, 512, 128]) assert np.array_equal(cv.mip_volume_size(2), [256, 256, 128]) assert np.array_equal(cv.mip_volume_size(3), [128, 128, 128]) assert np.array_equal(cv.mip_volume_size(4), [64, 64, 128]) slice64 = np.s_[0:64, 0:64, 0:64] cv.mip = 0 assert np.all(cv[slice64] == data[slice64]) data_ds1, = tinybrain.downsample_with_averaging(data, factor=[2, 2, 1, 1]) cv.mip = 1 assert np.all(cv[slice64] == data_ds1[slice64]) data_ds2, = tinybrain.downsample_with_averaging(data, factor=[4, 4, 1, 1]) cv.mip = 2 assert np.all(cv[slice64] == data_ds2[slice64]) data_ds3, = tinybrain.downsample_with_averaging(data, factor=[8, 8, 1, 1]) cv.mip = 3 assert np.all(cv[slice64] == data_ds3[slice64]) data_ds4, = tinybrain.downsample_with_averaging(data, factor=[16, 16, 1, 1]) cv.mip = 4 assert np.all(cv[slice64] == data_ds4[slice64])
def MultiResShardedMeshMergeTask(cloudpath: str, shard_no: str, draco_compression_level: int = 1, mesh_dir: Optional[str] = None, num_lod: int = 1, spatial_index_db: Optional[str] = None, progress: bool = False): cv = CloudVolume(cloudpath, spatial_index_db=spatial_index_db) cv.mip = cv.mesh.meta.mip if mesh_dir is None and 'mesh' in cv.info: mesh_dir = cv.info['mesh'] # This looks messy because we are trying to avoid retaining # unnecessary memory. In the original skeleton iteration, this was # using 50 GB+ memory on minnie65. So it makes sense to be just # as careful with a heavier type of object. locations = locations_for_labels(cv, labels_for_shard(cv, shard_no)) filenames = set(itertools.chain(*locations.values())) labels = set(locations.keys()) del locations meshes = collect_mesh_fragments(cv, labels, filenames, mesh_dir, progress) del labels del filenames meshes = { label: process_mesh(cv, label, mesh_frags, num_lod, draco_compression_level) for label, mesh_frags in tqdm(meshes.items(), disable=(not progress)) } data_offset = { label: len(manifest) for label, (manifest, mesh) in meshes.items() } meshes = { label: mesh + manifest.to_binary() for label, (manifest, mesh) in meshes.items() } if len(meshes) == 0: return shard_files = synthesize_shard_files(cv.mesh.reader.spec, meshes, data_offset) del meshes del data_offset if len(shard_files) != 1: raise ValueError("Only one shard file should be generated per task. " "Expected: {} Got: {} ".format( str(shard_no), ", ".join(shard_files.keys()))) cf = CloudFiles(cv.mesh.meta.layerpath, progress=progress) cf.puts( ((fname, data) for fname, data in shard_files.items()), compress=False, content_type='application/octet-stream', cache_control='no-cache', )
def execute(self): vol = CloudVolume(self.layer_path) highres_bbox = Bbox( self.offset, self.offset + self.shape ) for mip in vol.available_mips: vol.mip = mip slices = vol.slices_from_global_coords(highres_bbox.to_slices()) bbox = Bbox.from_slices(slices).round_to_chunk_size(vol.underlying, offset=vol.bounds.minpt) vol.delete(bbox)
def test_downsample_no_offset_2x2x2(): delete_layer() cf, data = create_layer(size=(512,512,512,1), offset=(0,0,0)) cv = CloudVolume(cf.cloudpath) assert len(cv.scales) == 1 assert len(cv.available_mips) == 1 cv.commit_info() tq = MockTaskQueue() tasks = create_downsampling_tasks( cf.cloudpath, mip=0, num_mips=3, compress=None, factor=(2,2,2) ) tq.insert_all(tasks) cv.refresh_info() assert len(cv.available_mips) == 4 assert np.array_equal(cv.mip_volume_size(0), [ 512, 512, 512 ]) assert np.array_equal(cv.mip_volume_size(1), [ 256, 256, 256 ]) assert np.array_equal(cv.mip_volume_size(2), [ 128, 128, 128 ]) assert np.array_equal(cv.mip_volume_size(3), [ 64, 64, 64 ]) slice64 = np.s_[0:64, 0:64, 0:64] cv.mip = 0 assert np.all(cv[slice64] == data[slice64]) data_ds1, = tinybrain.downsample_with_averaging(data, factor=[2, 2, 2, 1]) cv.mip = 1 assert np.all(cv[slice64] == data_ds1[slice64]) data_ds2, = tinybrain.downsample_with_averaging(data, factor=[4, 4, 4, 1]) cv.mip = 2 assert np.all(cv[slice64] == data_ds2[slice64]) data_ds3, = tinybrain.downsample_with_averaging(data, factor=[8, 8, 8, 1]) cv.mip = 3 assert np.all(cv[slice64] == data_ds3[slice64])
def test_downsample_no_offset(): delete_layer() storage, data = create_layer(size=(1024,1024,128,1), offset=(0,0,0)) cv = CloudVolume(storage.layer_path) assert len(cv.scales) == 5 assert len(cv.available_mips) == 5 cv.commit_info() create_downsampling_tasks(MockTaskQueue(), storage.layer_path, mip=0, num_mips=4) cv.refresh_info() assert len(cv.available_mips) == 5 assert np.array_equal(cv.mip_volume_size(0), [ 1024, 1024, 128 ]) assert np.array_equal(cv.mip_volume_size(1), [ 512, 512, 128 ]) assert np.array_equal(cv.mip_volume_size(2), [ 256, 256, 128 ]) assert np.array_equal(cv.mip_volume_size(3), [ 128, 128, 128 ]) assert np.array_equal(cv.mip_volume_size(4), [ 64, 64, 128 ]) slice64 = np.s_[0:64, 0:64, 0:64] cv.mip = 0 assert np.all(cv[slice64] == data[slice64]) data_ds1 = downsample.downsample_with_averaging(data, factor=[2, 2, 1, 1]) cv.mip = 1 assert np.all(cv[slice64] == data_ds1[slice64]) data_ds2 = downsample.downsample_with_averaging(data_ds1, factor=[2, 2, 1, 1]) cv.mip = 2 assert np.all(cv[slice64] == data_ds2[slice64]) data_ds3 = downsample.downsample_with_averaging(data_ds2, factor=[2, 2, 1, 1]) cv.mip = 3 assert np.all(cv[slice64] == data_ds3[slice64]) data_ds4 = downsample.downsample_with_averaging(data_ds3, factor=[2, 2, 1, 1]) cv.mip = 4 assert np.all(cv[slice64] == data_ds4[slice64])
def test_transfer_task_rechunk(tq, src_cv, transfer_data): tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath, chunk_size=(50, 50, 50)) tq.insert_all(tasks) dest_cv = CloudVolume(destpath) assert len(dest_cv.scales) == 4 assert np.all(src_cv[:] == dest_cv[:]) for mip in range(1, 4): dest_cv.mip = mip assert np.all(dest_cv[:] == transfer_data[mip]) rmsrc() rmdest()
def MultiResShardedMeshMergeTask( cloudpath:str, shard_no:str, draco_compression_level:int = 1, mesh_dir:Optional[str] = None, num_lod:int = 1, spatial_index_db:Optional[str] = None, progress:bool = False ): cv = CloudVolume(cloudpath, spatial_index_db=spatial_index_db) cv.mip = cv.mesh.meta.mip if mesh_dir is None and 'mesh' in cv.info: mesh_dir = cv.info['mesh'] # This looks messy because we are trying to avoid retaining # unnecessary memory. In the original skeleton iteration, this was # using 50 GB+ memory on minnie65. So it makes sense to be just # as careful with a heavier type of object. locations = locations_for_labels(cv, labels_for_shard(cv, shard_no)) filenames = set(itertools.chain(*locations.values())) labels = set(locations.keys()) del locations meshes = collect_mesh_fragments( cv, labels, filenames, mesh_dir, progress ) del filenames # important to iterate this way to avoid # creating a copy of meshes vs. { ... for in } for label in labels: meshes[label] = Mesh.concatenate(*meshes[label]) del labels fname, shard = create_mesh_shard( cv, meshes, num_lod, draco_compression_level, progress, shard_no ) del meshes if shard is None: return cf = CloudFiles(cv.mesh.meta.layerpath) cf.put( fname, shard, compress=False, content_type='application/octet-stream', cache_control='no-cache', )
def test_transfer_task_dest_offset(tq, src_cv, transfer_data): tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath, chunk_size=(50, 50, 50), dest_voxel_offset=(100, 100, 100)) tq.insert_all(tasks) dest_cv = CloudVolume(destpath) assert len(dest_cv.scales) == 4 assert tuple(dest_cv.voxel_offset) == (100, 100, 100) assert tuple(src_cv.voxel_offset) == (0, 0, 0) assert np.all(src_cv[:] == dest_cv[:]) for mip in range(1, 4): dest_cv.mip = mip assert np.all(dest_cv[:] == transfer_data[mip]) rmsrc() rmdest()
def test_downsample_higher_mip(): delete_layer() storage, data = create_layer(size=(512,512,64,1), offset=(3,7,11)) cv = CloudVolume(storage.layer_path) cv.info['scales'] = cv.info['scales'][:1] cv.commit_info() create_downsampling_tasks(MockTaskQueue(), storage.layer_path, mip=0, num_mips=2) cv.refresh_info() assert len(cv.available_mips) == 3 create_downsampling_tasks(MockTaskQueue(), storage.layer_path, mip=1, num_mips=2) cv.refresh_info() assert len(cv.available_mips) == 4 cv.mip = 3 assert cv[:,:,:].shape == (64,64,64,1)
def execute(self): vol = CloudVolume(self.layer_path, mip=self.mip) highres_bbox = Bbox( self.offset, self.offset + self.shape ) top_mip = min(vol.available_mips[-1], self.mip + self.num_mips) for mip in range(self.mip, top_mip + 1): vol.mip = mip bbox = vol.bbox_to_mip(highres_bbox, self.mip, mip) bbox = bbox.round_to_chunk_size(vol.underlying, offset=vol.bounds.minpt) bbox = Bbox.clamp(bbox, vol.bounds) if bbox.volume() == 0: continue vol.delete(bbox)
def test_quantize(): qpath = 'file:///tmp/removeme/quantized/' delete_layer() delete_layer(qpath) storage, _ = create_layer(size=(256, 256, 128, 3), offset=(0, 0, 0), layer_type="affinities") cv = CloudVolume(storage.layer_path) shape = (128, 128, 64) slices = np.s_[:shape[0], :shape[1], :shape[2], :1] data = cv[slices] data *= 255.0 data = data.astype(np.uint8) task = QuantizeTask( source_layer_path=storage.layer_path, dest_layer_path=qpath, shape=shape, offset=(0, 0, 0), mip=0, ) info = create_quantized_affinity_info(storage.layer_path, qpath, shape, mip=0, chunk_size=[64, 64, 64], encoding='raw') qcv = CloudVolume(qpath, info=info) qcv.commit_info() create_downsample_scales(qpath, mip=0, ds_shape=shape) task.execute() qcv.mip = 0 qdata = qcv[slices] assert np.all(data.shape == qdata.shape) assert np.all(data == qdata) assert data.dtype == np.uint8
def DeleteTask(layer_path:str, shape, offset, mip=0, num_mips=5): """Delete a block of images inside a layer on all mip levels.""" shape = Vec(*shape) offset = Vec(*offset) vol = CloudVolume(layer_path, mip=mip, max_redirects=0) highres_bbox = Bbox( offset, offset + shape ) top_mip = min(vol.available_mips[-1], mip + num_mips) for mip_i in range(mip, top_mip + 1): vol.mip = mip_i bbox = vol.bbox_to_mip(highres_bbox, mip, mip_i) bbox = bbox.round_to_chunk_size(vol.chunk_size, offset=vol.bounds.minpt) bbox = Bbox.clamp(bbox, vol.bounds) if bbox.volume() == 0: continue vol.delete(bbox)
def test_downsample_higher_mip(): delete_layer() cf, data = create_layer(size=(512,512,64,1), offset=(3,7,11)) cv = CloudVolume(cf.cloudpath) cv.info['scales'] = cv.info['scales'][:1] tq = MockTaskQueue() cv.commit_info() tasks = create_downsampling_tasks(cf.cloudpath, mip=0, num_mips=2) tq.insert_all(tasks) cv.refresh_info() assert len(cv.available_mips) == 3 tasks = create_downsampling_tasks(cf.cloudpath, mip=1, num_mips=2) tq.insert_all(tasks) cv.refresh_info() assert len(cv.available_mips) == 4 cv.mip = 3 assert cv[:,:,:].shape == (64,64,64,1)
def create_sharded_skeleton_merge_tasks(layer_path, dust_threshold, tick_threshold, shard_index_bytes=2**13, minishard_index_bytes=2**15, minishard_index_encoding='gzip', data_encoding='gzip', max_cable_length=None, spatial_index_db=None): cv = CloudVolume(layer_path, progress=True, spatial_index_db=spatial_index_db) cv.mip = cv.skeleton.meta.mip # 17 sec to download for pinky100 all_labels = cv.skeleton.spatial_index.query(cv.bounds * cv.resolution) (shard_bits, minishard_bits, preshift_bits) = \ compute_shard_params_for_hashed( num_labels=len(all_labels), shard_index_bytes=int(shard_index_bytes), minishard_index_bytes=int(minishard_index_bytes), ) spec = ShardingSpecification( type='neuroglancer_uint64_sharded_v1', preshift_bits=preshift_bits, hash='murmurhash3_x86_128', minishard_bits=minishard_bits, shard_bits=shard_bits, minishard_index_encoding=minishard_index_encoding, data_encoding=data_encoding, ) cv.skeleton.meta.info['sharding'] = spec.to_dict() cv.skeleton.meta.commit_info() # rebuild b/c sharding changes the skeleton source cv = CloudVolume(layer_path, progress=True, spatial_index_db=spatial_index_db) cv.mip = cv.skeleton.meta.mip # perf: ~36k hashes/sec shardfn = lambda lbl: cv.skeleton.reader.spec.compute_shard_location( lbl).shard_number shard_labels = defaultdict(list) for label in tqdm(all_labels, desc="Hashes"): shard_labels[shardfn(label)].append(label) cf = CloudFiles(cv.skeleton.meta.layerpath, progress=True) files = ((str(shardno) + '.labels', labels) for shardno, labels in shard_labels.items()) cf.put_jsons(files, compress="gzip", cache_control="no-cache", total=len(shard_labels)) cv.provenance.processing.append({ 'method': { 'task': 'ShardedSkeletonMergeTask', 'cloudpath': layer_path, 'mip': cv.skeleton.meta.mip, 'dust_threshold': dust_threshold, 'tick_threshold': tick_threshold, 'max_cable_length': max_cable_length, 'preshift_bits': preshift_bits, 'minishard_bits': minishard_bits, 'shard_bits': shard_bits, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) cv.commit_provenance() return [ ShardedSkeletonMergeTask(layer_path, shard_no, dust_threshold, tick_threshold, max_cable_length=max_cable_length) for shard_no in shard_labels.keys() ]
def create_sharded_skeletons_from_unsharded_tasks( src: str, dest: str, shard_index_bytes=2**13, minishard_index_bytes=2**15, min_shards: int = 1, minishard_index_encoding='gzip', data_encoding='gzip', skel_dir: Optional[str] = None, ) -> Iterator[ShardedFromUnshardedSkeletonMergeTask]: cv_src = CloudVolume(src) cv_src.mip = cv_src.skeleton.meta.mip cf = CloudFiles(cv_src.skeleton.meta.layerpath) all_labels = [] SEGID_RE = re.compile(r'(\d+)(?:\.gz|\.br|\.zstd)?$') for path in cf.list(): match = SEGID_RE.search(path) if match is None: continue (segid, ) = match.groups() all_labels.append(int(segid)) cv_dest = CloudVolume(dest, skel_dir=skel_dir) cv_dest.skeleton.meta.info = copy.deepcopy(cv_src.skeleton.meta.info) cv_dest.skeleton.meta.info["vertex_attributes"] = [ attr for attr in cv_dest.skeleton.meta.info["vertex_attributes"] if attr["data_type"] in ("float32", "float64") ] (shard_bits, minishard_bits, preshift_bits) = \ compute_shard_params_for_hashed( num_labels=len(all_labels), shard_index_bytes=int(shard_index_bytes), minishard_index_bytes=int(minishard_index_bytes), min_shards=int(min_shards), ) spec = ShardingSpecification( type='neuroglancer_uint64_sharded_v1', preshift_bits=preshift_bits, hash='murmurhash3_x86_128', minishard_bits=minishard_bits, shard_bits=shard_bits, minishard_index_encoding=minishard_index_encoding, data_encoding=data_encoding, ) cv_dest.skeleton.meta.info['sharding'] = spec.to_dict() cv_dest.skeleton.meta.commit_info() cv_dest = CloudVolume(dest, skel_dir=skel_dir) # perf: ~66.5k hashes/sec on M1 ARM64 shardfn = lambda lbl: cv_dest.skeleton.reader.spec.compute_shard_location( lbl).shard_number shard_labels = defaultdict(list) for label in tqdm(all_labels, desc="Hashes"): shard_labels[shardfn(label)].append(label) del all_labels cf = CloudFiles(cv_dest.skeleton.meta.layerpath, progress=True) files = ((str(shardno) + '.labels', labels) for shardno, labels in shard_labels.items()) cf.put_jsons(files, compress="gzip", cache_control="no-cache", total=len(shard_labels)) cv_dest.provenance.processing.append({ 'method': { 'task': 'ShardedFromUnshardedSkeletonMergeTask', 'src': src, 'dest': dest, 'preshift_bits': preshift_bits, 'minishard_bits': minishard_bits, 'shard_bits': shard_bits, 'skel_dir': skel_dir, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) cv_dest.commit_provenance() return [ partial( ShardedFromUnshardedSkeletonMergeTask, src=src, dest=dest, shard_no=shard_no, skel_dir=skel_dir, ) for shard_no in shard_labels.keys() ]
def create_sharded_multires_mesh_tasks( cloudpath:str, shard_index_bytes=2**13, minishard_index_bytes=2**15, num_lod:int = 1, draco_compression_level:int = 1, vertex_quantization_bits:int = 16, minishard_index_encoding="gzip", mesh_dir:Optional[str] = None, spatial_index_db:Optional[str] = None ) -> Iterator[MultiResShardedMeshMergeTask]: configure_multires_info( cloudpath, vertex_quantization_bits, mesh_dir ) # rebuild b/c sharding changes the mesh source class cv = CloudVolume(cloudpath, progress=True, spatial_index_db=spatial_index_db) cv.mip = cv.mesh.meta.mip # 17 sec to download for pinky100 all_labels = cv.mesh.spatial_index.query(cv.bounds * cv.resolution) (shard_bits, minishard_bits, preshift_bits) = \ compute_shard_params_for_hashed( num_labels=len(all_labels), shard_index_bytes=int(shard_index_bytes), minishard_index_bytes=int(minishard_index_bytes), ) spec = ShardingSpecification( type='neuroglancer_uint64_sharded_v1', preshift_bits=preshift_bits, hash='murmurhash3_x86_128', minishard_bits=minishard_bits, shard_bits=shard_bits, minishard_index_encoding=minishard_index_encoding, data_encoding="raw", # draco encoded meshes ) cv.mesh.meta.info['sharding'] = spec.to_dict() cv.mesh.meta.commit_info() cv = CloudVolume(cloudpath) # perf: ~66.5k hashes/sec on M1 ARM64 shardfn = lambda lbl: cv.mesh.reader.spec.compute_shard_location(lbl).shard_number shard_labels = defaultdict(list) for label in tqdm(all_labels, desc="Hashes"): shard_labels[shardfn(label)].append(label) del all_labels cf = CloudFiles(cv.skeleton.meta.layerpath, progress=True) files = ( (str(shardno) + '.labels', labels) for shardno, labels in shard_labels.items() ) cf.put_jsons( files, compress="gzip", cache_control="no-cache", total=len(shard_labels) ) cv.provenance.processing.append({ 'method': { 'task': 'MultiResShardedMeshMergeTask', 'cloudpath': cloudpath, 'mip': cv.mesh.meta.mip, 'num_lod': num_lod, 'vertex_quantization_bits': vertex_quantization_bits, 'preshift_bits': preshift_bits, 'minishard_bits': minishard_bits, 'shard_bits': shard_bits, 'mesh_dir': mesh_dir, 'draco_compression_level': draco_compression_level, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) cv.commit_provenance() return [ partial(MultiResShardedMeshMergeTask, cloudpath, shard_no, num_lod=num_lod, mesh_dir=mesh_dir, spatial_index_db=spatial_index_db, draco_compression_level=draco_compression_level, ) for shard_no in shard_labels.keys() ]