def _upload_batch(self, meshes, bbox): cf = CloudFiles(self.layer_path, progress=self.options['progress']) mbuf = MapBuffer(meshes, compress="br") cf.put( f"{self._mesh_dir}/{bbox.to_filename()}.frags", content=mbuf.tobytes(), compress=None, content_type="application/x.mapbuffer", cache_control=False, )
def get_unfused(self, labels, filenames, cv): skeldirfn = lambda loc: cv.meta.join(cv.skeleton.meta.skeleton_path, loc) filenames = [skeldirfn(loc) for loc in filenames] block_size = 50 if len(filenames) < block_size: blocks = [filenames] n_blocks = 1 else: n_blocks = max(len(filenames) // block_size, 1) blocks = sip(filenames, block_size) all_skels = defaultdict(list) for filenames_block in tqdm(blocks, desc="Filename Block", total=n_blocks, disable=(not self.progress)): if cv.meta.path.protocol == "file": all_files = {} prefix = cv.cloudpath.replace("file://", "") for filename in filenames_block: all_files[filename] = open(os.path.join(prefix, filename), "rb") else: all_files = cv.skeleton.cache.download(filenames_block, progress=self.progress) for filename, content in tqdm(all_files.items(), desc="Scanning Fragments", disable=(not self.progress)): try: fragment = MapBuffer( content, frombytesfn=PrecomputedSkeleton.from_precomputed) fragment.validate() except mapbuffer.ValidationError: fragment = pickle.loads(content) for label in labels: try: skel = fragment[label] skel.id = label all_skels[label].append(skel) except KeyError: continue if hasattr(content, "close"): content.close() return all_skels
def upload_batch(self, vol, path, bbox, skeletons): mbuf = MapBuffer(skeletons, compress="br", tobytesfn=lambda skel: skel.to_precomputed()) cf = CloudFiles(path, progress=vol.progress) cf.put( path="{}.frags".format(bbox.to_filename()), content=mbuf.tobytes(), compress=None, content_type="application/x-mapbuffer", cache_control=False, )
def test_empty(compress): mbuf = MapBuffer({}, compress=compress) assert len(mbuf) == 0 assert list(mbuf) == [] assert mbuf.validate() assert mbuf.compress == compress try: mbuf[1000] assert False except KeyError: pass
def collect_mesh_fragments( cv:CloudVolume, labels:List[int], filenames:List[str], mesh_dir:str, progress:bool = False ) -> Dict[int, List[Mesh]]: dirfn = lambda loc: cv.meta.join(mesh_dir, loc) filenames = [ dirfn(loc) for loc in filenames ] block_size = 50 if len(filenames) < block_size: blocks = [ filenames ] n_blocks = 1 else: n_blocks = max(len(filenames) // block_size, 1) blocks = sip(filenames, block_size) all_meshes = defaultdict(list) for filenames_block in tqdm(blocks, desc="Filename Block", total=n_blocks, disable=(not progress)): if cv.meta.path.protocol == "file": all_files = {} prefix = cv.cloudpath.replace("file://", "") for filename in filenames_block: all_files[filename] = open(os.path.join(prefix, filename), "rb") else: all_files = cv.mesh.cache.download(filenames_block, progress=progress) for filename, content in tqdm(all_files.items(), desc="Scanning Fragments", disable=(not progress)): fragment = MapBuffer(content, frombytesfn=Mesh.from_precomputed) fragment.validate() for label in labels: try: mesh = fragment[label] mesh.id = label all_meshes[label].append((filename, mesh)) except KeyError: continue if hasattr(content, "close"): content.close() # ensure consistent results across multiple runs # by sorting mesh fragments by filename for label in all_meshes: all_meshes[label].sort(key=lambda pair: pair[0]) all_meshes[label] = [ pair[1] for pair in all_meshes[label] ] return all_meshes
def test_full(compress): data = { random.randint(0, 1000000000): bytes([ random.randint(0,255) for __ in range(random.randint(0,50)) ]) for _ in range(10000) } mbuf = MapBuffer(data, compress=compress) assert set(data.keys()) == set(mbuf.keys()) assert set(data) == set(mbuf) assert set(data.values()) == set(mbuf.values()) for key in data: assert data[key] == mbuf[key] assert data[key] == mbuf.get(key) assert key in mbuf assert data == mbuf.todict() for i in range(2000): if i not in data: assert i not in mbuf try: mbuf[i] assert False except KeyError: pass mbuf.validate() assert len(mbuf.buffer) > HEADER_LENGTH
def test_mb(data): datasize = len(data) labels = list(data.keys()) random.shuffle(labels) labels = labels[:datasize // 10] mbuf = MapBuffer(data) buf = mbuf.tobytes() s = time.time() mbuf = MapBuffer(buf) for label in labels: mbuf[label] t = time.time() - s mf.write(f"{datasize}\t{t*1000:.5f}\n") mf.flush()
def test_object_access(compress): data = { 1: b"hello", 2: b"world", } mbuf = MapBuffer(data, compress=compress) class Reader: def __init__(self): self.lst = mbuf.tobytes() def __getitem__(self, slc): return self.lst[slc] mbuf2 = MapBuffer(Reader()) assert mbuf2[1] == b"hello" assert mbuf2[2] == b"world"
def test_crc32c(compress): data = { 1: b"hello", 2: b"world", } mbuf = MapBuffer(data, compress=compress) idx = mbuf.buffer.index(b"hello") buf = list(mbuf.buffer) buf[idx] = ord(b'H') mbuf.buffer = bytes(buf) try: mbuf[1] assert False except ValidationError: pass
def test_mmap_access(compress): data = { 1: b"hello", 2: b"world", } mbuf = MapBuffer(data, compress=compress) fileno = random.randint(0,2**32) filename = f"test_mmap-{fileno}.mb" with open(filename, "wb") as f: f.write(mbuf.tobytes()) with open(filename, "rb") as f: mb = MapBuffer(f) assert mb[1] == b"hello" assert mb[2] == b"world" try: os.remove(filename) except (PermissionError, FileNotFoundError): pass