def _cp(src, dst, compression, progress, block_size, paths): cfsrc = CloudFiles(src, green=True, progress=progress) cfdest = CloudFiles(dst, green=True, progress=progress) cfsrc.transfer_to(cfdest, paths=paths, reencode=compression, block_size=block_size)
def TransferMeshFilesTask(src: str, dest: str, prefix: str, mesh_dir: Optional[str] = None): cv_src = CloudVolume(src) cv_dest = CloudVolume(dest, mesh_dir=mesh_dir) cf_src = CloudFiles(cv_src.mesh.meta.layerpath) cf_dest = CloudFiles(cv_dest.mesh.meta.layerpath) cf_src.transfer_to(cf_dest, paths=cf_src.list(prefix=prefix))
def test_transfer_semantics(compression): from cloudfiles import CloudFiles, exceptions path = '/tmp/cloudfiles/xfer' rmtree(path) cff = CloudFiles('file://' + path) cfm = CloudFiles('mem://cloudfiles/xfer') N = 128 content = b'some_string' cff.puts(((str(i), content) for i in range(N)), compress=compression) assert sorted(list(cff)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cff[:]] == [content] * N assert sorted([f['path'] for f in cff[:100] ]) == sorted([str(i) for i in range(N)])[:100] assert [f['content'] for f in cff[:100]] == [content] * 100 cfm[:] = cff assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N cfm.delete(list(cfm)) assert list(cfm) == [] cfm.transfer_from('file://' + path) assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N cfm.delete(list(cfm)) cff.transfer_to(cfm.cloudpath) assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N cfm.delete(list(cfm)) cff.transfer_to(cfm.cloudpath, reencode='br') assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N data = cfm._get_connection()._data data = [os.path.splitext(d)[1] for d in data.keys()] assert all([ext == '.br' for ext in data]) cfm.delete(list(cfm)) cff.delete(list(cff))