def _cp(src, dst, compression, progress, block_size, paths):
cfsrc = CloudFiles(src, green=True, progress=progress)
cfdest = CloudFiles(dst, green=True, progress=progress)
cfsrc.transfer_to(cfdest,
paths=paths,
reencode=compression,
block_size=block_size)
def TransferMeshFilesTask(src: str,
dest: str,
prefix: str,
mesh_dir: Optional[str] = None):
cv_src = CloudVolume(src)
cv_dest = CloudVolume(dest, mesh_dir=mesh_dir)
cf_src = CloudFiles(cv_src.mesh.meta.layerpath)
cf_dest = CloudFiles(cv_dest.mesh.meta.layerpath)
cf_src.transfer_to(cf_dest, paths=cf_src.list(prefix=prefix))
def test_transfer_semantics(compression):
from cloudfiles import CloudFiles, exceptions
path = '/tmp/cloudfiles/xfer'
rmtree(path)
cff = CloudFiles('file://' + path)
cfm = CloudFiles('mem://cloudfiles/xfer')
N = 128
content = b'some_string'
cff.puts(((str(i), content) for i in range(N)), compress=compression)
assert sorted(list(cff)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cff[:]] == [content] * N
assert sorted([f['path'] for f in cff[:100]
]) == sorted([str(i) for i in range(N)])[:100]
assert [f['content'] for f in cff[:100]] == [content] * 100
cfm[:] = cff
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
cfm.delete(list(cfm))
assert list(cfm) == []
cfm.transfer_from('file://' + path)
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
cfm.delete(list(cfm))
cff.transfer_to(cfm.cloudpath)
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
cfm.delete(list(cfm))
cff.transfer_to(cfm.cloudpath, reencode='br')
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
data = cfm._get_connection()._data
data = [os.path.splitext(d)[1] for d in data.keys()]
assert all([ext == '.br' for ext in data])
cfm.delete(list(cfm))
cff.delete(list(cff))
