def test_exceptions_raised(green):
from cloudfiles import CloudFiles, exceptions
from cloudfiles.lib import mkdir
path = compute_url("file", "exceptions_raised")
cf = CloudFiles(path, green=green)
pth = mkdir(path.replace("file://", ""))
with open(f"{pth}/wontdecompress.gz", "wb") as f:
f.write(b"not a valid gzip stream")
try:
x = cf.get("wontdecompress")
assert False
except exceptions.DecompressionError:
pass
try:
x = cf.get(["wontdecompress"], raise_errors=True)
assert False
except exceptions.DecompressionError:
pass
try:
x = cf.get(["wontdecompress"], return_dict=True)
assert False
except exceptions.DecompressionError:
pass
cf.delete("wontdecompress")
def test_isdir(s3, protocol):
from cloudfiles import CloudFiles, exceptions
url = compute_url(protocol, "isdir")
cf = CloudFiles(url, num_threads=5)
assert not cf.isdir()
content = b'some_string'
cf.put('info', content, compress=None)
assert cf.isdir()
cf.delete('info')
def test_cli_rm_python(s3, protocol):
from cloudfiles_cli.cloudfiles_cli import _rm
from cloudfiles import CloudFiles, exceptions
test_dir = compute_url(protocol, "cli_rm_python")
cf = CloudFiles(test_dir)
N = 100
def mkfiles():
cf.delete(cf.list())
for i in range(N):
cf[str(i)] = b"hello world"
def run_rm(path, recursive=False):
_rm(
path, recursive=recursive, progress=False,
parallel=1, block_size=128
)
mkfiles()
run_rm(test_dir, recursive=True)
assert list(cf) == []
mkfiles()
run_rm(test_dir, recursive=False)
assert len(list(cf)) == N
mkfiles()
run_rm(test_dir + "/*")
print(list(cf))
assert list(cf) == []
mkfiles()
run_rm(test_dir + "/**")
assert list(cf) == []
mkfiles()
run_rm(test_dir + "/0")
assert set(list(cf)) == set([ str(_) for _ in range(1, N) ])
mkfiles()
run_rm(test_dir + "/1*")
res = set([ str(_) for _ in range(N) ])
res.remove("1")
for x in range(10, 20):
res.remove(str(x))
assert set(list(cf)) == res
cf.delete(cf.list())
def test_get_json_order(s3, protocol):
from cloudfiles import CloudFiles
url = compute_url(protocol, 'get_json_order')
cf = CloudFiles(url)
N = 5300
cf.put_jsons(( (str(z), [ z ]) for z in range(N) ))
contents = cf.get_json(( str(z) for z in range(N) ))
for z, content in enumerate(contents):
assert content[0] == z
cf.delete(( str(z) for z in range(N) ))
def test_exists(s3, protocol):
from cloudfiles import CloudFiles, exceptions
url = compute_url(protocol, "exists")
cf = CloudFiles(url, num_threads=5)
content = b'some_string'
cf.put('info', content, compress=None)
assert cf.exists('info')
assert not cf.exists('doesntexist')
assert cf.exists(['info'])['info']
assert not cf.exists(['doesntexist'])['doesntexist']
cf.delete('info')
def test_access_non_cannonical_minimal_path(s3, protocol):
from cloudfiles import CloudFiles, exceptions
if protocol == 'file':
url = "file:///tmp/"
else:
url = "{}://cloudfiles/".format(protocol)
cf = CloudFiles(url, num_threads=5)
content = b'some_string'
cf.put('info', content, compress=None)
# time.sleep(0.5) # sometimes it takes a moment for google to update the list
assert cf.get('info') == content
assert cf.get('nonexistentfile') is None
cf.delete('info')
def test_list(s3, protocol):
from cloudfiles import CloudFiles, exceptions
url = compute_url(protocol, "list")
cf = CloudFiles(url, num_threads=5)
content = b'some_string'
cf.put('info1', content, compress=None)
cf.put('info2', content, compress=None)
cf.put('build/info3', content, compress=None)
cf.put('level1/level2/info4', content, compress=None)
cf.put('info5', content, compress='gzip')
cf.put('info.txt', content, compress=None)
# time.sleep(1) # sometimes it takes a moment for google to update the list
assert set(cf.list(prefix='')) == set([
'build/info3', 'info1', 'info2', 'level1/level2/info4', 'info5',
'info.txt'
])
assert set(list(cf)) == set(cf.list(prefix=''))
assert set(cf.list(prefix='inf')) == set(
['info1', 'info2', 'info5', 'info.txt'])
assert set(cf.list(prefix='info1')) == set(['info1'])
assert set(cf.list(prefix='build')) == set(['build/info3'])
assert set(cf.list(prefix='build/')) == set(['build/info3'])
assert set(cf.list(prefix='level1/')) == set(['level1/level2/info4'])
assert set(cf.list(prefix='nofolder/')) == set([])
# Tests (1)
assert set(cf.list(prefix='', flat=True)) == set(
['info1', 'info2', 'info5', 'info.txt'])
assert set(cf.list(prefix='inf', flat=True)) == set(
['info1', 'info2', 'info5', 'info.txt'])
# Tests (2)
assert set(cf.list(prefix='build', flat=True)) == set([])
# Tests (3)
assert set(cf.list(prefix='level1/', flat=True)) == set([])
assert set(cf.list(prefix='build/', flat=True)) == set(['build/info3'])
# Tests (4)
assert set(cf.list(prefix='build/inf', flat=True)) == set(['build/info3'])
for file_path in ('info1', 'info2', 'build/info3', 'level1/level2/info4',
'info5', 'info.txt'):
cf.delete(file_path)
if protocol == 'file':
rmtree("/tmp/cloudfiles/list")
def test_transfer_semantics(compression):
from cloudfiles import CloudFiles, exceptions
path = '/tmp/cloudfiles/xfer'
rmtree(path)
cff = CloudFiles('file://' + path)
cfm = CloudFiles('mem://cloudfiles/xfer')
N = 128
content = b'some_string'
cff.puts(((str(i), content) for i in range(N)), compress=compression)
assert sorted(list(cff)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cff[:]] == [content] * N
assert sorted([f['path'] for f in cff[:100]
]) == sorted([str(i) for i in range(N)])[:100]
assert [f['content'] for f in cff[:100]] == [content] * 100
cfm[:] = cff
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
cfm.delete(list(cfm))
assert list(cfm) == []
cfm.transfer_from('file://' + path)
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
cfm.delete(list(cfm))
cff.transfer_to(cfm.cloudpath)
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
cfm.delete(list(cfm))
cff.transfer_to(cfm.cloudpath, reencode='br')
assert sorted(list(cfm)) == sorted([str(i) for i in range(N)])
assert [f['content'] for f in cfm[:]] == [content] * N
data = cfm._get_connection()._data
data = [os.path.splitext(d)[1] for d in data.keys()]
assert all([ext == '.br' for ext in data])
cfm.delete(list(cfm))
cff.delete(list(cff))
def test_read_write(s3, protocol, num_threads, green):
from cloudfiles import CloudFiles, exceptions
url = compute_url(protocol, "rw")
cf = CloudFiles(url, num_threads=num_threads, green=green)
content = b'some_string'
cf.put('info', content, compress=None, cache_control='no-cache')
cf['info2'] = content
assert cf.get('info') == content
assert cf['info2'] == content
assert cf['info2', 0:3] == content[0:3]
assert cf['info2', :] == content[:]
assert cf.get('nonexistentfile') is None
assert cf.get('info', return_dict=True) == {"info": content}
assert cf.get(['info', 'info2'], return_dict=True) == {
"info": content,
"info2": content
}
del cf['info2']
assert cf.exists('info2') == False
num_infos = max(num_threads, 1)
results = cf.get(['info' for i in range(num_infos)])
assert len(results) == num_infos
assert results[0]['path'] == 'info'
assert results[0]['content'] == content
assert all(map(lambda x: x['error'] is None, results))
assert cf.get(['nonexistentfile'])[0]['content'] is None
cf.delete('info')
cf.put_json('info', {'omg': 'wow'}, cache_control='no-cache')
results = cf.get_json('info')
assert results == {'omg': 'wow'}
cf.delete('info')
if protocol == 'file':
rmtree(url)
def test_compression(s3, protocol, method, green):
from cloudfiles import CloudFiles, exceptions
url = compute_url(protocol, "compress")
cf = CloudFiles(url, num_threads=5, green=green)
content = b'some_string'
cf.put('info', content, compress=method)
retrieved = cf.get('info')
assert content == retrieved
assert cf.get('nonexistentfile') is None
try:
cf.put('info', content, compress='nonexistent')
assert False
except ValueError:
pass
cf.delete(iter(cf))
def test_delete(s3, green, protocol):
from cloudfiles import CloudFiles, exceptions
if protocol == 'file':
url = "file:///tmp/cloudfiles/delete"
else:
url = "{}://cloudfiles/delete".format(protocol)
cf = CloudFiles(url, green=green, num_threads=1)
content = b'some_string'
cf.put('delete-test', content, compress=None, cache_control='no-cache')
cf.put('delete-test-compressed', content, compress='gzip', cache_control='no-cache')
assert cf.get('delete-test') == content
cf.delete('delete-test')
assert cf.get('delete-test') is None
assert cf.get('delete-test-compressed') == content
cf.delete('delete-test-compressed')
assert cf.get('delete-test-compressed') is None
# Reset for batch delete
cf.put('delete-test', content, compress=None, cache_control='no-cache')
cf.put('delete-test-compressed', content, compress='gzip', cache_control='no-cache')
assert cf.get('delete-test') == content
assert cf.get('delete-test-compressed') == content
cf.delete(['delete-test', 'delete-nonexistent', 'delete-test-compressed'])
assert cf.get('delete-test') is None
assert cf.get('delete-test-compressed') is None
def execute(self):
self.vol = CloudVolume(self.cloudpath, cdn_cache=False)
self.vol.mip = self.vol.skeleton.meta.mip
fragment_filenames = self.get_filenames()
skels = self.get_skeletons_by_segid(fragment_filenames)
skeletons = []
for segid, frags in skels.items():
skeleton = self.fuse_skeletons(frags)
if self.max_cable_length is None or skel.cable_length(
) <= self.max_cable_length:
skeleton = kimimaro.postprocess(skeleton, self.dust_threshold,
self.tick_threshold)
skeleton.id = segid
skeletons.append(skeleton)
self.vol.skeleton.upload(skeletons)
if self.delete_fragments:
cf = CloudFiles(self.cloudpath, progress=True)
cf.delete(fragment_filenames)
def test_size(s3, protocol, compress, green):
from cloudfiles import CloudFiles, exceptions, compression
url = compute_url(protocol, 'size')
cf = CloudFiles(url)
content = b'some_string'
cf.put('info', content, compress=compress, cache_control='no-cache')
cf['info2'] = content
cf.put('zero', b'', compress=None, cache_control='no-cache')
compressed_content = compression.compress(content, compress)
assert cf.size('info') == len(compressed_content)
assert cf.size(['info', 'info2']) == {
"info": len(compressed_content),
"info2": len(content)
}
assert cf.size('nonexistent') is None
assert cf.size('zero') == 0
cf.delete(['info', 'info2', 'zero'])
def test_get_generator(num_threads, green): from cloudfiles import CloudFiles, exceptions path = '/tmp/cloudfiles/gen' rmtree(path) url = 'file://' + path cf = CloudFiles(url, num_threads=num_threads, green=green) gen = ( (str(i), b'hello world') for i in range(100) ) cf.puts(gen) files = cf.get(( str(i) for i in range(100) ), total=100) assert all([ f['error'] is None for f in files ]) assert len(files) == 100 assert all([ f['content'] == b'hello world' for f in files ]) fnames = [ str(i) for i in range(100) ] assert sorted(list(cf.list())) == sorted(fnames) cf.delete(( str(i) for i in range(100) )) assert list(cf.list()) == []
def DeleteMeshFilesTask(cloudpath: str,
prefix: str,
mesh_dir: Optional[str] = None):
cv = CloudVolume(cloudpath, mesh_dir=mesh_dir)
cf = CloudFiles(cv.mesh.meta.layerpath)
cf.delete(cf.list(prefix=prefix))
def create_mesh_deletion_tasks(
layer_path:str,
magnitude:int = 3,
mesh_dir:Optional[str] = None
):
assert int(magnitude) == magnitude
assert magnitude >= 0
cv = CloudVolume(layer_path, mesh_dir=mesh_dir)
cf = CloudFiles(cv.mesh.meta.layerpath)
cf.delete('info')
try:
if cv.mesh.meta.is_sharded():
return [
partial(DeleteMeshFilesTask,
cloudpath=layer_path,
prefix="",
mesh_dir=mesh_dir,
)
]
start = 10 ** (magnitude - 1)
end = 10 ** magnitude
class MeshDeleteTaskIterator:
def __len__(self):
return (10 ** magnitude)
def __iter__(self):
# get spatial index files that start
# with 0 too.
yield partial(DeleteMeshFilesTask,
cloudpath=layer_path,
prefix="0",
mesh_dir=mesh_dir,
)
for prefix in range(1, start):
yield partial(DeleteMeshFilesTask,
cloudpath=layer_path,
prefix=str(prefix) + ':',
mesh_dir=mesh_dir
)
# enumerate from e.g. 100 to 999
for prefix in range(start, end):
yield partial(DeleteMeshFilesTask,
cloudpath=layer_path,
prefix=str(prefix),
mesh_dir=mesh_dir
)
return MeshDeleteTaskIterator()
finally:
cv.provenance.processing.append({
'method': {
'task': 'DeleteMeshFilesTask',
'layer_path': layer_path,
'mesh_dir': mesh_dir,
},
'by': operator_contact(),
'date': strftime('%Y-%m-%d %H:%M %Z'),
})
cv.commit_provenance()
def DeleteSkeletonFilesTask(cloudpath: str,
prefix: str,
skel_dir: Optional[str] = None):
cv = CloudVolume(cloudpath, skel_dir=skel_dir)
cf = CloudFiles(cv.skeleton.meta.layerpath)
cf.delete(cf.list(prefix=prefix))
