def test_exceptions_raised(green): from cloudfiles import CloudFiles, exceptions from cloudfiles.lib import mkdir path = compute_url("file", "exceptions_raised") cf = CloudFiles(path, green=green) pth = mkdir(path.replace("file://", "")) with open(f"{pth}/wontdecompress.gz", "wb") as f: f.write(b"not a valid gzip stream") try: x = cf.get("wontdecompress") assert False except exceptions.DecompressionError: pass try: x = cf.get(["wontdecompress"], raise_errors=True) assert False except exceptions.DecompressionError: pass try: x = cf.get(["wontdecompress"], return_dict=True) assert False except exceptions.DecompressionError: pass cf.delete("wontdecompress")
def test_isdir(s3, protocol): from cloudfiles import CloudFiles, exceptions url = compute_url(protocol, "isdir") cf = CloudFiles(url, num_threads=5) assert not cf.isdir() content = b'some_string' cf.put('info', content, compress=None) assert cf.isdir() cf.delete('info')
def test_cli_rm_python(s3, protocol): from cloudfiles_cli.cloudfiles_cli import _rm from cloudfiles import CloudFiles, exceptions test_dir = compute_url(protocol, "cli_rm_python") cf = CloudFiles(test_dir) N = 100 def mkfiles(): cf.delete(cf.list()) for i in range(N): cf[str(i)] = b"hello world" def run_rm(path, recursive=False): _rm( path, recursive=recursive, progress=False, parallel=1, block_size=128 ) mkfiles() run_rm(test_dir, recursive=True) assert list(cf) == [] mkfiles() run_rm(test_dir, recursive=False) assert len(list(cf)) == N mkfiles() run_rm(test_dir + "/*") print(list(cf)) assert list(cf) == [] mkfiles() run_rm(test_dir + "/**") assert list(cf) == [] mkfiles() run_rm(test_dir + "/0") assert set(list(cf)) == set([ str(_) for _ in range(1, N) ]) mkfiles() run_rm(test_dir + "/1*") res = set([ str(_) for _ in range(N) ]) res.remove("1") for x in range(10, 20): res.remove(str(x)) assert set(list(cf)) == res cf.delete(cf.list())
def test_get_json_order(s3, protocol): from cloudfiles import CloudFiles url = compute_url(protocol, 'get_json_order') cf = CloudFiles(url) N = 5300 cf.put_jsons(( (str(z), [ z ]) for z in range(N) )) contents = cf.get_json(( str(z) for z in range(N) )) for z, content in enumerate(contents): assert content[0] == z cf.delete(( str(z) for z in range(N) ))
def test_exists(s3, protocol): from cloudfiles import CloudFiles, exceptions url = compute_url(protocol, "exists") cf = CloudFiles(url, num_threads=5) content = b'some_string' cf.put('info', content, compress=None) assert cf.exists('info') assert not cf.exists('doesntexist') assert cf.exists(['info'])['info'] assert not cf.exists(['doesntexist'])['doesntexist'] cf.delete('info')
def test_access_non_cannonical_minimal_path(s3, protocol): from cloudfiles import CloudFiles, exceptions if protocol == 'file': url = "file:///tmp/" else: url = "{}://cloudfiles/".format(protocol) cf = CloudFiles(url, num_threads=5) content = b'some_string' cf.put('info', content, compress=None) # time.sleep(0.5) # sometimes it takes a moment for google to update the list assert cf.get('info') == content assert cf.get('nonexistentfile') is None cf.delete('info')
def test_list(s3, protocol): from cloudfiles import CloudFiles, exceptions url = compute_url(protocol, "list") cf = CloudFiles(url, num_threads=5) content = b'some_string' cf.put('info1', content, compress=None) cf.put('info2', content, compress=None) cf.put('build/info3', content, compress=None) cf.put('level1/level2/info4', content, compress=None) cf.put('info5', content, compress='gzip') cf.put('info.txt', content, compress=None) # time.sleep(1) # sometimes it takes a moment for google to update the list assert set(cf.list(prefix='')) == set([ 'build/info3', 'info1', 'info2', 'level1/level2/info4', 'info5', 'info.txt' ]) assert set(list(cf)) == set(cf.list(prefix='')) assert set(cf.list(prefix='inf')) == set( ['info1', 'info2', 'info5', 'info.txt']) assert set(cf.list(prefix='info1')) == set(['info1']) assert set(cf.list(prefix='build')) == set(['build/info3']) assert set(cf.list(prefix='build/')) == set(['build/info3']) assert set(cf.list(prefix='level1/')) == set(['level1/level2/info4']) assert set(cf.list(prefix='nofolder/')) == set([]) # Tests (1) assert set(cf.list(prefix='', flat=True)) == set( ['info1', 'info2', 'info5', 'info.txt']) assert set(cf.list(prefix='inf', flat=True)) == set( ['info1', 'info2', 'info5', 'info.txt']) # Tests (2) assert set(cf.list(prefix='build', flat=True)) == set([]) # Tests (3) assert set(cf.list(prefix='level1/', flat=True)) == set([]) assert set(cf.list(prefix='build/', flat=True)) == set(['build/info3']) # Tests (4) assert set(cf.list(prefix='build/inf', flat=True)) == set(['build/info3']) for file_path in ('info1', 'info2', 'build/info3', 'level1/level2/info4', 'info5', 'info.txt'): cf.delete(file_path) if protocol == 'file': rmtree("/tmp/cloudfiles/list")
def test_transfer_semantics(compression): from cloudfiles import CloudFiles, exceptions path = '/tmp/cloudfiles/xfer' rmtree(path) cff = CloudFiles('file://' + path) cfm = CloudFiles('mem://cloudfiles/xfer') N = 128 content = b'some_string' cff.puts(((str(i), content) for i in range(N)), compress=compression) assert sorted(list(cff)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cff[:]] == [content] * N assert sorted([f['path'] for f in cff[:100] ]) == sorted([str(i) for i in range(N)])[:100] assert [f['content'] for f in cff[:100]] == [content] * 100 cfm[:] = cff assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N cfm.delete(list(cfm)) assert list(cfm) == [] cfm.transfer_from('file://' + path) assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N cfm.delete(list(cfm)) cff.transfer_to(cfm.cloudpath) assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N cfm.delete(list(cfm)) cff.transfer_to(cfm.cloudpath, reencode='br') assert sorted(list(cfm)) == sorted([str(i) for i in range(N)]) assert [f['content'] for f in cfm[:]] == [content] * N data = cfm._get_connection()._data data = [os.path.splitext(d)[1] for d in data.keys()] assert all([ext == '.br' for ext in data]) cfm.delete(list(cfm)) cff.delete(list(cff))
def test_read_write(s3, protocol, num_threads, green): from cloudfiles import CloudFiles, exceptions url = compute_url(protocol, "rw") cf = CloudFiles(url, num_threads=num_threads, green=green) content = b'some_string' cf.put('info', content, compress=None, cache_control='no-cache') cf['info2'] = content assert cf.get('info') == content assert cf['info2'] == content assert cf['info2', 0:3] == content[0:3] assert cf['info2', :] == content[:] assert cf.get('nonexistentfile') is None assert cf.get('info', return_dict=True) == {"info": content} assert cf.get(['info', 'info2'], return_dict=True) == { "info": content, "info2": content } del cf['info2'] assert cf.exists('info2') == False num_infos = max(num_threads, 1) results = cf.get(['info' for i in range(num_infos)]) assert len(results) == num_infos assert results[0]['path'] == 'info' assert results[0]['content'] == content assert all(map(lambda x: x['error'] is None, results)) assert cf.get(['nonexistentfile'])[0]['content'] is None cf.delete('info') cf.put_json('info', {'omg': 'wow'}, cache_control='no-cache') results = cf.get_json('info') assert results == {'omg': 'wow'} cf.delete('info') if protocol == 'file': rmtree(url)
def test_compression(s3, protocol, method, green): from cloudfiles import CloudFiles, exceptions url = compute_url(protocol, "compress") cf = CloudFiles(url, num_threads=5, green=green) content = b'some_string' cf.put('info', content, compress=method) retrieved = cf.get('info') assert content == retrieved assert cf.get('nonexistentfile') is None try: cf.put('info', content, compress='nonexistent') assert False except ValueError: pass cf.delete(iter(cf))
def test_delete(s3, green, protocol): from cloudfiles import CloudFiles, exceptions if protocol == 'file': url = "file:///tmp/cloudfiles/delete" else: url = "{}://cloudfiles/delete".format(protocol) cf = CloudFiles(url, green=green, num_threads=1) content = b'some_string' cf.put('delete-test', content, compress=None, cache_control='no-cache') cf.put('delete-test-compressed', content, compress='gzip', cache_control='no-cache') assert cf.get('delete-test') == content cf.delete('delete-test') assert cf.get('delete-test') is None assert cf.get('delete-test-compressed') == content cf.delete('delete-test-compressed') assert cf.get('delete-test-compressed') is None # Reset for batch delete cf.put('delete-test', content, compress=None, cache_control='no-cache') cf.put('delete-test-compressed', content, compress='gzip', cache_control='no-cache') assert cf.get('delete-test') == content assert cf.get('delete-test-compressed') == content cf.delete(['delete-test', 'delete-nonexistent', 'delete-test-compressed']) assert cf.get('delete-test') is None assert cf.get('delete-test-compressed') is None
def execute(self): self.vol = CloudVolume(self.cloudpath, cdn_cache=False) self.vol.mip = self.vol.skeleton.meta.mip fragment_filenames = self.get_filenames() skels = self.get_skeletons_by_segid(fragment_filenames) skeletons = [] for segid, frags in skels.items(): skeleton = self.fuse_skeletons(frags) if self.max_cable_length is None or skel.cable_length( ) <= self.max_cable_length: skeleton = kimimaro.postprocess(skeleton, self.dust_threshold, self.tick_threshold) skeleton.id = segid skeletons.append(skeleton) self.vol.skeleton.upload(skeletons) if self.delete_fragments: cf = CloudFiles(self.cloudpath, progress=True) cf.delete(fragment_filenames)
def test_size(s3, protocol, compress, green): from cloudfiles import CloudFiles, exceptions, compression url = compute_url(protocol, 'size') cf = CloudFiles(url) content = b'some_string' cf.put('info', content, compress=compress, cache_control='no-cache') cf['info2'] = content cf.put('zero', b'', compress=None, cache_control='no-cache') compressed_content = compression.compress(content, compress) assert cf.size('info') == len(compressed_content) assert cf.size(['info', 'info2']) == { "info": len(compressed_content), "info2": len(content) } assert cf.size('nonexistent') is None assert cf.size('zero') == 0 cf.delete(['info', 'info2', 'zero'])
def test_get_generator(num_threads, green): from cloudfiles import CloudFiles, exceptions path = '/tmp/cloudfiles/gen' rmtree(path) url = 'file://' + path cf = CloudFiles(url, num_threads=num_threads, green=green) gen = ( (str(i), b'hello world') for i in range(100) ) cf.puts(gen) files = cf.get(( str(i) for i in range(100) ), total=100) assert all([ f['error'] is None for f in files ]) assert len(files) == 100 assert all([ f['content'] == b'hello world' for f in files ]) fnames = [ str(i) for i in range(100) ] assert sorted(list(cf.list())) == sorted(fnames) cf.delete(( str(i) for i in range(100) )) assert list(cf.list()) == []
def DeleteMeshFilesTask(cloudpath: str, prefix: str, mesh_dir: Optional[str] = None): cv = CloudVolume(cloudpath, mesh_dir=mesh_dir) cf = CloudFiles(cv.mesh.meta.layerpath) cf.delete(cf.list(prefix=prefix))
def create_mesh_deletion_tasks( layer_path:str, magnitude:int = 3, mesh_dir:Optional[str] = None ): assert int(magnitude) == magnitude assert magnitude >= 0 cv = CloudVolume(layer_path, mesh_dir=mesh_dir) cf = CloudFiles(cv.mesh.meta.layerpath) cf.delete('info') try: if cv.mesh.meta.is_sharded(): return [ partial(DeleteMeshFilesTask, cloudpath=layer_path, prefix="", mesh_dir=mesh_dir, ) ] start = 10 ** (magnitude - 1) end = 10 ** magnitude class MeshDeleteTaskIterator: def __len__(self): return (10 ** magnitude) def __iter__(self): # get spatial index files that start # with 0 too. yield partial(DeleteMeshFilesTask, cloudpath=layer_path, prefix="0", mesh_dir=mesh_dir, ) for prefix in range(1, start): yield partial(DeleteMeshFilesTask, cloudpath=layer_path, prefix=str(prefix) + ':', mesh_dir=mesh_dir ) # enumerate from e.g. 100 to 999 for prefix in range(start, end): yield partial(DeleteMeshFilesTask, cloudpath=layer_path, prefix=str(prefix), mesh_dir=mesh_dir ) return MeshDeleteTaskIterator() finally: cv.provenance.processing.append({ 'method': { 'task': 'DeleteMeshFilesTask', 'layer_path': layer_path, 'mesh_dir': mesh_dir, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) cv.commit_provenance()
def DeleteSkeletonFilesTask(cloudpath: str, prefix: str, skel_dir: Optional[str] = None): cv = CloudVolume(cloudpath, skel_dir=skel_dir) cf = CloudFiles(cv.skeleton.meta.layerpath) cf.delete(cf.list(prefix=prefix))