def test_class_methods(): assert MemoryFileSystem._strip_protocol("memory:stuff") == "stuff" assert MemoryFileSystem._strip_protocol("memory://stuff") == "stuff" assert MemoryFileSystem._strip_protocol("stuff") == "stuff" assert MemoryFileSystem._strip_protocol("other://stuff") == "other://stuff" assert MemoryFileSystem._get_kwargs_from_urls("memory://user@thing") == {}
class MemoryFileSystem(BaseFileSystem): scheme = "local" PARAM_CHECKSUM = "md5" def __init__(self, **kwargs): from fsspec.implementations.memory import MemoryFileSystem as MemFS super().__init__(**kwargs) self.fs = MemFS() def exists(self, path_info) -> bool: return self.fs.exists(path_info.fspath) def open(self, path_info, mode="r", encoding=None, **kwargs): return self.fs.open(path_info.fspath, mode=mode, encoding=encoding, **kwargs) def info(self, path_info): return self.fs.info(path_info.fspath) def stat(self, path_info): import os info = self.fs.info(path_info.fspath) return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0)) def walk_files(self, path_info, **kwargs): raise NotImplementedError
def test_move(): fs = MemoryFileSystem() with fs.open("/myfile", "wb") as f: f.write(b"I had a nice big cabbage") fs.move("/myfile", "/otherfile") assert not fs.exists("/myfile") assert fs.info("/otherfile") assert isinstance(fs.ukey("/otherfile"), str)
def test_pipe_cat(): fs = MemoryFileSystem() fs.pipe("afile", b"contents") assert fs.cat("afile") == b"contents" data = {"bfile": b"more", "cfile": b"stuff"} fs.pipe(data) assert fs.cat(list(data)) == data
def test_move(): fs = MemoryFileSystem() with fs.open('/myfile', 'wb') as f: f.write(b'I had a nice big cabbage') fs.move('/myfile', '/otherfile') assert not fs.exists('/myfile') assert fs.info('/otherfile') assert isinstance(fs.ukey('/otherfile'), str)
def test_read_block_delimiter(): fs = MemoryFileSystem() with fs.open("/myfile", "wb") as f: f.write(b"some\n" b"lines\n" b"of\n" b"text") assert fs.read_block("/myfile", 0, 2, b"\n") == b"some\n" assert fs.read_block("/myfile", 2, 6, b"\n") == b"lines\n" assert fs.read_block("/myfile", 6, 2, b"\n") == b"" assert fs.read_block("/myfile", 2, 9, b"\n") == b"lines\nof\n" assert fs.read_block("/myfile", 12, 6, b"\n") == b"text" assert fs.read_block("/myfile", 0, None) == fs.cat("/myfile")
def test_read_block_delimiter(): fs = MemoryFileSystem() with fs.open('/myfile', 'wb') as f: f.write(b'some\n' b'lines\n' b'of\n' b'text') assert fs.read_block('/myfile', 0, 2, b'\n') == b'some\n' assert fs.read_block('/myfile', 2, 6, b'\n') == b'lines\n' assert fs.read_block('/myfile', 6, 2, b'\n') == b'' assert fs.read_block('/myfile', 2, 9, b'\n') == b'lines\nof\n' assert fs.read_block('/myfile', 12, 6, b'\n') == b'text' assert fs.read_block('/myfile', 0, None) == fs.cat('/myfile')
def test_basic(tmpdir): tmpdir = str(tmpdir) fs = MemoryFileSystem() fs.touch('/mounted/testfile') th = run(fs, '/mounted/', tmpdir, False) timeout = 10 while True: try: # can fail with device not ready while waiting for fuse if 'testfile' in os.listdir(tmpdir): break except: pass timeout -= 1 time.sleep(1) assert timeout > 0, "Timeout" fn = os.path.join(tmpdir, 'test') with open(fn, 'wb') as f: f.write(b'data') assert fs.info("/mounted/test")['size'] == 4 assert open(fn).read() == "data" os.remove(fn) os.mkdir(fn) assert os.listdir(fn) == [] os.mkdir(fn + '/inner') with pytest.raises(OSError): os.rmdir(fn) os.rmdir(fn + '/inner') os.rmdir(fn) assert not fs.pseudo_dirs # should not normally kill a thread like this, but FUSE blocks, so we # cannot have thread listen for event. Alternative may be to .join() but # send a SIGINT th._tstate_lock.release() th._stop() th.join() fs.store.clear()
def test_idempotent(): MemoryFileSystem.clear_instance_cache() fs = MemoryFileSystem() fs2 = MemoryFileSystem() assert fs is fs2 assert MemoryFileSystem.current() is fs2 MemoryFileSystem.clear_instance_cache() assert not MemoryFileSystem._cache fs2 = MemoryFileSystem().current() assert fs == fs2
def test_generic_open_FSFile_MemoryFileSystem(self): """Test the generic_open method with FSFile in MemoryFileSystem.""" mem_fs = MemoryFileSystem() mem_file = MemoryFile(fs=mem_fs, path="{}test.DAT".format(mem_fs.root_marker), data=b"TEST") mem_file.commit() fsf = FSFile(mem_file) with hf.generic_open(fsf) as file_object: data = file_object.read() assert data == b'TEST'
def test_idempotent(): MemoryFileSystem.clear_instance_cache() fs = MemoryFileSystem() fs2 = MemoryFileSystem() assert fs is fs2 assert MemoryFileSystem.current() is fs2 fs2 = MemoryFileSystem(do_cache=False) assert fs is not fs2 assert hash(fs) == hash(fs2) assert fs == fs2 MemoryFileSystem.clear_instance_cache() assert not MemoryFileSystem._cache fs2 = MemoryFileSystem().current() assert fs == fs2
def test_du(): fs = MemoryFileSystem() fs.store = { '/dir/afile': MemoryFile(fs, '/afile', b'a'), '/dir/dirb/afile': MemoryFile(fs, '/afile', b'bb'), '/dir/dirb/bfile': MemoryFile(fs, '/afile', b'ccc'), } assert fs.du('/dir') == 6 assert fs.du('/dir', total=False)['/dir/dirb/afile'] == 2 assert fs.du('/dir', maxdepth=0) == 1
def test_du(): fs = MemoryFileSystem() fs.store = { "/dir/afile": MemoryFile(fs, "/afile", b"a"), "/dir/dirb/afile": MemoryFile(fs, "/afile", b"bb"), "/dir/dirb/bfile": MemoryFile(fs, "/afile", b"ccc"), } assert fs.du("/dir") == 6 assert fs.du("/dir", total=False)["/dir/dirb/afile"] == 2 assert fs.du("/dir", maxdepth=0) == 1
def test_recursive_get_put(tmpdir): fs = MemoryFileSystem() os.makedirs(f"{tmpdir}/nest") for file in ["one", "two", "nest/other"]: with open(f"{tmpdir}/{file}", "wb") as f: f.write(b"data") fs.put(str(tmpdir), "test", recursive=True) d = tempfile.mkdtemp() fs.get("test", d, recursive=True) for file in ["one", "two", "nest/other"]: with open(f"{d}/{file}", "rb") as f: f.read() == b"data"
def __init__(self, repo, config): from fsspec.implementations.memory import MemoryFileSystem as MemFS super().__init__(repo, config) self.fs = MemFS()
def test_get_put(tmpdir): tmpdir = str(tmpdir) fn = os.path.join(tmpdir, "one") open(fn, "wb").write(b"one") os.mkdir(os.path.join(tmpdir, "dir")) fn2 = os.path.join(tmpdir, "dir", "two") open(fn2, "wb").write(b"two") fs = MemoryFileSystem() fs.put(fn, "/afile") assert fs.cat("/afile") == b"one" fs.store["/bfile"] = MemoryFile(fs, "/bfile", b"data") fn3 = os.path.join(tmpdir, "three") fs.get("/bfile", fn3) assert open(fn3, "rb").read() == b"data" fs.put(tmpdir, "/more", recursive=True) assert fs.find("/more") == ["/more/dir/two", "/more/one", "/more/three"] @contextlib.contextmanager def tmp_chdir(path): curdir = os.getcwd() os.chdir(path) try: yield finally: os.chdir(curdir) with tmp_chdir(os.path.join(tmpdir, os.path.pardir)): fs.put(os.path.basename(tmpdir), "/moretwo", recursive=True) assert fs.find("/moretwo") == [ "/moretwo/dir/two", "/moretwo/one", "/moretwo/three", ] with tmp_chdir(tmpdir): fs.put(os.path.curdir, "/morethree", recursive=True) assert fs.find("/morethree") == [ "/morethree/dir/two", "/morethree/one", "/morethree/three", ] for f in [fn, fn2, fn3]: os.remove(f) os.rmdir(os.path.join(tmpdir, "dir")) fs.get("/more/", tmpdir + "/", recursive=True) assert open(fn3, "rb").read() == b"data" assert open(fn, "rb").read() == b"one"
def test_get_put(tmpdir): tmpdir = str(tmpdir) fn = os.path.join(tmpdir, 'one') open(fn, 'wb').write(b'one') os.mkdir(os.path.join(tmpdir, 'dir')) fn2 = os.path.join(tmpdir, 'dir', 'two') open(fn2, 'wb').write(b'two') fs = MemoryFileSystem() fs.put(fn, '/afile') assert fs.cat('/afile') == b'one' fs.store['/bfile'] = MemoryFile(fs, '/bfile', b'data') fn3 = os.path.join(tmpdir, 'three') fs.get('/bfile', fn3) assert open(fn3, 'rb').read() == b'data' fs.put(tmpdir, '/more', recursive=True) assert fs.find('/more') == ['/more/dir/two', '/more/one', '/more/three'] for f in [fn, fn2, fn3]: os.remove(f) os.rmdir(os.path.join(tmpdir, 'dir')) fs.get('/more/', tmpdir + '/', recursive=True) assert open(fn3, 'rb').read() == b'data' assert open(fn, 'rb').read() == b'one'
def test_pickle(): fs = MemoryFileSystem() fs2 = pickle.loads(pickle.dumps(fs)) assert fs == fs2
def test_open_text(): fs = MemoryFileSystem() with fs.open("/myfile", "wb") as f: f.write(b"some\n" b"lines\n" b"of\n" b"text") f = fs.open("/myfile", "r", encoding="latin1") assert f.encoding == "latin1"
def test_head_tail(): fs = MemoryFileSystem() with fs.open('/myfile', 'wb') as f: f.write(b'I had a nice big cabbage') assert fs.head('/myfile', 5) == b'I had' assert fs.tail('/myfile', 7) == b'cabbage'
def test_get_put(tmpdir): tmpdir = str(tmpdir) fn = os.path.join(tmpdir, "one") open(fn, "wb").write(b"one") os.mkdir(os.path.join(tmpdir, "dir")) fn2 = os.path.join(tmpdir, "dir", "two") open(fn2, "wb").write(b"two") fs = MemoryFileSystem() fs.put(fn, "/afile") assert fs.cat("/afile") == b"one" fs.store["/bfile"] = MemoryFile(fs, "/bfile", b"data") fn3 = os.path.join(tmpdir, "three") fs.get("/bfile", fn3) assert open(fn3, "rb").read() == b"data" fs.put(tmpdir, "/more", recursive=True) assert fs.find("/more") == ["/more/dir/two", "/more/one", "/more/three"] for f in [fn, fn2, fn3]: os.remove(f) os.rmdir(os.path.join(tmpdir, "dir")) fs.get("/more/", tmpdir + "/", recursive=True) assert open(fn3, "rb").read() == b"data" assert open(fn, "rb").read() == b"one"
def __init__(self, **kwargs): from fsspec.implementations.memory import MemoryFileSystem as MemFS super().__init__(**kwargs) self.fs = MemFS()
def test_head_tail(): fs = MemoryFileSystem() with fs.open("/myfile", "wb") as f: f.write(b"I had a nice big cabbage") assert fs.head("/myfile", 5) == b"I had" assert fs.tail("/myfile", 7) == b"cabbage"
def __init__(self, **kwargs: Dict[Any, Any]) -> None: self.fs = MemoryFileSystem(asynchronous=True)
def host_fuse(mountdir): fs = MemoryFileSystem() fs.touch("/mounted/testfile") run(fs, "/mounted/", mountdir)