def test_class_methods():
assert MemoryFileSystem._strip_protocol("memory:stuff") == "stuff"
assert MemoryFileSystem._strip_protocol("memory://stuff") == "stuff"
assert MemoryFileSystem._strip_protocol("stuff") == "stuff"
assert MemoryFileSystem._strip_protocol("other://stuff") == "other://stuff"
assert MemoryFileSystem._get_kwargs_from_urls("memory://user@thing") == {}
class MemoryFileSystem(BaseFileSystem):
scheme = "local"
PARAM_CHECKSUM = "md5"
def __init__(self, **kwargs):
from fsspec.implementations.memory import MemoryFileSystem as MemFS
super().__init__(**kwargs)
self.fs = MemFS()
def exists(self, path_info) -> bool:
return self.fs.exists(path_info.fspath)
def open(self, path_info, mode="r", encoding=None, **kwargs):
return self.fs.open(path_info.fspath,
mode=mode,
encoding=encoding,
**kwargs)
def info(self, path_info):
return self.fs.info(path_info.fspath)
def stat(self, path_info):
import os
info = self.fs.info(path_info.fspath)
return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0))
def walk_files(self, path_info, **kwargs):
raise NotImplementedError
def test_move():
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"I had a nice big cabbage")
fs.move("/myfile", "/otherfile")
assert not fs.exists("/myfile")
assert fs.info("/otherfile")
assert isinstance(fs.ukey("/otherfile"), str)
def test_pipe_cat():
fs = MemoryFileSystem()
fs.pipe("afile", b"contents")
assert fs.cat("afile") == b"contents"
data = {"bfile": b"more", "cfile": b"stuff"}
fs.pipe(data)
assert fs.cat(list(data)) == data
def test_move():
fs = MemoryFileSystem()
with fs.open('/myfile', 'wb') as f:
f.write(b'I had a nice big cabbage')
fs.move('/myfile', '/otherfile')
assert not fs.exists('/myfile')
assert fs.info('/otherfile')
assert isinstance(fs.ukey('/otherfile'), str)
def test_read_block_delimiter():
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"some\n" b"lines\n" b"of\n" b"text")
assert fs.read_block("/myfile", 0, 2, b"\n") == b"some\n"
assert fs.read_block("/myfile", 2, 6, b"\n") == b"lines\n"
assert fs.read_block("/myfile", 6, 2, b"\n") == b""
assert fs.read_block("/myfile", 2, 9, b"\n") == b"lines\nof\n"
assert fs.read_block("/myfile", 12, 6, b"\n") == b"text"
assert fs.read_block("/myfile", 0, None) == fs.cat("/myfile")
def test_read_block_delimiter():
fs = MemoryFileSystem()
with fs.open('/myfile', 'wb') as f:
f.write(b'some\n' b'lines\n' b'of\n' b'text')
assert fs.read_block('/myfile', 0, 2, b'\n') == b'some\n'
assert fs.read_block('/myfile', 2, 6, b'\n') == b'lines\n'
assert fs.read_block('/myfile', 6, 2, b'\n') == b''
assert fs.read_block('/myfile', 2, 9, b'\n') == b'lines\nof\n'
assert fs.read_block('/myfile', 12, 6, b'\n') == b'text'
assert fs.read_block('/myfile', 0, None) == fs.cat('/myfile')
def test_basic(tmpdir):
tmpdir = str(tmpdir)
fs = MemoryFileSystem()
fs.touch('/mounted/testfile')
th = run(fs, '/mounted/', tmpdir, False)
timeout = 10
while True:
try:
# can fail with device not ready while waiting for fuse
if 'testfile' in os.listdir(tmpdir):
break
except:
pass
timeout -= 1
time.sleep(1)
assert timeout > 0, "Timeout"
fn = os.path.join(tmpdir, 'test')
with open(fn, 'wb') as f:
f.write(b'data')
assert fs.info("/mounted/test")['size'] == 4
assert open(fn).read() == "data"
os.remove(fn)
os.mkdir(fn)
assert os.listdir(fn) == []
os.mkdir(fn + '/inner')
with pytest.raises(OSError):
os.rmdir(fn)
os.rmdir(fn + '/inner')
os.rmdir(fn)
assert not fs.pseudo_dirs
# should not normally kill a thread like this, but FUSE blocks, so we
# cannot have thread listen for event. Alternative may be to .join() but
# send a SIGINT
th._tstate_lock.release()
th._stop()
th.join()
fs.store.clear()
def test_idempotent():
MemoryFileSystem.clear_instance_cache()
fs = MemoryFileSystem()
fs2 = MemoryFileSystem()
assert fs is fs2
assert MemoryFileSystem.current() is fs2
MemoryFileSystem.clear_instance_cache()
assert not MemoryFileSystem._cache
fs2 = MemoryFileSystem().current()
assert fs == fs2
def test_generic_open_FSFile_MemoryFileSystem(self):
"""Test the generic_open method with FSFile in MemoryFileSystem."""
mem_fs = MemoryFileSystem()
mem_file = MemoryFile(fs=mem_fs,
path="{}test.DAT".format(mem_fs.root_marker),
data=b"TEST")
mem_file.commit()
fsf = FSFile(mem_file)
with hf.generic_open(fsf) as file_object:
data = file_object.read()
assert data == b'TEST'
def test_idempotent():
MemoryFileSystem.clear_instance_cache()
fs = MemoryFileSystem()
fs2 = MemoryFileSystem()
assert fs is fs2
assert MemoryFileSystem.current() is fs2
fs2 = MemoryFileSystem(do_cache=False)
assert fs is not fs2
assert hash(fs) == hash(fs2)
assert fs == fs2
MemoryFileSystem.clear_instance_cache()
assert not MemoryFileSystem._cache
fs2 = MemoryFileSystem().current()
assert fs == fs2
def test_du():
fs = MemoryFileSystem()
fs.store = {
'/dir/afile': MemoryFile(fs, '/afile', b'a'),
'/dir/dirb/afile': MemoryFile(fs, '/afile', b'bb'),
'/dir/dirb/bfile': MemoryFile(fs, '/afile', b'ccc'),
}
assert fs.du('/dir') == 6
assert fs.du('/dir', total=False)['/dir/dirb/afile'] == 2
assert fs.du('/dir', maxdepth=0) == 1
def test_du():
fs = MemoryFileSystem()
fs.store = {
"/dir/afile": MemoryFile(fs, "/afile", b"a"),
"/dir/dirb/afile": MemoryFile(fs, "/afile", b"bb"),
"/dir/dirb/bfile": MemoryFile(fs, "/afile", b"ccc"),
}
assert fs.du("/dir") == 6
assert fs.du("/dir", total=False)["/dir/dirb/afile"] == 2
assert fs.du("/dir", maxdepth=0) == 1
def test_recursive_get_put(tmpdir):
fs = MemoryFileSystem()
os.makedirs(f"{tmpdir}/nest")
for file in ["one", "two", "nest/other"]:
with open(f"{tmpdir}/{file}", "wb") as f:
f.write(b"data")
fs.put(str(tmpdir), "test", recursive=True)
d = tempfile.mkdtemp()
fs.get("test", d, recursive=True)
for file in ["one", "two", "nest/other"]:
with open(f"{d}/{file}", "rb") as f:
f.read() == b"data"
def __init__(self, repo, config):
from fsspec.implementations.memory import MemoryFileSystem as MemFS
super().__init__(repo, config)
self.fs = MemFS()
def test_get_put(tmpdir):
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, "one")
open(fn, "wb").write(b"one")
os.mkdir(os.path.join(tmpdir, "dir"))
fn2 = os.path.join(tmpdir, "dir", "two")
open(fn2, "wb").write(b"two")
fs = MemoryFileSystem()
fs.put(fn, "/afile")
assert fs.cat("/afile") == b"one"
fs.store["/bfile"] = MemoryFile(fs, "/bfile", b"data")
fn3 = os.path.join(tmpdir, "three")
fs.get("/bfile", fn3)
assert open(fn3, "rb").read() == b"data"
fs.put(tmpdir, "/more", recursive=True)
assert fs.find("/more") == ["/more/dir/two", "/more/one", "/more/three"]
@contextlib.contextmanager
def tmp_chdir(path):
curdir = os.getcwd()
os.chdir(path)
try:
yield
finally:
os.chdir(curdir)
with tmp_chdir(os.path.join(tmpdir, os.path.pardir)):
fs.put(os.path.basename(tmpdir), "/moretwo", recursive=True)
assert fs.find("/moretwo") == [
"/moretwo/dir/two",
"/moretwo/one",
"/moretwo/three",
]
with tmp_chdir(tmpdir):
fs.put(os.path.curdir, "/morethree", recursive=True)
assert fs.find("/morethree") == [
"/morethree/dir/two",
"/morethree/one",
"/morethree/three",
]
for f in [fn, fn2, fn3]:
os.remove(f)
os.rmdir(os.path.join(tmpdir, "dir"))
fs.get("/more/", tmpdir + "/", recursive=True)
assert open(fn3, "rb").read() == b"data"
assert open(fn, "rb").read() == b"one"
def test_get_put(tmpdir):
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, 'one')
open(fn, 'wb').write(b'one')
os.mkdir(os.path.join(tmpdir, 'dir'))
fn2 = os.path.join(tmpdir, 'dir', 'two')
open(fn2, 'wb').write(b'two')
fs = MemoryFileSystem()
fs.put(fn, '/afile')
assert fs.cat('/afile') == b'one'
fs.store['/bfile'] = MemoryFile(fs, '/bfile', b'data')
fn3 = os.path.join(tmpdir, 'three')
fs.get('/bfile', fn3)
assert open(fn3, 'rb').read() == b'data'
fs.put(tmpdir, '/more', recursive=True)
assert fs.find('/more') == ['/more/dir/two', '/more/one', '/more/three']
for f in [fn, fn2, fn3]:
os.remove(f)
os.rmdir(os.path.join(tmpdir, 'dir'))
fs.get('/more/', tmpdir + '/', recursive=True)
assert open(fn3, 'rb').read() == b'data'
assert open(fn, 'rb').read() == b'one'
def test_pickle():
fs = MemoryFileSystem()
fs2 = pickle.loads(pickle.dumps(fs))
assert fs == fs2
def test_open_text():
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"some\n" b"lines\n" b"of\n" b"text")
f = fs.open("/myfile", "r", encoding="latin1")
assert f.encoding == "latin1"
def test_head_tail():
fs = MemoryFileSystem()
with fs.open('/myfile', 'wb') as f:
f.write(b'I had a nice big cabbage')
assert fs.head('/myfile', 5) == b'I had'
assert fs.tail('/myfile', 7) == b'cabbage'
def test_get_put(tmpdir):
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, "one")
open(fn, "wb").write(b"one")
os.mkdir(os.path.join(tmpdir, "dir"))
fn2 = os.path.join(tmpdir, "dir", "two")
open(fn2, "wb").write(b"two")
fs = MemoryFileSystem()
fs.put(fn, "/afile")
assert fs.cat("/afile") == b"one"
fs.store["/bfile"] = MemoryFile(fs, "/bfile", b"data")
fn3 = os.path.join(tmpdir, "three")
fs.get("/bfile", fn3)
assert open(fn3, "rb").read() == b"data"
fs.put(tmpdir, "/more", recursive=True)
assert fs.find("/more") == ["/more/dir/two", "/more/one", "/more/three"]
for f in [fn, fn2, fn3]:
os.remove(f)
os.rmdir(os.path.join(tmpdir, "dir"))
fs.get("/more/", tmpdir + "/", recursive=True)
assert open(fn3, "rb").read() == b"data"
assert open(fn, "rb").read() == b"one"
def __init__(self, **kwargs):
from fsspec.implementations.memory import MemoryFileSystem as MemFS
super().__init__(**kwargs)
self.fs = MemFS()
def test_head_tail():
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"I had a nice big cabbage")
assert fs.head("/myfile", 5) == b"I had"
assert fs.tail("/myfile", 7) == b"cabbage"
def __init__(self, **kwargs: Dict[Any, Any]) -> None:
self.fs = MemoryFileSystem(asynchronous=True)
def host_fuse(mountdir):
fs = MemoryFileSystem()
fs.touch("/mounted/testfile")
run(fs, "/mounted/", mountdir)
