def test_resolve_home_directory():
uri = '~/myfile.parquet'
fs, path = filesystem.resolve_filesystem_and_path(uri)
assert isinstance(fs, filesystem.LocalFileSystem)
assert path == os.path.expanduser(uri)
local_fs = filesystem.LocalFileSystem()
fs, path = filesystem.resolve_filesystem_and_path(uri, local_fs)
assert path == os.path.expanduser(uri)
def get_fs(type="local"):
"""
get filesystem function.
Args:
type : string, default "local".
Returns:
fs : pyarrow.filesystem.LocalFileSystem, pyarrow.filesystem.S3FSWrapper.
pyarrow.filesystem module.
Examples:
>>> from renom_cn.api.filesystem import get_fs
>>> fs = get_fs(type="local")
>>> fs.ls(".")
['./a.txt']
>>> fs.mkdir("testdir")
>>> fs.ls(".")
['./a.txt', './testdir']
>>> fs.exists("./a.txt")
True
>>> fs.exists("./b.txt")
False
>>> fs.isdir("./testdir")
True
>>> fs.rm("./a.txt")
>>> fs = get_fs(type="s3")
>>> fs.ls("my-bucket")
FileNotFoundError: my-bucket # if bucket is empty, raise FileNotFoundError
>>> fs.mkdir("my-bucket/testdir")
>>> fs.ls("my-bucket")
['my-bucket/testdir']
>>> with fs.open('my-bucket/testdir/a.txt', 'wb') as f:
... f.write(b"hello")
...
5
>>> with fs.open('my-bucket/testdir/a.txt', 'rb') as f:
... print(f.read())
...
b'hello'
>>> fs.rm('my-bucket/testdir/a.txt')
"""
if type == "s3":
tmpfs = s3fs.S3FileSystem(anon=False)
fs = pafs.S3FSWrapper(fs=tmpfs)
fs.mkdir = types.MethodType(s3mkdir, fs)
else:
fs = pafs.LocalFileSystem()
fs.rm = types.MethodType(rm, fs)
return fs
def test_filesystem_deprecated():
with pytest.warns(FutureWarning):
filesystem.LocalFileSystem()
with pytest.warns(FutureWarning):
filesystem.LocalFileSystem.get_instance()