def py_fsspec_s3fs(request, s3_connection, s3_server):
s3fs = pytest.importorskip("s3fs")
host, port, access_key, secret_key = s3_connection
bucket = 'pyarrow-filesystem/'
fs = s3fs.S3FileSystem(
key=access_key,
secret=secret_key,
client_kwargs=dict(endpoint_url='http://{}:{}'.format(host, port))
)
fs = PyFileSystem(FSSpecHandler(fs))
try:
fs.create_dir(bucket)
except IOError:
# BucketAlreadyOwnedByYou on second test
pass
return dict(
fs=fs,
pathfn=bucket.__add__,
allow_copy_file=True,
allow_move_dir=False,
allow_append_to_file=True,
)
def py_fsspec_s3fs(request, s3_connection, s3_server):
s3fs = pytest.importorskip("s3fs")
if sys.version_info < (3, 7) and s3fs.__version__ >= LooseVersion("0.5"):
pytest.skip("s3fs>=0.5 version is async and requires Python >= 3.7")
host, port, access_key, secret_key = s3_connection
bucket = 'pyarrow-filesystem/'
fs = s3fs.S3FileSystem(
key=access_key,
secret=secret_key,
client_kwargs=dict(endpoint_url='http://{}:{}'.format(host, port)))
fs = PyFileSystem(FSSpecHandler(fs))
fs.create_dir(bucket)
yield dict(
fs=fs,
pathfn=bucket.__add__,
allow_copy_file=True,
allow_move_dir=False,
allow_append_to_file=True,
)
fs.delete_dir(bucket)
def test_py_filesystem_ops():
handler = DummyHandler()
fs = PyFileSystem(handler)
fs.create_dir("recursive", recursive=True)
fs.create_dir("non-recursive", recursive=False)
with pytest.raises(IOError):
fs.create_dir("foobar")
fs.delete_dir("delete_dir")
fs.delete_dir_contents("delete_dir_contents")
fs.delete_file("delete_file")
fs.move("move_from", "move_to")
fs.copy_file("copy_file_from", "copy_file_to")