def _ensure_filesystem(fs):
if not isinstance(fs, FileSystem):
if type(fs).__name__ == 'S3FileSystem':
from pyarrow.filesystem import S3FSWrapper
return S3FSWrapper(fs)
else:
raise IOError('Unrecognized filesystem: {0}'.format(type(fs)))
else:
return fs
def test_read_partitioned_directory_s3fs(s3_example):
from pyarrow.filesystem import S3FSWrapper
import pyarrow.parquet as pq
fs, bucket_uri = s3_example
wrapper = S3FSWrapper(fs)
_partition_test_for_filesystem(wrapper, bucket_uri)
# Check that we can auto-wrap
dataset = pq.ParquetDataset(bucket_uri, filesystem=fs)
dataset.read()
def _ensure_filesystem(fs):
fs_type = type(fs)
# If the arrow filesystem was subclassed, assume it supports the full
# interface and return it
if not issubclass(fs_type, FileSystem):
for mro in inspect.getmro(fs_type):
if mro.__name__ is 'S3FileSystem':
return S3FSWrapper(fs)
# In case its a simple LocalFileSystem (e.g. dask) use native arrow
# FS
elif mro.__name__ is 'LocalFileSystem':
return LocalFileSystem.get_instance()
raise IOError('Unrecognized filesystem: {0}'.format(fs_type))
else:
return fs
