def _ensure_filesystem(fs): if not isinstance(fs, FileSystem): if type(fs).__name__ == 'S3FileSystem': from pyarrow.filesystem import S3FSWrapper return S3FSWrapper(fs) else: raise IOError('Unrecognized filesystem: {0}'.format(type(fs))) else: return fs
def test_read_partitioned_directory_s3fs(s3_example): from pyarrow.filesystem import S3FSWrapper import pyarrow.parquet as pq fs, bucket_uri = s3_example wrapper = S3FSWrapper(fs) _partition_test_for_filesystem(wrapper, bucket_uri) # Check that we can auto-wrap dataset = pq.ParquetDataset(bucket_uri, filesystem=fs) dataset.read()
def _ensure_filesystem(fs): fs_type = type(fs) # If the arrow filesystem was subclassed, assume it supports the full # interface and return it if not issubclass(fs_type, FileSystem): for mro in inspect.getmro(fs_type): if mro.__name__ is 'S3FileSystem': return S3FSWrapper(fs) # In case its a simple LocalFileSystem (e.g. dask) use native arrow # FS elif mro.__name__ is 'LocalFileSystem': return LocalFileSystem.get_instance() raise IOError('Unrecognized filesystem: {0}'.format(fs_type)) else: return fs