def test_s3_options(minio_server): from pyarrow.fs import S3Options options = S3Options() assert options.region == 'us-east-1' options.region = 'us-west-1' assert options.region == 'us-west-1' assert options.scheme == 'https' options.scheme = 'http' assert options.scheme == 'http' assert options.endpoint_override == '' options.endpoint_override = 'localhost:8999' assert options.endpoint_override == 'localhost:8999' with pytest.raises(ValueError): S3Options(access_key='access') with pytest.raises(ValueError): S3Options(secret_key='secret') address, access_key, secret_key = minio_server options = S3Options( access_key=access_key, secret_key=secret_key, endpoint_override=address, scheme='http' ) assert options.scheme == 'http' assert options.endpoint_override == address
def s3fs(request, minio_server): request.config.pyarrow.requires('s3') from pyarrow.fs import S3Options, S3FileSystem address, access_key, secret_key = minio_server bucket = 'pyarrow-filesystem/' options = S3Options(endpoint_override=address, access_key=access_key, secret_key=secret_key, scheme='http') fs = S3FileSystem(options) fs.create_dir(bucket) return dict( fs=fs, pathfn=bucket.__add__, allow_copy_file=True, allow_move_dir=False, allow_append_to_file=False, )