def test_s3_options(minio_server):
from pyarrow.fs import S3Options
options = S3Options()
assert options.region == 'us-east-1'
options.region = 'us-west-1'
assert options.region == 'us-west-1'
assert options.scheme == 'https'
options.scheme = 'http'
assert options.scheme == 'http'
assert options.endpoint_override == ''
options.endpoint_override = 'localhost:8999'
assert options.endpoint_override == 'localhost:8999'
with pytest.raises(ValueError):
S3Options(access_key='access')
with pytest.raises(ValueError):
S3Options(secret_key='secret')
address, access_key, secret_key = minio_server
options = S3Options(
access_key=access_key,
secret_key=secret_key,
endpoint_override=address,
scheme='http'
)
assert options.scheme == 'http'
assert options.endpoint_override == address
def s3fs(request, minio_server):
request.config.pyarrow.requires('s3')
from pyarrow.fs import S3Options, S3FileSystem
address, access_key, secret_key = minio_server
bucket = 'pyarrow-filesystem/'
options = S3Options(endpoint_override=address,
access_key=access_key,
secret_key=secret_key,
scheme='http')
fs = S3FileSystem(options)
fs.create_dir(bucket)
return dict(
fs=fs,
pathfn=bucket.__add__,
allow_copy_file=True,
allow_move_dir=False,
allow_append_to_file=False,
)
