def test_get_s3():
s3 = DaskS3FileSystem(key="key", secret="secret")
assert s3.key == "key"
assert s3.secret == "secret"
s3 = DaskS3FileSystem(username="******", password="******")
assert s3.key == "key"
assert s3.secret == "secret"
with pytest.raises(KeyError):
DaskS3FileSystem(key="key", username="******")
with pytest.raises(KeyError):
DaskS3FileSystem(secret="key", password="******")
def s3_context(bucket, files):
with ensure_safe_environment_variables():
# temporary workaround as moto fails for botocore >= 1.11 otherwise,
# see https://github.com/spulec/moto/issues/1924 & 1952
os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
with moto.mock_s3():
client = boto3.client("s3")
client.create_bucket(Bucket=bucket, ACL="public-read-write")
for f, data in files.items():
client.put_object(Bucket=bucket, Key=f, Body=data)
fs = DaskS3FileSystem(anon=True)
fs.invalidate_cache()
yield fs
for f, data in files.items():
try:
client.delete_object(Bucket=bucket, Key=f, Body=data)
except Exception:
pass
finally:
httpretty.HTTPretty.disable()
httpretty.HTTPretty.reset()
def test_get_pyarrow_fs_s3(s3):
pa = pytest.importorskip("pyarrow")
fs = DaskS3FileSystem(anon=True)
assert isinstance(fs, pa.filesystem.FileSystem)
def test_get_pyarrow_fs_s3(s3):
pa = pytest.importorskip("pyarrow")
if pa.__version__ >= LooseVersion("2.0.0"):
pytest.skip("fsspec no loger inherits from pyarrow>=2.0.")
fs = DaskS3FileSystem(anon=True)
assert isinstance(fs, pa.filesystem.FileSystem)
