def test_get_s3(): s3 = DaskS3FileSystem(key="key", secret="secret") assert s3.key == "key" assert s3.secret == "secret" s3 = DaskS3FileSystem(username="******", password="******") assert s3.key == "key" assert s3.secret == "secret" with pytest.raises(KeyError): DaskS3FileSystem(key="key", username="******") with pytest.raises(KeyError): DaskS3FileSystem(secret="key", password="******")
def s3_context(bucket, files): with ensure_safe_environment_variables(): # temporary workaround as moto fails for botocore >= 1.11 otherwise, # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") with moto.mock_s3(): client = boto3.client("s3") client.create_bucket(Bucket=bucket, ACL="public-read-write") for f, data in files.items(): client.put_object(Bucket=bucket, Key=f, Body=data) fs = DaskS3FileSystem(anon=True) fs.invalidate_cache() yield fs for f, data in files.items(): try: client.delete_object(Bucket=bucket, Key=f, Body=data) except Exception: pass finally: httpretty.HTTPretty.disable() httpretty.HTTPretty.reset()
def test_get_pyarrow_fs_s3(s3): pa = pytest.importorskip("pyarrow") fs = DaskS3FileSystem(anon=True) assert isinstance(fs, pa.filesystem.FileSystem)
def test_get_pyarrow_fs_s3(s3): pa = pytest.importorskip("pyarrow") if pa.__version__ >= LooseVersion("2.0.0"): pytest.skip("fsspec no loger inherits from pyarrow>=2.0.") fs = DaskS3FileSystem(anon=True) assert isinstance(fs, pa.filesystem.FileSystem)