def test_copy_multipart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
S3FileSystem._copy(s3, from_info, to_info, {})
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
S3FileSystem._copy(s3, from_info, to_info, {})
def test_verify_ssl_default_param(dvc):
config = {"url": url}
fs = S3FileSystem(**config)
assert "client_kwargs" not in fs.fs_args
config = {
"url": url,
"endpointurl": "https://my.custom.s3:1234",
}
fs = S3FileSystem(**config)
assert "verify" not in fs.fs_args["client_kwargs"]
def test_get_s3_no_credentials(mocker):
from botocore.exceptions import NoCredentialsError
fs = S3FileSystem(None, {})
with pytest.raises(DvcException, match="Unable to find AWS credentials"):
with fs._get_s3():
raise NoCredentialsError
def test_verify_ssl_default_param(dvc):
config = {
"url": url,
}
fs = S3FileSystem(dvc, config)
assert fs.ssl_verify
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch):
config_file = tmp_dir / "aws_config.ini"
config_file.write_text(
textwrap.dedent(
"""\
[default]
extra = keys
s3 =
addressing_style = auto
use_accelerate_endpoint = true
multipart_threshold = ThisIsNotGoingToBeCasted!
[profile dev]
some_extra = keys
s3 =
addressing_style = virtual
multipart_threshold = 2GiB
"""
)
)
monkeypatch.setenv("AWS_CONFIG_FILE", config_file)
fs = S3FileSystem(profile="dev", **s3.config)
s3_config = fs.fs_args["config_kwargs"]["s3"]
assert s3_config["addressing_style"] == "virtual"
transfer_config = fs._transfer_config
assert transfer_config.multipart_threshold == 2 * GB
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch):
config_file = tmp_dir / "aws_config.ini"
config_file.write_text(
textwrap.dedent("""\
[default]
extra = keys
s3 =
addressing_style = auto
use_accelerate_endpoint = true
multipart_threshold = ThisIsNotGoingToBeCasted!
[profile dev]
some_extra = keys
s3 =
addresing_style = virtual
multipart_threshold = 2GiB
"""))
monkeypatch.setenv("AWS_CONFIG_FILE", config_file)
fs = S3FileSystem(dvc, {**s3.config, "profile": "dev"})
assert fs._transfer_config is None
with fs._get_s3() as s3:
s3_config = s3.meta.client.meta.config.s3
assert s3_config["addresing_style"] == "virtual"
assert "use_accelerate_endpoint" not in s3_config
transfer_config = fs._transfer_config
assert transfer_config.multipart_threshold == 2 * GB
def test_verify_ssl_default_param(dvc):
config = {
"url": url,
}
fs = S3FileSystem(**config)
assert fs.fs_args["client_kwargs"]["verify"]
def test_get_s3_connection_error_endpoint(mocker):
from botocore.exceptions import EndpointConnectionError
fs = S3FileSystem(None, {"endpointurl": "https://example.com"})
msg = "Unable to connect to 'https://example.com'."
with pytest.raises(DvcException, match=msg):
with fs._get_s3():
raise EndpointConnectionError(endpoint_url="url")
def test_get_s3_connection_error(mocker):
from botocore.exceptions import EndpointConnectionError
fs = S3FileSystem(None, {})
msg = "Unable to connect to 'AWS S3'."
with pytest.raises(DvcException, match=msg):
with fs._get_s3():
raise EndpointConnectionError(endpoint_url="url")
def test_s3_upload_fobj(tmp_dir, dvc, s3):
s3.gen({"data": {"foo": "foo"}})
fs = S3FileSystem(dvc, s3.config)
to_info = s3 / "data" / "bar"
with fs.open(s3 / "data" / "foo", "rb") as stream:
fs.upload_fobj(stream, to_info, 1)
assert to_info.read_text() == "foo"
def test_key_id_and_secret(dvc):
fs = S3FileSystem(
url=url,
access_key_id=key_id,
secret_access_key=key_secret,
session_token=session_token,
)
assert fs.fs_args["key"] == key_id
assert fs.fs_args["secret"] == key_secret
assert fs.fs_args["token"] == session_token
def test_key_id_and_secret(dvc):
fs = S3FileSystem(
dvc,
{
"url": url,
"access_key_id": key_id,
"secret_access_key": key_secret,
"session_token": session_token,
},
)
assert fs.access_key_id == key_id
assert fs.secret_access_key == key_secret
assert fs.session_token == session_token
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
from dvc.checkout import _link
fs = S3FileSystem(dvc, {"url": str(base_info.parent)})
cache = CloudCache(fs)
s3 = cache.fs.s3.meta.client
s3.create_bucket(Bucket=base_info.bucket)
s3.put_object(Bucket=base_info.bucket,
Key=(base_info / "from").path,
Body="data")
_link(cache, base_info / "from", base_info / "to")
assert cache.fs.exists(base_info / "from")
assert cache.fs.exists(base_info / "to")
def test_s3_config_credentialpath(dvc, monkeypatch):
environment = {}
monkeypatch.setattr(os, "environ", environment)
config = {"url": url, "credentialpath": "somewhere"}
S3FileSystem(**config)
assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere"
environment.clear()
config = {"url": url, "configpath": "somewhere"}
S3FileSystem(**config)
assert environment["AWS_CONFIG_FILE"] == "somewhere"
environment.clear()
config = {
"url": url,
"credentialpath": "somewhere",
"configpath": "elsewhere",
}
S3FileSystem(**config)
assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere"
assert environment["AWS_CONFIG_FILE"] == "elsewhere"
environment.clear()
def test_copy_preserve_etag_across_buckets(remote, dvc):
s3 = remote.fs.s3
s3.Bucket("another").create()
another = S3FileSystem(dvc, {"url": "s3://another", "region": "us-east-1"})
from_info = remote.fs.path_info / "foo"
to_info = another.path_info / "foo"
remote.fs.copy(from_info, to_info)
from_hash = remote.fs.info(from_info)["etag"]
to_hash = another.info(to_info)["etag"]
assert from_hash == to_hash
def test_grants(dvc):
config = {
"url": url,
"grant_read": "id=read-permission-id,id=other-read-permission-id",
"grant_read_acp": "id=read-acp-permission-id",
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
fs = S3FileSystem(dvc, config)
assert (fs.extra_args["GrantRead"] ==
"id=read-permission-id,id=other-read-permission-id")
assert fs.extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert fs.extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert fs.extra_args["GrantFullControl"] == "id=full-control-permission-id"
def test_grants(dvc):
config = {
"url": url,
"grant_read": "id=read-permission-id,id=other-read-permission-id",
"grant_read_acp": "id=read-acp-permission-id",
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
fs = S3FileSystem(**config)
extra_args = fs.fs_args["s3_additional_kwargs"]
assert (extra_args["GrantRead"] ==
"id=read-permission-id,id=other-read-permission-id")
assert extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert extra_args["GrantFullControl"] == "id=full-control-permission-id"
def test_copy_preserve_etag_across_buckets(cloud, dvc):
cloud.gen(FILE_WITH_CONTENTS)
rem = get_remote(dvc, **cloud.config)
s3 = rem.fs
s3.fs.mkdir("another/")
config = cloud.config.copy()
config["url"] = "s3://another"
config["region"] = "us-east-1"
another = S3FileSystem(**config)
from_info = rem.fs.path_info / "foo"
to_info = another.path_info / "foo"
rem.fs.copy(from_info, to_info)
from_hash = rem.fs.info(from_info)["etag"]
to_hash = another.info(to_info)["etag"]
assert from_hash == to_hash
def test_copy_preserve_etag_across_buckets(cloud, dvc):
cloud.gen(FILE_WITH_CONTENTS)
rem = _get_odb(dvc, cloud.config)
s3 = rem.fs.s3
s3.create_bucket(Bucket="another")
config = cloud.config.copy()
config["url"] = "s3://another"
config["region"] = "us-east-1"
another = S3FileSystem(**config)
from_info = rem.fs.path.join(rem.fs_path, "foo")
to_info = "another/foo"
rem.fs.copy(from_info, to_info)
from_hash = rem.fs.info(from_info)["ETag"].strip('"')
to_hash = another.info(to_info)["ETag"].strip('"')
assert from_hash == to_hash
def test_ssl_verify_path_param(dvc):
config = {"url": url, "ssl_verify": "/path/to/custom/cabundle.pem"}
fs = S3FileSystem(**config)
assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
def test_ssl_verify_bool_param(dvc):
config = {"url": url, "ssl_verify": False}
fs = S3FileSystem(**config)
assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
def _get_src_dst():
base_info = S3FileSystem.PATH_CLS(S3.get_url())
return base_info / "from", base_info / "to"
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
base_info = S3FileSystem.PATH_CLS("s3://bucket/")
fs = S3FileSystem(dvc, {"url": str(base_info)})
fs.makedirs(base_info)
def test_get_bucket():
fs = S3FileSystem(None, {"url": "s3://mybucket/path"})
with pytest.raises(DvcException, match="Bucket 'mybucket' does not exist"):
with fs._get_bucket("mybucket") as bucket:
raise bucket.meta.client.exceptions.NoSuchBucket({}, None)
def test_ssl_verify_bool_param(dvc):
config = {"url": url, "ssl_verify": False}
fs = S3FileSystem(dvc, config)
assert fs.ssl_verify == config["ssl_verify"]
def test_sse_kms_key_id(dvc):
fs = S3FileSystem(url=url, sse_kms_key_id="key")
assert fs.fs_args["s3_additional_kwargs"]["SSEKMSKeyId"] == "key"
def test_grants_mutually_exclusive_acl_error(dvc, grants):
for grant_option, grant_value in grants.items():
config = {"url": url, "acl": "public-read", grant_option: grant_value}
with pytest.raises(ConfigError):
S3FileSystem(**config)
def test_init(dvc):
config = {"url": url}
fs = S3FileSystem(**config)
assert fs.path_info == url
def test_sse_kms_key_id(dvc):
fs = S3FileSystem(dvc, {"url": url, "sse_kms_key_id": "key"})
assert fs.extra_args["SSEKMSKeyId"] == "key"
