def test_verify_ssl_default_param(dvc): config = {"url": url} fs = S3FileSystem(**config) assert "client_kwargs" not in fs.fs_args config = { "url": url, "endpointurl": "https://my.custom.s3:1234", } fs = S3FileSystem(**config) assert "verify" not in fs.fs_args["client_kwargs"]
def test_get_s3_no_credentials(mocker): from botocore.exceptions import NoCredentialsError fs = S3FileSystem(None, {}) with pytest.raises(DvcException, match="Unable to find AWS credentials"): with fs._get_s3(): raise NoCredentialsError
def test_verify_ssl_default_param(dvc): config = { "url": url, } fs = S3FileSystem(**config) assert fs.fs_args["client_kwargs"]["verify"]
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch): config_file = tmp_dir / "aws_config.ini" config_file.write_text( textwrap.dedent( """\ [default] extra = keys s3 = addressing_style = auto use_accelerate_endpoint = true multipart_threshold = ThisIsNotGoingToBeCasted! [profile dev] some_extra = keys s3 = addressing_style = virtual multipart_threshold = 2GiB """ ) ) monkeypatch.setenv("AWS_CONFIG_FILE", config_file) fs = S3FileSystem(profile="dev", **s3.config) s3_config = fs.fs_args["config_kwargs"]["s3"] assert s3_config["addressing_style"] == "virtual" transfer_config = fs._transfer_config assert transfer_config.multipart_threshold == 2 * GB
def test_verify_ssl_default_param(dvc): config = { "url": url, } fs = S3FileSystem(dvc, config) assert fs.ssl_verify
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch): config_file = tmp_dir / "aws_config.ini" config_file.write_text( textwrap.dedent("""\ [default] extra = keys s3 = addressing_style = auto use_accelerate_endpoint = true multipart_threshold = ThisIsNotGoingToBeCasted! [profile dev] some_extra = keys s3 = addresing_style = virtual multipart_threshold = 2GiB """)) monkeypatch.setenv("AWS_CONFIG_FILE", config_file) fs = S3FileSystem(dvc, {**s3.config, "profile": "dev"}) assert fs._transfer_config is None with fs._get_s3() as s3: s3_config = s3.meta.client.meta.config.s3 assert s3_config["addresing_style"] == "virtual" assert "use_accelerate_endpoint" not in s3_config transfer_config = fs._transfer_config assert transfer_config.multipart_threshold == 2 * GB
def test_get_s3_connection_error(mocker): from botocore.exceptions import EndpointConnectionError fs = S3FileSystem(None, {}) msg = "Unable to connect to 'AWS S3'." with pytest.raises(DvcException, match=msg): with fs._get_s3(): raise EndpointConnectionError(endpoint_url="url")
def test_get_s3_connection_error_endpoint(mocker): from botocore.exceptions import EndpointConnectionError fs = S3FileSystem(None, {"endpointurl": "https://example.com"}) msg = "Unable to connect to 'https://example.com'." with pytest.raises(DvcException, match=msg): with fs._get_s3(): raise EndpointConnectionError(endpoint_url="url")
def test_s3_upload_fobj(tmp_dir, dvc, s3): s3.gen({"data": {"foo": "foo"}}) fs = S3FileSystem(dvc, s3.config) to_info = s3 / "data" / "bar" with fs.open(s3 / "data" / "foo", "rb") as stream: fs.upload_fobj(stream, to_info, 1) assert to_info.read_text() == "foo"
def test_key_id_and_secret(dvc): fs = S3FileSystem( url=url, access_key_id=key_id, secret_access_key=key_secret, session_token=session_token, ) assert fs.fs_args["key"] == key_id assert fs.fs_args["secret"] == key_secret assert fs.fs_args["token"] == session_token
def test_key_id_and_secret(dvc): fs = S3FileSystem( dvc, { "url": url, "access_key_id": key_id, "secret_access_key": key_secret, "session_token": session_token, }, ) assert fs.access_key_id == key_id assert fs.secret_access_key == key_secret assert fs.session_token == session_token
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): from dvc.checkout import _link fs = S3FileSystem(dvc, {"url": str(base_info.parent)}) cache = CloudCache(fs) s3 = cache.fs.s3.meta.client s3.create_bucket(Bucket=base_info.bucket) s3.put_object(Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data") _link(cache, base_info / "from", base_info / "to") assert cache.fs.exists(base_info / "from") assert cache.fs.exists(base_info / "to")
def test_s3_config_credentialpath(dvc, monkeypatch): environment = {} monkeypatch.setattr(os, "environ", environment) config = {"url": url, "credentialpath": "somewhere"} S3FileSystem(**config) assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere" environment.clear() config = {"url": url, "configpath": "somewhere"} S3FileSystem(**config) assert environment["AWS_CONFIG_FILE"] == "somewhere" environment.clear() config = { "url": url, "credentialpath": "somewhere", "configpath": "elsewhere", } S3FileSystem(**config) assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere" assert environment["AWS_CONFIG_FILE"] == "elsewhere" environment.clear()
def test_grants(dvc): config = { "url": url, "grant_read": "id=read-permission-id,id=other-read-permission-id", "grant_read_acp": "id=read-acp-permission-id", "grant_write_acp": "id=write-acp-permission-id", "grant_full_control": "id=full-control-permission-id", } fs = S3FileSystem(dvc, config) assert (fs.extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id") assert fs.extra_args["GrantReadACP"] == "id=read-acp-permission-id" assert fs.extra_args["GrantWriteACP"] == "id=write-acp-permission-id" assert fs.extra_args["GrantFullControl"] == "id=full-control-permission-id"
def test_copy_preserve_etag_across_buckets(remote, dvc): s3 = remote.fs.s3 s3.Bucket("another").create() another = S3FileSystem(dvc, {"url": "s3://another", "region": "us-east-1"}) from_info = remote.fs.path_info / "foo" to_info = another.path_info / "foo" remote.fs.copy(from_info, to_info) from_hash = remote.fs.info(from_info)["etag"] to_hash = another.info(to_info)["etag"] assert from_hash == to_hash
def test_grants(dvc): config = { "url": url, "grant_read": "id=read-permission-id,id=other-read-permission-id", "grant_read_acp": "id=read-acp-permission-id", "grant_write_acp": "id=write-acp-permission-id", "grant_full_control": "id=full-control-permission-id", } fs = S3FileSystem(**config) extra_args = fs.fs_args["s3_additional_kwargs"] assert (extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id") assert extra_args["GrantReadACP"] == "id=read-acp-permission-id" assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id" assert extra_args["GrantFullControl"] == "id=full-control-permission-id"
def test_copy_preserve_etag_across_buckets(cloud, dvc): cloud.gen(FILE_WITH_CONTENTS) rem = _get_odb(dvc, cloud.config) s3 = rem.fs.s3 s3.create_bucket(Bucket="another") config = cloud.config.copy() config["url"] = "s3://another" config["region"] = "us-east-1" another = S3FileSystem(**config) from_info = rem.fs.path.join(rem.fs_path, "foo") to_info = "another/foo" rem.fs.copy(from_info, to_info) from_hash = rem.fs.info(from_info)["ETag"].strip('"') to_hash = another.info(to_info)["ETag"].strip('"') assert from_hash == to_hash
def test_copy_preserve_etag_across_buckets(cloud, dvc): cloud.gen(FILE_WITH_CONTENTS) rem = get_remote(dvc, **cloud.config) s3 = rem.fs s3.fs.mkdir("another/") config = cloud.config.copy() config["url"] = "s3://another" config["region"] = "us-east-1" another = S3FileSystem(**config) from_info = rem.fs.path_info / "foo" to_info = another.path_info / "foo" rem.fs.copy(from_info, to_info) from_hash = rem.fs.info(from_info)["etag"] to_hash = another.info(to_info)["etag"] assert from_hash == to_hash
def test_sse_kms_key_id(dvc): fs = S3FileSystem(dvc, {"url": url, "sse_kms_key_id": "key"}) assert fs.extra_args["SSEKMSKeyId"] == "key"
def test_ssl_verify_bool_param(dvc): config = {"url": url, "ssl_verify": False} fs = S3FileSystem(**config) assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
def test_ssl_verify_path_param(dvc): config = {"url": url, "ssl_verify": "/path/to/custom/cabundle.pem"} fs = S3FileSystem(**config) assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm): base_info = S3FileSystem.PATH_CLS("s3://bucket/") fs = S3FileSystem(dvc, {"url": str(base_info)}) fs.makedirs(base_info)
def test_s3_isdir(tmp_dir, dvc, s3): s3.gen({"data": {"foo": "foo"}}) fs = S3FileSystem(dvc, s3.config) assert not fs.isdir(s3 / "data" / "foo") assert fs.isdir(s3 / "data")
def test_get_bucket(): fs = S3FileSystem(None, {"url": "s3://mybucket/path"}) with pytest.raises(DvcException, match="Bucket 'mybucket' does not exist"): with fs._get_bucket("mybucket") as bucket: raise bucket.meta.client.exceptions.NoSuchBucket({}, None)
def test_ssl_verify_bool_param(dvc): config = {"url": url, "ssl_verify": False} fs = S3FileSystem(dvc, config) assert fs.ssl_verify == config["ssl_verify"]
def test_sse_kms_key_id(dvc): fs = S3FileSystem(url=url, sse_kms_key_id="key") assert fs.fs_args["s3_additional_kwargs"]["SSEKMSKeyId"] == "key"
def test_grants_mutually_exclusive_acl_error(dvc, grants): for grant_option, grant_value in grants.items(): config = {"url": url, "acl": "public-read", grant_option: grant_value} with pytest.raises(ConfigError): S3FileSystem(**config)
def test_init(dvc): config = {"url": url} fs = S3FileSystem(**config) assert fs.path_info == url