def test_copy_multipart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
S3RemoteTree._copy(s3, from_info, to_info, {})
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
S3RemoteTree._copy(s3, from_info, to_info, {})
def test_copy_preserve_etag_across_buckets(remote, dvc):
s3 = remote.tree.s3
s3.create_bucket(Bucket="another")
another = S3RemoteTree(dvc, {"url": "s3://another", "region": "us-east-1"})
from_info = remote.path_info / "foo"
to_info = another.path_info / "foo"
remote.tree.copy(from_info, to_info)
from_etag = S3RemoteTree.get_etag(s3, from_info.bucket, from_info.path)
to_etag = S3RemoteTree.get_etag(s3, "another", "foo")
assert from_etag == to_etag
def test_checkout_for_external_outputs(tmp_dir, dvc):
dvc.cache.s3 = CloudCache(S3RemoteTree(dvc, {"url": S3.get_url()}))
remote = Remote(S3RemoteTree(dvc, {"url": S3.get_url()}))
file_path = remote.path_info / "foo"
remote.tree.s3.put_object(
Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo"
)
dvc.add(str(remote.path_info / "foo"), external=True)
remote.tree.remove(file_path)
stats = dvc.checkout(force=True)
assert stats == {**empty_checkout, "added": [str(file_path)]}
assert remote.tree.exists(file_path)
remote.tree.s3.put_object(
Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo\nfoo"
)
stats = dvc.checkout(force=True)
assert stats == {**empty_checkout, "modified": [str(file_path)]}
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
tree = S3RemoteTree(dvc, {"url": str(base_info.parent)})
cache = CloudCache(tree)
s3 = cache.tree.s3
s3.create_bucket(Bucket=base_info.bucket)
s3.put_object(Bucket=base_info.bucket,
Key=(base_info / "from").path,
Body="data")
cache.link(base_info / "from", base_info / "to")
assert cache.tree.exists(base_info / "from")
assert cache.tree.exists(base_info / "to")
def test_grants(dvc):
config = {
"url": url,
"grant_read": "id=read-permission-id,id=other-read-permission-id",
"grant_read_acp": "id=read-acp-permission-id",
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
tree = S3RemoteTree(dvc, config)
assert (tree.extra_args["GrantRead"] ==
"id=read-permission-id,id=other-read-permission-id")
assert tree.extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert tree.extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert (
tree.extra_args["GrantFullControl"] == "id=full-control-permission-id")
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
base_info = S3RemoteTree.PATH_CLS("s3://bucket/")
remote = S3Remote(dvc, {"url": str(base_info)})
remote.tree.makedirs(base_info)
def _get_src_dst():
base_info = S3RemoteTree.PATH_CLS(S3.get_url())
return base_info / "from", base_info / "to"
@mock_s3
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
S3RemoteTree._copy(s3, from_info, to_info, {})
@mock_s3
@pytest.mark.parametrize(
"base_info",
[
S3RemoteTree.PATH_CLS("s3://bucket/"),
S3RemoteTree.PATH_CLS("s3://bucket/ns/"),
],
)
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
remote = S3Remote(dvc, {"url": str(base_info.parent)})
s3 = remote.tree.s3
s3.create_bucket(Bucket=base_info.bucket)
s3.put_object(Bucket=base_info.bucket,
Key=(base_info / "from").path,
Body="data")
remote.link(base_info / "from", base_info / "to")
assert remote.tree.exists(base_info / "from")
assert remote.tree.exists(base_info / "to")
def remote(cls, repo):
with mock_s3():
yield Remote(S3RemoteTree(repo, {"url": cls.get_url()}))
def test_sse_kms_key_id(dvc):
tree = S3RemoteTree(dvc, {"url": url, "sse_kms_key_id": "key"})
assert tree.extra_args["SSEKMSKeyId"] == "key"
def test_grants_mutually_exclusive_acl_error(dvc, grants):
for grant_option, grant_value in grants.items():
config = {"url": url, "acl": "public-read", grant_option: grant_value}
with pytest.raises(ConfigError):
S3RemoteTree(dvc, config)
def test_init(dvc):
config = {"url": url}
tree = S3RemoteTree(dvc, config)
assert tree.path_info == url
