def test_copy_multipart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_multipart_preserve_etag():
from_info, to_info = _get_src_dst()
if not _should_test_aws():
pytest.skip()
s3 = boto3.client("s3")
_upload_multipart(s3, from_info.bucket, from_info.path)
RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
if not _should_test_aws():
pytest.skip()
s3 = boto3.client("s3")
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_preserve_etag_across_buckets(remote, dvc):
s3 = remote.s3
s3.create_bucket(Bucket="another")
another = RemoteS3(dvc, {"url": "s3://another", "region": "us-east-1"})
from_info = remote.path_info / "foo"
to_info = another.path_info / "foo"
remote.copy(from_info, to_info)
from_etag = RemoteS3.get_etag(s3, from_info.bucket, from_info.path)
to_etag = RemoteS3.get_etag(s3, "another", "foo")
assert from_etag == to_etag
def remote():
"""Returns a RemoteS3 connected to a bucket with the following structure:
bucket
├── data
│ ├── alice
│ ├── alpha
│ └── subdir
│ ├── 1
│ ├── 2
│ └── 3
├── empty_dir
├── empty_file
└── foo
"""
with mock_s3():
remote = RemoteS3(None, {"url": "s3://bucket", "region": "us-east-1"})
s3 = remote.s3
s3.create_bucket(Bucket="bucket")
s3.put_object(Bucket="bucket", Key="empty_dir/")
s3.put_object(Bucket="bucket", Key="empty_file", Body=b"")
s3.put_object(Bucket="bucket", Key="foo", Body=b"foo")
s3.put_object(Bucket="bucket", Key="data/alice", Body=b"alice")
s3.put_object(Bucket="bucket", Key="data/alpha", Body=b"alpha")
s3.put_object(Bucket="bucket", Key="data/subdir/1", Body=b"1")
s3.put_object(Bucket="bucket", Key="data/subdir/2", Body=b"2")
s3.put_object(Bucket="bucket", Key="data/subdir/3", Body=b"3")
yield remote
def __init__(self, stage, path, info=None):
super(DependencyS3, self).__init__(stage, path)
self.info = info
self.remote = RemoteS3(stage.project, {Config.SECTION_REMOTE_URL: '/'})
self.path_info = {'scheme': 's3',
'bucket': urlparse(path).netloc,
'key': urlparse(path).path.lstrip('/')}
def __init__(self, stage, path, info=None, remote=None):
super(DependencyS3, self).__init__(stage, path, info=info)
self.remote = remote or RemoteS3(stage.project, {})
bucket = remote.bucket if remote else urlparse(path).netloc
path = urlparse(path).path.lstrip('/')
if remote:
path = posixpath.join(remote.prefix, path)
self.path_info = {'scheme': 's3', 'bucket': bucket, 'path': path}
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
remote = RemoteS3(dvc, {"url": str(base_info.parent)})
remote.s3.create_bucket(Bucket=base_info.bucket)
remote.s3.put_object(Bucket=base_info.bucket,
Key=(base_info / "from").path,
Body="data")
remote.link(base_info / "from", base_info / "to")
assert remote.exists(base_info / "from")
assert remote.exists(base_info / "to")
def test_grants(dvc):
config = {
"url": url,
"grant_read": "id=read-permission-id,id=other-read-permission-id",
"grant_read_acp": "id=read-acp-permission-id",
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
remote = RemoteS3(dvc, config)
assert (remote.extra_args["GrantRead"] ==
"id=read-permission-id,id=other-read-permission-id")
assert remote.extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert remote.extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert (remote.extra_args["GrantFullControl"] ==
"id=full-control-permission-id")
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
base_info = RemoteS3.path_cls("s3://bucket/")
remote = RemoteS3(dvc, {"url": str(base_info)})
remote.makedirs(base_info)
def _get_src_dst():
base_info = RemoteS3.path_cls(S3.get_url())
return base_info / "from", base_info / "to"
def remote(cls):
with mock_s3():
yield RemoteS3(None, {"url": cls.get_url()})
def remote(cls, repo):
with mock_s3():
yield RemoteS3(repo, {"url": cls.get_url()})
def test_init(dvc):
config = {"url": url}
remote = RemoteS3(dvc, config)
assert remote.path_info == url
@mock_s3
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
RemoteS3._copy(s3, from_info, to_info, {})
@mock_s3
@pytest.mark.parametrize(
"base_info",
[RemoteS3.path_cls("s3://bucket/"),
RemoteS3.path_cls("s3://bucket/ns/")],
)
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
remote = RemoteS3(dvc, {"url": str(base_info.parent)})
remote.s3.create_bucket(Bucket=base_info.bucket)
remote.s3.put_object(Bucket=base_info.bucket,
Key=(base_info / "from").path,
Body="data")
remote.link(base_info / "from", base_info / "to")
assert remote.exists(base_info / "from")
assert remote.exists(base_info / "to")
@mock_s3
def test_grants_mutually_exclusive_acl_error(dvc, grants):
for grant_option, grant_value in grants.items():
config = {"url": url, "acl": "public-read", grant_option: grant_value}
with pytest.raises(ConfigError):
RemoteS3(dvc, config)