def test_copy_multipart_preserve_etag(): from_info, to_info = _get_src_dst() s3 = boto3.client("s3") s3.create_bucket(Bucket=from_info.bucket) _upload_multipart(s3, from_info.bucket, from_info.path) RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_singlepart_preserve_etag(): from_info, to_info = _get_src_dst() s3 = boto3.client("s3") s3.create_bucket(Bucket=from_info.bucket) s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_multipart_preserve_etag(): from_info, to_info = _get_src_dst() if not _should_test_aws(): pytest.skip() s3 = boto3.client("s3") _upload_multipart(s3, from_info.bucket, from_info.path) RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_singlepart_preserve_etag(): from_info, to_info = _get_src_dst() if not _should_test_aws(): pytest.skip() s3 = boto3.client("s3") s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") RemoteS3._copy(s3, from_info, to_info, {})
def test_copy_preserve_etag_across_buckets(remote, dvc): s3 = remote.s3 s3.create_bucket(Bucket="another") another = RemoteS3(dvc, {"url": "s3://another", "region": "us-east-1"}) from_info = remote.path_info / "foo" to_info = another.path_info / "foo" remote.copy(from_info, to_info) from_etag = RemoteS3.get_etag(s3, from_info.bucket, from_info.path) to_etag = RemoteS3.get_etag(s3, "another", "foo") assert from_etag == to_etag
def remote(): """Returns a RemoteS3 connected to a bucket with the following structure: bucket ├── data │ ├── alice │ ├── alpha │ └── subdir │ ├── 1 │ ├── 2 │ └── 3 ├── empty_dir ├── empty_file └── foo """ with mock_s3(): remote = RemoteS3(None, {"url": "s3://bucket", "region": "us-east-1"}) s3 = remote.s3 s3.create_bucket(Bucket="bucket") s3.put_object(Bucket="bucket", Key="empty_dir/") s3.put_object(Bucket="bucket", Key="empty_file", Body=b"") s3.put_object(Bucket="bucket", Key="foo", Body=b"foo") s3.put_object(Bucket="bucket", Key="data/alice", Body=b"alice") s3.put_object(Bucket="bucket", Key="data/alpha", Body=b"alpha") s3.put_object(Bucket="bucket", Key="data/subdir/1", Body=b"1") s3.put_object(Bucket="bucket", Key="data/subdir/2", Body=b"2") s3.put_object(Bucket="bucket", Key="data/subdir/3", Body=b"3") yield remote
def __init__(self, stage, path, info=None): super(DependencyS3, self).__init__(stage, path) self.info = info self.remote = RemoteS3(stage.project, {Config.SECTION_REMOTE_URL: '/'}) self.path_info = {'scheme': 's3', 'bucket': urlparse(path).netloc, 'key': urlparse(path).path.lstrip('/')}
def __init__(self, stage, path, info=None, remote=None): super(DependencyS3, self).__init__(stage, path, info=info) self.remote = remote or RemoteS3(stage.project, {}) bucket = remote.bucket if remote else urlparse(path).netloc path = urlparse(path).path.lstrip('/') if remote: path = posixpath.join(remote.prefix, path) self.path_info = {'scheme': 's3', 'bucket': bucket, 'path': path}
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): remote = RemoteS3(dvc, {"url": str(base_info.parent)}) remote.s3.create_bucket(Bucket=base_info.bucket) remote.s3.put_object(Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data") remote.link(base_info / "from", base_info / "to") assert remote.exists(base_info / "from") assert remote.exists(base_info / "to")
def test_grants(dvc): config = { "url": url, "grant_read": "id=read-permission-id,id=other-read-permission-id", "grant_read_acp": "id=read-acp-permission-id", "grant_write_acp": "id=write-acp-permission-id", "grant_full_control": "id=full-control-permission-id", } remote = RemoteS3(dvc, config) assert (remote.extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id") assert remote.extra_args["GrantReadACP"] == "id=read-acp-permission-id" assert remote.extra_args["GrantWriteACP"] == "id=write-acp-permission-id" assert (remote.extra_args["GrantFullControl"] == "id=full-control-permission-id")
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm): base_info = RemoteS3.path_cls("s3://bucket/") remote = RemoteS3(dvc, {"url": str(base_info)}) remote.makedirs(base_info)
def _get_src_dst(): base_info = RemoteS3.path_cls(S3.get_url()) return base_info / "from", base_info / "to"
def remote(cls): with mock_s3(): yield RemoteS3(None, {"url": cls.get_url()})
def remote(cls, repo): with mock_s3(): yield RemoteS3(repo, {"url": cls.get_url()})
def test_init(dvc): config = {"url": url} remote = RemoteS3(dvc, config) assert remote.path_info == url
@mock_s3 def test_copy_singlepart_preserve_etag(): from_info, to_info = _get_src_dst() s3 = boto3.client("s3") s3.create_bucket(Bucket=from_info.bucket) s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") RemoteS3._copy(s3, from_info, to_info, {}) @mock_s3 @pytest.mark.parametrize( "base_info", [RemoteS3.path_cls("s3://bucket/"), RemoteS3.path_cls("s3://bucket/ns/")], ) def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): remote = RemoteS3(dvc, {"url": str(base_info.parent)}) remote.s3.create_bucket(Bucket=base_info.bucket) remote.s3.put_object(Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data") remote.link(base_info / "from", base_info / "to") assert remote.exists(base_info / "from") assert remote.exists(base_info / "to") @mock_s3
def test_grants_mutually_exclusive_acl_error(dvc, grants): for grant_option, grant_value in grants.items(): config = {"url": url, "acl": "public-read", grant_option: grant_value} with pytest.raises(ConfigError): RemoteS3(dvc, config)