def test_save_blob_s3(blob, monkeypatch, dask_client):
region_name = "us-west-2"
blob2 = blob + blob
dask_blob = dask.delayed(blob)
dask_blob2 = dask.delayed(blob2)
url = "s3://bucket/file.txt"
url2 = "s3://bucket/file-2.txt"
bucket, _ = s3_url_parse(url)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
with moto.mock_s3():
s3 = s3_client(region_name=region_name)
s3.create_bucket(Bucket=bucket)
rr = save_blob_to_s3(dask_blob, url, region_name=region_name)
assert rr.compute() == (url, True)
rr = save_blob_to_s3(dask_blob2, url2, region_name=region_name)
assert dask_client.compute(rr).result() == (url2, True)
bb1 = s3_fetch(url, s3=s3)
bb2 = s3_fetch(url2, s3=s3)
if isinstance(blob, str):
bb1 = bb1.decode("utf8")
bb2 = bb2.decode("utf8")
assert bb1 == blob
assert bb2 == blob2
def _write_blob(self,
data,
url: str,
ContentType: Optional[str] = None,
with_deps=None) -> Delayed:
"""
Returns Delayed WriteResult[path, sha1, error=None]
"""
_u = urlparse(url)
sha1 = _dask_sha1(data)
if _u.scheme == "s3":
kw = dict(creds=self._get_creds())
if ContentType is not None:
kw["ContentType"] = ContentType
if self._acl is not None:
kw["ACL"] = self._acl
return _pack_write_result(
save_blob_to_s3(data, url, with_deps=with_deps, **kw), sha1)
elif _u.scheme == "file":
_dir = Path(_u.path).parent
if not _dir.exists():
_dir.mkdir(parents=True, exist_ok=True)
return _pack_write_result(
save_blob_to_file(data, _u.path, with_deps=with_deps), sha1)
else:
raise ValueError(f"Don't know how to save to '{url}'")
def _write_blob(
self, data, url: str, ContentType: Optional[str] = None, with_deps=None
) -> Delayed:
_u = urlparse(url)
if _u.scheme == "s3":
kw = dict(creds=self._get_creds())
if ContentType is not None:
kw["ContentType"] = ContentType
if self._public:
kw["ACL"] = "public-read"
return save_blob_to_s3(data, url, with_deps=with_deps, **kw)
elif _u.scheme == "file":
_dir = Path(_u.path).parent
if not _dir.exists():
_dir.mkdir(parents=True, exist_ok=True)
return save_blob_to_file(data, _u.path, with_deps=with_deps)
else:
raise ValueError(f"Don't know how to save to '{url}'")