def test_save_blob_file(tmpdir, blob, dask_client):
tmpdir = Path(str(tmpdir))
fname = str(tmpdir / "file.txt")
dask_blob = dask.delayed(blob)
mode = "rt" if isinstance(blob, str) else "rb"
rr = save_blob_to_file(dask_blob, fname)
assert dask_client.compute(rr).result() == (fname, True)
assert slurp(fname, mode=mode) == blob
fname = str(tmpdir / "missing" / "file.txt")
rr = save_blob_to_file(dask_blob, fname)
assert dask_client.compute(rr).result() == (fname, False)
def _write_blob(self,
data,
url: str,
ContentType: Optional[str] = None,
with_deps=None) -> Delayed:
"""
Returns Delayed WriteResult[path, sha1, error=None]
"""
_u = urlparse(url)
sha1 = _dask_sha1(data)
if _u.scheme == "s3":
kw = dict(creds=self._get_creds())
if ContentType is not None:
kw["ContentType"] = ContentType
if self._acl is not None:
kw["ACL"] = self._acl
return _pack_write_result(
save_blob_to_s3(data, url, with_deps=with_deps, **kw), sha1)
elif _u.scheme == "file":
_dir = Path(_u.path).parent
if not _dir.exists():
_dir.mkdir(parents=True, exist_ok=True)
return _pack_write_result(
save_blob_to_file(data, _u.path, with_deps=with_deps), sha1)
else:
raise ValueError(f"Don't know how to save to '{url}'")
def _write_blob(
self, data, url: str, ContentType: Optional[str] = None, with_deps=None
) -> Delayed:
_u = urlparse(url)
if _u.scheme == "s3":
kw = dict(creds=self._get_creds())
if ContentType is not None:
kw["ContentType"] = ContentType
if self._public:
kw["ACL"] = "public-read"
return save_blob_to_s3(data, url, with_deps=with_deps, **kw)
elif _u.scheme == "file":
_dir = Path(_u.path).parent
if not _dir.exists():
_dir.mkdir(parents=True, exist_ok=True)
return save_blob_to_file(data, _u.path, with_deps=with_deps)
else:
raise ValueError(f"Don't know how to save to '{url}'")