def test_save_blob_file(tmpdir, blob, dask_client): tmpdir = Path(str(tmpdir)) fname = str(tmpdir / "file.txt") dask_blob = dask.delayed(blob) mode = "rt" if isinstance(blob, str) else "rb" rr = save_blob_to_file(dask_blob, fname) assert dask_client.compute(rr).result() == (fname, True) assert slurp(fname, mode=mode) == blob fname = str(tmpdir / "missing" / "file.txt") rr = save_blob_to_file(dask_blob, fname) assert dask_client.compute(rr).result() == (fname, False)
def _write_blob(self, data, url: str, ContentType: Optional[str] = None, with_deps=None) -> Delayed: """ Returns Delayed WriteResult[path, sha1, error=None] """ _u = urlparse(url) sha1 = _dask_sha1(data) if _u.scheme == "s3": kw = dict(creds=self._get_creds()) if ContentType is not None: kw["ContentType"] = ContentType if self._acl is not None: kw["ACL"] = self._acl return _pack_write_result( save_blob_to_s3(data, url, with_deps=with_deps, **kw), sha1) elif _u.scheme == "file": _dir = Path(_u.path).parent if not _dir.exists(): _dir.mkdir(parents=True, exist_ok=True) return _pack_write_result( save_blob_to_file(data, _u.path, with_deps=with_deps), sha1) else: raise ValueError(f"Don't know how to save to '{url}'")
def _write_blob( self, data, url: str, ContentType: Optional[str] = None, with_deps=None ) -> Delayed: _u = urlparse(url) if _u.scheme == "s3": kw = dict(creds=self._get_creds()) if ContentType is not None: kw["ContentType"] = ContentType if self._public: kw["ACL"] = "public-read" return save_blob_to_s3(data, url, with_deps=with_deps, **kw) elif _u.scheme == "file": _dir = Path(_u.path).parent if not _dir.exists(): _dir.mkdir(parents=True, exist_ok=True) return save_blob_to_file(data, _u.path, with_deps=with_deps) else: raise ValueError(f"Don't know how to save to '{url}'")