def download_files(self,
target: Union[Path, str] = None,
include_stac_metadata: bool = True) -> List[Path]:
"""
Download all assets to given folder.
:param target: path to folder to download to (must be a folder if it already exists)
:param include_stac_metadata: whether to download the job result metadata as a STAC (JSON) file.
:return: list of paths to the downloaded assets.
"""
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
f"Target argument {target} exists but isn't a folder.")
ensure_dir(target)
downloaded = [a.download(target) for a in self.get_assets()]
if include_stac_metadata:
# TODO #184: convention for metadata file name?
metadata_file = target / "job-results.json"
# TODO #184: rewrite references to locally downloaded assets?
metadata_file.write_text(json.dumps(self.get_metadata()))
downloaded.append(metadata_file)
return downloaded
def _create_job_dir(job_dir: Path):
logger.info("creating job dir {j!r} (parent dir: {p}))".format(
j=job_dir, p=describe_path(job_dir.parent)))
ensure_dir(job_dir)
if not ConfigParams().is_kube_deploy:
shutil.chown(job_dir, user=None, group='eodata')
_add_permissions(job_dir, stat.S_ISGID
| stat.S_IWGRP) # make children inherit this group
def _download_url(self, url: str, path: Path):
ensure_dir(path.parent)
with path.open('wb') as handle:
# TODO: finetune download parameters/chunking?
response = self.job.connection.get(url, stream=True)
for block in response.iter_content(1024):
if not block:
break
handle.write(block)
return path
def download_files(self, target: Union[Path, str] = None) -> List[Path]:
"""
Download all assets to given folder.
:param target: path to folder to download to (must be a folder if it already exists)
:return: list of paths to the downloaded assets.
"""
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
"The target argument must be a folder. Got {t!r}".format(
t=str(target)))
ensure_dir(target)
return [a.download(target) for a in self.get_assets()]
def test_ensure_dir_str(tmp_path):
work_dir = str(tmp_path / "work/data/foo")
assert not os.path.exists(work_dir)
p = ensure_dir(work_dir)
assert os.path.exists(work_dir)
assert isinstance(p, pathlib.Path)
assert p.exists()
assert str(p) == str(work_dir)
def tmp_openeo_config_home(tmp_path):
"""
Fixture to set `OPENEO_CONFIG_HOME` env var to temp path,
which is used as default for get_user_config_dir, get_user_data_dir, AuthConfig, PrivateJsonFile, ...
"""
path = ensure_dir(Path(str(tmp_path)) / "openeo-conf")
with mock.patch.dict("os.environ", {"OPENEO_CONFIG_HOME": str(path)}):
yield path
def test_ensure_dir_pathlib(tmp_path):
# Note: tmp_path might be pathlib2
work_dir = pathlib.Path(str(tmp_path / "work/data/foo"))
assert not work_dir.exists()
p = ensure_dir(work_dir)
assert work_dir.exists()
assert isinstance(p, pathlib.Path)
assert p.exists()
assert str(p) == str(work_dir)
def download(self,
target: Optional[Union[Path, str]] = None,
chunk_size=None) -> Path:
"""
Download asset to given location
:param target: download target path
"""
target = Path(target or Path.cwd())
if target.is_dir():
target = target / self.name
ensure_dir(target.parent)
logger.info(
"Downloading Job result asset {n!r} from {h!s} to {t!s}".format(
n=self.name, h=self.href, t=target))
with target.open("wb") as f:
response = self._get_response(stream=True)
for block in response.iter_content(chunk_size=chunk_size):
f.write(block)
return target
def download(self,
target: Optional[Union[Path, str]] = None,
chunk_size=None) -> Path:
"""
Download asset to given location
:param target: download target path. Can be an existing folder
(in which case the filename advertised by backend will be used)
or full file name. By default, the working directory will be used.
"""
target = Path(target or Path.cwd())
if target.is_dir():
target = target / self.name
ensure_dir(target.parent)
logger.info(
"Downloading Job result asset {n!r} from {h!s} to {t!s}".format(
n=self.name, h=self.href, t=target))
with target.open("wb") as f:
response = self._get_response(stream=True)
for block in response.iter_content(chunk_size=chunk_size):
f.write(block)
return target
def write_assets(self,
directory: Union[str, Path],
format: str,
options: Optional[dict] = None) -> Dict[str, StacAsset]:
directory = ensure_dir(directory)
format_info = IOFORMATS.get(format)
# TODO: check if format can be used for vector data?
path = directory / f"vectorcube.{format_info.extension}"
self._as_geopandas_df().to_file(path, driver=format_info.fiona_driver)
if not format_info.multi_file:
# single file format
return {
path.name: {
"href": path,
"title": "Vector cube",
"type": format_info.mimetype,
"roles": ["data"],
}
}
else:
# Multi-file format
components = list(directory.glob("vectorcube.*"))
if options.get("zip_multi_file"):
# TODO: automatically zip shapefile components?
zip_path = path.with_suffix(f".{format_info.extension}.zip")
with zipfile.ZipFile(zip_path, "w") as zip_file:
for component in components:
zip_file.write(component, arcname=component.name)
return {
path.name: {
"href": zip_path,
"title": "Vector cube",
"type": "application/zip",
"roles": ["data"],
}
}
else:
# TODO: better multi-file support?
return {p.name: {"href": p} for p in components}
def tmp_openeo_config(tmp_path):
path = ensure_dir(Path(str(tmp_path)) / "openeo_conf")
with mock.patch.dict("os.environ", {"OPENEO_CONFIG_HOME": str(path)}):
yield path
def _get_job_output_dir(self, job_id: str) -> Path:
return ensure_dir(self._output_root_dir / job_id)