def from_file(cls, filename: PathLike) -> "Configuration":
if not isinstance(filename, Path):
filename = Path(filename)
filename = filename.expanduser()
with open(filename) as input_file:
configuration = yaml.safe_load(input_file)
return cls(**configuration)
def ensure_directory(directory: PathLike) -> Path:
"""
ensure that a directory exists
:param directory: directory path to ensure
:returns: path to ensured directory
"""
if not isinstance(directory, Path):
directory = Path(directory)
directory = directory.expanduser()
if directory.is_file():
directory = directory.parent
if not directory.exists():
directory.mkdir(parents=True, exist_ok=True)
return directory
def fetch(self,
query_results: QueryResponseTable,
*,
path: os.PathLike = None,
downloader: parfive.Downloader,
**kwargs):
"""
Fetch asdf files describing the datasets.
Parameters
----------
query_results:
Results to download.
path : `str` or `pathlib.Path`, optional
Path to the download directory
downloader : `parfive.Downloader`
The download manager to use.
"""
# This logic is being upstreamed into Fido hopefully in 2.1rc4
if path is None:
path = Path(config.get(
'downloads', 'download_dir')) / '{file}' # pragma: no cover
elif isinstance(path,
(str, os.PathLike)) and '{file}' not in str(path):
path = Path(path) / '{file}' # pragma: no cover
else:
path = Path(path) # pragma: no cover
path = path.expanduser()
if not len(query_results):
return
for row in query_results:
url = f"{self._BASE_DOWNLOAD_URL}/asdf?datasetId={row['Dataset ID']}"
# Set max_splits here as the metadata streamer doesn't like accept-range at the moment.
downloader.enqueue_file(url,
filename=partial(self._make_filename, path,
row),
max_splits=1)
