def fetch_dataset(self, var, d_key):
"""Copy files to temporary directory.
(GCP can't copy to home dir, so always copy to a temp dir)
"""
tmpdir = core.TempDirManager().make_tempdir()
self.log.debug("Created GCP fetch temp dir at %s.", tmpdir)
(cp_command, smartsite) = self._get_fetch_method(self._fetch_method)
paths = d_key.remote_data()
if isinstance(paths, pd.Series):
paths = paths.to_list()
if not util.is_iterable(paths):
paths = (paths, )
local_paths = []
for path in paths:
# exceptions caught in parent loop in data_manager.DataSourceBase
local_path = os.path.join(tmpdir, os.path.basename(path))
self.log.info(f"\tFetching {path[len(self.attrs.CASE_ROOT_DIR):]}")
util.run_command(
cp_command + [
smartsite + path,
# gcp requires trailing slash, ln ignores it
smartsite + tmpdir + os.sep
],
timeout=self.timeout,
dry_run=self.dry_run,
log=self.log)
local_paths.append(local_path)
d_key.local_data = local_paths
def lookup(self, source_items, source, dest):
"""Lookup the corresponding *dest* values for *source_items* (keys).
Args:
source_items (str or list): one or more keys
source (str): the CV category that the items in *source_items*
belong to.
dest (str): the CV category we'd like the corresponding values for.
Returns: list of *dest* values corresponding to each entry in *source_items*.
"""
_lookup = self.get_lookup(source, dest)
if util.is_iterable(source_items):
return [util.from_iter(_lookup[item]) for item in source_items]
else:
return util.from_iter(_lookup[source_items])
def is_in_cv(self, category, items):
"""Determine if *items* take values that are valid for the CV category
*category*.
Args:
category (str): the CV category to use to validate values.
items (str or list of str): Entries whose validity we'd like to
check.
Returns: boolean or list of booleans, corresponding to the validity of
the entries in *items*.
"""
self._make_cv()
assert category in self.cv
if util.is_iterable(items):
return [(item in self.cv[category]) for item in items]
else:
return (items in self.cv[category])
def is_in_cv(self, category, items):
"""Determine if *items* take values that are valid for the CV category
*category*.
Args:
category (str): The CV category to use to validate values.
items (str or list of str): Entries whose validity we'd like to
check.
Returns:
Boolean or list of booleans, corresponding to the validity of
the entries in *items*.
"""
self._make_cv()
if category not in self.cv:
raise KeyError(f"Unrecognized CMIP6 CV category {category}.")
if util.is_iterable(items):
return [(item in self.cv[category]) for item in items]
else:
return (items in self.cv[category])
