def lookup(self, source_items, source, dest):
_lookup = self.get_lookup(source, dest)
if hasattr(source_items, '__iter__'):
return [
util.coerce_from_iter(_lookup[item]) for item in source_items
]
else:
return util.coerce_from_iter(_lookup[source_items])
def __call__(self, parser, namespace, values, option_string=None):
if self.nargs == 0 and self.const is not None:
setattr(namespace, self.dest, self.const)
elif self.nargs == 1:
setattr(namespace, self.dest, util.coerce_from_iter(values))
else:
setattr(namespace, self.dest, values)
# set flag to indicate user has set this argument
setattr(namespace, self.dest + self.flag_suffix, False)
def _init_path(self, key, d, env=None):
if self._unittest_flag: # use in unit testing only
return 'TEST_' + key
else:
# need to check existence in case we're being called directly
assert key in d, 'Error: {} not initialized.'.format(key)
return util.resolve_path(util.coerce_from_iter(d[key]),
root_path=self.CODE_ROOT,
env=env)
def _dispatch(setting, class_suffix):
class_prefix = config.config.get(setting, '')
class_prefix = util.coerce_from_iter(class_prefix)
# drop '_' and title-case class name
class_prefix = ''.join(class_prefix.split('_')).title()
for mod in self._dispatch_search:
try:
return getattr(mod, class_prefix + class_suffix)
except:
continue
print("No class named {}.".format(class_prefix + class_suffix))
raise Exception('no_class')
def toCF(self, convention, varname_in):
if convention == 'CF':
return varname_in
assert convention in self.variables, \
"Variable name translation doesn't recognize {}.".format(convention)
inv_lookup = self.variables[convention].inverse()
try:
return util.coerce_from_iter(inv_lookup[varname_in])
except KeyError:
print("ERROR: name {} not defined for convention {}.".format(
varname_in, convention))
raise
def fetch_dataset(self, d_key, method='auto'):
"""Copy files to temporary directory and combine chunks.
"""
# pylint: disable=maybe-no-member
(cp_command, smartsite) = self._determine_fetch_method(method)
dest_path = self.local_path(d_key)
dest_dir = os.path.dirname(dest_path)
# ncrcat will error instead of creating destination directories
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# GCP can't copy to home dir, so always copy to temp
tmpdirs = util_mdtf.TempDirManager()
work_dir = tmpdirs.make_tempdir(hash_obj=d_key)
remote_files = sorted( # cast from set to list so we can go in chrono order
list(self.data_files[d_key]),
key=lambda ds: ds.date_range.start)
# copy remote files
# TODO: Do something intelligent with logging, caught OSErrors
for f in remote_files:
print("\tcopying ...{} to {}".format(
f._remote_data[len(self.root_dir):], work_dir))
util.run_command(
cp_command + [
smartsite + f._remote_data,
# gcp requires trailing slash, ln ignores it
smartsite + work_dir + os.sep
],
timeout=self.file_transfer_timeout,
dry_run=self.dry_run)
# ----------------------------------------
# Processing of copied files: TODO: refactor individual steps into
# separate functions
# set axis names from header info
# only look at first file; if other chunks for same var differ, NCO will
# raise error when we try to concat them
file_name = os.path.basename(remote_files[0]._remote_data)
var_name = remote_files[0].name_in_model
file_axes = self.nc_get_axes_attributes(var_name,
in_file=file_name,
cwd=work_dir,
dry_run=self.dry_run)
for fax, fax_attrs in file_axes.iteritems():
# update DataSets with axis info - need to loop since multiple PODs
# may reference this file (warning will be repeated; TODO fix that)
error_flag = 0
for var in self.data_keys[d_key]:
if fax in var.axes:
# file's axis in list of case's axis names; check their
# axis attributes match if they're both defined
if 'axis' in fax_attrs and 'axis' in var.axes[fax] \
and fax_attrs['axis'].lower() != var.axes[fax]['axis'].lower() \
and error_flag != 1:
print((
"\tWarning: unexpected axis attribute for {0} in "
"{1} (found {2}, {3} convention is {4})").format(
fax, file_name, fax_attrs['axis'],
self.convention, var.axes[fax]['axis']))
error_flag = 1
var.axes[fax]['MDTF_set_from_axis'] = False
else:
# file has different axis name, try to match by attribute
for vax, vax_attrs in var.axes.iteritems():
if 'axis' not in fax_attrs or 'axis' not in vax_attrs:
continue
elif vax_attrs['axis'].lower(
) == fax_attrs['axis'].lower():
# matched axis attributes: log warning & reassign
if error_flag != 2:
print((
"\tWarning: unexpected {0} axis name in {1} "
"(found {2}, {3} convention is {4})"
).format(fax_attrs['axis'], file_name, fax,
self.convention, vax))
error_flag = 2
# only update so we don't overwrite the envvar name
var.axes[fax] = vax_attrs.copy()
var.axes[fax].update(fax_attrs)
var.axes[fax]['MDTF_set_from_axis'] = True
del var.axes[vax]
break
else:
# get here if we didn't hit 'break' above -- give up
if error_flag != 3:
print(("\tWarning: unable to assign {0} axis "
"in {1}.").format(fax, file_name))
error_flag = 3
# crop time axis to requested range
# do this *before* combining chunks to reduce disk activity
for vax, vax_attrs in var.axes.iteritems():
if 'axis' not in vax_attrs or vax_attrs['axis'].lower() != 't':
continue
else:
time_var_name = vax
break
else:
print("\tCan't determine time axis for {}.".format(file_name))
time_var_name = 'time' # will probably give KeyError
trim_count = 0
for f in remote_files:
file_name = os.path.basename(f._remote_data)
if not self.date_range.overlaps(f.date_range):
print(("\tWarning: {} has dates {} outside of requested "
"range {}.").format(file_name, f.date_range,
self.date_range))
continue
if not self.date_range.contains(f.date_range):
# file overlaps analysis range but is not strictly contained
# in it means we need to trim either start or end or both
trimmed_range = f.date_range.intersection(
self.date_range, precision=f.date_range.precision)
print("\ttrimming '{}' of {} from {} to {}".format(
time_var_name, file_name, f.date_range, trimmed_range))
trim_count = trim_count + 1
self.nc_crop_time_axis(time_var_name,
trimmed_range,
in_file=file_name,
cwd=work_dir,
dry_run=self.dry_run)
if trim_count > 2:
print("trimmed {} files!".format(trim_count))
raise AssertionError()
# cat chunks to destination, if more than one
if len(remote_files) > 1:
# not running in shell, so can't use glob expansion.
print("\tcatting {} chunks to {}".format(d_key.name_in_model,
dest_path))
chunks = [os.path.basename(f._remote_data) for f in remote_files]
self.nc_cat_chunks(chunks,
dest_path,
cwd=work_dir,
dry_run=self.dry_run)
else:
f = util.coerce_from_iter(remote_files)
file_name = os.path.basename(f._remote_data)
print("\tsymlinking {} to {}".format(d_key.name_in_model,
dest_path))
util.run_command(['ln', '-fs', \
os.path.join(work_dir, file_name), dest_path],
dry_run=self.dry_run
)