def parse_pod_list(self, cli_obj, pod_info_tuple):
pod_data = pod_info_tuple.pod_data
all_realms = pod_info_tuple.sorted_lists.get('realms', [])
pod_realms = pod_info_tuple.realm_data
args = util.to_iter(cli_obj.config.pop('pods', []), set)
if 'example' in args or 'examples' in args:
self.pod_list = [p for p in pod_data if p.startswith('example')]
elif 'all' in args:
self.pod_list = [p for p in pod_data if not p.startswith('example')]
else:
# specify pods by realm
realms = args.intersection(all_realms)
args = args.difference(all_realms) # remainder
for key in pod_realms:
if util.to_iter(key, set).issubset(realms):
self.pod_list.extend(pod_realms[key])
# specify pods by name
pods = args.intersection(set(pod_data))
self.pod_list.extend(list(pods))
for arg in args.difference(set(pod_data)): # remainder:
print("WARNING: Didn't recognize POD {}, ignoring".format(arg))
# exclude examples
self.pod_list = [p for p in pod_data if not p.startswith('example')]
if not self.pod_list:
_log.critical(("ERROR: no PODs selected to be run. Do `./mdtf info pods`"
" for a list of available PODs, and check your -p/--pods argument."
f"\nReceived --pods = {str(list(args))}"))
exit(1)
def framework_test(code_root, output_dir, cli_config):
print("Starting framework test run")
abs_out_dir = util.resolve_path(output_dir,
root_path=code_root,
env=os.environ)
try:
log_str = shell_command_wrapper('./mdtf -f {input_file}'.format(
input_file=os.path.join(code_root, cli_config['config_out'])),
cwd=code_root)
log_str = util.to_iter(log_str)
# write to most recent directory in output_dir
runs = [
d for d in glob.glob(os.path.join(abs_out_dir, '*'))
if os.path.isdir(d)
]
if not runs:
raise IOError(
"Can't find framework output in {}".format(abs_out_dir))
run_output = max(runs, key=os.path.getmtime)
with io.open(os.path.join(run_output, 'mdtf_test.log'),
'w',
encoding='utf-8') as f:
f.write('\n'.join(log_str))
except Exception as exc:
fatal_exception_handler(exc, "ERROR: framework test run failed.")
print("Finished framework test run at {}".format(run_output))
return run_output
def __post_init__(self):
core.MDTFObjectBase.__post_init__(self)
# set up log (PODLoggerMixin)
self.init_log()
for k, v in self.runtime_requirements.items():
self.runtime_requirements[k] = util.to_iter(v)
def guess_attr(name,
expected_val,
options,
default=None,
comparison_func=None):
"""Return element of *options* equal to *expected_val*. If none are equal,
try a looser, case-insensititve match. (All arguments expected to be strings.)
"""
def str_munge(s):
# comparison function: lowercase, drop non-alphanumeric chars
return re.sub(r'[^a-z0-9]+', '', s.lower())
if comparison_func is None:
comparison_func = (lambda x, y: x == y)
options = util.to_iter(options)
test_count = sum(comparison_func(opt, expected_val) for opt in options)
if test_count > 1:
_log.debug("Found multiple values of '%s' set for '%s'.",
expected_val, name)
if test_count >= 1:
return expected_val
munged_opts = [
(comparison_func(str_munge(opt), str_munge(expected_val)), opt) \
for opt in options
]
if sum(tup[0] for tup in munged_opts) == 1:
guessed_val = [tup[1] for tup in munged_opts if tup[0]][0]
_log.debug(
("Correcting '%s' to '%s' as the intended value for '%s'."),
expected_val, guessed_val, name)
return guessed_val
if default is None:
raise KeyError(expected_val)
else:
return default
def _print_pod_info(self, pod, verbose):
ds = self.pods[pod]['settings']
dv = self.pods[pod]['varlist']
if verbose == 1:
print(' {}: {}.'.format(pod, ds['long_name']))
elif verbose == 2:
print(' {}: {}.'.format(pod, ds['long_name']))
print(' {}'.format(ds['description']))
print(' Variables: {}'.format(
', '.join([v['var_name'].replace('_var','') for v in dv])
))
elif verbose == 3:
print('{}: {}.'.format(pod, ds['long_name']))
print(' Realm: {}.'.format(' and '.join(util.to_iter(ds['realm']))))
print(' {}'.format(ds['description']))
print(' Variables:')
for var in dv:
var_str = ' {} ({}) @ {} frequency'.format(
var['var_name'].replace('_var',''),
var.get('requirement',''),
var['freq']
)
if 'alternates' in var:
var_str = var_str + '; alternates: {}'.format(
', '.join([s.replace('_var','') for s in var['alternates']])
)
print(var_str)
def get_lookup(self, source, dest):
"""Find the appropriate lookup table to convert values in *source* (keys)
to values in *dest* (values), generating it if necessary.
Args:
source (str): the CV category to use for the keys.
dest (str): the CV category to use for the values.
Returns: :class:`util.MultiMap` providing a dict-like lookup interface,
ie dest_value = d[source_key].
"""
if (source, dest) in self._lookups:
return self._lookups[(source, dest)]
elif (dest, source) in self._lookups:
return self._lookups[(dest, source)].inverse()
elif source in self._contents:
k = list(self._contents[source])[0]
if dest not in self._contents[source][k]:
raise KeyError(f"Can't find {dest} in attributes for {source}.")
mm = util.MultiMap()
for k in self._contents[source]:
mm[k].update(
util.to_iter(self._contents[source][k][dest], set)
)
self._lookups[(source, dest)] = mm
return mm
elif dest in self._contents:
return self._lookups[(dest, source)].inverse()
else:
raise KeyError(f"Neither {source} or {dest} in CV table list.")
def _print_pod_info(self, pod, verbose):
"""Handler which prints information about PODs as taken from their
settings.json files, at configurable levels of verbosity.
"""
ds = self.pods[pod]['settings']
dv = self.pods[pod]['varlist']
if verbose == 1:
print(' {}: {}.'.format(pod, ds['long_name']))
elif verbose == 2:
print(' {}: {}.'.format(pod, ds['long_name']))
print(' {}'.format(ds['description']))
print(' Variables: {}'.format(', '.join(
[v['var_name'].replace('_var', '') for v in dv])))
elif verbose == 3:
print('{}: {}.'.format(pod, ds['long_name']))
print(' Realm: {}.'.format(' and '.join(util.to_iter(
ds['realm']))))
print(' {}'.format(ds['description']))
print(' Variables:')
for var in dv:
var_str = ' {} ({}) @ {} frequency'.format(
var['var_name'].replace('_var', ''),
var.get('requirement', ''), var['freq'])
if 'alternates' in var:
var_str = var_str + '; alternates: {}'.format(', '.join(
[s.replace('_var', '') for s in var['alternates']]))
print(var_str)
def makedirs(self, path_keys, delete_existing):
path_keys = util.to_iter(path_keys)
for key in path_keys:
path = self.config[key]
if path:
if not os.path.isdir(path):
os.makedirs(path) # recursive mkdir if needed
elif delete_existing:
shutil.rmtree(path) # overwrite everything
def _make_cv(self):
"""Populate the *cv* attribute of :class:`CMIP6_CVs` with the tables
read in during __init__().
Do this on-demand rather than in __init__, in case this information isn't
needed for this run of the framework.
"""
if self.cv:
return
for k in self._contents:
self.cv[k] = util.to_iter(self._contents[k])
def parse_case_list(self, cli_obj):
d = cli_obj.config # abbreviate
if 'CASENAME' in d and d['CASENAME']:
# defined case from CLI
cli_d = self._populate_from_cli(cli_obj, 'MODEL')
if 'CASE_ROOT_DIR' not in cli_d and d.get('root_dir', None):
# CASE_ROOT was set positionally
cli_d['CASE_ROOT_DIR'] = d['root_dir']
case_list_in = [cli_d]
else:
case_list_in = util.to_iter(cli_obj.file_case_list)
case_list = []
for i, case_d in enumerate(case_list_in):
case_list.append(self.parse_case(i, case_d, cli_obj))
self.case_list = [case for case in case_list if case]
if not self.case_list:
_log.critical(("ERROR: no valid entries in case_list. Please specify "
"model run information.\nReceived:"
f"\n{util.pretty_print_json(case_list_in)}"))
exit(1)
def parse_case(self, n, d, cli_obj):
# really need to move this into init of DataManager
if 'CASE_ROOT_DIR' not in d and 'root_dir' in d:
d['CASE_ROOT_DIR'] = d.pop('root_dir')
case_convention = d.get('convention', '')
if case_convention:
d['convention'] = case_convention
if not ('CASENAME' in d or ('model' in d and 'experiment' in d)):
_log.warning(("Need to specify either CASENAME or model/experiment "
"in caselist entry %s, skipping."), n+1)
return None
_ = d.setdefault('model', d.get('convention', ''))
_ = d.setdefault('experiment', '')
_ = d.setdefault('CASENAME', '{}_{}'.format(d['model'], d['experiment']))
for field in ['FIRSTYR', 'LASTYR', 'convention']:
if not d.get(field, None):
_log.warning(("No value set for %s in caselist entry %s, "
"skipping."), field, n+1)
return None
# if pods set from CLI, overwrite pods in case list
d['pod_list'] = self.set_case_pod_list(d, cli_obj)
return d
def set_case_pod_list(self, case, cli_obj):
# if pods set from CLI, overwrite pods in case list
# already finalized self.pod-list by the time we get here
if not cli_obj.is_default['pods'] or not case.get('pod_list', None):
return self.pod_list
def parse_case_list(self, cli_obj, pod_info_tuple):
d = cli_obj.config # abbreviate
if 'CASENAME' in d and d['CASENAME']:
# defined case from CLI
cli_d = self._populate_from_cli(cli_obj, 'MODEL')
if 'CASE_ROOT_DIR' not in cli_d and d.get('root_dir', None):
# CASE_ROOT was set positionally
cli_d['CASE_ROOT_DIR'] = d['root_dir']
case_list_in = [cli_d]
else:
case_list_in = util.to_iter(cli_obj.file_case_list)
self.cases = dict()
for i, case_d in enumerate(case_list_in):
case = self.parse_case(i, case_d, cli_obj, pod_info_tuple)
if case:
self.cases[case['CASENAME']] = case
def parse_pod_list(self, pod_list, pod_info_tuple):
pod_data = pod_info_tuple.pod_data # pod names -> contents of settings file
args = util.to_iter(pod_list, set)
bad_args = []
pods = []
for arg in args:
if arg == 'all':
# add all PODs except example PODs
pods.extend(
[p for p in pod_data if not p.startswith('example')])
elif arg == 'example' or arg == 'examples':
# add example PODs
pods.extend([p for p in pod_data if p.startswith('example')])
elif arg in pod_info_tuple.realm_data:
# realm_data: realm name -> list of POD names
# add all PODs for this realm
pods.extend(pod_info_tuple.realm_data[arg])
elif arg in pod_data:
# add POD by name
pods.append(arg)
else:
# unrecognized argument
_log.error("POD identifier '%s' not recognized.", arg)
bad_args.append(arg)
if bad_args:
valid_args = ['all', 'examples'] \
+ pod_info_tuple.sorted_realms \
+ pod_info_tuple.sorted_pods
_log.critical((
"The following POD identifiers were not recognized: "
"[%s].\nRecognized identifiers are: [%s].\n(Received --pods = %s)."
), ', '.join(f"'{p}'" for p in bad_args),
', '.join(f"'{p}'" for p in valid_args),
str(list(args)))
util.exit_handler(code=1)
pods = list(set(pods)) # delete duplicates
if not pods:
_log.critical((
"ERROR: no PODs selected to be run. Do `./mdtf info pods`"
" for a list of available PODs, and check your -p/--pods argument."
f"\nReceived --pods = {str(list(args))}"))
util.exit_handler(code=1)
return pods
def load_pod_settings(code_root, pod=None, pod_list=None):
"""Wrapper to load POD settings files, used by ConfigManager and CLIInfoHandler.
"""
# only place we can put it would be util.py if we want to avoid circular imports
_pod_dir = 'diagnostics'
_pod_settings = 'settings.jsonc'
def _load_one_json(pod):
try:
d = util.read_json(
os.path.join(code_root, _pod_dir, pod, _pod_settings)
)
assert 'settings' in d
except Exception as exc:
raise util.PodConfigError(
"Syntax error encountered when reading settings.jsonc.", pod) from exc
return d
# get list of pods
if not pod_list:
pod_list = os.listdir(os.path.join(code_root, _pod_dir))
pod_list = [s for s in pod_list if not s.startswith(('_','.'))]
pod_list.sort(key=str.lower)
if pod == 'list':
return pod_list
# load one settings.jsonc file
if pod is not None:
if pod not in pod_list:
print(f"Couldn't recognize '{pod}' out of the following diagnostics:")
print(', '.join(pod_list))
return dict()
return _load_one_json(pod)
# load all of them
pods = dict()
realm_list = set()
bad_pods = []
realms = collections.defaultdict(list)
for p in pod_list:
try:
d = _load_one_json(p)
assert d
except Exception as exc:
bad_pods.append(p)
continue
pods[p] = d
# PODs requiring data from multiple realms get stored in the dict
# under a tuple of those realms; realms stored indivudally in realm_list
_realm = util.to_iter(d['settings'].get('realm', None), tuple)
if len(_realm) == 0:
continue
elif len(_realm) == 1:
_realm = _realm[0]
realm_list.add(_realm)
else:
realm_list.update(_realm)
realms[_realm].append(p)
for p in bad_pods:
pod_list.remove(p)
return PodDataTuple(
pod_data=pods, realm_data=realms,
sorted_lists={
"pods": pod_list,
"realms": sorted(list(realm_list), key=str.lower)
}
)
def __post_init__(self):
self.exceptions = util.ExceptionQueue()
for k, v in self.runtime_requirements.items():
self.runtime_requirements[k] = util.to_iter(v)
def load_pod_settings(code_root, pod=None, pod_list=None):
"""Wrapper to load and parse the contents of POD settings files, used by
:class:`~src.core.MDTFFramework` and :class:`InfoCLIHandler`.
Args:
code_root (str): Absolute path to t
pod (str, optional):
pod_list (list, optional): List of POD names to load settings files.
Raises:
:class:`~src.util.PodConfigError`: If an error is raised opening or
parsing the contents of a settings file. In normal operation, this
is treated as a fatal error and will cause package exit.
Returns:
Instance of :data:`PodDataTuple`.
"""
_pod_dir = 'diagnostics'
_file_name = 'settings.jsonc'
def _load_one_json(pod_):
pod_dir = os.path.join(code_root, _pod_dir, pod_)
settings_path = os.path.join(pod_dir, _file_name)
try:
d = util.read_json(settings_path)
for section in ['settings', 'varlist']:
if section not in d:
raise AssertionError(
f"'{section}' entry not found in '{_file_name}'.")
except util.MDTFFileNotFoundError as exc:
if not os.path.isdir(pod_dir):
raise util.PodConfigError(
(f"'{pod_}' directory not found in "
f"'{os.path.join(code_root, _pod_dir)}'."), pod_)
elif not os.path.isfile(settings_path):
raise util.PodConfigError((f"'{_file_name}' file not found in "
f"'{pod_dir}'."), pod_)
else:
raise exc
except (JSONDecodeError, AssertionError) as exc:
raise util.PodConfigError((f"Syntax error in '{_file_name}': "
f"{str(exc)}."), pod_)
except Exception as exc:
raise util.PodConfigError(
(f"Error encountered in reading '{_file_name}': "
f"{repr(exc)}."), pod_)
return d
# get list of pods
if not pod_list:
pod_list = os.listdir(os.path.join(code_root, _pod_dir))
pod_list = [s for s in pod_list if not s.startswith(('_', '.'))]
pod_list.sort(key=str.lower)
if pod == 'list':
return pod_list
# load one settings.jsonc file
if pod is not None:
if pod not in pod_list:
print(
f"Couldn't recognize '{pod}' out of the following diagnostics:"
)
print(', '.join(pod_list))
return dict()
return _load_one_json(pod)
# load all of them
pods = dict()
realm_list = set()
bad_pods = []
realms = collections.defaultdict(list)
for p in pod_list:
try:
d = _load_one_json(p)
except Exception as exc:
_log.error(exc)
bad_pods.append(p)
continue
pods[p] = d
# PODs requiring data from multiple realms get stored in the dict
# under a tuple of those realms; realms stored indivudally in realm_list
_realm = util.to_iter(d['settings'].get('realm', None), tuple)
if len(_realm) == 0:
continue
elif len(_realm) == 1:
_realm = _realm[0]
realm_list.add(_realm)
else:
realm_list.update(_realm)
realms[_realm].append(p)
if bad_pods:
_log.critical(
("Errors were encountered when finding the following PODS: "
"[%s]."), ', '.join(f"'{p}'" for p in bad_pods))
util.exit_handler(code=1)
return PodDataTuple(pod_data=pods,
realm_data=realms,
sorted_pods=pod_list,
sorted_realms=sorted(list(realm_list), key=str.lower))
def _add_topic_handler(keywords, function):
# keep cmd_list ordered
keywords = util.to_iter(keywords)
self.cmd_list.extend(keywords)
for k in keywords:
self.cmds[k] = function
def load_pod_settings(code_root, pod=None, pod_list=None):
"""Wrapper to load POD settings files, used by ConfigManager and CLIInfoHandler.
"""
# only place we can put it would be util.py if we want to avoid circular imports
_pod_dir = 'diagnostics'
_file_name = 'settings.jsonc'
def _load_one_json(pod_):
pod_dir = os.path.join(code_root, _pod_dir, pod_)
settings_path = os.path.join(pod_dir, _file_name)
try:
d = util.read_json(settings_path)
for section in ['settings', 'varlist']:
if section not in d:
raise AssertionError(
f"'{section}' entry not found in '{_file_name}'.")
except util.MDTFFileNotFoundError as exc:
if not os.path.isdir(pod_dir):
raise util.PodConfigError(
(f"'{pod_}' directory not found in "
f"'{os.path.join(code_root, _pod_dir)}'."), pod_)
elif not os.path.isfile(settings_path):
raise util.PodConfigError((f"'{_file_name}' file not found in "
f"'{pod_dir}'."), pod_)
else:
raise exc
except (JSONDecodeError, AssertionError) as exc:
raise util.PodConfigError((f"Syntax error in '{_file_name}': "
f"{str(exc)}."), pod_)
except Exception as exc:
raise util.PodConfigError(
(f"Error encountered in reading '{_file_name}': "
f"{repr(exc)}."), pod_)
return d
# get list of pods
if not pod_list:
pod_list = os.listdir(os.path.join(code_root, _pod_dir))
pod_list = [s for s in pod_list if not s.startswith(('_', '.'))]
pod_list.sort(key=str.lower)
if pod == 'list':
return pod_list
# load one settings.jsonc file
if pod is not None:
if pod not in pod_list:
print(
f"Couldn't recognize '{pod}' out of the following diagnostics:"
)
print(', '.join(pod_list))
return dict()
return _load_one_json(pod)
# load all of them
pods = dict()
realm_list = set()
bad_pods = []
realms = collections.defaultdict(list)
for p in pod_list:
try:
d = _load_one_json(p)
except Exception as exc:
_log.error(exc)
bad_pods.append(p)
continue
pods[p] = d
# PODs requiring data from multiple realms get stored in the dict
# under a tuple of those realms; realms stored indivudally in realm_list
_realm = util.to_iter(d['settings'].get('realm', None), tuple)
if len(_realm) == 0:
continue
elif len(_realm) == 1:
_realm = _realm[0]
realm_list.add(_realm)
else:
realm_list.update(_realm)
realms[_realm].append(p)
if bad_pods:
_log.critical(
("Errors were encountered when finding the following PODS: "
"[%s]."), ', '.join(f"'{p}'" for p in bad_pods))
util.exit_handler(code=1)
return PodDataTuple(pod_data=pods,
realm_data=realms,
sorted_pods=pod_list,
sorted_realms=sorted(list(realm_list), key=str.lower))
