def parse_pod_list(self, cli_obj, config):
self.pod_list = []
args = util.coerce_to_iter(config.config.pop('pods', []), set)
if 'example' in args or 'examples' in args:
self.pod_list = [pod for pod in config.pods.keys() \
if pod.startswith('example')]
elif 'all' in args:
self.pod_list = [pod for pod in config.pods.keys() \
if not pod.startswith('example')]
else:
# specify pods by realm
realms = args.intersection(set(config.all_realms))
args = args.difference(set(config.all_realms)) # remainder
for key in config.pod_realms:
if util.coerce_to_iter(key, set).issubset(realms):
self.pod_list.extend(config.pod_realms[key])
# specify pods by name
pods = args.intersection(set(config.pods.keys()))
self.pod_list.extend(list(pods))
for arg in args.difference(set(config.pods.keys())): # remainder:
print("WARNING: Didn't recognize POD {}, ignoring".format(arg))
# exclude examples
self.pod_list = [pod for pod in self.pod_list \
if not pod.startswith('example')]
if not self.pod_list:
print(("WARNING: no PODs selected to be run. Do `./mdtf info pods`"
" for a list of available PODs, and check your -p/--pods argument."))
print('Received --pods = {}'.format(list(args)))
exit()
def make_parser(self, d):
args = util.coerce_to_iter(d.pop('arguments', None))
arg_groups = util.coerce_to_iter(d.pop('argument_groups', None))
d['formatter_class'] = CustomHelpFormatter
p_kwargs = util.filter_kwargs(d, argparse.ArgumentParser.__init__)
p = argparse.ArgumentParser(**p_kwargs)
for arg in args:
# add arguments not in any group
self.add_parser_argument(arg, p, 'parser')
for group in arg_groups:
# add groups and arguments therein
self.add_parser_group(group, p)
return p
def spawn_subprocess(self,
cmd_list,
env_name,
env=None,
cwd=None,
stdout=None,
stderr=None):
if stdout is None:
stdout = subprocess.STDOUT
if stderr is None:
stderr = subprocess.STDOUT
run_cmds = util.coerce_to_iter(cmd_list, list)
if self.test_mode:
run_cmds = [
'echo "TEST MODE: call {}"'.format('; '.join(run_cmds))
]
commands = self.activate_env_commands(env_name) \
+ run_cmds \
+ self.deactivate_env_commands(env_name)
# '&&' so we abort if any command in the sequence fails.
if self.test_mode:
for cmd in commands:
print('TEST MODE: call {}'.format(cmd))
else:
print("Calling : {}".format(run_cmds[-1]))
commands = ' && '.join([s for s in commands if s])
# Need to run bash explicitly because 'conda activate' sources
# env vars (can't do that in posix sh). tcsh could also work.
return subprocess.Popen(['bash', '-c', commands],
env=env,
cwd=cwd,
stdout=stdout,
stderr=stderr)
def parse_cli(self, args=None):
# call preparse_cli if child class hasn't done so already
if not self.config:
self.preparse_cli(args)
# if no additional defaults were set, that's sufficient, otherwise need
# to take into account their intermediate priority
if isinstance(self.partial_defaults, dict):
# not handled correctly by coerce_to_iter
self.partial_defaults = [self.partial_defaults]
self.partial_defaults = util.coerce_to_iter(self.partial_defaults)
partial_defaults = []
for d in self.partial_defaults:
# drop empty strings
partial_defaults.append(
{k: v
for k, v in d.iteritems() if v != ""})
# self.config was populated by preparse_cli()
# Options explicitly set by user on CLI; is_default = None if no default
cli_opts = {k:v for k,v in self.config.iteritems() \
if not self.is_default.get(k, True)}
# full set of defaults from cli.jsonc, from running parser on empty input
defaults = vars(self.parser.parse_args([]))
chained_dict_list = [cli_opts] + partial_defaults + [defaults]
# CLI opts override options set from file, which override defaults
self.config = dict(ChainMap(*chained_dict_list))
def framework_test(code_root, output_dir, cli_config):
print("Starting framework test run")
abs_out_dir = util.resolve_path(output_dir,
root_path=code_root,
env=os.environ)
try:
log_str = shell_command_wrapper('./mdtf -f {input_file}'.format(
input_file=os.path.join(code_root, cli_config['config_out'])),
cwd=code_root)
log_str = util.coerce_to_iter(log_str)
# write to most recent directory in output_dir
runs = [
d for d in glob.glob(os.path.join(abs_out_dir, '*'))
if os.path.isdir(d)
]
if not runs:
raise IOError(
"Can't find framework output in {}".format(abs_out_dir))
run_output = max(runs, key=os.path.getmtime)
with open(os.path.join(run_output, 'mdtf_test.log'), 'w') as f:
f.write('\n'.join(log_str))
except Exception as exc:
fatal_exception_handler(exc, "ERROR: framework test run failed.")
print("Finished framework test run at {}".format(run_output))
return run_output
def bump_version(path, new_v=None, extra_dirs=[]):
# return a filename that doesn't conflict with existing files.
# if extra_dirs supplied, make sure path doesn't conflict with pre-existing
# files at those locations either.
def _split_version(file_):
match = re.match(
r"""
^(?P<file_base>.*?) # arbitrary characters (lazy match)
(\.v(?P<version>\d+)) # literal '.v' followed by digits
? # previous group may occur 0 or 1 times
$ # end of string
""", file_, re.VERBOSE)
if match:
return (match.group('file_base'), match.group('version'))
else:
return (file_, '')
def _reassemble(dir_, file_, version, ext_, final_sep):
if version:
file_ = ''.join([file_, '.v', str(version), ext_])
else:
# get here for version == 0, '' or None
file_ = ''.join([file_, ext_])
return os.path.join(dir_, file_) + final_sep
def _path_exists(dir_list, file_, new_v, ext_, sep):
new_paths = [_reassemble(d, file_, new_v, ext_, sep) for d in dir_list]
return any([os.path.exists(p) for p in new_paths])
if path.endswith(os.sep):
# remove any terminating slash on directory
path = path.rstrip(os.sep)
final_sep = os.sep
else:
final_sep = ''
dir_, file_ = os.path.split(path)
dir_list = util.coerce_to_iter(extra_dirs)
dir_list.append(dir_)
file_, old_v = _split_version(file_)
if not old_v:
# maybe it has an extension and then a version number
file_, ext_ = os.path.splitext(file_)
file_, old_v = _split_version(file_)
else:
ext_ = ''
if new_v is not None:
# removes version if new_v ==0
new_path = _reassemble(dir_, file_, new_v, ext_, final_sep)
else:
if not old_v:
new_v = 0
else:
new_v = int(old_v)
while _path_exists(dir_list, file_, new_v, ext_, final_sep):
new_v = new_v + 1
new_path = _reassemble(dir_, file_, new_v, ext_, final_sep)
return (new_path, new_v)
def iteritems_cli(self, group_nm=None):
if not group_nm:
_groups = self.parser_groups
else:
_groups = util.coerce_to_iter(group_nm)
for group in _groups:
for action in self.parser_args_from_group[group]:
key = action.dest
yield (key, self.config[key])
def makedirs(self, path_keys, delete_existing):
path_keys = util.coerce_to_iter(path_keys)
for key in path_keys:
path = self.config[key]
if path:
if not os.path.isdir(path):
os.makedirs(path) # recursive mkdir if needed
elif delete_existing:
shutil.rmtree(path) # overwrite everything
def add_parser_argument(self, d, target_obj, target_name):
# set flags:
arg_nm = self.canonical_arg_name(d.pop('name'))
assert arg_nm, "No argument name found in {}".format(d)
arg_flags = [arg_nm]
if d.pop('is_positional', False):
# code to handle positional arguments
pass
else:
# argument is a command-line flag (default)
if 'dest' not in d:
d['dest'] = arg_nm
if '_' in arg_nm:
# recognize both --hyphen_opt and --hyphen-opt (GNU CLI convention)
arg_flags = [arg_nm.replace('_', '-'), arg_nm]
arg_flags = ['--' + s for s in arg_flags]
if 'short_name' in d:
# recognize both --option and -O, if short_name defined
arg_flags.append('-' + d.pop('short_name'))
# type conversion of default value
if 'type' in d:
d['type'] = eval(d['type'])
if 'default' in d:
d['default'] = d['type'](d['default'])
if d.get('action', '') == 'count' and 'default' in d:
d['default'] = int(d['default'])
if d.get('parse_type', None):
# make list of args requiring custom post-parsing later
self.custom_types[d.pop('parse_type')].append(d['dest'])
# TODO: what if following require env vars, etc??
if d.get('eval', None):
for attr in util.coerce_to_iter(d.pop('eval')):
if attr in d:
d[attr] = eval(d[attr])
# set more technical argparse options based on default value
if 'default' in d:
if 'action' not in d:
d['action'] = RecordDefaultsAction
elif isinstance(d['default'], basestring) and 'nargs' not in d:
# unless explicitly specified, string options accept 1 argument
d['nargs'] = 1
# change help string based on default value
if d.pop('hidden', False):
# do not list argument in "mdtf --help", but recognize it
d['help'] = argparse.SUPPRESS
elif 'default' in d:
# display default value in help string
#self._append_to_entry(d, 'help', "(default: %(default)s)")
pass
# d = util.filter_kwargs(d, argparse.ArgumentParser.add_argument)
self.parser_args_from_group[target_name].append(
target_obj.add_argument(*arg_flags, **d))
def add_parser_group(self, d, target_obj):
gp_nm = d.pop('name')
if 'title' not in d:
d['title'] = gp_nm
args = util.coerce_to_iter(d.pop('arguments', None))
gp_kwargs = util.filter_kwargs(d, argparse._ArgumentGroup.__init__)
gp_obj = target_obj.add_argument_group(**gp_kwargs)
self.parser_groups[gp_nm] = gp_obj
for arg in args:
self.add_parser_argument(arg, gp_obj, gp_nm)
def add_parser_group(self, d, target_obj):
gp_nm = d.pop('name')
_ = d.setdefault('title', gp_nm)
args = util.coerce_to_iter(d.pop('arguments', None))
if args:
# only add group if it has > 0 arguments
gp_kwargs = util.filter_kwargs(d, argparse._ArgumentGroup.__init__)
gp_obj = target_obj.add_argument_group(**gp_kwargs)
self.parser_groups[gp_nm] = gp_obj
for arg in args:
self.add_parser_argument(arg, gp_obj, gp_nm)
def parse_case_list(self, cli_obj, config):
case_list_in = util.coerce_to_iter(cli_obj.case_list)
cli_d = self._populate_from_cli(cli_obj, 'MODEL')
if 'CASE_ROOT_DIR' not in cli_d and cli_obj.config.get('root_dir', None):
# CASE_ROOT was set positionally
cli_d['CASE_ROOT_DIR'] = cli_obj.config['root_dir']
if not case_list_in:
case_list_in = [cli_d]
case_list = []
for case_tup in enumerate(case_list_in):
case_list.append(self.parse_case(case_tup, cli_d, cli_obj, config))
self.case_list = [case for case in case_list if case is not None]
def parse_case_list(self, cli_obj, config):
d = config.config # abbreviate
self.case_list = []
if d.get('CASENAME', None) \
or (d.get('model', None) and d.get('experiment', None)):
case_list = self.caselist_from_args(cli_obj)
else:
case_list = util.coerce_to_iter(cli_obj.case_list)
for case_dict in case_list:
# remove empty entries
case = {k: v for k, v in case_dict.iteritems() if v}
if not case.get('CASE_ROOT_DIR', None) and case.get(
'root_dir', None):
case['CASE_ROOT_DIR'] = case['root_dir']
del case['root_dir']
def parse_pod_list(self, cli_obj, config):
self.pod_list = []
args = util.coerce_to_iter(config.config.pop('pods', []), set)
if 'example' in args or 'examples' in args:
self.pod_list = [pod for pod in config.pods.keys() \
if pod.startswith('example')]
elif 'all' in args:
self.pod_list = [pod for pod in config.pods.keys() \
if not pod.startswith('example')]
else:
# specify pods by realm
realms = args.intersection(set(config.all_realms))
args = args.difference(set(config.all_realms)) # remainder
for key in config.pod_realms:
if util.coerce_to_iter(key, set).issubset(realms):
self.pod_list.extend(config.pod_realms[key])
# specify pods by name
pods = args.intersection(set(config.pods.keys()))
self.pod_list.extend(list(pods))
for arg in args.difference(set(config.pods.keys())): # remainder:
print("WARNING: Didn't recognize POD {}, ignoring".format(arg))
# exclude examples
self.pod_list = [pod for pod in self.pod_list \
if not pod.startswith('example')]
def add_parser_argument(self, d, target_obj, target_name):
# set flags:
if 'name' not in d:
raise ValueError("No argument name found in {}".format(d))
arg_nm = self.canonical_arg_name(d.pop('name'))
arg_flags = [arg_nm]
if d.pop('is_positional', False):
# code to handle positional arguments
pass
else:
# argument is a command-line flag (default)
if 'dest' not in d:
d['dest'] = arg_nm
if '_' in arg_nm:
# recognize both --hyphen_opt and --hyphen-opt (GNU CLI convention)
arg_flags = [arg_nm.replace('_', '-'), arg_nm]
arg_flags = ['--' + s for s in arg_flags]
if 'short_name' in d:
# recognize both --option and -O, if short_name defined
arg_flags.append('-' + d.pop('short_name'))
# type conversion of default value
if 'type' in d:
d['type'] = eval(d['type'])
if 'default' in d:
d['default'] = d['type'](d['default'])
if d.get('action', '') == 'count' and 'default' in d:
d['default'] = int(d['default'])
if d.get('parse_type', None):
# make list of args requiring custom post-parsing later
self.custom_types[d.pop('parse_type')].append(d['dest'])
# TODO: what if following require env vars, etc??
if d.get('eval', None):
for attr in util.coerce_to_iter(d.pop('eval')):
if attr in d:
d[attr] = eval(d[attr])
_ = d.setdefault('action', RecordDefaultsAction)
# change help string based on default value
if d.pop('hidden', False):
# do not list argument in "mdtf --help", but recognize it
d['help'] = argparse.SUPPRESS
self.parser_args_from_group[target_name].append(
target_obj.add_argument(*arg_flags, **d))
def _list_filtered_subdirs(self, dirs_in, subdir_filter=None):
subdir_filter = util.coerce_to_iter(subdir_filter)
found_dirs = []
for dir_ in dirs_in:
found_subdirs = {d for d \
in self._listdir(os.path.join(self.root_dir, dir_)) \
if not (d.startswith('.') or d.endswith('.nc'))
}
if subdir_filter:
found_subdirs = found_subdirs.intersection(subdir_filter)
if not found_subdirs:
print("\tCouldn't find subdirs (in {}) at {}, skipping".format(
subdir_filter, os.path.join(self.root_dir, dir_)))
continue
found_dirs.extend([
os.path.join(dir_, subdir_) for subdir_ in found_subdirs \
if os.path.isdir(os.path.join(self.root_dir, dir_, subdir_))
])
return found_dirs
def __init__(self, unittest_flag=False, verbose=0):
if unittest_flag:
# value not used, when we're testing will mock out call to read_json
# below with actual translation table to use for test
config_files = ['dummy_filename']
else:
config = ConfigManager()
glob_pattern = os.path.join(config.paths.CODE_ROOT, 'src',
'fieldlist_*.jsonc')
config_files = glob.glob(glob_pattern)
# always have CF-compliant option, which does no translation
self.axes = {
'CF': {
"lon": {
"axis": "X",
"MDTF_envvar": "lon_coord"
},
"lat": {
"axis": "Y",
"MDTF_envvar": "lat_coord"
},
"lev": {
"axis": "Z",
"MDTF_envvar": "lev_coord"
},
"time": {
"axis": "T",
"MDTF_envvar": "time_coord"
}
}
}
self.variables = {'CF': dict()}
self.units = {'CF': dict()}
for filename in config_files:
d = util.read_json(filename)
for conv in util.coerce_to_iter(d['convention_name']):
if verbose > 0:
print('XXX found ', conv)
self.axes[conv] = d.get('axes', dict())
self.variables[conv] = util.MultiMap(d.get(
'var_names', dict()))
self.units[conv] = util.MultiMap(d.get('units', dict()))
def make_default_parser(self, d, config_path):
# add more standard options to top-level parser
if 'usage' not in d:
d['usage'] = ("%(prog)s [options] CASE_ROOT_DIR\n"
"{}%(prog)s info [INFO_TOPIC]").format(
len('usage: ') * ' ')
self._append_to_entry(d, 'description', (
"The second form ('mdtf info') prints information about available "
"diagnostics."))
d['arguments'] = util.coerce_to_iter(d.get('arguments', None))
d['arguments'].extend([
{
"name": "root_dir",
"is_positional": True,
"nargs":
"?", # 0 or 1 occurences: might have set this with CASE_ROOT_DIR
"help": "Root directory of model data to analyze.",
"metavar": "CASE_ROOT_DIR"
},
{
'name': 'version',
'action': 'version',
'version': '%(prog)s 2.2'
},
{
'name': 'config_file',
'short_name': 'f',
'help': """
Path to a user configuration file. This can be a JSON
file (a simple list of key:value pairs, or a modified copy of
the defaults file), or a text file containing command-line flags.
Other options set via the command line will still override
settings in this file.
""",
'metavar': 'FILE'
}
])
self._append_to_entry(
d, 'epilog',
"The default values above are set in {}.".format(config_path))
return self.make_parser(d)
def get_lookup(self, source, dest):
if (source, dest) in self._lookups:
return self._lookups[(source, dest)]
elif (dest, source) in self._lookups:
return self._lookups[(dest, source)].inverse()
elif source in self._contents:
k = self._contents[source].keys()[0]
if dest not in self._contents[source][k]:
raise KeyError("Can't find {} in attributes for {}.".format(
dest, source))
mm = util.MultiMap()
for k in self._contents[source]:
mm[k].update(
util.coerce_to_iter(self._contents[source][k][dest], set))
self._lookups[(source, dest)] = mm
return mm
elif dest in self._contents:
return self._lookups[(dest, source)].inverse()
else:
raise KeyError('Neither {} or {} in CV table list.'.format(
source, dest))
def _print_pod_info(self, pod, verbose):
ds = self.pods[pod]['settings']
dv = self.pods[pod]['varlist']
if verbose == 1:
print(' {}: {}.'.format(pod, ds['long_name']))
elif verbose == 2:
print(' {}: {}.'.format(pod, ds['long_name']))
print(' {}'.format(ds['description']))
print(' Variables: {}'.format(', '.join(
[v['var_name'].replace('_var', '') for v in dv])))
elif verbose == 3:
print('{}: {}.'.format(pod, ds['long_name']))
print(' Realm: {}.'.format(' and '.join(
util.coerce_to_iter(ds['realm']))))
print(' {}'.format(ds['description']))
print(' Variables:')
for var in dv:
var_str = ' {} ({}) @ {} frequency'.format(
var['var_name'].replace('_var', ''),
var.get('requirement', ''), var['freq'])
if 'alternates' in var:
var_str = var_str + '; alternates: {}'.format(', '.join(
[s.replace('_var', '') for s in var['alternates']]))
print(var_str)
def _add_topic_handler(keywords, function):
# keep cmd_list ordered
keywords = util.coerce_to_iter(keywords)
self.cmd_list.extend(keywords)
for k in keywords:
self.cmds[k] = function
def load_pod_settings(code_root, pod=None, pod_list=None):
"""Wrapper to load POD settings files, used by ConfigManager and CLIInfoHandler.
"""
# only place we can put it would be util.py if we want to avoid circular imports
_pod_dir = 'diagnostics'
_pod_settings = 'settings.jsonc'
def _load_one_json(pod):
d = dict()
try:
d = util.read_json(
os.path.join(code_root, _pod_dir, pod, _pod_settings))
assert 'settings' in d
except Exception:
pass # better error handling?
return d
# get list of pods
if not pod_list:
pod_list = os.listdir(os.path.join(code_root, _pod_dir))
pod_list = [s for s in pod_list if not s.startswith(('_', '.'))]
pod_list.sort(key=str.lower)
if pod == 'list':
return pod_list
# load JSON files
if not pod:
# load all of them
pods = dict()
realm_list = set()
bad_pods = []
realms = collections.defaultdict(list)
for p in pod_list:
d = _load_one_json(p)
if not d:
bad_pods.append(p)
continue
pods[p] = d
# PODs requiring data from multiple realms get stored in the dict
# under a tuple of those realms; realms stored indivudally in realm_list
_realm = util.coerce_to_iter(d['settings'].get('realm', None),
tuple)
if len(_realm) == 0:
continue
elif len(_realm) == 1:
_realm = _realm[0]
realm_list.add(_realm)
else:
realm_list.update(_realm)
realms[_realm].append(p)
for p in bad_pods:
pod_list.remove(p)
return PodDataTuple(pod_data=pods,
realm_data=realms,
sorted_lists={
"pods": pod_list,
"realms": sorted(list(realm_list),
key=str.lower)
})
else:
if pod not in pod_list:
print(
"Couldn't recognize POD {} out of the following diagnostics:".
format(pod))
print(', '.join(pod_list))
return dict()
return _load_one_json(pod)
def _make_cv(self):
# make on-demand
if self.cv:
return
for k in self._contents:
self.cv[k] = util.coerce_to_iter(self._contents[k])
