def parse_pod_list(self, cli_obj, pod_info_tuple): pod_data = pod_info_tuple.pod_data all_realms = pod_info_tuple.sorted_lists.get('realms', []) pod_realms = pod_info_tuple.realm_data args = util.to_iter(cli_obj.config.pop('pods', []), set) if 'example' in args or 'examples' in args: self.pod_list = [p for p in pod_data if p.startswith('example')] elif 'all' in args: self.pod_list = [p for p in pod_data if not p.startswith('example')] else: # specify pods by realm realms = args.intersection(all_realms) args = args.difference(all_realms) # remainder for key in pod_realms: if util.to_iter(key, set).issubset(realms): self.pod_list.extend(pod_realms[key]) # specify pods by name pods = args.intersection(set(pod_data)) self.pod_list.extend(list(pods)) for arg in args.difference(set(pod_data)): # remainder: print("WARNING: Didn't recognize POD {}, ignoring".format(arg)) # exclude examples self.pod_list = [p for p in pod_data if not p.startswith('example')] if not self.pod_list: _log.critical(("ERROR: no PODs selected to be run. Do `./mdtf info pods`" " for a list of available PODs, and check your -p/--pods argument." f"\nReceived --pods = {str(list(args))}")) exit(1)
def framework_test(code_root, output_dir, cli_config): print("Starting framework test run") abs_out_dir = util.resolve_path(output_dir, root_path=code_root, env=os.environ) try: log_str = shell_command_wrapper('./mdtf -f {input_file}'.format( input_file=os.path.join(code_root, cli_config['config_out'])), cwd=code_root) log_str = util.to_iter(log_str) # write to most recent directory in output_dir runs = [ d for d in glob.glob(os.path.join(abs_out_dir, '*')) if os.path.isdir(d) ] if not runs: raise IOError( "Can't find framework output in {}".format(abs_out_dir)) run_output = max(runs, key=os.path.getmtime) with io.open(os.path.join(run_output, 'mdtf_test.log'), 'w', encoding='utf-8') as f: f.write('\n'.join(log_str)) except Exception as exc: fatal_exception_handler(exc, "ERROR: framework test run failed.") print("Finished framework test run at {}".format(run_output)) return run_output
def __post_init__(self): core.MDTFObjectBase.__post_init__(self) # set up log (PODLoggerMixin) self.init_log() for k, v in self.runtime_requirements.items(): self.runtime_requirements[k] = util.to_iter(v)
def guess_attr(name, expected_val, options, default=None, comparison_func=None): """Return element of *options* equal to *expected_val*. If none are equal, try a looser, case-insensititve match. (All arguments expected to be strings.) """ def str_munge(s): # comparison function: lowercase, drop non-alphanumeric chars return re.sub(r'[^a-z0-9]+', '', s.lower()) if comparison_func is None: comparison_func = (lambda x, y: x == y) options = util.to_iter(options) test_count = sum(comparison_func(opt, expected_val) for opt in options) if test_count > 1: _log.debug("Found multiple values of '%s' set for '%s'.", expected_val, name) if test_count >= 1: return expected_val munged_opts = [ (comparison_func(str_munge(opt), str_munge(expected_val)), opt) \ for opt in options ] if sum(tup[0] for tup in munged_opts) == 1: guessed_val = [tup[1] for tup in munged_opts if tup[0]][0] _log.debug( ("Correcting '%s' to '%s' as the intended value for '%s'."), expected_val, guessed_val, name) return guessed_val if default is None: raise KeyError(expected_val) else: return default
def _print_pod_info(self, pod, verbose): ds = self.pods[pod]['settings'] dv = self.pods[pod]['varlist'] if verbose == 1: print(' {}: {}.'.format(pod, ds['long_name'])) elif verbose == 2: print(' {}: {}.'.format(pod, ds['long_name'])) print(' {}'.format(ds['description'])) print(' Variables: {}'.format( ', '.join([v['var_name'].replace('_var','') for v in dv]) )) elif verbose == 3: print('{}: {}.'.format(pod, ds['long_name'])) print(' Realm: {}.'.format(' and '.join(util.to_iter(ds['realm'])))) print(' {}'.format(ds['description'])) print(' Variables:') for var in dv: var_str = ' {} ({}) @ {} frequency'.format( var['var_name'].replace('_var',''), var.get('requirement',''), var['freq'] ) if 'alternates' in var: var_str = var_str + '; alternates: {}'.format( ', '.join([s.replace('_var','') for s in var['alternates']]) ) print(var_str)
def get_lookup(self, source, dest): """Find the appropriate lookup table to convert values in *source* (keys) to values in *dest* (values), generating it if necessary. Args: source (str): the CV category to use for the keys. dest (str): the CV category to use for the values. Returns: :class:`util.MultiMap` providing a dict-like lookup interface, ie dest_value = d[source_key]. """ if (source, dest) in self._lookups: return self._lookups[(source, dest)] elif (dest, source) in self._lookups: return self._lookups[(dest, source)].inverse() elif source in self._contents: k = list(self._contents[source])[0] if dest not in self._contents[source][k]: raise KeyError(f"Can't find {dest} in attributes for {source}.") mm = util.MultiMap() for k in self._contents[source]: mm[k].update( util.to_iter(self._contents[source][k][dest], set) ) self._lookups[(source, dest)] = mm return mm elif dest in self._contents: return self._lookups[(dest, source)].inverse() else: raise KeyError(f"Neither {source} or {dest} in CV table list.")
def _print_pod_info(self, pod, verbose): """Handler which prints information about PODs as taken from their settings.json files, at configurable levels of verbosity. """ ds = self.pods[pod]['settings'] dv = self.pods[pod]['varlist'] if verbose == 1: print(' {}: {}.'.format(pod, ds['long_name'])) elif verbose == 2: print(' {}: {}.'.format(pod, ds['long_name'])) print(' {}'.format(ds['description'])) print(' Variables: {}'.format(', '.join( [v['var_name'].replace('_var', '') for v in dv]))) elif verbose == 3: print('{}: {}.'.format(pod, ds['long_name'])) print(' Realm: {}.'.format(' and '.join(util.to_iter( ds['realm'])))) print(' {}'.format(ds['description'])) print(' Variables:') for var in dv: var_str = ' {} ({}) @ {} frequency'.format( var['var_name'].replace('_var', ''), var.get('requirement', ''), var['freq']) if 'alternates' in var: var_str = var_str + '; alternates: {}'.format(', '.join( [s.replace('_var', '') for s in var['alternates']])) print(var_str)
def makedirs(self, path_keys, delete_existing): path_keys = util.to_iter(path_keys) for key in path_keys: path = self.config[key] if path: if not os.path.isdir(path): os.makedirs(path) # recursive mkdir if needed elif delete_existing: shutil.rmtree(path) # overwrite everything
def _make_cv(self): """Populate the *cv* attribute of :class:`CMIP6_CVs` with the tables read in during __init__(). Do this on-demand rather than in __init__, in case this information isn't needed for this run of the framework. """ if self.cv: return for k in self._contents: self.cv[k] = util.to_iter(self._contents[k])
def parse_case_list(self, cli_obj): d = cli_obj.config # abbreviate if 'CASENAME' in d and d['CASENAME']: # defined case from CLI cli_d = self._populate_from_cli(cli_obj, 'MODEL') if 'CASE_ROOT_DIR' not in cli_d and d.get('root_dir', None): # CASE_ROOT was set positionally cli_d['CASE_ROOT_DIR'] = d['root_dir'] case_list_in = [cli_d] else: case_list_in = util.to_iter(cli_obj.file_case_list) case_list = [] for i, case_d in enumerate(case_list_in): case_list.append(self.parse_case(i, case_d, cli_obj)) self.case_list = [case for case in case_list if case] if not self.case_list: _log.critical(("ERROR: no valid entries in case_list. Please specify " "model run information.\nReceived:" f"\n{util.pretty_print_json(case_list_in)}")) exit(1) def parse_case(self, n, d, cli_obj): # really need to move this into init of DataManager if 'CASE_ROOT_DIR' not in d and 'root_dir' in d: d['CASE_ROOT_DIR'] = d.pop('root_dir') case_convention = d.get('convention', '') if case_convention: d['convention'] = case_convention if not ('CASENAME' in d or ('model' in d and 'experiment' in d)): _log.warning(("Need to specify either CASENAME or model/experiment " "in caselist entry %s, skipping."), n+1) return None _ = d.setdefault('model', d.get('convention', '')) _ = d.setdefault('experiment', '') _ = d.setdefault('CASENAME', '{}_{}'.format(d['model'], d['experiment'])) for field in ['FIRSTYR', 'LASTYR', 'convention']: if not d.get(field, None): _log.warning(("No value set for %s in caselist entry %s, " "skipping."), field, n+1) return None # if pods set from CLI, overwrite pods in case list d['pod_list'] = self.set_case_pod_list(d, cli_obj) return d def set_case_pod_list(self, case, cli_obj): # if pods set from CLI, overwrite pods in case list # already finalized self.pod-list by the time we get here if not cli_obj.is_default['pods'] or not case.get('pod_list', None): return self.pod_list
def parse_case_list(self, cli_obj, pod_info_tuple): d = cli_obj.config # abbreviate if 'CASENAME' in d and d['CASENAME']: # defined case from CLI cli_d = self._populate_from_cli(cli_obj, 'MODEL') if 'CASE_ROOT_DIR' not in cli_d and d.get('root_dir', None): # CASE_ROOT was set positionally cli_d['CASE_ROOT_DIR'] = d['root_dir'] case_list_in = [cli_d] else: case_list_in = util.to_iter(cli_obj.file_case_list) self.cases = dict() for i, case_d in enumerate(case_list_in): case = self.parse_case(i, case_d, cli_obj, pod_info_tuple) if case: self.cases[case['CASENAME']] = case
def parse_pod_list(self, pod_list, pod_info_tuple): pod_data = pod_info_tuple.pod_data # pod names -> contents of settings file args = util.to_iter(pod_list, set) bad_args = [] pods = [] for arg in args: if arg == 'all': # add all PODs except example PODs pods.extend( [p for p in pod_data if not p.startswith('example')]) elif arg == 'example' or arg == 'examples': # add example PODs pods.extend([p for p in pod_data if p.startswith('example')]) elif arg in pod_info_tuple.realm_data: # realm_data: realm name -> list of POD names # add all PODs for this realm pods.extend(pod_info_tuple.realm_data[arg]) elif arg in pod_data: # add POD by name pods.append(arg) else: # unrecognized argument _log.error("POD identifier '%s' not recognized.", arg) bad_args.append(arg) if bad_args: valid_args = ['all', 'examples'] \ + pod_info_tuple.sorted_realms \ + pod_info_tuple.sorted_pods _log.critical(( "The following POD identifiers were not recognized: " "[%s].\nRecognized identifiers are: [%s].\n(Received --pods = %s)." ), ', '.join(f"'{p}'" for p in bad_args), ', '.join(f"'{p}'" for p in valid_args), str(list(args))) util.exit_handler(code=1) pods = list(set(pods)) # delete duplicates if not pods: _log.critical(( "ERROR: no PODs selected to be run. Do `./mdtf info pods`" " for a list of available PODs, and check your -p/--pods argument." f"\nReceived --pods = {str(list(args))}")) util.exit_handler(code=1) return pods
def load_pod_settings(code_root, pod=None, pod_list=None): """Wrapper to load POD settings files, used by ConfigManager and CLIInfoHandler. """ # only place we can put it would be util.py if we want to avoid circular imports _pod_dir = 'diagnostics' _pod_settings = 'settings.jsonc' def _load_one_json(pod): try: d = util.read_json( os.path.join(code_root, _pod_dir, pod, _pod_settings) ) assert 'settings' in d except Exception as exc: raise util.PodConfigError( "Syntax error encountered when reading settings.jsonc.", pod) from exc return d # get list of pods if not pod_list: pod_list = os.listdir(os.path.join(code_root, _pod_dir)) pod_list = [s for s in pod_list if not s.startswith(('_','.'))] pod_list.sort(key=str.lower) if pod == 'list': return pod_list # load one settings.jsonc file if pod is not None: if pod not in pod_list: print(f"Couldn't recognize '{pod}' out of the following diagnostics:") print(', '.join(pod_list)) return dict() return _load_one_json(pod) # load all of them pods = dict() realm_list = set() bad_pods = [] realms = collections.defaultdict(list) for p in pod_list: try: d = _load_one_json(p) assert d except Exception as exc: bad_pods.append(p) continue pods[p] = d # PODs requiring data from multiple realms get stored in the dict # under a tuple of those realms; realms stored indivudally in realm_list _realm = util.to_iter(d['settings'].get('realm', None), tuple) if len(_realm) == 0: continue elif len(_realm) == 1: _realm = _realm[0] realm_list.add(_realm) else: realm_list.update(_realm) realms[_realm].append(p) for p in bad_pods: pod_list.remove(p) return PodDataTuple( pod_data=pods, realm_data=realms, sorted_lists={ "pods": pod_list, "realms": sorted(list(realm_list), key=str.lower) } )
def __post_init__(self): self.exceptions = util.ExceptionQueue() for k, v in self.runtime_requirements.items(): self.runtime_requirements[k] = util.to_iter(v)
def load_pod_settings(code_root, pod=None, pod_list=None): """Wrapper to load and parse the contents of POD settings files, used by :class:`~src.core.MDTFFramework` and :class:`InfoCLIHandler`. Args: code_root (str): Absolute path to t pod (str, optional): pod_list (list, optional): List of POD names to load settings files. Raises: :class:`~src.util.PodConfigError`: If an error is raised opening or parsing the contents of a settings file. In normal operation, this is treated as a fatal error and will cause package exit. Returns: Instance of :data:`PodDataTuple`. """ _pod_dir = 'diagnostics' _file_name = 'settings.jsonc' def _load_one_json(pod_): pod_dir = os.path.join(code_root, _pod_dir, pod_) settings_path = os.path.join(pod_dir, _file_name) try: d = util.read_json(settings_path) for section in ['settings', 'varlist']: if section not in d: raise AssertionError( f"'{section}' entry not found in '{_file_name}'.") except util.MDTFFileNotFoundError as exc: if not os.path.isdir(pod_dir): raise util.PodConfigError( (f"'{pod_}' directory not found in " f"'{os.path.join(code_root, _pod_dir)}'."), pod_) elif not os.path.isfile(settings_path): raise util.PodConfigError((f"'{_file_name}' file not found in " f"'{pod_dir}'."), pod_) else: raise exc except (JSONDecodeError, AssertionError) as exc: raise util.PodConfigError((f"Syntax error in '{_file_name}': " f"{str(exc)}."), pod_) except Exception as exc: raise util.PodConfigError( (f"Error encountered in reading '{_file_name}': " f"{repr(exc)}."), pod_) return d # get list of pods if not pod_list: pod_list = os.listdir(os.path.join(code_root, _pod_dir)) pod_list = [s for s in pod_list if not s.startswith(('_', '.'))] pod_list.sort(key=str.lower) if pod == 'list': return pod_list # load one settings.jsonc file if pod is not None: if pod not in pod_list: print( f"Couldn't recognize '{pod}' out of the following diagnostics:" ) print(', '.join(pod_list)) return dict() return _load_one_json(pod) # load all of them pods = dict() realm_list = set() bad_pods = [] realms = collections.defaultdict(list) for p in pod_list: try: d = _load_one_json(p) except Exception as exc: _log.error(exc) bad_pods.append(p) continue pods[p] = d # PODs requiring data from multiple realms get stored in the dict # under a tuple of those realms; realms stored indivudally in realm_list _realm = util.to_iter(d['settings'].get('realm', None), tuple) if len(_realm) == 0: continue elif len(_realm) == 1: _realm = _realm[0] realm_list.add(_realm) else: realm_list.update(_realm) realms[_realm].append(p) if bad_pods: _log.critical( ("Errors were encountered when finding the following PODS: " "[%s]."), ', '.join(f"'{p}'" for p in bad_pods)) util.exit_handler(code=1) return PodDataTuple(pod_data=pods, realm_data=realms, sorted_pods=pod_list, sorted_realms=sorted(list(realm_list), key=str.lower))
def _add_topic_handler(keywords, function): # keep cmd_list ordered keywords = util.to_iter(keywords) self.cmd_list.extend(keywords) for k in keywords: self.cmds[k] = function
def load_pod_settings(code_root, pod=None, pod_list=None): """Wrapper to load POD settings files, used by ConfigManager and CLIInfoHandler. """ # only place we can put it would be util.py if we want to avoid circular imports _pod_dir = 'diagnostics' _file_name = 'settings.jsonc' def _load_one_json(pod_): pod_dir = os.path.join(code_root, _pod_dir, pod_) settings_path = os.path.join(pod_dir, _file_name) try: d = util.read_json(settings_path) for section in ['settings', 'varlist']: if section not in d: raise AssertionError( f"'{section}' entry not found in '{_file_name}'.") except util.MDTFFileNotFoundError as exc: if not os.path.isdir(pod_dir): raise util.PodConfigError( (f"'{pod_}' directory not found in " f"'{os.path.join(code_root, _pod_dir)}'."), pod_) elif not os.path.isfile(settings_path): raise util.PodConfigError((f"'{_file_name}' file not found in " f"'{pod_dir}'."), pod_) else: raise exc except (JSONDecodeError, AssertionError) as exc: raise util.PodConfigError((f"Syntax error in '{_file_name}': " f"{str(exc)}."), pod_) except Exception as exc: raise util.PodConfigError( (f"Error encountered in reading '{_file_name}': " f"{repr(exc)}."), pod_) return d # get list of pods if not pod_list: pod_list = os.listdir(os.path.join(code_root, _pod_dir)) pod_list = [s for s in pod_list if not s.startswith(('_', '.'))] pod_list.sort(key=str.lower) if pod == 'list': return pod_list # load one settings.jsonc file if pod is not None: if pod not in pod_list: print( f"Couldn't recognize '{pod}' out of the following diagnostics:" ) print(', '.join(pod_list)) return dict() return _load_one_json(pod) # load all of them pods = dict() realm_list = set() bad_pods = [] realms = collections.defaultdict(list) for p in pod_list: try: d = _load_one_json(p) except Exception as exc: _log.error(exc) bad_pods.append(p) continue pods[p] = d # PODs requiring data from multiple realms get stored in the dict # under a tuple of those realms; realms stored indivudally in realm_list _realm = util.to_iter(d['settings'].get('realm', None), tuple) if len(_realm) == 0: continue elif len(_realm) == 1: _realm = _realm[0] realm_list.add(_realm) else: realm_list.update(_realm) realms[_realm].append(p) if bad_pods: _log.critical( ("Errors were encountered when finding the following PODS: " "[%s]."), ', '.join(f"'{p}'" for p in bad_pods)) util.exit_handler(code=1) return PodDataTuple(pod_data=pods, realm_data=realms, sorted_pods=pod_list, sorted_realms=sorted(list(realm_list), key=str.lower))