def get_node_state_attrs(config, section, option=None, allowed_sections=None): """Get Graphviz node attributes like color for a given setting.""" node_attrs = {} if option is None: node_attrs["shape"] = SHAPE_NODE_SECTION if not config.value.keys(): # Empty configuration - we can assume pure metadata. return node_attrs if allowed_sections is None: allowed_sections = [] config_section_node = config.get([section]) id_ = metomi.rose.macro.get_id_from_section_option(section, option) state = "" config_node = config.get([section, option]) node_attrs["color"] = COLOUR_IGNORED if (config_section_node is not None and config_section_node.state != STATE_NORMAL and option is not None): state = metomi.rose.config.STATE_SECT_IGNORED if config_node is None: node_attrs["color"] = COLOUR_MISSING elif config_node.state != STATE_NORMAL: state += config_node.state if config_node.state == config_node.STATE_USER_IGNORED: node_attrs["color"] = COLOUR_USER_IGNORED elif not state: node_attrs["color"] = COLOUR_ENABLED if allowed_sections and section not in allowed_sections: node_attrs["shape"] = SHAPE_NODE_EXTERNAL if state: node_attrs["label"] = state + id_ return node_attrs
def rename_setting(self, config, keys, new_keys, info=None): """Rename a setting in the configuration. Args: config (metomi.rose.config.ConfigNode): The application configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. new_keys (list): The new hierarchy of node.value 'keys'. info (str): A short string containing no new lines, describing the addition of the setting. Returns: None """ section, option = self._get_section_option_from_keys(keys) new_section, new_option = self._get_section_option_from_keys(new_keys) if option is None: if new_option is not None: raise TypeError(self.ERROR_RENAME_SECT_TO_OPT.format( section, new_section, new_option)) elif new_option is None: raise TypeError(self.ERROR_RENAME_OPT_TO_SECT.format( section, option, new_section)) node = config.get(keys) if node is None: return if info is None: if option is None: info = self.INFO_RENAMED_SECT.format(section, new_section) else: info = self.INFO_RENAMED_VAR.format(section, option, new_section, new_option) if option is None: if config.get([new_section]) is not None: self.remove_setting(config, [new_section]) self.add_setting(config, [new_section], value=None, forced=True, state=node.state, comments=node.comments, info=info) for option_keys, opt_node in config.walk([section]): renamed_option = option_keys[1] self.add_setting(config, [new_section, renamed_option], value=opt_node.value, forced=True, state=opt_node.state, comments=opt_node.comments, info=info) else: self.add_setting(config, new_keys, value=node.value, forced=True, state=node.state, comments=node.comments, info=info) self.remove_setting(config, keys)
def get_setting_value(self, config, keys, no_ignore=False): """Return the value of a setting or ``None`` if not set. Args: config (metomi.rose.config.ConfigNode): The application configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. no_ignore (bool - optional): If ``True`` return ``None`` if the setting is ignored (else return the value). Returns: object - The setting value or ``None`` if not defined. """ section, option = self._get_section_option_from_keys(keys) if config.get([section, option], no_ignore=no_ignore) is None: return None return config.get([section, option]).value
def transform(self, config, meta_config=None): """Apply metadata trigger expressions to variables.""" self.reports = [] meta_config = self._load_meta_config(config, meta_config) self._setup_triggers(meta_config) self.enabled_dict = {} self.ignored_dict = {} enabled = metomi.rose.config.ConfigNode.STATE_NORMAL trig_ignored = metomi.rose.config.ConfigNode.STATE_SYST_IGNORED user_ignored = metomi.rose.config.ConfigNode.STATE_USER_IGNORED state_map = { enabled: 'enabled ', trig_ignored: 'trig-ignored', user_ignored: 'user-ignored', } id_list = [] prev_ignoreds = {trig_ignored: [], user_ignored: []} for keylist, node in config.walk(): if len(keylist) == 1: n_id = keylist[0] else: n_id = self._get_id_from_section_option(*keylist) id_list.append(n_id) if node.state in prev_ignoreds: prev_ignoreds[node.state].append(n_id) ranked_ids = self._get_ranked_trigger_ids() for _, var_id in sorted(ranked_ids): self.update(var_id, config, meta_config) # Report any discrepancies in ignored status. for var_id in id_list: section, option = self._get_section_option_from_id(var_id) node = config.get([section, option]) old, new = None, None if var_id in self.ignored_dict: node.state = trig_ignored if not any(var_id in v for v in prev_ignoreds.values()): old, new = state_map[enabled], state_map[trig_ignored] elif var_id in prev_ignoreds[trig_ignored]: node.state = enabled old, new = state_map[trig_ignored], state_map[enabled] elif (var_id in prev_ignoreds[user_ignored] and var_id in self._trigger_involved_ids): node.state = enabled old, new = state_map[user_ignored], state_map[enabled] if old != new: info = self.WARNING_STATE_CHANGED.format(old, new) if option is None: value = None else: value = node.value self.add_report(section, option, value, info) return config, self.reports
def _remove_setting(self, config, keys, info=None): """Remove a setting from the configuration, if it exists.""" section, option = self._get_section_option_from_keys(keys) if config.get([section, option]) is None: return False if info is None: info = self.INFO_REMOVED node = config.unset([section, option]) value = "" if node.value: value = node.value self.add_report(section, option, value, info)
def transform( self, config, meta_config=None, opt_non_interactive=False, custom_inspector=False, ): """Transform a configuration by looping over upgrade macros.""" self.reports = [] for macro in self.get_macros(): if self.downgrade: func = macro.downgrade else: func = macro.upgrade res = {} if not opt_non_interactive: arglist = inspect.getfullargspec(func).args defaultlist = inspect.getfullargspec(func).defaults optionals = {} while defaultlist is not None and len(defaultlist) > 0: if arglist[-1] not in ["self", "config", "meta_config"]: optionals[arglist[-1]] = defaultlist[-1] arglist = arglist[0:-1] defaultlist = defaultlist[0:-1] else: break if optionals: if custom_inspector: res = custom_inspector(optionals, "upgrade_macro") else: res = metomi.rose.macro.get_user_values(optionals) upgrade_macro_result = func(config, meta_config, **res) config, i_changes = upgrade_macro_result self.reports += i_changes opt_node = config.get( [metomi.rose.CONFIG_SECT_TOP, metomi.rose.CONFIG_OPT_META_TYPE], no_ignore=True, ) new_value = self.meta_flag_no_tag + "/" + self.new_tag opt_node.value = new_value if self.downgrade: info = INFO_DOWNGRADED.format(self.tag, self.new_tag) else: info = INFO_UPGRADED.format(self.tag, self.new_tag) report = metomi.rose.macro.MacroReport( metomi.rose.CONFIG_SECT_TOP, metomi.rose.CONFIG_OPT_META_TYPE, new_value, info, ) self.reports += [report] return config, self.reports
def _check_suite_version(self, fname): """Check the suite is compatible with this version of rose-stem.""" if not os.path.isfile(fname): raise RoseSuiteConfNotFoundException(os.path.dirname(fname)) config = metomi.rose.config.load(fname) suite_rose_stem_version = config.get(['ROSE_STEM_VERSION']) if suite_rose_stem_version: suite_rose_stem_version = int(suite_rose_stem_version.value) else: suite_rose_stem_version = None if not suite_rose_stem_version == ROSE_STEM_VERSION: raise RoseStemVersionException(suite_rose_stem_version)
def change_setting_value( self, config, keys, value, forced=False, comments=None, info=None ): """Change a setting (option) value in the configuration. Args: config (metomi.rose.config.ConfigNode): The application configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. value (str): The new value. Required for options, can be ``None`` for sections. forced (bool): Create the setting if it is not present in config. comments (list): List of comment lines (strings) for the new setting or ``None``. info (str): A short string containing no new lines, describing the addition of the setting. Returns: None """ section, option = self._get_section_option_from_keys(keys) id_ = self._get_id_from_section_option(section, option) node = config.get([section, option]) if node is None: if forced: return self.add_setting( config, keys, value=value, comments=comments, info=info ) return False if node.value == value: return False if option is None: text = "Not valid for value change: {0}".format(id_) raise TypeError(text) if info is None: info = self.INFO_CHANGED_VAR.format(repr(node.value), repr(value)) if value is not None and not isinstance(value, str): text = "New value {0} for {1} is not a string" raise ValueError(text.format(repr(value), id_)) node.value = value if comments is not None: node.comments = comments self.add_report(section, option, value, info)
def _ignore_setting(self, config, keys, info=None, state=None): """Set the ignored state of a setting, if it exists.""" section, option = self._get_section_option_from_keys(keys) node = config.get([section, option]) if node is None or state is None: return False if option is None: value = None else: value = node.value info_text = self.INFO_STATE.format(IGNORE_MAP[node.state], IGNORE_MAP[state]) if node.state == state: return False if info is None: info = info_text node.state = state self.add_report(section, option, value, info)
def remove_setting(self, config, keys, info=None): """Remove a setting from the configuration. Args: config (metomi.rose.config.ConfigNode): The application configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. info (string - optional): A short string containing no new lines, describing the addition of the setting. Returns: None """ section, option = self._get_section_option_from_keys(keys) if option is None: if config.get([section]) is None: return False option_node_pairs = config.walk([section]) for opt_keys, _ in option_node_pairs: opt = opt_keys[1] self._remove_setting(config, [section, opt], info) return self._remove_setting(config, [section, option], info)
def add_setting(self, config, keys, value=None, forced=False, state=None, comments=None, info=None): """Add a setting to the configuration. Args: config (metomi.rose.config.ConfigNode): The application configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. value (string - optional): String denoting the new setting value. Required for options but not for settings. forced (bool - optional) If True override value if the setting already exists. state (str - optional): The state of the new setting - should be one of the ``rose.config.ConfigNode`` states e.g. ``rose.config.ConfigNode.STATE_USER_IGNORED``. Defaults to ``rose.config.ConfigNode.STATE_NORMAL``. comments (list - optional): List of comment lines (strings) for the new setting or ``None``. info (string - optional): A short string containing no new lines, describing the addition of the setting. Returns: None """ section, option = self._get_section_option_from_keys(keys) id_ = self._get_id_from_section_option(section, option) if option is not None and value is None: value = "" if info is None: if option is None: info = self.INFO_ADDED_SECT else: info = self.INFO_ADDED_VAR.format(repr(value)) # Search for existing conflicting settings. conflict_id = None found_setting = False if config.get([section, option]) is None: for key in config.get_value(): existing_section = key if not existing_section.startswith(section): continue existing_base_section = ( metomi.rose.macro.REC_ID_STRIP.sub("", existing_section)) if option is None: # For section 'foo', look for 'foo', 'foo{bar}', 'foo(1)'. found_setting = (existing_section == section or existing_base_section == section) else: # For 'foo=bar', don't allow sections 'foo(1)', 'foo{bar}'. found_setting = (existing_section != section and existing_base_section == section) if found_setting: conflict_id = existing_section break if option is not None: for keys, _ in config.walk([existing_section]): existing_option = keys[1] existing_base_option = ( metomi.rose.macro.REC_ID_STRIP_DUPL.sub( "", existing_option) ) # For option 'foo', look for 'foo', 'foo(1)'. if (existing_section == section and (existing_option == option or existing_base_option == option)): found_setting = True conflict_id = self._get_id_from_section_option( existing_section, existing_option) break if found_setting: break else: found_setting = True conflict_id = None # If already added, quit, unless "forced". if found_setting: if forced and (conflict_id is None or id_ == conflict_id): # If forced, override settings for an identical id. return self.change_setting_value( config, keys, value, state, comments, info) if conflict_id: self.add_report( section, option, value, self.WARNING_ADD_CLASH.format(id_, conflict_id), is_warning=True ) return False # Add parent section if missing. if option is not None and config.get([section]) is None: self.add_setting(config, [section]) if value is not None and not isinstance(value, str): text = "New value {0} for {1} is not a string" raise ValueError(text.format(repr(value), id_)) # Set (add) the section/option. config.set([section, option], value=value, state=state, comments=comments) self.add_report(section, option, value, info)
def add_trigger_graph(graph, config, meta_config, err_reporter, allowed_sections=None): """Add trigger-related nodes and edges to the graph.""" trigger = metomi.rose.macros.trigger.TriggerMacro() bad_reports = trigger.validate_dependencies(config, meta_config) if bad_reports: err_reporter( metomi.rose.macro.get_reports_as_text(bad_reports, "trigger.TriggerMacro")) return None ids = [] for keylist, node in meta_config.walk(no_ignore=True): id_ = keylist[0] if (id_.startswith(metomi.rose.META_PROP_NS + metomi.rose.CONFIG_DELIMITER) or id_.startswith(metomi.rose.SUB_CONFIG_FILE_DIR + ":*")): continue if isinstance(node.value, dict): section, option = ( metomi.rose.macro.get_section_option_from_id(id_)) if not allowed_sections or (allowed_sections and section in allowed_sections): ids.append(id_) ids.sort(key=cmp_to_key(metomi.rose.config.sort_settings)) for id_ in ids: section, option = metomi.rose.macro.get_section_option_from_id(id_) node_attrs = get_node_state_attrs(config, section, option, allowed_sections=allowed_sections) graph.add_node(id_, **node_attrs) for setting_id, id_value_dict in sorted( trigger.trigger_family_lookup.items()): section, option = metomi.rose.macro.get_section_option_from_id( setting_id) section_node = config.get([section], no_ignore=True) node = config.get([section, option]) if node is None: setting_value = None else: setting_value = node.value setting_is_section_ignored = (option is None and section_node is None) for dependent_id, values in sorted(id_value_dict.items()): dep_section, dep_option = \ metomi.rose.macro.get_section_option_from_id( dependent_id) if (allowed_sections and (section not in allowed_sections and dep_section not in allowed_sections)): continue if not values: values = [None] has_success = False if setting_value is not None: for value in values: if value is None: if (node.state == node.STATE_NORMAL and not setting_is_section_ignored): has_success = True break elif trigger._check_values_ok(setting_value, setting_id, [value]): has_success = True break for value in values: value_id = setting_id + "=" + str(value) dependent_attrs = {} if setting_value is None: dependent_attrs["color"] = COLOUR_MISSING else: dependent_attrs["color"] = COLOUR_IGNORED if value is None: if (node.state == node.STATE_NORMAL and not setting_is_section_ignored): dependent_attrs["color"] = COLOUR_ENABLED elif trigger._check_values_ok(setting_value, setting_id, [value]): dependent_attrs["color"] = COLOUR_ENABLED if not graph.has_node(setting_id): node_attrs = get_node_state_attrs( config, section, option, allowed_sections=allowed_sections) graph.add_node(setting_id, **node_attrs) if not graph.has_node(dependent_id): node_attrs = get_node_state_attrs( config, dep_section, dep_option, allowed_sections=allowed_sections) graph.add_node(dependent_id, **node_attrs) if not graph.has_node(value_id): node_attrs = { "style": "filled", "label": value, "shape": "box" } node_attrs.update(dependent_attrs) graph.add_node(value_id, **node_attrs) edge_attrs = {} edge_attrs.update(dependent_attrs) if setting_value is not None: edge_attrs["label"] = setting_value graph.add_edge(setting_id, value_id, **edge_attrs) if dependent_attrs["color"] == COLOUR_IGNORED and has_success: dependent_attrs["arrowhead"] = STYLE_ARROWHEAD_EMPTY graph.add_edge(value_id, dependent_id, **dependent_attrs)
def _get_error_report_for_id(self, variable_id, config, error_string): section, option = self._get_section_option_from_id(variable_id) node = config.get([section, option]) value = None if node is None else node.value self.add_report(section, option, value, error_string) return self.reports
def validate_dependencies(self, config, meta_config): """Validate the trigger setup - e.g. check for cyclic dependencies.""" self.reports = [] if meta_config is None: meta_config = metomi.rose.config.ConfigNode() if not hasattr(self, 'trigger_family_lookup'): self._setup_triggers(meta_config) config_sections = config.value meta_settings = [ k for k in meta_config.value if not meta_config.value[k].is_ignored() ] allowed_repetitions = {} trigger_ids = list(self.trigger_family_lookup) trigger_ids.sort() for var_id in trigger_ids: allowed_repetitions[var_id] = 0 for id_value_dict in self.trigger_family_lookup.values(): for var_id in id_value_dict: allowed_repetitions.setdefault(var_id, 0) allowed_repetitions[var_id] += 1 for start_id in trigger_ids: id_value_dict = self._get_family_dict(start_id, config, meta_config) triggered_ids = list(id_value_dict) triggered_ids.sort() if self._check_is_id_dupl(start_id, meta_config): st_sect = self._get_section_option_from_id(start_id)[0] for tr_id in triggered_ids: tr_sect = self._get_section_option_from_id(tr_id)[0] if tr_sect != st_sect: return self._get_error_report_for_id( start_id, config, self.ERROR_DUPL_TRIG.format(st_sect), ) for value_list in id_value_dict.values(): for string in [s for s in value_list if s is not None]: if self.rec_rule.search(string): try: self.evaluate_trig_rule(string, start_id, '') except metomi.rose.macros.rule.RuleValueError: continue except Exception: return self._get_error_report_for_id( start_id, config, self.ERROR_BAD_EXPR.format(string), ) stack = [(start_id, triggered_ids)] id_list = [] while stack: var_id, child_ids = stack[0] base_id = self._get_stripped_id(var_id, meta_config) if base_id not in meta_settings: return self._get_error_report_for_id( var_id, config, self.ERROR_MISSING_METADATA) id_list.append(var_id) child_ids.sort() if var_id in config_sections: child_ids += config.get([var_id]).value.keys() for child_id in child_ids: base_id = self._get_stripped_id(child_id, meta_config) if base_id not in meta_settings: return self._get_error_report_for_id( child_id, config, self.ERROR_MISSING_METADATA) if child_id in self.trigger_family_lookup: grandchildren = list( self.trigger_family_lookup[child_id]) grandchildren.sort() stack.insert(1, (child_id, grandchildren)) if (id_list.count(child_id) + 1 > allowed_repetitions[child_id] and id_list.count(child_id) >= 2): # Then it may be looping cyclically. duplicate_seq = self._get_dup_sequence( id_list, child_id) if duplicate_seq: return self._get_error_report_for_id( var_id, config, self.ERROR_CYCLIC.format(child_id, var_id), ) stack.pop(0) return []