def get_node_state_attrs(config, section, option=None, allowed_sections=None):
"""Get Graphviz node attributes like color for a given setting."""
node_attrs = {}
if option is None:
node_attrs["shape"] = SHAPE_NODE_SECTION
if not config.value.keys():
# Empty configuration - we can assume pure metadata.
return node_attrs
if allowed_sections is None:
allowed_sections = []
config_section_node = config.get([section])
id_ = metomi.rose.macro.get_id_from_section_option(section, option)
state = ""
config_node = config.get([section, option])
node_attrs["color"] = COLOUR_IGNORED
if (config_section_node is not None
and config_section_node.state != STATE_NORMAL
and option is not None):
state = metomi.rose.config.STATE_SECT_IGNORED
if config_node is None:
node_attrs["color"] = COLOUR_MISSING
elif config_node.state != STATE_NORMAL:
state += config_node.state
if config_node.state == config_node.STATE_USER_IGNORED:
node_attrs["color"] = COLOUR_USER_IGNORED
elif not state:
node_attrs["color"] = COLOUR_ENABLED
if allowed_sections and section not in allowed_sections:
node_attrs["shape"] = SHAPE_NODE_EXTERNAL
if state:
node_attrs["label"] = state + id_
return node_attrs
def rename_setting(self, config, keys, new_keys, info=None):
"""Rename a setting in the configuration.
Args:
config (metomi.rose.config.ConfigNode): The application
configuration.
keys (list): A list defining a hierarchy of node.value 'keys'.
A section will be a list of one keys, an option will have two.
new_keys (list): The new hierarchy of node.value 'keys'.
info (str): A short string containing no new lines,
describing the addition of the setting.
Returns:
None
"""
section, option = self._get_section_option_from_keys(keys)
new_section, new_option = self._get_section_option_from_keys(new_keys)
if option is None:
if new_option is not None:
raise TypeError(self.ERROR_RENAME_SECT_TO_OPT.format(
section, new_section, new_option))
elif new_option is None:
raise TypeError(self.ERROR_RENAME_OPT_TO_SECT.format(
section, option, new_section))
node = config.get(keys)
if node is None:
return
if info is None:
if option is None:
info = self.INFO_RENAMED_SECT.format(section, new_section)
else:
info = self.INFO_RENAMED_VAR.format(section, option,
new_section, new_option)
if option is None:
if config.get([new_section]) is not None:
self.remove_setting(config, [new_section])
self.add_setting(config, [new_section], value=None, forced=True,
state=node.state, comments=node.comments,
info=info)
for option_keys, opt_node in config.walk([section]):
renamed_option = option_keys[1]
self.add_setting(config, [new_section, renamed_option],
value=opt_node.value, forced=True,
state=opt_node.state,
comments=opt_node.comments, info=info)
else:
self.add_setting(config, new_keys, value=node.value, forced=True,
state=node.state, comments=node.comments,
info=info)
self.remove_setting(config, keys)
def get_setting_value(self, config, keys, no_ignore=False):
"""Return the value of a setting or ``None`` if not set.
Args:
config (metomi.rose.config.ConfigNode): The application
configuration.
keys (list): A list defining a hierarchy of node.value 'keys'.
A section will be a list of one keys, an option will have two.
no_ignore (bool - optional): If ``True`` return ``None`` if the
setting is ignored (else return the value).
Returns:
object - The setting value or ``None`` if not defined.
"""
section, option = self._get_section_option_from_keys(keys)
if config.get([section, option], no_ignore=no_ignore) is None:
return None
return config.get([section, option]).value
def transform(self, config, meta_config=None):
"""Apply metadata trigger expressions to variables."""
self.reports = []
meta_config = self._load_meta_config(config, meta_config)
self._setup_triggers(meta_config)
self.enabled_dict = {}
self.ignored_dict = {}
enabled = metomi.rose.config.ConfigNode.STATE_NORMAL
trig_ignored = metomi.rose.config.ConfigNode.STATE_SYST_IGNORED
user_ignored = metomi.rose.config.ConfigNode.STATE_USER_IGNORED
state_map = {
enabled: 'enabled ',
trig_ignored: 'trig-ignored',
user_ignored: 'user-ignored',
}
id_list = []
prev_ignoreds = {trig_ignored: [], user_ignored: []}
for keylist, node in config.walk():
if len(keylist) == 1:
n_id = keylist[0]
else:
n_id = self._get_id_from_section_option(*keylist)
id_list.append(n_id)
if node.state in prev_ignoreds:
prev_ignoreds[node.state].append(n_id)
ranked_ids = self._get_ranked_trigger_ids()
for _, var_id in sorted(ranked_ids):
self.update(var_id, config, meta_config)
# Report any discrepancies in ignored status.
for var_id in id_list:
section, option = self._get_section_option_from_id(var_id)
node = config.get([section, option])
old, new = None, None
if var_id in self.ignored_dict:
node.state = trig_ignored
if not any(var_id in v for v in prev_ignoreds.values()):
old, new = state_map[enabled], state_map[trig_ignored]
elif var_id in prev_ignoreds[trig_ignored]:
node.state = enabled
old, new = state_map[trig_ignored], state_map[enabled]
elif (var_id in prev_ignoreds[user_ignored]
and var_id in self._trigger_involved_ids):
node.state = enabled
old, new = state_map[user_ignored], state_map[enabled]
if old != new:
info = self.WARNING_STATE_CHANGED.format(old, new)
if option is None:
value = None
else:
value = node.value
self.add_report(section, option, value, info)
return config, self.reports
def _remove_setting(self, config, keys, info=None):
"""Remove a setting from the configuration, if it exists."""
section, option = self._get_section_option_from_keys(keys)
if config.get([section, option]) is None:
return False
if info is None:
info = self.INFO_REMOVED
node = config.unset([section, option])
value = ""
if node.value:
value = node.value
self.add_report(section, option, value, info)
def transform(
self,
config,
meta_config=None,
opt_non_interactive=False,
custom_inspector=False,
):
"""Transform a configuration by looping over upgrade macros."""
self.reports = []
for macro in self.get_macros():
if self.downgrade:
func = macro.downgrade
else:
func = macro.upgrade
res = {}
if not opt_non_interactive:
arglist = inspect.getfullargspec(func).args
defaultlist = inspect.getfullargspec(func).defaults
optionals = {}
while defaultlist is not None and len(defaultlist) > 0:
if arglist[-1] not in ["self", "config", "meta_config"]:
optionals[arglist[-1]] = defaultlist[-1]
arglist = arglist[0:-1]
defaultlist = defaultlist[0:-1]
else:
break
if optionals:
if custom_inspector:
res = custom_inspector(optionals, "upgrade_macro")
else:
res = metomi.rose.macro.get_user_values(optionals)
upgrade_macro_result = func(config, meta_config, **res)
config, i_changes = upgrade_macro_result
self.reports += i_changes
opt_node = config.get(
[metomi.rose.CONFIG_SECT_TOP, metomi.rose.CONFIG_OPT_META_TYPE],
no_ignore=True,
)
new_value = self.meta_flag_no_tag + "/" + self.new_tag
opt_node.value = new_value
if self.downgrade:
info = INFO_DOWNGRADED.format(self.tag, self.new_tag)
else:
info = INFO_UPGRADED.format(self.tag, self.new_tag)
report = metomi.rose.macro.MacroReport(
metomi.rose.CONFIG_SECT_TOP,
metomi.rose.CONFIG_OPT_META_TYPE,
new_value,
info,
)
self.reports += [report]
return config, self.reports
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of rose-stem."""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = metomi.rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def change_setting_value(
self, config, keys, value, forced=False, comments=None, info=None
):
"""Change a setting (option) value in the configuration.
Args:
config (metomi.rose.config.ConfigNode): The application
configuration.
keys (list): A list defining a hierarchy of node.value 'keys'.
A section will be a list of one keys, an option will have two.
value (str): The new value. Required for options, can be
``None`` for sections.
forced (bool): Create the setting if it is not present in config.
comments (list): List of comment lines (strings) for
the new setting or ``None``.
info (str): A short string containing no new lines,
describing the addition of the setting.
Returns:
None
"""
section, option = self._get_section_option_from_keys(keys)
id_ = self._get_id_from_section_option(section, option)
node = config.get([section, option])
if node is None:
if forced:
return self.add_setting(
config, keys, value=value, comments=comments, info=info
)
return False
if node.value == value:
return False
if option is None:
text = "Not valid for value change: {0}".format(id_)
raise TypeError(text)
if info is None:
info = self.INFO_CHANGED_VAR.format(repr(node.value), repr(value))
if value is not None and not isinstance(value, str):
text = "New value {0} for {1} is not a string"
raise ValueError(text.format(repr(value), id_))
node.value = value
if comments is not None:
node.comments = comments
self.add_report(section, option, value, info)
def _ignore_setting(self, config, keys, info=None, state=None):
"""Set the ignored state of a setting, if it exists."""
section, option = self._get_section_option_from_keys(keys)
node = config.get([section, option])
if node is None or state is None:
return False
if option is None:
value = None
else:
value = node.value
info_text = self.INFO_STATE.format(IGNORE_MAP[node.state],
IGNORE_MAP[state])
if node.state == state:
return False
if info is None:
info = info_text
node.state = state
self.add_report(section, option, value, info)
def remove_setting(self, config, keys, info=None):
"""Remove a setting from the configuration.
Args:
config (metomi.rose.config.ConfigNode): The application
configuration.
keys (list): A list defining a hierarchy of node.value 'keys'.
A section will be a list of one keys, an option will have two.
info (string - optional): A short string containing no new lines,
describing the addition of the setting.
Returns:
None
"""
section, option = self._get_section_option_from_keys(keys)
if option is None:
if config.get([section]) is None:
return False
option_node_pairs = config.walk([section])
for opt_keys, _ in option_node_pairs:
opt = opt_keys[1]
self._remove_setting(config, [section, opt], info)
return self._remove_setting(config, [section, option], info)
def add_setting(self, config, keys, value=None, forced=False,
state=None, comments=None, info=None):
"""Add a setting to the configuration.
Args:
config (metomi.rose.config.ConfigNode): The application
configuration.
keys (list): A list defining a hierarchy of node.value 'keys'.
A section will be a list of one keys, an option will have two.
value (string - optional): String denoting the new setting value.
Required for options but not for settings.
forced (bool - optional)
If True override value if the setting already exists.
state (str - optional):
The state of the new setting - should be one of the
``rose.config.ConfigNode`` states e.g.
``rose.config.ConfigNode.STATE_USER_IGNORED``. Defaults to
``rose.config.ConfigNode.STATE_NORMAL``.
comments (list - optional): List of comment lines (strings) for
the new setting or ``None``.
info (string - optional): A short string containing no new lines,
describing the addition of the setting.
Returns:
None
"""
section, option = self._get_section_option_from_keys(keys)
id_ = self._get_id_from_section_option(section, option)
if option is not None and value is None:
value = ""
if info is None:
if option is None:
info = self.INFO_ADDED_SECT
else:
info = self.INFO_ADDED_VAR.format(repr(value))
# Search for existing conflicting settings.
conflict_id = None
found_setting = False
if config.get([section, option]) is None:
for key in config.get_value():
existing_section = key
if not existing_section.startswith(section):
continue
existing_base_section = (
metomi.rose.macro.REC_ID_STRIP.sub("", existing_section))
if option is None:
# For section 'foo', look for 'foo', 'foo{bar}', 'foo(1)'.
found_setting = (existing_section == section or
existing_base_section == section)
else:
# For 'foo=bar', don't allow sections 'foo(1)', 'foo{bar}'.
found_setting = (existing_section != section and
existing_base_section == section)
if found_setting:
conflict_id = existing_section
break
if option is not None:
for keys, _ in config.walk([existing_section]):
existing_option = keys[1]
existing_base_option = (
metomi.rose.macro.REC_ID_STRIP_DUPL.sub(
"", existing_option)
)
# For option 'foo', look for 'foo', 'foo(1)'.
if (existing_section == section and
(existing_option == option or
existing_base_option == option)):
found_setting = True
conflict_id = self._get_id_from_section_option(
existing_section, existing_option)
break
if found_setting:
break
else:
found_setting = True
conflict_id = None
# If already added, quit, unless "forced".
if found_setting:
if forced and (conflict_id is None or id_ == conflict_id):
# If forced, override settings for an identical id.
return self.change_setting_value(
config, keys, value, state, comments, info)
if conflict_id:
self.add_report(
section, option, value,
self.WARNING_ADD_CLASH.format(id_, conflict_id),
is_warning=True
)
return False
# Add parent section if missing.
if option is not None and config.get([section]) is None:
self.add_setting(config, [section])
if value is not None and not isinstance(value, str):
text = "New value {0} for {1} is not a string"
raise ValueError(text.format(repr(value), id_))
# Set (add) the section/option.
config.set([section, option], value=value, state=state,
comments=comments)
self.add_report(section, option, value, info)
def add_trigger_graph(graph,
config,
meta_config,
err_reporter,
allowed_sections=None):
"""Add trigger-related nodes and edges to the graph."""
trigger = metomi.rose.macros.trigger.TriggerMacro()
bad_reports = trigger.validate_dependencies(config, meta_config)
if bad_reports:
err_reporter(
metomi.rose.macro.get_reports_as_text(bad_reports,
"trigger.TriggerMacro"))
return None
ids = []
for keylist, node in meta_config.walk(no_ignore=True):
id_ = keylist[0]
if (id_.startswith(metomi.rose.META_PROP_NS +
metomi.rose.CONFIG_DELIMITER)
or id_.startswith(metomi.rose.SUB_CONFIG_FILE_DIR + ":*")):
continue
if isinstance(node.value, dict):
section, option = (
metomi.rose.macro.get_section_option_from_id(id_))
if not allowed_sections or (allowed_sections
and section in allowed_sections):
ids.append(id_)
ids.sort(key=cmp_to_key(metomi.rose.config.sort_settings))
for id_ in ids:
section, option = metomi.rose.macro.get_section_option_from_id(id_)
node_attrs = get_node_state_attrs(config,
section,
option,
allowed_sections=allowed_sections)
graph.add_node(id_, **node_attrs)
for setting_id, id_value_dict in sorted(
trigger.trigger_family_lookup.items()):
section, option = metomi.rose.macro.get_section_option_from_id(
setting_id)
section_node = config.get([section], no_ignore=True)
node = config.get([section, option])
if node is None:
setting_value = None
else:
setting_value = node.value
setting_is_section_ignored = (option is None and section_node is None)
for dependent_id, values in sorted(id_value_dict.items()):
dep_section, dep_option = \
metomi.rose.macro.get_section_option_from_id(
dependent_id)
if (allowed_sections
and (section not in allowed_sections
and dep_section not in allowed_sections)):
continue
if not values:
values = [None]
has_success = False
if setting_value is not None:
for value in values:
if value is None:
if (node.state == node.STATE_NORMAL
and not setting_is_section_ignored):
has_success = True
break
elif trigger._check_values_ok(setting_value, setting_id,
[value]):
has_success = True
break
for value in values:
value_id = setting_id + "=" + str(value)
dependent_attrs = {}
if setting_value is None:
dependent_attrs["color"] = COLOUR_MISSING
else:
dependent_attrs["color"] = COLOUR_IGNORED
if value is None:
if (node.state == node.STATE_NORMAL
and not setting_is_section_ignored):
dependent_attrs["color"] = COLOUR_ENABLED
elif trigger._check_values_ok(setting_value, setting_id,
[value]):
dependent_attrs["color"] = COLOUR_ENABLED
if not graph.has_node(setting_id):
node_attrs = get_node_state_attrs(
config,
section,
option,
allowed_sections=allowed_sections)
graph.add_node(setting_id, **node_attrs)
if not graph.has_node(dependent_id):
node_attrs = get_node_state_attrs(
config,
dep_section,
dep_option,
allowed_sections=allowed_sections)
graph.add_node(dependent_id, **node_attrs)
if not graph.has_node(value_id):
node_attrs = {
"style": "filled",
"label": value,
"shape": "box"
}
node_attrs.update(dependent_attrs)
graph.add_node(value_id, **node_attrs)
edge_attrs = {}
edge_attrs.update(dependent_attrs)
if setting_value is not None:
edge_attrs["label"] = setting_value
graph.add_edge(setting_id, value_id, **edge_attrs)
if dependent_attrs["color"] == COLOUR_IGNORED and has_success:
dependent_attrs["arrowhead"] = STYLE_ARROWHEAD_EMPTY
graph.add_edge(value_id, dependent_id, **dependent_attrs)
def _get_error_report_for_id(self, variable_id, config, error_string):
section, option = self._get_section_option_from_id(variable_id)
node = config.get([section, option])
value = None if node is None else node.value
self.add_report(section, option, value, error_string)
return self.reports
def validate_dependencies(self, config, meta_config):
"""Validate the trigger setup - e.g. check for cyclic dependencies."""
self.reports = []
if meta_config is None:
meta_config = metomi.rose.config.ConfigNode()
if not hasattr(self, 'trigger_family_lookup'):
self._setup_triggers(meta_config)
config_sections = config.value
meta_settings = [
k for k in meta_config.value
if not meta_config.value[k].is_ignored()
]
allowed_repetitions = {}
trigger_ids = list(self.trigger_family_lookup)
trigger_ids.sort()
for var_id in trigger_ids:
allowed_repetitions[var_id] = 0
for id_value_dict in self.trigger_family_lookup.values():
for var_id in id_value_dict:
allowed_repetitions.setdefault(var_id, 0)
allowed_repetitions[var_id] += 1
for start_id in trigger_ids:
id_value_dict = self._get_family_dict(start_id, config,
meta_config)
triggered_ids = list(id_value_dict)
triggered_ids.sort()
if self._check_is_id_dupl(start_id, meta_config):
st_sect = self._get_section_option_from_id(start_id)[0]
for tr_id in triggered_ids:
tr_sect = self._get_section_option_from_id(tr_id)[0]
if tr_sect != st_sect:
return self._get_error_report_for_id(
start_id,
config,
self.ERROR_DUPL_TRIG.format(st_sect),
)
for value_list in id_value_dict.values():
for string in [s for s in value_list if s is not None]:
if self.rec_rule.search(string):
try:
self.evaluate_trig_rule(string, start_id, '')
except metomi.rose.macros.rule.RuleValueError:
continue
except Exception:
return self._get_error_report_for_id(
start_id,
config,
self.ERROR_BAD_EXPR.format(string),
)
stack = [(start_id, triggered_ids)]
id_list = []
while stack:
var_id, child_ids = stack[0]
base_id = self._get_stripped_id(var_id, meta_config)
if base_id not in meta_settings:
return self._get_error_report_for_id(
var_id, config, self.ERROR_MISSING_METADATA)
id_list.append(var_id)
child_ids.sort()
if var_id in config_sections:
child_ids += config.get([var_id]).value.keys()
for child_id in child_ids:
base_id = self._get_stripped_id(child_id, meta_config)
if base_id not in meta_settings:
return self._get_error_report_for_id(
child_id, config, self.ERROR_MISSING_METADATA)
if child_id in self.trigger_family_lookup:
grandchildren = list(
self.trigger_family_lookup[child_id])
grandchildren.sort()
stack.insert(1, (child_id, grandchildren))
if (id_list.count(child_id) + 1 >
allowed_repetitions[child_id]
and id_list.count(child_id) >= 2):
# Then it may be looping cyclically.
duplicate_seq = self._get_dup_sequence(
id_list, child_id)
if duplicate_seq:
return self._get_error_report_for_id(
var_id,
config,
self.ERROR_CYCLIC.format(child_id, var_id),
)
stack.pop(0)
return []
