class Configuration(object):
"""A full Spack configuration, from a hierarchy of config files.
This class makes it easy to add a new scope on top of an existing one.
"""
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
def push_scope(self, scope):
"""Add a higher precedence scope to the Configuration."""
cmd_line_scope = None
if self.scopes:
highest_precedence_scope = list(self.scopes.values())[-1]
if highest_precedence_scope.name == 'command_line':
# If the command-line scope is present, it should always
# be the scope of highest precedence
cmd_line_scope = self.pop_scope()
self.scopes[scope.name] = scope
if cmd_line_scope:
self.scopes['command_line'] = cmd_line_scope
def pop_scope(self):
"""Remove the highest precedence scope and return it."""
name, scope = self.scopes.popitem(last=True)
return scope
def remove_scope(self, scope_name):
return self.scopes.pop(scope_name)
@property
def file_scopes(self):
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values() if type(s) == ConfigScope]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
return next(reversed(self.file_scopes), None)
def matching_scopes(self, reg_expr):
"""
List of all scopes whose names match the provided regular expression.
For example, matching_scopes(r'^command') will return all scopes
whose names begin with `command`.
"""
return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]
def _validate_scope(self, scope):
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError("Invalid config scope: '%s'. Must be one of %s"
% (scope, self.scopes.keys()))
def get_config_filename(self, scope, section):
"""For some scope and section, get the name of the configuration file.
"""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
def clear_caches(self):
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
def update_config(self, section, update_data, scope=None):
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
"""
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# read only the requested section's data.
scope.sections[section] = {section: update_data}
scope.write_section(section)
def get_config(self, section, scope=None):
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the confiugration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree: $spack/opt/spack
module_roots:
lmod: $spack/share/spack/lmod
``get_config('config')`` will return::
{ 'install_tree': '$spack/opt/spack',
'module_roots: {
'lmod': '$spack/share/spack/lmod'
}
}
"""
_validate_section_name(section)
if scope is None:
scopes = self.scopes.values()
else:
scopes = [self._validate_scope(scope)]
merged_section = syaml.syaml_dict()
for scope in scopes:
# read potentially cached data from the scope.
data = scope.get_section(section)
# Skip empty configs
if not data or not isinstance(data, dict):
continue
if section not in data:
continue
merged_section = _merge_yaml(merged_section, data)
# no config files -- empty config.
if section not in merged_section:
return {}
# take the top key off before returning.
return merged_section[section]
def get(self, path, default=None, scope=None):
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the 'config' section would look like::
spack.config.get('config')
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get('config:dirty')
We use ``:`` as the separator, like YAML objects.
"""
# TODO: Currently only handles maps. Think about lists if neded.
section, _, rest = path.partition(':')
value = self.get_config(section, scope=scope)
if not rest:
return value
parts = rest.split(':')
while parts:
key = parts.pop(0)
value = value.get(key, default)
return value
def set(self, path, value, scope=None):
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
section, _, rest = path.partition(':')
if not rest:
self.update_config(section, value, scope=scope)
else:
section_data = self.get_config(section, scope=scope)
parts = rest.split(':')
data = section_data
while len(parts) > 1:
key = parts.pop(0)
data = data[key]
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
for scope in self.scopes.values():
yield scope
def print_section(self, section, blame=False):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump_config(
data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
class Configuration(object):
"""A full Spack configuration, from a hierarchy of config files.
This class makes it easy to add a new scope on top of an existing one.
"""
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
self.format_updates = collections.defaultdict(list)
@_config_mutator
def push_scope(self, scope):
"""Add a higher precedence scope to the Configuration."""
cmd_line_scope = None
if self.scopes:
highest_precedence_scope = list(self.scopes.values())[-1]
if highest_precedence_scope.name == 'command_line':
# If the command-line scope is present, it should always
# be the scope of highest precedence
cmd_line_scope = self.pop_scope()
self.scopes[scope.name] = scope
if cmd_line_scope:
self.scopes['command_line'] = cmd_line_scope
@_config_mutator
def pop_scope(self):
"""Remove the highest precedence scope and return it."""
name, scope = self.scopes.popitem(last=True)
return scope
@_config_mutator
def remove_scope(self, scope_name):
return self.scopes.pop(scope_name)
@property
def file_scopes(self):
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values()
if (type(s) == ConfigScope
or type(s) == SingleFileScope)]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
return next(reversed(self.file_scopes), None)
def highest_precedence_non_platform_scope(self):
"""Non-internal non-platform scope with highest precedence
Platform-specific scopes are of the form scope/platform"""
generator = reversed(self.file_scopes)
highest = next(generator, None)
while highest and highest.is_platform_dependent:
highest = next(generator, None)
return highest
def matching_scopes(self, reg_expr):
"""
List of all scopes whose names match the provided regular expression.
For example, matching_scopes(r'^command') will return all scopes
whose names begin with `command`.
"""
return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]
def _validate_scope(self, scope):
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError("Invalid config scope: '%s'. Must be one of %s"
% (scope, self.scopes.keys()))
def get_config_filename(self, scope, section):
"""For some scope and section, get the name of the configuration file.
"""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
@_config_mutator
def clear_caches(self):
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
@_config_mutator
def update_config(self, section, update_data, scope=None, force=False):
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
Configuration scopes that are still written in an old schema
format will fail to update unless ``force`` is True.
Args:
section (str): section of the configuration to be updated
update_data (dict): data to be used for the update
scope (str): scope to be updated
force (str): force the update
"""
if self.format_updates.get(section) and not force:
msg = ('The "{0}" section of the configuration needs to be written'
' to disk, but is currently using a deprecated format. '
'Please update it using:\n\n'
'\tspack config [--scope=<scope] update {0}\n\n'
'Note that previous versions of Spack will not be able to '
'use the updated configuration.')
msg = msg.format(section)
raise RuntimeError(msg)
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# manually preserve comments
need_comment_copy = (section in scope.sections and
scope.sections[section] is not None)
if need_comment_copy:
comments = getattr(scope.sections[section][section],
yaml.comments.Comment.attrib,
None)
# read only the requested section's data.
scope.sections[section] = syaml.syaml_dict({section: update_data})
if need_comment_copy and comments:
setattr(scope.sections[section][section],
yaml.comments.Comment.attrib,
comments)
scope._write_section(section)
def get_config(self, section, scope=None):
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the configuration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree: $spack/opt/spack
module_roots:
lmod: $spack/share/spack/lmod
``get_config('config')`` will return::
{ 'install_tree': '$spack/opt/spack',
'module_roots: {
'lmod': '$spack/share/spack/lmod'
}
}
"""
return self._get_config_memoized(section, scope)
@llnl.util.lang.memoized
def _get_config_memoized(self, section, scope):
_validate_section_name(section)
if scope is None:
scopes = self.scopes.values()
else:
scopes = [self._validate_scope(scope)]
merged_section = syaml.syaml_dict()
for scope in scopes:
# read potentially cached data from the scope.
data = scope.get_section(section)
# Skip empty configs
if not data or not isinstance(data, dict):
continue
if section not in data:
continue
# We might be reading configuration files in an old format,
# thus read data and update it in memory if need be.
changed = _update_in_memory(data, section)
if changed:
self.format_updates[section].append(scope)
merged_section = merge_yaml(merged_section, data)
# no config files -- empty config.
if section not in merged_section:
return syaml.syaml_dict()
# take the top key off before returning.
ret = merged_section[section]
if isinstance(ret, dict):
ret = syaml.syaml_dict(ret)
return ret
def get(self, path, default=None, scope=None):
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the 'config' section would look like::
spack.config.get('config')
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get('config:dirty')
We use ``:`` as the separator, like YAML objects.
"""
# TODO: Currently only handles maps. Think about lists if needed.
parts = process_config_path(path)
section = parts.pop(0)
value = self.get_config(section, scope=scope)
while parts:
key = parts.pop(0)
value = value.get(key, default)
return value
@_config_mutator
def set(self, path, value, scope=None):
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
if ':' not in path:
# handle bare section name as path
self.update_config(path, value, scope=scope)
return
parts = process_config_path(path)
section = parts.pop(0)
section_data = self.get_config(section, scope=scope)
data = section_data
while len(parts) > 1:
key = parts.pop(0)
if _override(key):
new = type(data[key])()
del data[key]
else:
new = data[key]
if isinstance(new, dict):
# Make it an ordered dict
new = syaml.syaml_dict(new)
# reattach to parent object
data[key] = new
data = new
if _override(parts[0]):
data.pop(parts[0], None)
# update new value
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
for scope in self.scopes.values():
yield scope
def print_section(self, section, blame=False):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump_config(
data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
class Configuration(object):
"""A full Spack configuration, from a hierarchy of config files.
This class makes it easy to add a new scope on top of an existing one.
"""
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
def push_scope(self, scope):
"""Add a higher precedence scope to the Configuration."""
cmd_line_scope = None
if self.scopes:
highest_precedence_scope = list(self.scopes.values())[-1]
if highest_precedence_scope.name == 'command_line':
# If the command-line scope is present, it should always
# be the scope of highest precedence
cmd_line_scope = self.pop_scope()
self.scopes[scope.name] = scope
if cmd_line_scope:
self.scopes['command_line'] = cmd_line_scope
def pop_scope(self):
"""Remove the highest precedence scope and return it."""
name, scope = self.scopes.popitem(last=True)
return scope
def remove_scope(self, scope_name):
return self.scopes.pop(scope_name)
@property
def file_scopes(self):
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values() if type(s) == ConfigScope]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
return next(reversed(self.file_scopes), None)
def _validate_scope(self, scope):
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError("Invalid config scope: '%s'. Must be one of %s"
% (scope, self.scopes.keys()))
def get_config_filename(self, scope, section):
"""For some scope and section, get the name of the configuration file.
"""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
def clear_caches(self):
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
def update_config(self, section, update_data, scope=None):
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
"""
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# read only the requested section's data.
scope.sections[section] = {section: update_data}
scope.write_section(section)
def get_config(self, section, scope=None):
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the confiugration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree: $spack/opt/spack
module_roots:
lmod: $spack/share/spack/lmod
``get_config('config')`` will return::
{ 'install_tree': '$spack/opt/spack',
'module_roots: {
'lmod': '$spack/share/spack/lmod'
}
}
"""
_validate_section_name(section)
if scope is None:
scopes = self.scopes.values()
else:
scopes = [self._validate_scope(scope)]
merged_section = syaml.syaml_dict()
for scope in scopes:
# read potentially cached data from the scope.
data = scope.get_section(section)
# Skip empty configs
if not data or not isinstance(data, dict):
continue
if section not in data:
continue
merged_section = _merge_yaml(merged_section, data)
# no config files -- empty config.
if section not in merged_section:
return {}
# take the top key off before returning.
return merged_section[section]
def get(self, path, default=None, scope=None):
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the 'config' section would look like::
spack.config.get('config')
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get('config:dirty')
We use ``:`` as the separator, like YAML objects.
"""
# TODO: Currently only handles maps. Think about lists if neded.
section, _, rest = path.partition(':')
value = self.get_config(section, scope=scope)
if not rest:
return value
parts = rest.split(':')
while parts:
key = parts.pop(0)
value = value.get(key, default)
return value
def set(self, path, value, scope=None):
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
section, _, rest = path.partition(':')
if not rest:
self.update_config(section, value, scope=scope)
else:
section_data = self.get_config(section, scope=scope)
parts = rest.split(':')
data = section_data
while len(parts) > 1:
key = parts.pop(0)
data = data[key]
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
for scope in self.scopes.values():
yield scope
def print_section(self, section, blame=False):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump(
data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)