class Configuration(object): """A full Spack configuration, from a hierarchy of config files. This class makes it easy to add a new scope on top of an existing one. """ def __init__(self, *scopes): """Initialize a configuration with an initial list of scopes. Args: scopes (list of ConfigScope): list of scopes to add to this Configuration, ordered from lowest to highest precedence """ self.scopes = OrderedDict() for scope in scopes: self.push_scope(scope) def push_scope(self, scope): """Add a higher precedence scope to the Configuration.""" cmd_line_scope = None if self.scopes: highest_precedence_scope = list(self.scopes.values())[-1] if highest_precedence_scope.name == 'command_line': # If the command-line scope is present, it should always # be the scope of highest precedence cmd_line_scope = self.pop_scope() self.scopes[scope.name] = scope if cmd_line_scope: self.scopes['command_line'] = cmd_line_scope def pop_scope(self): """Remove the highest precedence scope and return it.""" name, scope = self.scopes.popitem(last=True) return scope def remove_scope(self, scope_name): return self.scopes.pop(scope_name) @property def file_scopes(self): """List of writable scopes with an associated file.""" return [s for s in self.scopes.values() if type(s) == ConfigScope] def highest_precedence_scope(self): """Non-internal scope with highest precedence.""" return next(reversed(self.file_scopes), None) def matching_scopes(self, reg_expr): """ List of all scopes whose names match the provided regular expression. For example, matching_scopes(r'^command') will return all scopes whose names begin with `command`. """ return [s for s in self.scopes.values() if re.search(reg_expr, s.name)] def _validate_scope(self, scope): """Ensure that scope is valid in this configuration. This should be used by routines in ``config.py`` to validate scope name arguments, and to determine a default scope where no scope is specified. Raises: ValueError: if ``scope`` is not valid Returns: ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid """ if scope is None: # default to the scope with highest precedence. return self.highest_precedence_scope() elif scope in self.scopes: return self.scopes[scope] else: raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, self.scopes.keys())) def get_config_filename(self, scope, section): """For some scope and section, get the name of the configuration file. """ scope = self._validate_scope(scope) return scope.get_section_filename(section) def clear_caches(self): """Clears the caches for configuration files, This will cause files to be re-read upon the next request.""" for scope in self.scopes.values(): scope.clear() def update_config(self, section, update_data, scope=None): """Update the configuration file for a particular scope. Overwrites contents of a section in a scope with update_data, then writes out the config file. update_data should have the top-level section name stripped off (it will be re-added). Data itself can be a list, dict, or any other yaml-ish structure. """ _validate_section_name(section) # validate section name scope = self._validate_scope(scope) # get ConfigScope object # read only the requested section's data. scope.sections[section] = {section: update_data} scope.write_section(section) def get_config(self, section, scope=None): """Get configuration settings for a section. If ``scope`` is ``None`` or not provided, return the merged contents of all of Spack's configuration scopes. If ``scope`` is provided, return only the confiugration as specified in that scope. This off the top-level name from the YAML section. That is, for a YAML config file that looks like this:: config: install_tree: $spack/opt/spack module_roots: lmod: $spack/share/spack/lmod ``get_config('config')`` will return:: { 'install_tree': '$spack/opt/spack', 'module_roots: { 'lmod': '$spack/share/spack/lmod' } } """ _validate_section_name(section) if scope is None: scopes = self.scopes.values() else: scopes = [self._validate_scope(scope)] merged_section = syaml.syaml_dict() for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) # Skip empty configs if not data or not isinstance(data, dict): continue if section not in data: continue merged_section = _merge_yaml(merged_section, data) # no config files -- empty config. if section not in merged_section: return {} # take the top key off before returning. return merged_section[section] def get(self, path, default=None, scope=None): """Get a config section or a single value from one. Accepts a path syntax that allows us to grab nested config map entries. Getting the 'config' section would look like:: spack.config.get('config') and the ``dirty`` section in the ``config`` scope would be:: spack.config.get('config:dirty') We use ``:`` as the separator, like YAML objects. """ # TODO: Currently only handles maps. Think about lists if neded. section, _, rest = path.partition(':') value = self.get_config(section, scope=scope) if not rest: return value parts = rest.split(':') while parts: key = parts.pop(0) value = value.get(key, default) return value def set(self, path, value, scope=None): """Convenience function for setting single values in config files. Accepts the path syntax described in ``get()``. """ section, _, rest = path.partition(':') if not rest: self.update_config(section, value, scope=scope) else: section_data = self.get_config(section, scope=scope) parts = rest.split(':') data = section_data while len(parts) > 1: key = parts.pop(0) data = data[key] data[parts[0]] = value self.update_config(section, section_data, scope=scope) def __iter__(self): """Iterate over scopes in this configuration.""" for scope in self.scopes.values(): yield scope def print_section(self, section, blame=False): """Print a configuration to stdout.""" try: data = syaml.syaml_dict() data[section] = self.get_config(section) syaml.dump_config( data, stream=sys.stdout, default_flow_style=False, blame=blame) except (yaml.YAMLError, IOError): raise ConfigError("Error reading configuration: %s" % section)
class Configuration(object): """A full Spack configuration, from a hierarchy of config files. This class makes it easy to add a new scope on top of an existing one. """ def __init__(self, *scopes): """Initialize a configuration with an initial list of scopes. Args: scopes (list of ConfigScope): list of scopes to add to this Configuration, ordered from lowest to highest precedence """ self.scopes = OrderedDict() for scope in scopes: self.push_scope(scope) self.format_updates = collections.defaultdict(list) @_config_mutator def push_scope(self, scope): """Add a higher precedence scope to the Configuration.""" cmd_line_scope = None if self.scopes: highest_precedence_scope = list(self.scopes.values())[-1] if highest_precedence_scope.name == 'command_line': # If the command-line scope is present, it should always # be the scope of highest precedence cmd_line_scope = self.pop_scope() self.scopes[scope.name] = scope if cmd_line_scope: self.scopes['command_line'] = cmd_line_scope @_config_mutator def pop_scope(self): """Remove the highest precedence scope and return it.""" name, scope = self.scopes.popitem(last=True) return scope @_config_mutator def remove_scope(self, scope_name): return self.scopes.pop(scope_name) @property def file_scopes(self): """List of writable scopes with an associated file.""" return [s for s in self.scopes.values() if (type(s) == ConfigScope or type(s) == SingleFileScope)] def highest_precedence_scope(self): """Non-internal scope with highest precedence.""" return next(reversed(self.file_scopes), None) def highest_precedence_non_platform_scope(self): """Non-internal non-platform scope with highest precedence Platform-specific scopes are of the form scope/platform""" generator = reversed(self.file_scopes) highest = next(generator, None) while highest and highest.is_platform_dependent: highest = next(generator, None) return highest def matching_scopes(self, reg_expr): """ List of all scopes whose names match the provided regular expression. For example, matching_scopes(r'^command') will return all scopes whose names begin with `command`. """ return [s for s in self.scopes.values() if re.search(reg_expr, s.name)] def _validate_scope(self, scope): """Ensure that scope is valid in this configuration. This should be used by routines in ``config.py`` to validate scope name arguments, and to determine a default scope where no scope is specified. Raises: ValueError: if ``scope`` is not valid Returns: ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid """ if scope is None: # default to the scope with highest precedence. return self.highest_precedence_scope() elif scope in self.scopes: return self.scopes[scope] else: raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, self.scopes.keys())) def get_config_filename(self, scope, section): """For some scope and section, get the name of the configuration file. """ scope = self._validate_scope(scope) return scope.get_section_filename(section) @_config_mutator def clear_caches(self): """Clears the caches for configuration files, This will cause files to be re-read upon the next request.""" for scope in self.scopes.values(): scope.clear() @_config_mutator def update_config(self, section, update_data, scope=None, force=False): """Update the configuration file for a particular scope. Overwrites contents of a section in a scope with update_data, then writes out the config file. update_data should have the top-level section name stripped off (it will be re-added). Data itself can be a list, dict, or any other yaml-ish structure. Configuration scopes that are still written in an old schema format will fail to update unless ``force`` is True. Args: section (str): section of the configuration to be updated update_data (dict): data to be used for the update scope (str): scope to be updated force (str): force the update """ if self.format_updates.get(section) and not force: msg = ('The "{0}" section of the configuration needs to be written' ' to disk, but is currently using a deprecated format. ' 'Please update it using:\n\n' '\tspack config [--scope=<scope] update {0}\n\n' 'Note that previous versions of Spack will not be able to ' 'use the updated configuration.') msg = msg.format(section) raise RuntimeError(msg) _validate_section_name(section) # validate section name scope = self._validate_scope(scope) # get ConfigScope object # manually preserve comments need_comment_copy = (section in scope.sections and scope.sections[section] is not None) if need_comment_copy: comments = getattr(scope.sections[section][section], yaml.comments.Comment.attrib, None) # read only the requested section's data. scope.sections[section] = syaml.syaml_dict({section: update_data}) if need_comment_copy and comments: setattr(scope.sections[section][section], yaml.comments.Comment.attrib, comments) scope._write_section(section) def get_config(self, section, scope=None): """Get configuration settings for a section. If ``scope`` is ``None`` or not provided, return the merged contents of all of Spack's configuration scopes. If ``scope`` is provided, return only the configuration as specified in that scope. This off the top-level name from the YAML section. That is, for a YAML config file that looks like this:: config: install_tree: $spack/opt/spack module_roots: lmod: $spack/share/spack/lmod ``get_config('config')`` will return:: { 'install_tree': '$spack/opt/spack', 'module_roots: { 'lmod': '$spack/share/spack/lmod' } } """ return self._get_config_memoized(section, scope) @llnl.util.lang.memoized def _get_config_memoized(self, section, scope): _validate_section_name(section) if scope is None: scopes = self.scopes.values() else: scopes = [self._validate_scope(scope)] merged_section = syaml.syaml_dict() for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) # Skip empty configs if not data or not isinstance(data, dict): continue if section not in data: continue # We might be reading configuration files in an old format, # thus read data and update it in memory if need be. changed = _update_in_memory(data, section) if changed: self.format_updates[section].append(scope) merged_section = merge_yaml(merged_section, data) # no config files -- empty config. if section not in merged_section: return syaml.syaml_dict() # take the top key off before returning. ret = merged_section[section] if isinstance(ret, dict): ret = syaml.syaml_dict(ret) return ret def get(self, path, default=None, scope=None): """Get a config section or a single value from one. Accepts a path syntax that allows us to grab nested config map entries. Getting the 'config' section would look like:: spack.config.get('config') and the ``dirty`` section in the ``config`` scope would be:: spack.config.get('config:dirty') We use ``:`` as the separator, like YAML objects. """ # TODO: Currently only handles maps. Think about lists if needed. parts = process_config_path(path) section = parts.pop(0) value = self.get_config(section, scope=scope) while parts: key = parts.pop(0) value = value.get(key, default) return value @_config_mutator def set(self, path, value, scope=None): """Convenience function for setting single values in config files. Accepts the path syntax described in ``get()``. """ if ':' not in path: # handle bare section name as path self.update_config(path, value, scope=scope) return parts = process_config_path(path) section = parts.pop(0) section_data = self.get_config(section, scope=scope) data = section_data while len(parts) > 1: key = parts.pop(0) if _override(key): new = type(data[key])() del data[key] else: new = data[key] if isinstance(new, dict): # Make it an ordered dict new = syaml.syaml_dict(new) # reattach to parent object data[key] = new data = new if _override(parts[0]): data.pop(parts[0], None) # update new value data[parts[0]] = value self.update_config(section, section_data, scope=scope) def __iter__(self): """Iterate over scopes in this configuration.""" for scope in self.scopes.values(): yield scope def print_section(self, section, blame=False): """Print a configuration to stdout.""" try: data = syaml.syaml_dict() data[section] = self.get_config(section) syaml.dump_config( data, stream=sys.stdout, default_flow_style=False, blame=blame) except (yaml.YAMLError, IOError): raise ConfigError("Error reading configuration: %s" % section)
class Configuration(object): """A full Spack configuration, from a hierarchy of config files. This class makes it easy to add a new scope on top of an existing one. """ def __init__(self, *scopes): """Initialize a configuration with an initial list of scopes. Args: scopes (list of ConfigScope): list of scopes to add to this Configuration, ordered from lowest to highest precedence """ self.scopes = OrderedDict() for scope in scopes: self.push_scope(scope) def push_scope(self, scope): """Add a higher precedence scope to the Configuration.""" cmd_line_scope = None if self.scopes: highest_precedence_scope = list(self.scopes.values())[-1] if highest_precedence_scope.name == 'command_line': # If the command-line scope is present, it should always # be the scope of highest precedence cmd_line_scope = self.pop_scope() self.scopes[scope.name] = scope if cmd_line_scope: self.scopes['command_line'] = cmd_line_scope def pop_scope(self): """Remove the highest precedence scope and return it.""" name, scope = self.scopes.popitem(last=True) return scope def remove_scope(self, scope_name): return self.scopes.pop(scope_name) @property def file_scopes(self): """List of writable scopes with an associated file.""" return [s for s in self.scopes.values() if type(s) == ConfigScope] def highest_precedence_scope(self): """Non-internal scope with highest precedence.""" return next(reversed(self.file_scopes), None) def _validate_scope(self, scope): """Ensure that scope is valid in this configuration. This should be used by routines in ``config.py`` to validate scope name arguments, and to determine a default scope where no scope is specified. Raises: ValueError: if ``scope`` is not valid Returns: ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid """ if scope is None: # default to the scope with highest precedence. return self.highest_precedence_scope() elif scope in self.scopes: return self.scopes[scope] else: raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, self.scopes.keys())) def get_config_filename(self, scope, section): """For some scope and section, get the name of the configuration file. """ scope = self._validate_scope(scope) return scope.get_section_filename(section) def clear_caches(self): """Clears the caches for configuration files, This will cause files to be re-read upon the next request.""" for scope in self.scopes.values(): scope.clear() def update_config(self, section, update_data, scope=None): """Update the configuration file for a particular scope. Overwrites contents of a section in a scope with update_data, then writes out the config file. update_data should have the top-level section name stripped off (it will be re-added). Data itself can be a list, dict, or any other yaml-ish structure. """ _validate_section_name(section) # validate section name scope = self._validate_scope(scope) # get ConfigScope object # read only the requested section's data. scope.sections[section] = {section: update_data} scope.write_section(section) def get_config(self, section, scope=None): """Get configuration settings for a section. If ``scope`` is ``None`` or not provided, return the merged contents of all of Spack's configuration scopes. If ``scope`` is provided, return only the confiugration as specified in that scope. This off the top-level name from the YAML section. That is, for a YAML config file that looks like this:: config: install_tree: $spack/opt/spack module_roots: lmod: $spack/share/spack/lmod ``get_config('config')`` will return:: { 'install_tree': '$spack/opt/spack', 'module_roots: { 'lmod': '$spack/share/spack/lmod' } } """ _validate_section_name(section) if scope is None: scopes = self.scopes.values() else: scopes = [self._validate_scope(scope)] merged_section = syaml.syaml_dict() for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) # Skip empty configs if not data or not isinstance(data, dict): continue if section not in data: continue merged_section = _merge_yaml(merged_section, data) # no config files -- empty config. if section not in merged_section: return {} # take the top key off before returning. return merged_section[section] def get(self, path, default=None, scope=None): """Get a config section or a single value from one. Accepts a path syntax that allows us to grab nested config map entries. Getting the 'config' section would look like:: spack.config.get('config') and the ``dirty`` section in the ``config`` scope would be:: spack.config.get('config:dirty') We use ``:`` as the separator, like YAML objects. """ # TODO: Currently only handles maps. Think about lists if neded. section, _, rest = path.partition(':') value = self.get_config(section, scope=scope) if not rest: return value parts = rest.split(':') while parts: key = parts.pop(0) value = value.get(key, default) return value def set(self, path, value, scope=None): """Convenience function for setting single values in config files. Accepts the path syntax described in ``get()``. """ section, _, rest = path.partition(':') if not rest: self.update_config(section, value, scope=scope) else: section_data = self.get_config(section, scope=scope) parts = rest.split(':') data = section_data while len(parts) > 1: key = parts.pop(0) data = data[key] data[parts[0]] = value self.update_config(section, section_data, scope=scope) def __iter__(self): """Iterate over scopes in this configuration.""" for scope in self.scopes.values(): yield scope def print_section(self, section, blame=False): """Print a configuration to stdout.""" try: data = syaml.syaml_dict() data[section] = self.get_config(section) syaml.dump( data, stream=sys.stdout, default_flow_style=False, blame=blame) except (yaml.YAMLError, IOError): raise ConfigError("Error reading configuration: %s" % section)