def get_default(self, type_name):
"""Finds the configuration specified default obj of type_name.
Returns C{None} if no defaults.
"""
try:
defaults = self.types.get(type_name, {}).items()
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'Collapsing defaults for {type_name!r}') from e
defaults = [(name, section) for name, section in defaults
if section.default]
if not defaults:
return None
if len(defaults) > 1:
defaults = ', '.join(map(repr, sorted(x[0] for x in defaults)))
raise errors.ConfigurationError(
f'type {type_name} incorrectly has multiple default sections: {defaults}'
)
try:
return defaults[0][1].instantiate()
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'failed instantiating default {type_name} {defaults[0][0]!r}'
) from e
return None
def convert_string(central, value, arg_type):
"""Conversion func for a string-based DictConfigSection."""
if not isinstance(value, str):
raise ValueError('convert_string invoked with non str instance: '
f'val({value!r}), arg_type({arg_type!r})')
if arg_type == 'callable':
try:
func = modules.load_attribute(value)
except modules.FailedImport as e:
raise errors.ConfigurationError(f'cannot import {value!r}') from e
if not callable(func):
raise errors.ConfigurationError(f'{value!r} is not callable')
return func
elif arg_type.startswith('refs:'):
return list(
LazyNamedSectionRef(central, arg_type, ref)
for ref in str_to_list(value))
elif arg_type.startswith('ref:'):
return LazyNamedSectionRef(central, arg_type, str_to_str(value))
elif arg_type == 'repr':
return 'str', value
func = _str_converters.get(arg_type)
if func is None:
raise errors.ConfigurationError(f'unknown type {arg_type!r}')
return func(value)
def collapse_named_section(self, name, raise_on_missing=True):
"""Collapse a config by name, possibly returning a cached instance.
@returns: :obj:`CollapsedConfig`.
If there is no section with this name a ConfigurationError is raised,
unless raise_on_missing is False in which case None is returned.
"""
if name in self._refs:
raise errors.ConfigurationError('Reference to %r is recursive' %
(name, ))
self._refs.add(name)
try:
result = self.rendered_sections.get(name)
if result is not None:
return result
section_stack = self.sections_lookup.get(name)
if section_stack is None:
if not raise_on_missing:
return None
raise errors.ConfigurationError('no section called %r' %
(name, ))
try:
result = self.collapse_section(section_stack, name)
result.name = name
except compatibility.IGNORED_EXCEPTIONS:
raise
except Exception:
compatibility.raise_from(
errors.ConfigurationError("Collapsing section named %r" %
(name, )))
self.rendered_sections[name] = result
return result
finally:
self._refs.remove(name)
def convert_string(central, value, arg_type):
"""Conversion func for a string-based DictConfigSection."""
if not isinstance(value, basestring):
raise ValueError('convert_string invoked with non basestring instance:'
' val(%r), arg_type(%r)' % (value, arg_type))
if arg_type == 'callable':
try:
func = modules.load_attribute(value)
except modules.FailedImport:
compatibility.raise_from(
errors.ConfigurationError('Cannot import %r' % (value, )))
if not callable(func):
raise errors.ConfigurationError('%r is not callable' % (value, ))
return func
elif arg_type.startswith('refs:'):
return list(
LazyNamedSectionRef(central, arg_type, ref)
for ref in str_to_list(value))
elif arg_type.startswith('ref:'):
return LazyNamedSectionRef(central, arg_type, str_to_str(value))
elif arg_type == 'repr':
return 'str', value
func = _str_converters.get(arg_type)
if func is None:
raise errors.ConfigurationError('Unknown type %r' % (arg_type, ))
return func(value)
def get_default(self, type_name):
"""Finds the configuration specified default obj of type_name.
Returns C{None} if no defaults.
"""
try:
defaults = self.types.get(type_name, {}).iteritems()
except compatibility.IGNORED_EXCEPTIONS:
raise
except Exception:
compatibility.raise_from(
errors.ConfigurationError("Collapsing defaults for %r" %
(type_name, )))
defaults = [(name, section) for name, section in defaults
if section.default]
if not defaults:
return None
if len(defaults) > 1:
defaults = sorted([x[0] for x in defaults])
raise errors.ConfigurationError(
'type %s incorrectly has multiple default sections: %s' %
(type_name, ', '.join(map(repr, defaults))))
try:
return defaults[0][1].instantiate()
except compatibility.IGNORED_EXCEPTIONS:
raise
except Exception:
compatibility.raise_from(
errors.ConfigurationError(
"Failed instantiating default %s %r" %
(type_name, defaults[0][0])))
return None
def _add_config_source(self, config):
"""Pull extra type and config sections from configs and use them.
Things loaded this way are added after already loaded things
(meaning the config containing the autoload section overrides
the config(s) added by that section).
"""
config_data = config.sections()
collision = set(self.rendered_sections)
collision.intersection_update(config_data)
if collision:
# If this matches something we previously instantiated
# we should probably blow up to prevent massive
# amounts of confusion (and recursive autoloads)
raise errors.ConfigurationError(
"New config is trying to "
"modify existing section(s) %s that was already instantiated."
% (', '.join(repr(x) for x in sorted(collision)), ))
self.configs.append(config_data)
self.config_sources.append(config)
for name in config_data:
self.sections_lookup[name].appendleft(config_data[name])
# Do not even touch the ConfigSection if it's not an autoload.
if not name.startswith('autoload'):
continue
try:
collapsed = self.collapse_named_section(name)
except compatibility.IGNORED_EXCEPTIONS:
raise
except Exception:
compatibility.raise_from(
errors.ConfigurationError(
"Failed collapsing autoload section %r" % (name, )))
if collapsed.type.name != 'configsection':
raise errors.ConfigurationError(
'Section %r is marked as autoload but type is %s, not '
'configsection' % (name, collapsed.type.name))
try:
instance = collapsed.instantiate()
except compatibility.IGNORED_EXCEPTIONS:
compatibility.raise_from(
errors.AutoloadInstantiationError(name))
except Exception:
compatibility.raise_from(
errors.AutoloadInstantiationError(name))
if collapsed.type.name == 'configsection':
self.add_config_source(instance)
def _render_config_stack(self, type_obj, config_stack):
conf = {}
for key in config_stack:
typename = type_obj.types.get(key)
if typename is None:
if not type_obj.allow_unknowns:
raise errors.ConfigurationError('Type of %r unknown' %
(key, ))
typename = 'str'
is_ref = typename.startswith('ref:')
is_refs = typename.startswith('refs:')
if typename.startswith('lazy_'):
typename = typename[5:]
if typename.startswith('refs:') or typename in ('list', 'str'):
result = config_stack.render_prepends(
self, key, typename, flatten=(typename != 'str'))
if typename == 'str':
result = ' '.join(result)
else:
result = config_stack.render_val(self, key, typename)
if is_ref:
result = [result]
is_refs = True
if is_refs:
try:
result = [ref.collapse() for ref in result]
except compatibility.IGNORED_EXCEPTIONS:
raise
except Exception:
compatibility.raise_from(
errors.ConfigurationError(
"Failed collapsing section key %r" % (key, )))
if is_ref:
result = result[0]
conf[key] = result
# Check if we got all values required to instantiate.
missing = set(type_obj.required) - set(conf)
if missing:
raise errors.ConfigurationError(
'type %s.%s needs settings for %s' %
(type_obj.callable.__module__, type_obj.callable.__name__,
', '.join(repr(var) for var in missing)))
return mappings.ImmutableDict(conf)
def _render_config_stack(self, type_obj, config_stack):
conf = {}
for key in config_stack:
typename = type_obj.types.get(key)
if typename is None:
if not type_obj.allow_unknowns:
raise errors.ConfigurationError(f'Type of {key!r} unknown')
typename = 'str'
is_ref = typename.startswith('ref:')
is_refs = typename.startswith('refs:')
if typename.startswith('lazy_'):
typename = typename[5:]
if typename.startswith('refs:') or typename in ('list', 'str'):
result = config_stack.render_prepends(
self, key, typename, flatten=(typename != 'str'))
if typename == 'str':
result = ' '.join(result)
else:
result = config_stack.render_val(self, key, typename)
if is_ref:
result = [result]
is_refs = True
if is_refs:
try:
result = [ref.collapse() for ref in result]
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'Failed collapsing section key {key!r}') from e
if is_ref:
result = result[0]
conf[key] = result
# Check if we got all values required to instantiate.
missing = set(type_obj.required) - set(conf)
if missing:
module = type_obj.callable.__module__
name = type_obj.callable.__name__
missing_vars = ', '.join(map(repr, missing))
raise errors.ConfigurationError(
f'type {module}.{name} needs settings for {missing_vars}')
return mappings.ImmutableDict(conf)
def str_to_int(string):
"""convert a string to a integer"""
string = str_to_str(string)
try:
return int(string)
except ValueError:
raise errors.ConfigurationError('%r is not an integer' % string)
def collapse_section(self, sections, _name=None):
"""Collapse a ConfigSection to a :obj:`CollapsedConfig`."""
if self._section_is_inherit_only(sections[0]):
if sections[0].render_value(self, 'inherit-only', 'bool'):
raise errors.CollapseInheritOnly(
'cannot collapse inherit-only section')
relevant_sections = self._get_inherited_sections(_name, sections)
config_stack = _ConfigStack()
for data in relevant_sections:
for key in data.section.keys():
config_stack[key].append(data)
kls = config_stack.render_val(self, 'class', 'callable')
if kls is None:
raise errors.ConfigurationError('no class specified')
type_obj = basics.ConfigType(kls)
is_default = bool(config_stack.render_val(self, 'default', 'bool'))
for key in ('inherit', 'inherit-only', 'class', 'default'):
config_stack.pop(key, None)
collapsed = CollapsedConfig(type_obj,
self._render_config_stack(
type_obj, config_stack),
self,
default=is_default,
debug=self.debug)
return collapsed
def str_to_int(string):
"""Convert a string to a integer."""
string = str_to_str(string)
try:
return int(string)
except ValueError:
raise errors.ConfigurationError(f'{string!r} is not an integer')
def render_value(self, central, name, arg_type):
if name != 'sects':
raise KeyError(name)
if arg_type != 'repr':
raise errors.ConfigurationError('%r unsupported' % (arg_type,))
return 'refs', [
['spork', basics.HardCodedConfigSection({'foo': 'bar'})],
None, None]
def render_value(self, central, name, arg_type):
try:
return self.func(central, self.dict[name], arg_type)
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'Failed converting argument {name!r} to {arg_type}') from e
def str_to_bool(string):
"""Convert a string to a boolean."""
s = str_to_str(string).lower()
if s in ("no", "false", "0"):
return False
if s in ("yes", "true", "1"):
return True
raise errors.ConfigurationError(f'{s!r} is not a boolean')
def collapse(self):
""":return: :obj:`pkgcore.config.central.CollapsedConfig`."""
if self.cached_config is None:
config = self.cached_config = self._collapse()
if self.typename is not None and config.type.name != self.typename:
raise errors.ConfigurationError(
'reference %r should be of type %r, got %r' %
(self.name, self.typename, config.type.name))
return self.cached_config
def config_from_file(file_obj):
try:
config = parser.parseFile(file_obj)
except pyp.ParseException as e:
name = getattr(file_obj, 'name', file_obj)
raise errors.ConfigurationError('%s: %s' % (name, e))
def build_section(name):
return ConfigSection(config[name])
return mappings.LazyValDict(config.keys, build_section)
def render_value(self, central, name, arg_type):
try:
return self.func(central, self.dict[name], arg_type)
except compatibility.IGNORED_EXCEPTIONS:
raise
except Exception:
compatibility.raise_from(
errors.ConfigurationError(
"Failed converting argument %r to %s" % (name, arg_type)))
def collapse(self):
""":return: :obj:`pkgcore.config.central.CollapsedConfig`."""
if self.cached_config is None:
config = self.cached_config = self._collapse()
if self.typename is not None and config.type.name != self.typename:
raise errors.ConfigurationError(
f'reference {self.name!r} should be of type '
f'{self.typename!r}, got {config.type.name!r}')
return self.cached_config
def convert_asis(central, value, arg_type):
""""Conversion" func assuming the types are already correct."""
if arg_type == 'callable':
if not callable(value):
raise errors.ConfigurationError(f'{value!r} is not callable')
return value
elif arg_type.startswith('ref:'):
if not isinstance(value, ConfigSection):
raise errors.ConfigurationError(
f'{value!r} is not a config section')
return LazyUnnamedSectionRef(central, arg_type, value)
elif arg_type.startswith('refs:'):
l = []
for section in value:
if not isinstance(section, ConfigSection):
raise errors.ConfigurationError(
f'{value!r} is not a config section')
l.append(LazyUnnamedSectionRef(central, arg_type, section))
return l
elif arg_type == 'repr':
if callable(value):
return 'callable', value
if isinstance(value, ConfigSection):
return 'ref', value
if isinstance(value, str):
return 'str', value
if isinstance(value, bool):
return 'bool', value
if isinstance(value, (list, tuple)):
if not value or isinstance(value[0], str):
return 'list', value
if isinstance(value[0], ConfigSection):
return 'refs', value
raise errors.ConfigurationError(f'unsupported type for {value!r}')
elif not isinstance(value, {
'list': (list, tuple),
'str': str,
'bool': bool
}[arg_type]):
raise errors.ConfigurationError(
f'{value!r} does not have type {arg_type!r}')
return value
def config_from_file(file_obj):
try:
config = parser.parseFile(file_obj)
except pyp.ParseException as e:
name = getattr(file_obj, 'name', file_obj)
raise errors.ConfigurationError(f'{name}: {e}') from e
def build_section(name):
return dhcpformat.ConfigSection(config[name])
return mappings.LazyValDict(config.keys, build_section)
def _get_inherited_sections(self, name, sections):
# List of (name, ConfigSection, index) tuples, most specific first.
slist = [(name, sections)]
# first map out inherits.
inherit_names = set([name])
for current_section, section_stack in slist:
current_conf = section_stack[0]
if 'inherit' not in current_conf:
continue
prepend, inherits, append = current_conf.render_value(
self, 'inherit', 'list')
if prepend is not None or append is not None:
raise errors.ConfigurationError(
'Prepending or appending to the inherit list makes no sense'
)
for inherit in inherits:
if inherit == current_section:
# self-inherit. Mkae use of section_stack to handle this.
if len(section_stack) == 1:
# nothing else to self inherit.
raise errors.ConfigurationError(
f'Self-inherit {inherit!r} cannot be found')
if isinstance(section_stack, deque):
slist.append((inherit, list(section_stack)[1:]))
else:
slist.append((inherit, section_stack[1:]))
else:
if inherit in inherit_names:
raise errors.ConfigurationError(
f'Inherit {inherit!r} is recursive')
inherit_names.add(inherit)
target = self.sections_lookup.get(inherit)
if target is None:
raise errors.ConfigurationError(
f'Inherit target {inherit!r} cannot be found')
slist.append((inherit, target))
return [_section_data(name, stack[0]) for (name, stack) in slist]
def __init__(self, type_obj, config, manager, debug=False, default=False):
"""Initialize instance vars."""
# Check if we got all values required to instantiate.
missing = set(type_obj.required) - set(config)
if missing:
raise errors.ConfigurationError(
'type %s.%s needs settings for %s' %
(type_obj.callable.__module__, type_obj.callable.__name__,
', '.join(repr(var) for var in missing)))
self.name = None
self.default = default
self.debug = debug
self.type = type_obj
self.config = config
# Cached instance if we have one.
self._instance = None
if manager is not None:
manager = weakref.ref(manager)
self.manager = manager
def error_main(options, out, err):
raise errors.ConfigurationError('bork')
def _fail(central, value, arg_type):
raise errors.ConfigurationError('fail')
def render_value(self, central, name, arg_type):
# Check if we need our special incremental magic.
if arg_type in ('list', 'str', 'repr') or arg_type.startswith('refs:'):
result = []
# Careful: None is a valid dict value, so use something else here.
missing = object()
for subname in (name + '.prepend', name, name + '.append'):
val = self.dict.get(subname, missing)
if val is missing:
val = None
else:
try:
val = self.func(central, val, arg_type)
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'Failed converting argument {subname!r} to {arg_type}'
) from e
result.append(val)
if result[0] is result[1] is result[2] is None:
raise KeyError(name)
if arg_type != 'repr':
# Done.
return result
# If "kind" is of some incremental-ish kind or we have
# .prepend or .append for this key then we need to
# convert everything we have to the same kind and
# return all three.
#
# (we do not get called for separate reprs for the
# .prepend or .append because those are filtered from
# .keys(). If we do not filter those from .keys()
# central gets upset because it does not know their
# type. Perhaps this means we should have a separate
# .keys() used together with repr, not sure yet
# --marienz)
#
# The problem here is that we may get unsuitable for
# incremental or differing types for the three reprs
# we run, so we need to convert to a suitable common
# kind.
if result[0] is None and result[2] is None:
# Simple case: no extra data, so no need for any
# conversions.
kind, val = result[1]
if kind in ('list', 'str') or kind == 'refs':
# Caller expects a three-tuple.
return kind, (None, val, None)
else:
# non-incremental, just return as-is.
return kind, val
# We have more than one return value. Figure out what
# target to convert to. Choices are list, str and refs.
kinds = set(v[0] for v in result if v is not None)
if 'refs' in kinds or 'ref' in kinds:
# If we have any refs we have to convert to refs.
target_kind = 'refs'
elif kinds == set(['str']):
# If we have only str we can just use that.
target_kind = 'str'
else:
# Convert to list. May not make any sense, but is
# the best we can do.
target_kind = 'list'
converted = []
for val in result:
if val is None:
converted.append(None)
continue
kind, val = val
if kind == 'ref':
if target_kind != 'refs':
raise ValueError(
'Internal issue detected: kind(ref), '
f'target_kind({target_kind!r}), name({name!r}), '
f'val({val!r}), arg_type({arg_type!r})')
converted.append([val])
elif kind == 'refs':
if target_kind != 'refs':
raise ValueError(
'Internal issue detected: kind(refs), '
f'target_kind({target_kind!r}), name({name!r}), '
f'val({val!r}), arg_type({arg_type!r})')
converted.append(val)
elif kind == 'list':
if target_kind == 'str':
raise ValueError(
'Internal issue detected: kind(str), '
f'target_kind({target_kind!r}), name({name!r}), '
f'val({val!r}), arg_type({arg_type!r})')
converted.append(val)
else:
# Everything else gets converted to a string first.
if kind == 'callable':
val = '%s.%s' % (val.__module__, val.__name__)
elif kind in ('bool', 'int', 'str'):
val = str(val)
else:
raise errors.ConfigurationError(
f'unsupported type {kind!r}')
# Then convert the str to list if needed.
if target_kind == 'str':
converted.append(val)
else:
converted.append([val])
return target_kind, converted
# Not incremental.
try:
return self.func(central, self.dict[name], arg_type)
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'Failed converting argument {name!r} to {arg_type}') from e
def collapse_named_section(self, section):
try:
return {'target': target_config}[section]
except KeyError:
raise errors.ConfigurationError(section)
def collapse_named_section(self, section):
try:
return {'1': config1, '2': config2}[section]
except KeyError:
raise errors.ConfigurationError(section)
def _instantiate(self):
"""Call our type's callable, cache and return the result.
Calling instantiate more than once will return the cached value.
"""
# Needed because this code can run twice even with instance
# caching if we trigger an ComplexInstantiationError.
config = mappings.ProtectedDict(self.config)
# Instantiate section refs.
# Careful: not everything we have for needs to be in the conf dict
# (because of default values) and not everything in the conf dict
# needs to have a type (because of allow_unknowns).
for name, val in config.items():
typename = self.type.types.get(name)
if typename is None:
continue
# central already checked the type, no need to repeat that here.
unlist_it = False
if typename.startswith('ref:'):
val = [val]
unlist_it = True
if typename.startswith('refs:') or unlist_it:
try:
final_val = []
for ref in val:
final_val.append(ref.instantiate())
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise errors.ConfigurationError(
f'Instantiating reference {name!r} pointing at {ref.name!r}'
) from e
if unlist_it:
final_val = final_val[0]
config[name] = final_val
if self.type.requires_config:
if self.manager is None:
raise Exception('configuration internal error; '
'requires_config is enabled '
'but we have no config manager to return ')
manager = self.manager()
if manager is None:
raise Exception(
'Configuration internal error, potentially '
'client code error; manager requested, but the config '
'manager is no longer in memory')
config[self.type.requires_config] = manager
callable_obj = self.type.callable
pargs = []
for var in self.type.positional:
pargs.append(config.pop(var))
# Python is basically the worst language ever:
# TypeError: repo() argument after ** must be a dictionary
configdict = dict(config)
try:
self._instance = callable_obj(*pargs, **configdict)
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
source = errors._identify_functor_source(self.type.callable)
raise errors.InstantiationError(
self.name, f'exception caught from {source!r}') from e
if self._instance is None:
raise errors.ComplexInstantiationError('No object returned',
callable_obj=callable_obj,
pargs=pargs,
kwargs=configdict)
return self._instance
def render_value(self, central, name, arg_type):
value = self.section[name]
if arg_type == 'callable':
if len(value) != 1:
raise errors.ConfigurationError('only one argument required')
value = value[0]
if not isinstance(value, basestring):
raise errors.ConfigurationError(
'need a callable, not a section')
try:
value = modules.load_attribute(value)
except modules.FailedImport:
raise errors.ConfigurationError('cannot import %r' % (value,))
if not callable(value):
raise errors.ConfigurationError('%r is not callable' % value)
return value
elif arg_type.startswith('ref:'):
if len(value) != 1:
raise errors.ConfigurationError('only one argument required')
value = value[0]
if isinstance(value, basestring):
# it's a section ref
return basics.LazyNamedSectionRef(central, arg_type, value)
else:
# it's an anonymous inline section
return basics.LazyUnnamedSectionRef(central, arg_type,
ConfigSection(value))
elif arg_type.startswith('refs:'):
result = []
for ref in value:
if isinstance(ref, basestring):
# it's a section ref
result.append(basics.LazyNamedSectionRef(
central, arg_type, ref))
else:
# it's an anonymous inline section
result.append(basics.LazyUnnamedSectionRef(
central, arg_type, ConfigSection(ref)))
return None, result, None
elif arg_type == 'list':
if not isinstance(value, basestring):
# sequence
value = ' '.join(value)
return None, basics.str_to_list(value), None
elif arg_type == 'repr':
if len(value) == 1:
value = value[0]
if isinstance(value, basestring):
return 'str', value
return 'ref', ConfigSection(value)
if all(isinstance(v, basestring) for v in value):
return 'list', list(value)
result = []
for v in value:
if not isinstance(v, basestring):
v = ConfigSection(v)
result.append(v)
return 'refs', result
else:
if len(value) != 1:
raise errors.ConfigurationError('only one argument required')
if not isinstance(value[0], basestring):
raise errors.ConfigurationError(
'%r should be a string' % value)
if arg_type == 'str':
return [None, basics.str_to_str(value[0]), None]
elif arg_type == 'bool':
return basics.str_to_bool(value[0])
else:
raise errors.ConfigurationError(
'unsupported type %r' % (arg_type,))