def validate_config(self, configspec, suppress_warnings=False):
"""
Validate this config with the given configspec
"""
self._handle_configspec(configspec)
errors = self.validate(validator, preserve_errors=True)
for entry in flatten_errors(self, errors):
section_list, key, error = entry
if not error:
LOGGER.error("Missing parameter %s",
'.'.join(section_list + [key]))
else:
LOGGER.error("Configuration error %s: %s",
'.'.join(section_list + [key]), error)
# warn about any unknown parameters before we potentially abort on
# validation errors
if not suppress_warnings:
for sections, name in get_extra_values(self):
LOGGER.warn("Unknown parameter '%s' in section '%s'", name,
".".join(sections))
if errors is not True:
raise ConfigError(
"Configuration errors were encountered while validating %r" %
self.filename)
return errors
def test_get_extra_values(self):
log.info("Test getting extra values.")
from configobj import ConfigObj, get_extra_values
from validate import Validator
log.debug("Using ini file %r.", self.ini_file)
log.debug("Using spec file %r.", self.spec_file)
conf = ConfigObj(self.ini_file, configspec = self.spec_file)
conf.validate(Validator(), preserve_errors = True)
log.debug("ConfigObj after validate: %s", str(conf))
expected = sorted([
((), 'extra'),
((), 'extra-section'),
(('section', 'sub-section'), 'extra'),
(('section',), 'extra-sub-section'),
])
log.debug("Expected extra values: %r.", expected)
extra_values = get_extra_values(conf)
log.debug("Got extra values: %r.", extra_values)
self.assertEqual(sorted(extra_values), expected)
def collect_validation_messages(conf, error_list):
validator = validate.Validator()
conf.validate(validator, preserve_errors=True)
message = []
cm_extras = {
'hadoop_hdfs_home': [('hadoop', 'hdfs_clusters', 'default')],
'hadoop_bin': [('hadoop', 'hdfs_clusters', 'default'), ('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')],
'hadoop_mapred_home': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')],
'hadoop_conf_dir': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')],
'ssl_cacerts': [('beeswax', 'ssl'), ('impala', 'ssl')],
'remote_data_dir': [('liboozie', )],
'shell': [()]
}
whitelist_extras = ((sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or (name in cm_extras.keys() and sections in cm_extras[name])))
for sections, name in whitelist_extras:
the_section = conf
hierarchy_sections_string = ''
try:
parent = conf
for section in sections:
the_section = parent[section]
hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " "
parent = the_section
except KeyError, ex:
LOG.warn("Section %s not found: %s" % (section, str(ex)))
the_value = ''
try:
# the_value may be a section or a value
the_value = the_section[name]
except KeyError, ex:
LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
def get_config(config_path=None):
"""reads the config file, validates it and return a config dict
:param config_path: path to a custom config file, if none is given the
default locations will be searched
:type config_path: str
:returns: configuration
:rtype: dict
"""
if config_path is None:
config_path = _find_configuration_file()
logger.debug('using the config file at {}'.format(config_path))
config = ConfigObj(DEFAULTSPATH, interpolation=False)
try:
user_config = ConfigObj(config_path,
configspec=SPECPATH,
interpolation=False,
file_error=True,
)
except ConfigObjError as error:
logger.fatal('parsing the config file file with the following error: '
'{}'.format(error))
logger.fatal('if you recently updated khal, the config file format '
'might have changed, in that case please consult the '
'CHANGELOG or other documentation')
sys.exit(1)
fdict = {'timezone': is_timezone,
'expand_path': expand_path,
}
validator = Validator(fdict)
results = user_config.validate(validator, preserve_errors=True)
if not results:
for entry in flatten_errors(config, results):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error is False:
error = 'Missing value or section.'
print(section_string, ' = ', error)
raise ValueError # TODO own error class
config.merge(user_config)
config_checks(config)
extras = get_extra_values(user_config)
for section, value in extras:
if section == ():
logger.warn('unknown section "{}" in config file'.format(value))
else:
section = sectionize(section)
logger.warn('unknown key or subsection "{}" in '
'section "{}"'.format(value, section))
return config
def collect_validation_messages(conf, error_list):
validator = validate.Validator()
conf.validate(validator, preserve_errors=True)
message = []
cm_extras = {
'hadoop_hdfs_home': [('hadoop', 'hdfs_clusters', 'default')],
'hadoop_bin': [('hadoop', 'hdfs_clusters', 'default'),
('hadoop', 'yarn_clusters', 'default'),
('hadoop', 'yarn_clusters', 'ha')],
'hadoop_mapred_home': [('hadoop', 'yarn_clusters', 'default'),
('hadoop', 'yarn_clusters', 'ha')],
'hadoop_conf_dir': [('hadoop', 'yarn_clusters', 'default'),
('hadoop', 'yarn_clusters', 'ha')],
'ssl_cacerts': [('beeswax', 'ssl'), ('impala', 'ssl')],
'remote_data_dir': [('liboozie', )],
'shell': [()],
}
whitelist_extras = (
(sections, name) for sections, name in get_extra_values(conf)
if not (name in desktop.conf.APP_BLACKLIST.get() or (
name in list(cm_extras.keys()) and sections in cm_extras[name])))
for sections, name in whitelist_extras:
the_section = conf
hierarchy_sections_string = ''
try:
parent = conf
for section in sections:
the_section = parent[section]
hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " "
parent = the_section
except KeyError as ex:
LOG.warning("Section %s not found: %s" % (section, str(ex)))
the_value = ''
try:
# the_value may be a section or a value
the_value = the_section[name]
except KeyError as ex:
LOG.warning("Error in accessing Section or Value %s: %s" %
(name, str(ex)))
section_or_value = 'keyvalue'
if isinstance(the_value, dict):
# Sections are subclasses of dict
section_or_value = 'section'
section_string = hierarchy_sections_string or "top level"
message.append('Extra %s, %s in the section: %s' %
(section_or_value, name, section_string))
if message:
error = {
'name': 'ini configuration',
'message': ', '.join(message),
}
error_list.append(error)
def test_get_extra_values(conf):
conf.validate(Validator(), preserve_errors=True)
extra_values = get_extra_values(conf)
expected = sorted([
((), 'extra'),
((), 'extra-section'),
(('section', 'sub-section'), 'extra'),
(('section',), 'extra-sub-section'),
])
assert sorted(extra_values) == expected
def test_get_extra_values(conf):
conf.validate(Validator(), preserve_errors=True)
extra_values = get_extra_values(conf)
expected = sorted([
((), 'extra'),
((), 'extra-section'),
(('section', 'sub-section'), 'extra'),
(('section',), 'extra-sub-section'),
])
assert sorted(extra_values) == expected
def test_get_extra_values(self):
conf = ConfigObj(inipath, configspec=specpath)
conf.validate(Validator(), preserve_errors=True)
extra_values = get_extra_values(conf)
expected = sorted([
((), 'extra'),
((), 'extra-section'),
(('section', 'sub-section'), 'extra'),
(('section',), 'extra-sub-section'),
])
self.assertEqual(sorted(extra_values), expected)
def __get_confobj(cls, config):
spec_lines = resource_stream("vpnporthole", "resources/settings.spec").readlines()
confobj = ConfigObj(config, configspec=spec_lines, raise_errors=True)
result = confobj.validate(Validator())
if result is not True:
# print(result)
pass
extra = get_extra_values(confobj)
if extra:
# print(extra)
pass
# print (confobj)
return confobj
def validate_config(config, spec):
"""
Perform validation against <config> using <spec>, returning type-casted ConfigObj, any errors,
and any extra values.
:param config: [ConfigObj]
:param spec: [string] filepath to config specification file
:return: [dict] {config: ConfigObj,
errors: {<section>: <error>, ...},
missing: [<section>, ...]
extras: {<section>: <value>, ...}
}
"""
from validate import Validator
from configobj import (ConfigObj, flatten_errors, get_extra_values)
_return = {'config': None, 'errors': {}, 'missing': [], 'extras': {}}
_validator = Validator()
_validator.functions['filepath'] = filepath
_config = ConfigObj(config, configspec=spec, stringify=True)
_result = _config.validate(_validator, preserve_errors=True)
# http://configobj.readthedocs.io/en/latest/configobj.html#flatten-errors
for _entry in flatten_errors(_config, _result):
_section_list, _key, _error = _entry
if _key is not None:
_return['errors'][_key] = _error
_section_list.append(_key)
elif _error is False:
_return['missing'].append(_key)
# http://configobj.readthedocs.io/en/latest/configobj.html#get-extra-values
for _sections, _name in get_extra_values(_config):
# this code gets the extra values themselves
_the_section = _config
for _subsection in _sections:
_the_section = _the_section[_subsection]
# the_value may be a section or a value
_the_value = _the_section[_name]
_return['extras'][_name] = _the_value
_return['config'] = _config
return _return
def _validate_config(self):
"""Check for errors in the configuration file. Bail out if required
sections/values are missing, or if extra sections/values are
present."""
result = self.config.validate(Validator(), preserve_errors=True,
copy=True)
error_msgs = []
for sections, name in get_extra_values(self.config):
the_section = self.config
try:
for section in sections:
the_section = self.config[section]
except KeyError:
continue
the_value = the_section[name]
section_or_value = 'value'
if isinstance(the_value, dict):
section_or_value = 'section'
section_string = ', '.join(sections) or 'top level'
msg = 'E: extra %s entry in %s section: %r' % \
(section_or_value, section_string, name)
error_msgs.append(msg)
for entry in flatten_errors(self.config, result):
section_list, key, error = entry
if key is None:
section_list.append('section missing')
else:
section_list.append(key)
if error == False:
error = 'value or section missing'
msg = 'E: config%s: %s' % \
(''.join(["['%s']" % s for s in section_list]), error)
error_msgs.append(msg)
if error_msgs:
error_msgs.insert(0, 'config file failed validation: %s' %
self.config_file.name)
raise ConfigError('\n'.join(error_msgs))
def load_config(config_file,
schema,
postprocessing=None,
retain=None,
drop=None):
"""
Loads a configobj-type configuration file. It is also
possible to specify an application-specific postprocessing function which
has three arguments: the configuration object and the warnings and errors
lists. These are also the return values from the function.
It is possible to specify a list of section titles that are retained (in
this case, or others are dropped) or dropped. This allows more fine grain
control over which sections need to be validated and which do not.
"""
def keep_section(sections, retained, dropped):
"""Tells if the section path in question should be kept."""
secs = set(sections)
if retained and not secs & retained:
return False
if dropped and secs & dropped:
return False
return True
config = configobj.ConfigObj(get_config_file(config_file),
configspec=get_config_file(schema))
warnings, errors = [], []
v = Validator()
results = config.validate(v, preserve_errors=True)
if postprocessing:
postprocessing(config, warnings, errors)
retained = set(retain) if retain else set()
dropped = set(drop) if drop else set()
for sections, key in configobj.get_extra_values(config):
if keep_section(sections, retained, dropped):
warnings.append('Undefined key {}'.format('.'.join(
(chain(sections, [key])))))
for sections, key, error in configobj.flatten_errors(config, results):
if keep_section(sections, retained, dropped):
errors.append('{}: {}'.format('.'.join((chain(sections, [key]))),
error))
return config, warnings, errors
def validate(self, config, value_sources):
""" Validate configuration
Args:
config (:obj:`ConfigObj`): configuration
Raises:
:obj:`InvalidConfigError`: if configuration doesn't validate against schema
:obj:`ValueError`: if no configuration is found
"""
validator = Validator()
validator.functions['any'] = any_checker
result = config.validate(validator, copy=True, preserve_errors=True)
if result is not True:
raise InvalidConfigError(value_sources, config, result)
if get_extra_values(config):
raise ExtraValuesError(value_sources, config)
def validateconfig(self):
""" Test config against configspec and print errors if it doesn't
conform.
This code is currently not used
Things that need to be fixed:
* Config validation should not overwrite default value from
paramlist in steps
* If a keyword is missing in the [HEADER] section config validation
shouldn't put it in.
"""
errFlag = False # set to true if errors encountered
results = self.config.validate(validate.Validator(), copy=True)
extra = configobj.get_extra_values(self.config) # keywords not in spec
# List configuration keywords that failed validation
if results != True:
for (section_list, key,
_) in configobj.flatten_errors(self.config, results):
blah = ', '.join(section_list)
if key is not None:
msg = "ValidateConfig: key '%s' in section '%s' failed"
self.log.error(msg % (key, blah))
errFlag = True
else:
self.log.error("ValidateConfig: Section '%s' failed" %
blah)
errFlag = True
# Warn for keywords not found in validation
for s, k in extra:
if len(s) == 0:
msg = 'ValidateConfig: Skippping unknown global keyword "%s"'
self.log.warning(msg % k)
else:
msg = 'ValidateConfig: Skippping unknown keyword "%s" in section "%s"'
self.log.warning(msg % (k, s[0]))
if errFlag:
raise validate.ValidateError
self.log.debug('ValidateConfig: done')
def __init__(self, sources, config):
"""
Args:
sources (:obj:`list` of :obj:`str`): list of sources of configuration values
config (:obj:`configobj.ConfigObj`): configuration
"""
self.sources = sources
self.config = config
messages = []
# todo: ensure that self.msg is generated even if this for loop raises another exception
for section_list, name in get_extra_values(config):
# this code gets the extra values themselves
the_section = config
for i_section, section in enumerate(section_list):
if section in config:
the_section = config[section]
else:
section_list = section_list[0:i_section]
name = section
break
# the_value may be a section or a value
the_value = the_section[name]
section_or_value = 'value'
if isinstance(the_value, dict):
# Sections are subclasses of dict
section_or_value = 'section'
section_string = ', '.join(section_list) or "top level"
messages.append(
"Extra entry in section '{:s}'. Entry '{}' is a {:s}.".format(
section_string, name, section_or_value))
self.msg = ('The following configuration sources\n {}\n\n'
'contain the following configuration errors\n {}').format(
'\n '.join(sources), '\n '.join(messages))
def collect_validation_messages(conf, error_list):
validator = validate.Validator()
conf.validate(validator, preserve_errors=True)
message = []
for sections, name in get_extra_values(conf):
the_section = conf
hierarchy_sections_string = ''
try:
parent = conf
for section in sections:
the_section = parent[section]
hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " "
parent = the_section
except KeyError, ex:
LOG.warn("Section %s not found: %s" % (section, str(ex)))
the_value = ''
try:
# the_value may be a section or a value
the_value = the_section[name]
except KeyError, ex:
LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
def check_extra_values(config, logger):
"""
Checks the config and warns the user if there are any extra entries
in their config file. This function is based on suggested usage in the
ConfigObj manual.
Args:
config (ConfigObj): A ConfigObj instance.
logger (logger): The logger to which to write complaints.
Returns:
Nothing: Nothing.
"""
warnings = 0
for sections, name in get_extra_values(config):
# this code gets the extra values themselves
the_section = config
for section in sections:
the_section = the_section[section]
# the_value may be a section or a value
the_value = the_section[name]
section_or_value = 'value'
if isinstance(the_value, dict):
# Sections are subclasses of dict
section_or_value = 'section'
section_string = ', '.join(sections) or "top level"
logger.warning('Extra entry in section: %s: %s %r is not in spec.' %
(section_string, section_or_value, name))
warnings += 1
if warnings:
logger.warning('The extra value(s) may be the result of deprecated '
'entries or other changes to the config files; please '
'check the conifg files in shakemap/data for the most '
'up to date specs.')
def test_extra(self):
# test to __str__
config_specification = configobj.ConfigObj(
debug_logs_default_paths.schema, list_values=False, _inspec=True)
config = configobj.ConfigObj(configspec=config_specification)
config.merge({'__extra__': True})
validator = Validator()
result = config.validate(validator, preserve_errors=True)
self.assertNotEqual(configobj.get_extra_values(config), None)
# extra section
extra = {'__extra__': True}
with self.assertRaisesRegex(ExtraValuesError,
"The following configuration sources"):
ConfigManager(debug_logs_default_paths).get_config(extra)
with self.assertRaisesRegex(ExtraValuesError, " 'extra' argument"):
ConfigManager(debug_logs_default_paths).get_config(extra)
with self.assertRaisesRegex(
ExtraValuesError,
"Extra entry in section 'top level'. Entry '__extra__' is a value"
):
ConfigManager(debug_logs_default_paths).get_config(extra)
# extra subsection, extra key
extra = {
'debug_logs': {
'__extra__': True,
'__extra__2': {
'val': 'is_dict'
}
}
}
with self.assertRaisesRegex(ExtraValuesError,
"Entry '__extra__2' is a section"):
ConfigManager(debug_logs_default_paths).get_config(extra)
def validate_config(self, configspec, suppress_warnings=False):
"""
Validate this config with the given configspec
"""
self._handle_configspec(configspec)
errors = self.validate(validator, preserve_errors=True)
for entry in flatten_errors(self, errors):
section_list, key, error = entry
if not error:
LOGGER.error("Missing parameter %s", '.'.join(section_list + [key]))
else:
LOGGER.error("Configuration error %s: %s", '.'.join(section_list + [key]), error)
# warn about any unknown parameters before we potentially abort on
# validation errors
if not suppress_warnings:
for sections, name in get_extra_values(self):
LOGGER.warn("Unknown parameter '%s' in section '%s'",
name, ".".join(sections))
if errors is not True:
raise ConfigError("Configuration errors were encountered while validating %r" % self.filename)
return errors
def check_settings(settings):
validator = MorphologistConfigValidator()
validation = settings.validate(validator, preserve_errors=True)
settings_issues = flatten_errors(settings, validation)
if settings_issues:
print("Error: loading settings from file '%s'" % settings.filename)
print("The following items have wrong values:")
for sections, key, error in settings_issues:
sections = '/'.join(sections)
if key is None:
key = '[Missing section]'
if error == False:
error = 'Missing value or section.'
print(" - %s/%s: %s" % (sections, key, error))
return False
extra_values = get_extra_values(settings)
if extra_values:
print("Error: unknown additionnal items in setting file '%s'" %
settings.filename)
for sections, name in extra_values:
item = '/'.join(list(sections) + [name])
print(" -", item)
return False
return True
def get_cfg(cfg_file_name="Parameters.in", kvargs={}, err=sys.stderr):
""" Parse a config """
def validate_convert(value, func, name=None):
"""Helper function for validate parameter conversion"""
if value is None:
return None
try:
return func(value)
except (TypeError, ValueError):
if name is None:
raise validate.VdtValueError(value)
else:
raise validate.VdtParamError(name, value)
def sci_int(val):
"""Converts type to int, honouring scientific notation if possible"""
try:
i = int(val)
except ValueError:
val = float(val)
i = int(val)
if type(val)(i) != val:
raise ValueError("Not exactly representable as integer")
return i
def is_sci_integer(value, min=None, max=None):
"""Function for the validate module to support scientific notation"""
value = validate_convert(value, sci_int)
minval = validate_convert(min, sci_int, "min")
maxval = validate_convert(max, sci_int, "max")
if min is not None and value < minval:
raise validate.VdtValueTooSmallError(value)
if max is not None and value > maxval:
raise validate.VdtValueTooBigError(value)
return value
def is_option_or_float_list(value, *options):
try:
return validate.is_option(value, *options)
except validate.VdtTypeError:
return validate.is_float_list(value)
# config
configspec = file(os.path.dirname(__file__) + '/configspec','r')
cfg = configobj.ConfigObj(infile=file(cfg_file_name, 'r'),
configspec=configspec.readlines(),
indent_type="\t")
# update command line parameters
for key, value in kvargs.iteritems():
groups = key.split(".")
parent = cfg
for group in groups[:-1]:
parent = parent[group]
parent[groups[-1]] = value
validator = validate.Validator()
validator.functions["integer"] = is_sci_integer
validator.functions["option_or_list"] = is_option_or_float_list
valid = cfg.validate(validator, copy=True)
try:
pairs = configobj.get_extra_values(cfg)
except AttributeError:
print >> err, "WARNING: cannot check unknown entries in config"
pairs = False
if pairs:
print >> err, "error: unknown entries in config: %s" % cfg_file_name
print >> err, ">>>", ", ".join(".".join(e[0] + e[1:]) for e in pairs)
raise CfgException()
if valid is not True:
print >> err, "error: invalid entries in config: %s" % cfg_file_name
print >> err, ">>>", ", ".join(str(entry) for entry, ok
in flat_items(valid) if not ok)
raise CfgException()
return cfg
def call(self, filename):
# Get config
config = self.read(filename)
# Validate top level config
validator = Validator({
'dict':
self._is_dict,
'integer_or_none':
self._is_int_or_none,
'float_or_none':
self._is_float_or_none,
'dict_list':
self._is_dict_list,
'string_list':
self._is_string_list,
'int_list':
self._is_int_list,
'float_list':
self._is_float_list,
'int_list_or_none':
self._is_int_list_or_none,
'float_list_or_none':
self._is_float_list_or_none,
'string_list_or_none':
self._is_string_list_or_none
})
results = config.validate(validator, preserve_errors=True)
# Insert timestamp
config['DEFAULT']['timestamp'] = config['DEFAULT'].get(
'timestamp', time.strftime("%Y%m%d-%H%M%S", time.gmtime()))
# Get default values for the default section
config['DEFAULT'] = {**self.configspec['DEFAULT'], **config['DEFAULT']}
self.configspec['DEFAULT'] = config['DEFAULT']
# Report extra (not defined) sections
if self.allow_extra_values < 1:
# Get extra values
extra_values = get_extra_values(config)
# Filter out non top level values
extra_values = list(filter(lambda x: len(x[0]) < 2, extra_values))
# Print errors
msg = 'Section is not defined in the configspec file!'
self._print_extra_val_results(extra_values, msg=msg)
# Remove extra values form config
for extra_value in extra_values:
if not extra_value[0]:
del config[extra_value[-1]]
else:
section_mane = extra_value[0][0]
del config.get(section_mane)[extra_value[-1]]
# Report validation errors
self._print_val_results(config, results)
# Validate preprocessing
config['PREPROCESS'] = self._check_preprocess(section=ConfigObj(
config['PREPROCESS']),
validator=validator)
# Validate generate
config['GENERATE'] = self._check_generate(section=ConfigObj(
config['GENERATE']),
validator=validator)
# Validate model
config['MODEL'] = self._check_model(section=ConfigObj(config['MODEL']),
validator=validator)
# Validate train
config['TRAIN'] = self._check_train(section=ConfigObj(config['TRAIN']),
validator=validator)
# Parse None values (destringify)
config = self._destringify_none(config)
# Raise exception if validation failed
if not self.valid:
raise ValidateError(
'Validation faild! Please see validation error messages!')
return config
def _recursive_validation(self, config, validator, module, key: str,
section_list: list):
"""
Returns a recursively validated config object.
Inputs:
config: Config object, <dict or ConfigObj>.
Arguments:
validator: Validator instance used to validate the config object, <Validator>.
module: Module to get the configspec from, <class>.
key: Key/name of the current config object, <str>.
section_list: Section list ('path') of the current config object, <list[str, ..., str]>.
Return:
config: Validated config object, <dict>.
"""
# Recursive call of the validation
def _recursive_call(self, value, validator, module, key, section_list):
if isinstance(value, (dict, list)):
return self._recursive_validation(value, validator, module,
key, section_list)
else:
if hasattr(module, 'validate_' + key.lower()):
return self._recursive_validation(value, validator, module,
key, section_list)
else:
return value
# Remove non alphanumeric characters, except underscores
key = re.sub(r'\W+', '', key)
# Check layer
if isinstance(config, dict):
# Get layer configspec
try:
configspec = getattr(module, 'validate_' + key.lower())()
except AttributeError as a:
self._print_errors([(section_list, key, str(a))])
raise a
except Exception as e:
raise e
else:
# Set default section to allow string interpolation
configspec['DEFAULT'] = self.configspec['DEFAULT']
# Validate layer
config = ConfigObj(config, configspec=configspec)
results = config.validate(validator,
copy=True,
preserve_errors=True)
# Remove default section
del config['DEFAULT']
# Report extra (not defined) value errors
if self.allow_extra_values < 1:
# Get extra values
extra_values = get_extra_values(config)
# Filter out wrongly assigned dict values which are interpreted as sections
extra_values = list(
filter(lambda x: x[1] not in configspec, extra_values))
extra_values = list(
filter(lambda x: len(x[0]) < 1, extra_values))
# Set error message
msg = 'Value is not defined in the model validation file!'
# Print errors
self._print_extra_val_results(extra_values, section_list, msg)
# Remove extra values form config
list(
map(
config.__delitem__,
frozenset(map(lambda x: x[1], extra_values)) -
frozenset(configspec)))
# Report validation errors
self._print_val_results(config, results, section_list=section_list)
# Cast config
config = config.dict()
# Validate
for key, value in config.items():
section_list.append(key)
config[key] = _recursive_call(self, value, validator, module,
key, section_list)
del section_list[-1]
elif isinstance(config, (list, tuple)):
for i, value in enumerate(config):
if isinstance(value, str):
try:
value = ast.literal_eval(value)
except ValueError:
pass
except Exception as e:
raise e
config[i] = _recursive_call(self, value, validator, module,
key, section_list)
else:
config[i] = _recursive_call(self, value, validator, module,
key, section_list)
elif hasattr(module, 'validate_' + key.lower()):
# Get layer configspec
check = getattr(module, 'validate_' + key.lower())()
# Check configuration value
try:
return validator.check(check, config, missing=False)
except ValidateError as v:
self._print_val_results({}, {key: v},
section_list=section_list)
except Exception as e:
raise e
return config
def get_config(
config_path=None,
_get_color_from_vdir=get_color_from_vdir,
_get_vdir_type=get_vdir_type):
"""reads the config file, validates it and return a config dict
:param config_path: path to a custom config file, if none is given the
default locations will be searched
:type config_path: str
:param _get_color_from_vdir: override get_color_from_vdir for testing purposes
:param _get_vdir_type: override get_vdir_type for testing purposes
:returns: configuration
:rtype: dict
"""
if config_path is None:
config_path = find_configuration_file()
if config_path is None or not os.path.exists(config_path):
raise NoConfigFile()
logger.debug('using the config file at {}'.format(config_path))
try:
user_config = ConfigObj(config_path,
configspec=SPECPATH,
interpolation=False,
file_error=True,
)
except ConfigObjError as error:
logger.fatal('parsing the config file with the following error: '
'{}'.format(error))
logger.fatal('if you recently updated khal, the config file format '
'might have changed, in that case please consult the '
'CHANGELOG or other documentation')
raise CannotParseConfigFileError()
fdict = {'timezone': is_timezone,
'timedelta': is_timedelta,
'expand_path': expand_path,
'expand_db_path': expand_db_path,
'weeknumbers': weeknumber_option,
'monthdisplay': monthdisplay_option,
'color': is_color,
}
validator = Validator(fdict)
results = user_config.validate(validator, preserve_errors=True)
abort = False
for section, subsection, config_error in flatten_errors(user_config, results):
abort = True
if isinstance(config_error, Exception):
logger.fatal(
'config error:\n'
'in [{}] {}: {}'.format(section[0], subsection, config_error))
else:
for key in config_error:
if isinstance(config_error[key], Exception):
logger.fatal('config error:\nin {} {}: {}'.format(
sectionize(section + [subsection]),
key,
str(config_error[key]))
)
if abort or not results:
raise InvalidSettingsError()
config_checks(user_config, _get_color_from_vdir, _get_vdir_type)
extras = get_extra_values(user_config)
for section, value in extras:
if section == ():
logger.warning('unknown section "{}" in config file'.format(value))
else:
section = sectionize(section)
logger.warning(
'unknown key or subsection "{}" in section "{}"'.format(value, section))
deprecated = [{'value': 'default_command', 'section': 'default'}]
for d in deprecated:
if (value == d['value']) and (section == d['section']):
logger.warning('Key "{}" in section "{}" was deprecated. '
'See the FAQ to find out when and why!'.format(value, section))
return user_config
def __init__(self, config='pools.conf'):
"""
Аргументы:
config: путь к файлу конфигурации"""
# Шаблон файла конфигурации
template = """
[Script]
Debug = boolean(default=False)
[Zabbix]
Server = ip_addr(default=127.0.0.1)
Port = integer(min=1, max=65535, default=10051)
Send = boolean(default=True)
Log = string(default=False)
[__many__]
Pool = option('FlyPool', 'NanoPool')
Coin = option('Etherium', 'Monero', 'ZCash')
Host = string
Port = integer(min=1, max=65535, default=443)
Timeout= integer(min=1, max=60, default=5)
Account = string
Workers = string_list
[[Request]]
Name = string
[[[Args]]]
worker = string(default=None)
hours = integer(min=1, max=24, default=None)
"""
# Загружаем конфигурацию
try:
self.__config = ConfigObj(config,
configspec=template.split('\n'),
file_error=True)
except (ConfigObjError, IOError) as e:
# Ошибка загрузки конфигурации
raise IOError("could not read config file'{file}': {error}".format(
file=config, error=e)) from None
# Проверяем файл конфигурации
validator = Validator()
result = self.__config.validate(validator, preserve_errors=True)
# Если возникли ошибки
if result is not True:
for (sections, key,
result) in flatten_errors(self.__config, result):
if key is not None:
raise ValueError(
"the key '{key}' in the section '{section}' of config file failed validation: {result}"
.format(key=key,
section=', '.join(sections),
result=result if result else "missing key"))
else:
raise ValueError(
"the following section in config file was missing: '{section}'"
.format(section=', '.join(sections)))
# Проверяем на наличие дополнительных параметров
for section, key in get_extra_values(self.__config):
raise ValueError(
"extra section or key '{key}' in section '{section}' of config file"
.format(key=key,
section=', '.join(section) if section else None))
def parse_config(test_config=None):
"""
Parse config file. Exits on error.
:return:
"""
def dict_to_class(the_class: str, section: str, list: Tuple):
"""
Set class attributes from parsed config data
:param the_class: Name of class
:param section: Name of config section
:param list: list of section keywords
:return:
"""
for item in list:
setattr(globals()[the_class], item, config[section][item])
the_config_spec = configobj.ConfigObj(io.StringIO(initial_value=configspec,
newline='\n'),
_inspec=True,
encoding='UTF8')
config = None
for config_file in (test_config, options.config_file,
sys.prefix + '/etc/serverpki.conf',
'/usr/local/etc/serverPKI/serverpki.conf'):
if not config_file:
continue
sld('Trying config file {}'.format(config_file))
if not os.path.isfile(config_file):
continue
sli('Using config file {}'.format(config_file))
try:
config = configobj.ConfigObj(config_file,
encoding='UTF8',
interpolation='Template',
configspec=the_config_spec)
except SyntaxError as e:
sle('Configuration file errors found. Can'
't continue. List of errors:')
for err in e.errors:
sle('{}'.format(err))
sys.exit(1)
break
if not config:
sle('No config file found. Can' 't continue.')
sys.exit(1)
try:
vtor = validate.Validator()
result = config.validate(vtor, preserve_errors=True)
except configobj.MissingInterpolationOption as e:
sle('Substitution error: {}. Can' 't continue.'.format(e.msg))
sys.exit(1)
if result != True:
sle('Config validation failed:')
for entry in configobj.flatten_errors(config, result):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error == False:
error = 'Missing value or section.'
sle(section_string + ' = ' + str(error))
for sections, name in configobj.get_extra_values(config):
# this code gets the extra values themselves
the_section = config
for section in sections:
the_section = the_section[section]
# the_value may be a section or a value
the_value = the_section[name]
section_or_value = 'value'
if isinstance(the_value, dict):
# Sections are subclasses of dict
section_or_value = 'section'
section_string = ', '.join(sections) or "top level"
print('Extra entry in section: %s. Entry %r is a %s' %
(section_string, name, section_or_value))
sld(str(result))
def test_walk(section, key):
sld('{}[{}] {} = {}'.format(
'[' + section.parent.name + ']' if section.parent.name else '',
section.name, key, section[key]))
config.walk(test_walk)
def add_attribute_to_class(section: configobj, key: str):
"""
If called by config.walk, filles corresponding container class with class variable
:param section: The section
:param key: The key of attribute
:return:
"""
def flatten_list(value: List) -> str:
"""
Make string from list
:param value: list
:return:
"""
result = ''
first = True
for e in value:
if not first:
result += ', '
first = False
result += e
return result
value = section[key]
if section.name == 'Pathes' and key == 'zone_tlsa_inc_mode':
value = int(value, 8) # file permission is octal
elif (section.name == 'Misc'
and key == 'MAIL_RECIPIENT') or (section.name == 'DBAccount'
and key == 'dbSearchPath'):
result = ''
first = True
for e in value:
if not first:
result += ', '
first = False
result += e
value = result
if not section.parent.name:
setattr(globals()[section.name], key, value)
else:
try:
d = getattr(globals()[section.parent.name], section.name)
except AttributeError:
d = {}
setattr(globals()[section.parent.name], section.name, d)
d[key] = value
config.walk(add_attribute_to_class)
## re_init_syslog() FIXME: openlog works only once
sld(DBAccount.dbHost + ' ' + str(DBAccount.dbPort) + ' ' +
DBAccount.dbUser + ' ' + DBAccount.dbCert)
return
def get_config(config_path=None):
"""reads the config file, validates it and return a config dict
:param config_path: path to a custom config file, if none is given the
default locations will be searched
:type config_path: str
:returns: configuration
:rtype: dict
"""
if config_path is None:
config_path = _find_configuration_file()
logger.debug('using the config file at {}'.format(config_path))
try:
user_config = ConfigObj(config_path,
configspec=SPECPATH,
interpolation=False,
file_error=True,
)
except ConfigObjError as error:
logger.fatal('parsing the config file file with the following error: '
'{}'.format(error))
logger.fatal('if you recently updated khal, the config file format '
'might have changed, in that case please consult the '
'CHANGELOG or other documentation')
raise CannotParseConfigFileError()
fdict = {'timezone': is_timezone,
'expand_path': expand_path,
'expand_db_path': expand_db_path,
'weeknumbers': weeknumber_option,
}
validator = Validator(fdict)
results = user_config.validate(validator, preserve_errors=True)
abort = False
for section, subsection, error in flatten_errors(user_config, results):
abort = True
if isinstance(error, Exception):
logger.fatal(
'config error:\n'
'in [{}] {}: {}'.format(section[0], subsection, error))
else:
for key in error:
if isinstance(error[key], Exception):
logger.fatal('config error:\nin {} {}: {}'.format(
sectionize(section + [subsection]),
key,
str(error[key]))
)
if abort or not results:
raise InvalidSettingsError()
config_checks(user_config)
extras = get_extra_values(user_config)
for section, value in extras:
if section == ():
logger.warn('unknown section "{}" in config file'.format(value))
else:
section = sectionize(section)
logger.warn('unknown key or subsection "{}" in '
'section "{}"'.format(value, section))
return user_config
def read_config(configpath=None, specpath=None, checks=None,
report_extra=False):
"""
get a (validated) config object for given config file path.
:param configpath: path to config-file or a list of lines as its content
:type configpath: str or list(str)
:param specpath: path to spec-file
:type specpath: str
:param checks: custom checks to use for validator.
see `validate docs <http://www.voidspace.org.uk/python/validate.html>`_
:type checks: dict str->callable,
:param report_extra: log if a setting is not present in the spec file
:type report_extra: boolean
:raises: :class:`~alot.settings.errors.ConfigError`
:rtype: `configobj.ConfigObj`
"""
checks = checks or {}
try:
config = ConfigObj(infile=configpath, configspec=specpath,
file_error=True, encoding='UTF8')
except ConfigObjError as e:
msg = 'Error when parsing `%s`:\n%s' % (configpath, e)
logging.error(msg)
raise ConfigError(msg)
except IOError:
raise ConfigError('Could not read %s and/or %s'
% (configpath, specpath))
except UnboundLocalError:
# this works around a bug in configobj
msg = '%s is malformed. Check for sections without parents..'
raise ConfigError(msg % configpath)
if specpath:
validator = Validator()
validator.functions.update(checks)
try:
results = config.validate(validator, preserve_errors=True)
except ConfigObjError as e:
raise ConfigError(str(e))
if results is not True:
error_msg = ''
for (section_list, key, res) in flatten_errors(config, results):
if key is not None:
if res is False:
msg = 'key "%s" in section "%s" is missing.'
msg = msg % (key, ', '.join(section_list))
else:
msg = 'key "%s" in section "%s" failed validation: %s'
msg = msg % (key, ', '.join(section_list), res)
else:
msg = 'section "%s" is missing' % '.'.join(section_list)
error_msg += msg + '\n'
raise ConfigError(error_msg)
extra_values = get_extra_values(config) if report_extra else None
if extra_values:
msg = ['Unknown values were found in `%s`. Please check for '
'typos if a specified setting does not seem to work:'
% configpath]
for sections, val in extra_values:
if sections:
msg.append('%s: %s' % ('->'.join(sections), val))
else:
msg.append(str(val))
logging.info('\n'.join(msg))
return config
def read_config(configpath=None,
specpath=None,
checks=None,
report_extra=False):
"""
get a (validated) config object for given config file path.
:param configpath: path to config-file or a list of lines as its content
:type configpath: str or list(str)
:param specpath: path to spec-file
:type specpath: str
:param checks: custom checks to use for validator.
see `validate docs <http://www.voidspace.org.uk/python/validate.html>`_
:type checks: dict str->callable,
:param report_extra: log if a setting is not present in the spec file
:type report_extra: boolean
:raises: :class:`~alot.settings.errors.ConfigError`
:rtype: `configobj.ConfigObj`
"""
checks = checks or {}
try:
config = ConfigObj(infile=configpath,
configspec=specpath,
file_error=True,
encoding='UTF8')
except ConfigObjError as e:
msg = 'Error when parsing `%s`:\n%s' % (configpath, e)
logging.error(msg)
raise ConfigError(msg)
except IOError:
raise ConfigError('Could not read %s and/or %s' %
(configpath, specpath))
except UnboundLocalError:
# this works around a bug in configobj
msg = '%s is malformed. Check for sections without parents..'
raise ConfigError(msg % configpath)
if specpath:
validator = Validator()
validator.functions.update(checks)
try:
results = config.validate(validator, preserve_errors=True)
except ConfigObjError as e:
raise ConfigError(str(e))
if results is not True:
error_msg = ''
for (section_list, key, res) in flatten_errors(config, results):
if key is not None:
if res is False:
msg = 'key "%s" in section "%s" is missing.'
msg = msg % (key, ', '.join(section_list))
else:
msg = 'key "%s" in section "%s" failed validation: %s'
msg = msg % (key, ', '.join(section_list), res)
else:
msg = 'section "%s" is missing' % '.'.join(section_list)
error_msg += msg + '\n'
raise ConfigError(error_msg)
extra_values = get_extra_values(config) if report_extra else None
if extra_values:
msg = [
'Unknown values were found in `%s`. Please check for '
'typos if a specified setting does not seem to work:' %
configpath
]
for sections, val in extra_values:
if sections:
msg.append('%s: %s' % ('->'.join(sections), val))
else:
msg.append(str(val))
logging.info('\n'.join(msg))
return config
# Validation failed
failed_items = flatten_errors( cfg, test )
# Always print reason for validation failure
for item in failed_items:
sections, key, result = item
print >> sys.stderr, ' ',
for sec in sections:
print >> sys.stderr, sec, ' / ',
print >> sys.stderr, key
if result == False:
print >> sys.stderr, "ERROR, required item missing."
else:
print >> sys.stderr, result
raise SystemExit( "ERROR gcylc.rc validation failed")
extras = []
for sections, name in get_extra_values( cfg ):
extra = ' '
for sec in sections:
extra += sec + ' / '
extras.append( extra + name )
if len(extras) != 0:
for extra in extras:
print >> sys.stderr, ' ERROR, illegal entry:', extra
raise SystemExit( "ERROR illegal gcylc.rc entry(s) found" )
# combine user config into default config
self.inherit( dcfg, ucfg )
# theme inheritance
if force_theme:
# override
def __init__(self, path=None):
if path is None:
path = os.getcwd()
res_path = resource_path()
self.path = path
self.resource_path = res_path
settings_path = os.path.join(path, 'settings.ini')
default_settings_path = os.path.join(res_path, 'settings.ini.default')
settings = ConfigObj(settings_path,
configspec=default_settings_path,
encoding='utf-8')
settings.filename = settings_path
class Boolean:
def __init__(self, value):
self._str = value
self._bool = is_boolean(value)
def __bool__(self):
return self._bool
def __str__(self):
return self._str
validator = Validator(dict(boolean=lambda x: Boolean(x)))
result = settings.validate(validator, preserve_errors=True, copy=True)
for sections, key in get_extra_values(settings):
section = settings
for section_name in sections:
section = section[section_name]
del section[key]
for sections, key, result in flatten_errors(settings, result):
section = settings
for section_name in sections:
section = section[section_name]
del section[key]
assert settings.validate(validator, preserve_errors=False, copy=True)
default_settings = ConfigObj(default_settings_path,
configspec=default_settings_path,
encoding='utf-8')
default_settings.merge(settings)
settings = default_settings
settings.filename = settings_path
settings.write()
self.settings = settings
self.black_flash = settings["Don't Blind Me!"]['black_flash']
self.black_smoke = settings["Don't Blind Me!"]['black_smoke']
self.ignore_temperature = False
self.host = settings['Game State Integration']['host']
self.port = settings['Game State Integration'].as_int('port')
gamestate_integration_cfg_template_path = os.path.join(
res_path, 'gamestate_integration_dont_blind_me.cfg.template')
with open(gamestate_integration_cfg_template_path) as f:
gamestate_integration_cfg_template = f.read()
gamestate_integration_cfg_path = os.path.join(
path, 'gamestate_integration_dont_blind_me.cfg')
with open(gamestate_integration_cfg_path, mode='w') as f:
f.write(
gamestate_integration_cfg_template.format(host=self.host,
port=self.port))
self.mat_monitorgamma = settings['Video Settings'].as_float(
'mat_monitorgamma')
self.mat_monitorgamma_tv_enabled = settings['Video Settings'].as_bool(
'mat_monitorgamma_tv_enabled')
self.temperature = [None, 6500]
self.round_phase = [None, None]
self.player_alive = None
self.player_flashed = [None, 0]
self.player_smoked = [None, 0]
self.context = Context.open()
def __init__(self, config='miners.conf'):
"""Аргументы:
config: путь к файлу конфигурации
"""
# Шаблон файла конфигурации
template = """
[Script]
Template = string(default=miners.template.html)
Refresh = integer(min=10, max=3600, default=30)
Url = string(default=cgi-bin/miners.py/)
Colorize = boolean(default=False)
Debug = boolean(default=False)
[Zabbix]
Server = ip_addr(default=127.0.0.1)
Port = integer(default=10051)
Log = string(default=False)
Send = boolean(default=False)
[__many__]
Host = ip_addr
Port = integer(min=1, max=65535)
Miner = string
Request = force_list
Timeout= integer(min=1, max=60, default=5)
Description = string
"""
# Загружаем конфигурацию
try:
self.__config = ConfigObj(
config,
configspec=template.split('\n'),
file_error=True,
)
except (ConfigObjError, IOError) as e:
# Ошибка загрузки конфигурации
raise IOError(
"could not read config file'{file}': {error}".format(
file=config,
error=e,
), ) from None
# Проверяем файл конфигурации
validator = Validator()
result = self.__config.validate(validator, preserve_errors=True)
# Если возникли ошибки
if result is not True:
for (sections, key,
result) in flatten_errors(self.__config, result):
if key is not None:
if key != 'Description':
raise ValueError(
"the key '{key}' in the section '{section}' "
"of config file failed validation: "
"{result}".format(
key=key,
section=', '.join(sections),
result=result if result else "missing key",
), )
else:
raise ValueError(
"the following section in "
"config file was missing: '{section}'".format(
section=', '.join(sections), ), )
# Проверяем на наличие дополнительных параметров
for section, key in get_extra_values(self.__config):
raise ValueError(
"extra section or key '{key}' "
"in section '{section}' of config file".format(
key=key,
section=', '.join(section) if section else None,
), )
def read_options_file(KOLIBRI_HOME, ini_filename="options.ini"):
logger = _get_logger(KOLIBRI_HOME)
ini_path = os.path.join(KOLIBRI_HOME, ini_filename)
conf = ConfigObj(ini_path, configspec=get_configspec())
# validate once up front to ensure section structure is in place
conf.validate(_get_validator())
# keep track of which options were overridden using environment variables, to support error reporting
using_env_vars = {}
# override any values from their environment variables (if set)
for section, opts in option_spec.items():
for optname, attrs in opts.items():
for envvar in attrs.get("envvars", []):
if os.environ.get(envvar):
logger.info(
"Option {optname} in section [{section}] being overridden by environment variable {envvar}".format(
optname=optname, section=section, envvar=envvar
)
)
conf[section][optname] = os.environ[envvar]
using_env_vars[optname] = envvar
break
conf = clean_conf(conf)
validation = conf.validate(_get_validator(), preserve_errors=True)
# loop over and display any errors with config values, and then bail
if validation is not True:
for section_list, optname, error in flatten_errors(conf, validation):
section = section_list[0]
if optname in using_env_vars:
logger.error(
"Error processing environment variable option {envvar}: {error}".format(
envvar=using_env_vars[optname], error=error
)
)
else:
logger.error(
"Error processing {file} under section [{section}] for option {option}: {error}".format(
file=ini_path, section=section, option=optname, error=error
)
)
logger.critical(
"Aborting: Could not process options config (see errors above for more details)"
)
raise SystemExit(1)
# loop over any extraneous options and warn the user that we're ignoring them
for sections, name in get_extra_values(conf):
# this code gets the extra values themselves
the_section = conf
for section in sections:
the_section = the_section[section]
# the_value may be a section or a value
the_value = the_section.pop(name)
# determine whether the extra item is a section (dict) or value
kind = "section" if isinstance(the_value, dict) else "option"
logger.warn(
"Ignoring unknown {kind} in options file {file} under {section}: {name}.".format(
kind=kind,
file=ini_path,
section=sections[0] if sections else "top level",
name=name,
)
)
# run validation once again to fill in any default values for options we deleted due to issues
conf.validate(_get_validator())
# ensure all arguments under section "Paths" are fully resolved and expanded, relative to KOLIBRI_HOME
_expand_paths(KOLIBRI_HOME, conf.get("Paths", {}))
return conf
def get_config(config_path=None):
"""reads the config file, validates it and return a config dict
:param config_path: path to a custom config file, if none is given the
default locations will be searched
:type config_path: str
:returns: configuration
:rtype: dict
"""
if config_path is None:
config_path = _find_configuration_file()
logger.debug('using the config file at {}'.format(config_path))
try:
user_config = ConfigObj(config_path,
configspec=SPECPATH,
interpolation=False,
file_error=True,
)
except ConfigObjError as error:
logger.fatal('parsing the config file file with the following error: '
'{}'.format(error))
logger.fatal('if you recently updated khal, the config file format '
'might have changed, in that case please consult the '
'CHANGELOG or other documentation')
raise CannotParseConfigFileError()
fdict = {'timezone': is_timezone,
'expand_path': expand_path,
'expand_db_path': expand_db_path,
'weeknumbers': weeknumber_option,
}
validator = Validator(fdict)
results = user_config.validate(validator, preserve_errors=True)
abort = False
for section, subsection, error in flatten_errors(user_config, results):
abort = True
if isinstance(error, Exception):
logger.fatal(
'config error:\n'
'in [{}] {}: {}'.format(section[0], subsection, error))
else:
for key in error:
if isinstance(error[key], Exception):
logger.fatal('config error:\nin {} {}: {}'.format(
sectionize(section + [subsection]),
key,
str(error[key]))
)
if abort or not results:
raise InvalidSettingsError()
config_checks(user_config)
extras = get_extra_values(user_config)
for section, value in extras:
if section == ():
logger.warn('unknown section "{}" in config file'.format(value))
else:
section = sectionize(section)
logger.warn('unknown key or subsection "{}" in '
'section "{}"'.format(value, section))
return user_config
