def validate_config(self, configspec, suppress_warnings=False): """ Validate this config with the given configspec """ self._handle_configspec(configspec) errors = self.validate(validator, preserve_errors=True) for entry in flatten_errors(self, errors): section_list, key, error = entry if not error: LOGGER.error("Missing parameter %s", '.'.join(section_list + [key])) else: LOGGER.error("Configuration error %s: %s", '.'.join(section_list + [key]), error) # warn about any unknown parameters before we potentially abort on # validation errors if not suppress_warnings: for sections, name in get_extra_values(self): LOGGER.warn("Unknown parameter '%s' in section '%s'", name, ".".join(sections)) if errors is not True: raise ConfigError( "Configuration errors were encountered while validating %r" % self.filename) return errors
def test_get_extra_values(self): log.info("Test getting extra values.") from configobj import ConfigObj, get_extra_values from validate import Validator log.debug("Using ini file %r.", self.ini_file) log.debug("Using spec file %r.", self.spec_file) conf = ConfigObj(self.ini_file, configspec = self.spec_file) conf.validate(Validator(), preserve_errors = True) log.debug("ConfigObj after validate: %s", str(conf)) expected = sorted([ ((), 'extra'), ((), 'extra-section'), (('section', 'sub-section'), 'extra'), (('section',), 'extra-sub-section'), ]) log.debug("Expected extra values: %r.", expected) extra_values = get_extra_values(conf) log.debug("Got extra values: %r.", extra_values) self.assertEqual(sorted(extra_values), expected)
def collect_validation_messages(conf, error_list): validator = validate.Validator() conf.validate(validator, preserve_errors=True) message = [] cm_extras = { 'hadoop_hdfs_home': [('hadoop', 'hdfs_clusters', 'default')], 'hadoop_bin': [('hadoop', 'hdfs_clusters', 'default'), ('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_mapred_home': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_conf_dir': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'ssl_cacerts': [('beeswax', 'ssl'), ('impala', 'ssl')], 'remote_data_dir': [('liboozie', )], 'shell': [()] } whitelist_extras = ((sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or (name in cm_extras.keys() and sections in cm_extras[name]))) for sections, name in whitelist_extras: the_section = conf hierarchy_sections_string = '' try: parent = conf for section in sections: the_section = parent[section] hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " " parent = the_section except KeyError, ex: LOG.warn("Section %s not found: %s" % (section, str(ex))) the_value = '' try: # the_value may be a section or a value the_value = the_section[name] except KeyError, ex: LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
def get_config(config_path=None): """reads the config file, validates it and return a config dict :param config_path: path to a custom config file, if none is given the default locations will be searched :type config_path: str :returns: configuration :rtype: dict """ if config_path is None: config_path = _find_configuration_file() logger.debug('using the config file at {}'.format(config_path)) config = ConfigObj(DEFAULTSPATH, interpolation=False) try: user_config = ConfigObj(config_path, configspec=SPECPATH, interpolation=False, file_error=True, ) except ConfigObjError as error: logger.fatal('parsing the config file file with the following error: ' '{}'.format(error)) logger.fatal('if you recently updated khal, the config file format ' 'might have changed, in that case please consult the ' 'CHANGELOG or other documentation') sys.exit(1) fdict = {'timezone': is_timezone, 'expand_path': expand_path, } validator = Validator(fdict) results = user_config.validate(validator, preserve_errors=True) if not results: for entry in flatten_errors(config, results): # each entry is a tuple section_list, key, error = entry if key is not None: section_list.append(key) else: section_list.append('[missing section]') section_string = ', '.join(section_list) if error is False: error = 'Missing value or section.' print(section_string, ' = ', error) raise ValueError # TODO own error class config.merge(user_config) config_checks(config) extras = get_extra_values(user_config) for section, value in extras: if section == (): logger.warn('unknown section "{}" in config file'.format(value)) else: section = sectionize(section) logger.warn('unknown key or subsection "{}" in ' 'section "{}"'.format(value, section)) return config
def collect_validation_messages(conf, error_list): validator = validate.Validator() conf.validate(validator, preserve_errors=True) message = [] cm_extras = { 'hadoop_hdfs_home': [('hadoop', 'hdfs_clusters', 'default')], 'hadoop_bin': [('hadoop', 'hdfs_clusters', 'default'), ('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_mapred_home': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_conf_dir': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'ssl_cacerts': [('beeswax', 'ssl'), ('impala', 'ssl')], 'remote_data_dir': [('liboozie', )], 'shell': [()], } whitelist_extras = ( (sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or ( name in list(cm_extras.keys()) and sections in cm_extras[name]))) for sections, name in whitelist_extras: the_section = conf hierarchy_sections_string = '' try: parent = conf for section in sections: the_section = parent[section] hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " " parent = the_section except KeyError as ex: LOG.warning("Section %s not found: %s" % (section, str(ex))) the_value = '' try: # the_value may be a section or a value the_value = the_section[name] except KeyError as ex: LOG.warning("Error in accessing Section or Value %s: %s" % (name, str(ex))) section_or_value = 'keyvalue' if isinstance(the_value, dict): # Sections are subclasses of dict section_or_value = 'section' section_string = hierarchy_sections_string or "top level" message.append('Extra %s, %s in the section: %s' % (section_or_value, name, section_string)) if message: error = { 'name': 'ini configuration', 'message': ', '.join(message), } error_list.append(error)
def test_get_extra_values(conf): conf.validate(Validator(), preserve_errors=True) extra_values = get_extra_values(conf) expected = sorted([ ((), 'extra'), ((), 'extra-section'), (('section', 'sub-section'), 'extra'), (('section',), 'extra-sub-section'), ]) assert sorted(extra_values) == expected
def test_get_extra_values(self): conf = ConfigObj(inipath, configspec=specpath) conf.validate(Validator(), preserve_errors=True) extra_values = get_extra_values(conf) expected = sorted([ ((), 'extra'), ((), 'extra-section'), (('section', 'sub-section'), 'extra'), (('section',), 'extra-sub-section'), ]) self.assertEqual(sorted(extra_values), expected)
def __get_confobj(cls, config): spec_lines = resource_stream("vpnporthole", "resources/settings.spec").readlines() confobj = ConfigObj(config, configspec=spec_lines, raise_errors=True) result = confobj.validate(Validator()) if result is not True: # print(result) pass extra = get_extra_values(confobj) if extra: # print(extra) pass # print (confobj) return confobj
def validate_config(config, spec): """ Perform validation against <config> using <spec>, returning type-casted ConfigObj, any errors, and any extra values. :param config: [ConfigObj] :param spec: [string] filepath to config specification file :return: [dict] {config: ConfigObj, errors: {<section>: <error>, ...}, missing: [<section>, ...] extras: {<section>: <value>, ...} } """ from validate import Validator from configobj import (ConfigObj, flatten_errors, get_extra_values) _return = {'config': None, 'errors': {}, 'missing': [], 'extras': {}} _validator = Validator() _validator.functions['filepath'] = filepath _config = ConfigObj(config, configspec=spec, stringify=True) _result = _config.validate(_validator, preserve_errors=True) # http://configobj.readthedocs.io/en/latest/configobj.html#flatten-errors for _entry in flatten_errors(_config, _result): _section_list, _key, _error = _entry if _key is not None: _return['errors'][_key] = _error _section_list.append(_key) elif _error is False: _return['missing'].append(_key) # http://configobj.readthedocs.io/en/latest/configobj.html#get-extra-values for _sections, _name in get_extra_values(_config): # this code gets the extra values themselves _the_section = _config for _subsection in _sections: _the_section = _the_section[_subsection] # the_value may be a section or a value _the_value = _the_section[_name] _return['extras'][_name] = _the_value _return['config'] = _config return _return
def _validate_config(self): """Check for errors in the configuration file. Bail out if required sections/values are missing, or if extra sections/values are present.""" result = self.config.validate(Validator(), preserve_errors=True, copy=True) error_msgs = [] for sections, name in get_extra_values(self.config): the_section = self.config try: for section in sections: the_section = self.config[section] except KeyError: continue the_value = the_section[name] section_or_value = 'value' if isinstance(the_value, dict): section_or_value = 'section' section_string = ', '.join(sections) or 'top level' msg = 'E: extra %s entry in %s section: %r' % \ (section_or_value, section_string, name) error_msgs.append(msg) for entry in flatten_errors(self.config, result): section_list, key, error = entry if key is None: section_list.append('section missing') else: section_list.append(key) if error == False: error = 'value or section missing' msg = 'E: config%s: %s' % \ (''.join(["['%s']" % s for s in section_list]), error) error_msgs.append(msg) if error_msgs: error_msgs.insert(0, 'config file failed validation: %s' % self.config_file.name) raise ConfigError('\n'.join(error_msgs))
def load_config(config_file, schema, postprocessing=None, retain=None, drop=None): """ Loads a configobj-type configuration file. It is also possible to specify an application-specific postprocessing function which has three arguments: the configuration object and the warnings and errors lists. These are also the return values from the function. It is possible to specify a list of section titles that are retained (in this case, or others are dropped) or dropped. This allows more fine grain control over which sections need to be validated and which do not. """ def keep_section(sections, retained, dropped): """Tells if the section path in question should be kept.""" secs = set(sections) if retained and not secs & retained: return False if dropped and secs & dropped: return False return True config = configobj.ConfigObj(get_config_file(config_file), configspec=get_config_file(schema)) warnings, errors = [], [] v = Validator() results = config.validate(v, preserve_errors=True) if postprocessing: postprocessing(config, warnings, errors) retained = set(retain) if retain else set() dropped = set(drop) if drop else set() for sections, key in configobj.get_extra_values(config): if keep_section(sections, retained, dropped): warnings.append('Undefined key {}'.format('.'.join( (chain(sections, [key]))))) for sections, key, error in configobj.flatten_errors(config, results): if keep_section(sections, retained, dropped): errors.append('{}: {}'.format('.'.join((chain(sections, [key]))), error)) return config, warnings, errors
def validate(self, config, value_sources): """ Validate configuration Args: config (:obj:`ConfigObj`): configuration Raises: :obj:`InvalidConfigError`: if configuration doesn't validate against schema :obj:`ValueError`: if no configuration is found """ validator = Validator() validator.functions['any'] = any_checker result = config.validate(validator, copy=True, preserve_errors=True) if result is not True: raise InvalidConfigError(value_sources, config, result) if get_extra_values(config): raise ExtraValuesError(value_sources, config)
def validateconfig(self): """ Test config against configspec and print errors if it doesn't conform. This code is currently not used Things that need to be fixed: * Config validation should not overwrite default value from paramlist in steps * If a keyword is missing in the [HEADER] section config validation shouldn't put it in. """ errFlag = False # set to true if errors encountered results = self.config.validate(validate.Validator(), copy=True) extra = configobj.get_extra_values(self.config) # keywords not in spec # List configuration keywords that failed validation if results != True: for (section_list, key, _) in configobj.flatten_errors(self.config, results): blah = ', '.join(section_list) if key is not None: msg = "ValidateConfig: key '%s' in section '%s' failed" self.log.error(msg % (key, blah)) errFlag = True else: self.log.error("ValidateConfig: Section '%s' failed" % blah) errFlag = True # Warn for keywords not found in validation for s, k in extra: if len(s) == 0: msg = 'ValidateConfig: Skippping unknown global keyword "%s"' self.log.warning(msg % k) else: msg = 'ValidateConfig: Skippping unknown keyword "%s" in section "%s"' self.log.warning(msg % (k, s[0])) if errFlag: raise validate.ValidateError self.log.debug('ValidateConfig: done')
def __init__(self, sources, config): """ Args: sources (:obj:`list` of :obj:`str`): list of sources of configuration values config (:obj:`configobj.ConfigObj`): configuration """ self.sources = sources self.config = config messages = [] # todo: ensure that self.msg is generated even if this for loop raises another exception for section_list, name in get_extra_values(config): # this code gets the extra values themselves the_section = config for i_section, section in enumerate(section_list): if section in config: the_section = config[section] else: section_list = section_list[0:i_section] name = section break # the_value may be a section or a value the_value = the_section[name] section_or_value = 'value' if isinstance(the_value, dict): # Sections are subclasses of dict section_or_value = 'section' section_string = ', '.join(section_list) or "top level" messages.append( "Extra entry in section '{:s}'. Entry '{}' is a {:s}.".format( section_string, name, section_or_value)) self.msg = ('The following configuration sources\n {}\n\n' 'contain the following configuration errors\n {}').format( '\n '.join(sources), '\n '.join(messages))
def collect_validation_messages(conf, error_list): validator = validate.Validator() conf.validate(validator, preserve_errors=True) message = [] for sections, name in get_extra_values(conf): the_section = conf hierarchy_sections_string = '' try: parent = conf for section in sections: the_section = parent[section] hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " " parent = the_section except KeyError, ex: LOG.warn("Section %s not found: %s" % (section, str(ex))) the_value = '' try: # the_value may be a section or a value the_value = the_section[name] except KeyError, ex: LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
def check_extra_values(config, logger): """ Checks the config and warns the user if there are any extra entries in their config file. This function is based on suggested usage in the ConfigObj manual. Args: config (ConfigObj): A ConfigObj instance. logger (logger): The logger to which to write complaints. Returns: Nothing: Nothing. """ warnings = 0 for sections, name in get_extra_values(config): # this code gets the extra values themselves the_section = config for section in sections: the_section = the_section[section] # the_value may be a section or a value the_value = the_section[name] section_or_value = 'value' if isinstance(the_value, dict): # Sections are subclasses of dict section_or_value = 'section' section_string = ', '.join(sections) or "top level" logger.warning('Extra entry in section: %s: %s %r is not in spec.' % (section_string, section_or_value, name)) warnings += 1 if warnings: logger.warning('The extra value(s) may be the result of deprecated ' 'entries or other changes to the config files; please ' 'check the conifg files in shakemap/data for the most ' 'up to date specs.')
def test_extra(self): # test to __str__ config_specification = configobj.ConfigObj( debug_logs_default_paths.schema, list_values=False, _inspec=True) config = configobj.ConfigObj(configspec=config_specification) config.merge({'__extra__': True}) validator = Validator() result = config.validate(validator, preserve_errors=True) self.assertNotEqual(configobj.get_extra_values(config), None) # extra section extra = {'__extra__': True} with self.assertRaisesRegex(ExtraValuesError, "The following configuration sources"): ConfigManager(debug_logs_default_paths).get_config(extra) with self.assertRaisesRegex(ExtraValuesError, " 'extra' argument"): ConfigManager(debug_logs_default_paths).get_config(extra) with self.assertRaisesRegex( ExtraValuesError, "Extra entry in section 'top level'. Entry '__extra__' is a value" ): ConfigManager(debug_logs_default_paths).get_config(extra) # extra subsection, extra key extra = { 'debug_logs': { '__extra__': True, '__extra__2': { 'val': 'is_dict' } } } with self.assertRaisesRegex(ExtraValuesError, "Entry '__extra__2' is a section"): ConfigManager(debug_logs_default_paths).get_config(extra)
def validate_config(self, configspec, suppress_warnings=False): """ Validate this config with the given configspec """ self._handle_configspec(configspec) errors = self.validate(validator, preserve_errors=True) for entry in flatten_errors(self, errors): section_list, key, error = entry if not error: LOGGER.error("Missing parameter %s", '.'.join(section_list + [key])) else: LOGGER.error("Configuration error %s: %s", '.'.join(section_list + [key]), error) # warn about any unknown parameters before we potentially abort on # validation errors if not suppress_warnings: for sections, name in get_extra_values(self): LOGGER.warn("Unknown parameter '%s' in section '%s'", name, ".".join(sections)) if errors is not True: raise ConfigError("Configuration errors were encountered while validating %r" % self.filename) return errors
def check_settings(settings): validator = MorphologistConfigValidator() validation = settings.validate(validator, preserve_errors=True) settings_issues = flatten_errors(settings, validation) if settings_issues: print("Error: loading settings from file '%s'" % settings.filename) print("The following items have wrong values:") for sections, key, error in settings_issues: sections = '/'.join(sections) if key is None: key = '[Missing section]' if error == False: error = 'Missing value or section.' print(" - %s/%s: %s" % (sections, key, error)) return False extra_values = get_extra_values(settings) if extra_values: print("Error: unknown additionnal items in setting file '%s'" % settings.filename) for sections, name in extra_values: item = '/'.join(list(sections) + [name]) print(" -", item) return False return True
def get_cfg(cfg_file_name="Parameters.in", kvargs={}, err=sys.stderr): """ Parse a config """ def validate_convert(value, func, name=None): """Helper function for validate parameter conversion""" if value is None: return None try: return func(value) except (TypeError, ValueError): if name is None: raise validate.VdtValueError(value) else: raise validate.VdtParamError(name, value) def sci_int(val): """Converts type to int, honouring scientific notation if possible""" try: i = int(val) except ValueError: val = float(val) i = int(val) if type(val)(i) != val: raise ValueError("Not exactly representable as integer") return i def is_sci_integer(value, min=None, max=None): """Function for the validate module to support scientific notation""" value = validate_convert(value, sci_int) minval = validate_convert(min, sci_int, "min") maxval = validate_convert(max, sci_int, "max") if min is not None and value < minval: raise validate.VdtValueTooSmallError(value) if max is not None and value > maxval: raise validate.VdtValueTooBigError(value) return value def is_option_or_float_list(value, *options): try: return validate.is_option(value, *options) except validate.VdtTypeError: return validate.is_float_list(value) # config configspec = file(os.path.dirname(__file__) + '/configspec','r') cfg = configobj.ConfigObj(infile=file(cfg_file_name, 'r'), configspec=configspec.readlines(), indent_type="\t") # update command line parameters for key, value in kvargs.iteritems(): groups = key.split(".") parent = cfg for group in groups[:-1]: parent = parent[group] parent[groups[-1]] = value validator = validate.Validator() validator.functions["integer"] = is_sci_integer validator.functions["option_or_list"] = is_option_or_float_list valid = cfg.validate(validator, copy=True) try: pairs = configobj.get_extra_values(cfg) except AttributeError: print >> err, "WARNING: cannot check unknown entries in config" pairs = False if pairs: print >> err, "error: unknown entries in config: %s" % cfg_file_name print >> err, ">>>", ", ".join(".".join(e[0] + e[1:]) for e in pairs) raise CfgException() if valid is not True: print >> err, "error: invalid entries in config: %s" % cfg_file_name print >> err, ">>>", ", ".join(str(entry) for entry, ok in flat_items(valid) if not ok) raise CfgException() return cfg
def call(self, filename): # Get config config = self.read(filename) # Validate top level config validator = Validator({ 'dict': self._is_dict, 'integer_or_none': self._is_int_or_none, 'float_or_none': self._is_float_or_none, 'dict_list': self._is_dict_list, 'string_list': self._is_string_list, 'int_list': self._is_int_list, 'float_list': self._is_float_list, 'int_list_or_none': self._is_int_list_or_none, 'float_list_or_none': self._is_float_list_or_none, 'string_list_or_none': self._is_string_list_or_none }) results = config.validate(validator, preserve_errors=True) # Insert timestamp config['DEFAULT']['timestamp'] = config['DEFAULT'].get( 'timestamp', time.strftime("%Y%m%d-%H%M%S", time.gmtime())) # Get default values for the default section config['DEFAULT'] = {**self.configspec['DEFAULT'], **config['DEFAULT']} self.configspec['DEFAULT'] = config['DEFAULT'] # Report extra (not defined) sections if self.allow_extra_values < 1: # Get extra values extra_values = get_extra_values(config) # Filter out non top level values extra_values = list(filter(lambda x: len(x[0]) < 2, extra_values)) # Print errors msg = 'Section is not defined in the configspec file!' self._print_extra_val_results(extra_values, msg=msg) # Remove extra values form config for extra_value in extra_values: if not extra_value[0]: del config[extra_value[-1]] else: section_mane = extra_value[0][0] del config.get(section_mane)[extra_value[-1]] # Report validation errors self._print_val_results(config, results) # Validate preprocessing config['PREPROCESS'] = self._check_preprocess(section=ConfigObj( config['PREPROCESS']), validator=validator) # Validate generate config['GENERATE'] = self._check_generate(section=ConfigObj( config['GENERATE']), validator=validator) # Validate model config['MODEL'] = self._check_model(section=ConfigObj(config['MODEL']), validator=validator) # Validate train config['TRAIN'] = self._check_train(section=ConfigObj(config['TRAIN']), validator=validator) # Parse None values (destringify) config = self._destringify_none(config) # Raise exception if validation failed if not self.valid: raise ValidateError( 'Validation faild! Please see validation error messages!') return config
def _recursive_validation(self, config, validator, module, key: str, section_list: list): """ Returns a recursively validated config object. Inputs: config: Config object, <dict or ConfigObj>. Arguments: validator: Validator instance used to validate the config object, <Validator>. module: Module to get the configspec from, <class>. key: Key/name of the current config object, <str>. section_list: Section list ('path') of the current config object, <list[str, ..., str]>. Return: config: Validated config object, <dict>. """ # Recursive call of the validation def _recursive_call(self, value, validator, module, key, section_list): if isinstance(value, (dict, list)): return self._recursive_validation(value, validator, module, key, section_list) else: if hasattr(module, 'validate_' + key.lower()): return self._recursive_validation(value, validator, module, key, section_list) else: return value # Remove non alphanumeric characters, except underscores key = re.sub(r'\W+', '', key) # Check layer if isinstance(config, dict): # Get layer configspec try: configspec = getattr(module, 'validate_' + key.lower())() except AttributeError as a: self._print_errors([(section_list, key, str(a))]) raise a except Exception as e: raise e else: # Set default section to allow string interpolation configspec['DEFAULT'] = self.configspec['DEFAULT'] # Validate layer config = ConfigObj(config, configspec=configspec) results = config.validate(validator, copy=True, preserve_errors=True) # Remove default section del config['DEFAULT'] # Report extra (not defined) value errors if self.allow_extra_values < 1: # Get extra values extra_values = get_extra_values(config) # Filter out wrongly assigned dict values which are interpreted as sections extra_values = list( filter(lambda x: x[1] not in configspec, extra_values)) extra_values = list( filter(lambda x: len(x[0]) < 1, extra_values)) # Set error message msg = 'Value is not defined in the model validation file!' # Print errors self._print_extra_val_results(extra_values, section_list, msg) # Remove extra values form config list( map( config.__delitem__, frozenset(map(lambda x: x[1], extra_values)) - frozenset(configspec))) # Report validation errors self._print_val_results(config, results, section_list=section_list) # Cast config config = config.dict() # Validate for key, value in config.items(): section_list.append(key) config[key] = _recursive_call(self, value, validator, module, key, section_list) del section_list[-1] elif isinstance(config, (list, tuple)): for i, value in enumerate(config): if isinstance(value, str): try: value = ast.literal_eval(value) except ValueError: pass except Exception as e: raise e config[i] = _recursive_call(self, value, validator, module, key, section_list) else: config[i] = _recursive_call(self, value, validator, module, key, section_list) elif hasattr(module, 'validate_' + key.lower()): # Get layer configspec check = getattr(module, 'validate_' + key.lower())() # Check configuration value try: return validator.check(check, config, missing=False) except ValidateError as v: self._print_val_results({}, {key: v}, section_list=section_list) except Exception as e: raise e return config
def get_config( config_path=None, _get_color_from_vdir=get_color_from_vdir, _get_vdir_type=get_vdir_type): """reads the config file, validates it and return a config dict :param config_path: path to a custom config file, if none is given the default locations will be searched :type config_path: str :param _get_color_from_vdir: override get_color_from_vdir for testing purposes :param _get_vdir_type: override get_vdir_type for testing purposes :returns: configuration :rtype: dict """ if config_path is None: config_path = find_configuration_file() if config_path is None or not os.path.exists(config_path): raise NoConfigFile() logger.debug('using the config file at {}'.format(config_path)) try: user_config = ConfigObj(config_path, configspec=SPECPATH, interpolation=False, file_error=True, ) except ConfigObjError as error: logger.fatal('parsing the config file with the following error: ' '{}'.format(error)) logger.fatal('if you recently updated khal, the config file format ' 'might have changed, in that case please consult the ' 'CHANGELOG or other documentation') raise CannotParseConfigFileError() fdict = {'timezone': is_timezone, 'timedelta': is_timedelta, 'expand_path': expand_path, 'expand_db_path': expand_db_path, 'weeknumbers': weeknumber_option, 'monthdisplay': monthdisplay_option, 'color': is_color, } validator = Validator(fdict) results = user_config.validate(validator, preserve_errors=True) abort = False for section, subsection, config_error in flatten_errors(user_config, results): abort = True if isinstance(config_error, Exception): logger.fatal( 'config error:\n' 'in [{}] {}: {}'.format(section[0], subsection, config_error)) else: for key in config_error: if isinstance(config_error[key], Exception): logger.fatal('config error:\nin {} {}: {}'.format( sectionize(section + [subsection]), key, str(config_error[key])) ) if abort or not results: raise InvalidSettingsError() config_checks(user_config, _get_color_from_vdir, _get_vdir_type) extras = get_extra_values(user_config) for section, value in extras: if section == (): logger.warning('unknown section "{}" in config file'.format(value)) else: section = sectionize(section) logger.warning( 'unknown key or subsection "{}" in section "{}"'.format(value, section)) deprecated = [{'value': 'default_command', 'section': 'default'}] for d in deprecated: if (value == d['value']) and (section == d['section']): logger.warning('Key "{}" in section "{}" was deprecated. ' 'See the FAQ to find out when and why!'.format(value, section)) return user_config
def __init__(self, config='pools.conf'): """ Аргументы: config: путь к файлу конфигурации""" # Шаблон файла конфигурации template = """ [Script] Debug = boolean(default=False) [Zabbix] Server = ip_addr(default=127.0.0.1) Port = integer(min=1, max=65535, default=10051) Send = boolean(default=True) Log = string(default=False) [__many__] Pool = option('FlyPool', 'NanoPool') Coin = option('Etherium', 'Monero', 'ZCash') Host = string Port = integer(min=1, max=65535, default=443) Timeout= integer(min=1, max=60, default=5) Account = string Workers = string_list [[Request]] Name = string [[[Args]]] worker = string(default=None) hours = integer(min=1, max=24, default=None) """ # Загружаем конфигурацию try: self.__config = ConfigObj(config, configspec=template.split('\n'), file_error=True) except (ConfigObjError, IOError) as e: # Ошибка загрузки конфигурации raise IOError("could not read config file'{file}': {error}".format( file=config, error=e)) from None # Проверяем файл конфигурации validator = Validator() result = self.__config.validate(validator, preserve_errors=True) # Если возникли ошибки if result is not True: for (sections, key, result) in flatten_errors(self.__config, result): if key is not None: raise ValueError( "the key '{key}' in the section '{section}' of config file failed validation: {result}" .format(key=key, section=', '.join(sections), result=result if result else "missing key")) else: raise ValueError( "the following section in config file was missing: '{section}'" .format(section=', '.join(sections))) # Проверяем на наличие дополнительных параметров for section, key in get_extra_values(self.__config): raise ValueError( "extra section or key '{key}' in section '{section}' of config file" .format(key=key, section=', '.join(section) if section else None))
def parse_config(test_config=None): """ Parse config file. Exits on error. :return: """ def dict_to_class(the_class: str, section: str, list: Tuple): """ Set class attributes from parsed config data :param the_class: Name of class :param section: Name of config section :param list: list of section keywords :return: """ for item in list: setattr(globals()[the_class], item, config[section][item]) the_config_spec = configobj.ConfigObj(io.StringIO(initial_value=configspec, newline='\n'), _inspec=True, encoding='UTF8') config = None for config_file in (test_config, options.config_file, sys.prefix + '/etc/serverpki.conf', '/usr/local/etc/serverPKI/serverpki.conf'): if not config_file: continue sld('Trying config file {}'.format(config_file)) if not os.path.isfile(config_file): continue sli('Using config file {}'.format(config_file)) try: config = configobj.ConfigObj(config_file, encoding='UTF8', interpolation='Template', configspec=the_config_spec) except SyntaxError as e: sle('Configuration file errors found. Can' 't continue. List of errors:') for err in e.errors: sle('{}'.format(err)) sys.exit(1) break if not config: sle('No config file found. Can' 't continue.') sys.exit(1) try: vtor = validate.Validator() result = config.validate(vtor, preserve_errors=True) except configobj.MissingInterpolationOption as e: sle('Substitution error: {}. Can' 't continue.'.format(e.msg)) sys.exit(1) if result != True: sle('Config validation failed:') for entry in configobj.flatten_errors(config, result): # each entry is a tuple section_list, key, error = entry if key is not None: section_list.append(key) else: section_list.append('[missing section]') section_string = ', '.join(section_list) if error == False: error = 'Missing value or section.' sle(section_string + ' = ' + str(error)) for sections, name in configobj.get_extra_values(config): # this code gets the extra values themselves the_section = config for section in sections: the_section = the_section[section] # the_value may be a section or a value the_value = the_section[name] section_or_value = 'value' if isinstance(the_value, dict): # Sections are subclasses of dict section_or_value = 'section' section_string = ', '.join(sections) or "top level" print('Extra entry in section: %s. Entry %r is a %s' % (section_string, name, section_or_value)) sld(str(result)) def test_walk(section, key): sld('{}[{}] {} = {}'.format( '[' + section.parent.name + ']' if section.parent.name else '', section.name, key, section[key])) config.walk(test_walk) def add_attribute_to_class(section: configobj, key: str): """ If called by config.walk, filles corresponding container class with class variable :param section: The section :param key: The key of attribute :return: """ def flatten_list(value: List) -> str: """ Make string from list :param value: list :return: """ result = '' first = True for e in value: if not first: result += ', ' first = False result += e return result value = section[key] if section.name == 'Pathes' and key == 'zone_tlsa_inc_mode': value = int(value, 8) # file permission is octal elif (section.name == 'Misc' and key == 'MAIL_RECIPIENT') or (section.name == 'DBAccount' and key == 'dbSearchPath'): result = '' first = True for e in value: if not first: result += ', ' first = False result += e value = result if not section.parent.name: setattr(globals()[section.name], key, value) else: try: d = getattr(globals()[section.parent.name], section.name) except AttributeError: d = {} setattr(globals()[section.parent.name], section.name, d) d[key] = value config.walk(add_attribute_to_class) ## re_init_syslog() FIXME: openlog works only once sld(DBAccount.dbHost + ' ' + str(DBAccount.dbPort) + ' ' + DBAccount.dbUser + ' ' + DBAccount.dbCert) return
def get_config(config_path=None): """reads the config file, validates it and return a config dict :param config_path: path to a custom config file, if none is given the default locations will be searched :type config_path: str :returns: configuration :rtype: dict """ if config_path is None: config_path = _find_configuration_file() logger.debug('using the config file at {}'.format(config_path)) try: user_config = ConfigObj(config_path, configspec=SPECPATH, interpolation=False, file_error=True, ) except ConfigObjError as error: logger.fatal('parsing the config file file with the following error: ' '{}'.format(error)) logger.fatal('if you recently updated khal, the config file format ' 'might have changed, in that case please consult the ' 'CHANGELOG or other documentation') raise CannotParseConfigFileError() fdict = {'timezone': is_timezone, 'expand_path': expand_path, 'expand_db_path': expand_db_path, 'weeknumbers': weeknumber_option, } validator = Validator(fdict) results = user_config.validate(validator, preserve_errors=True) abort = False for section, subsection, error in flatten_errors(user_config, results): abort = True if isinstance(error, Exception): logger.fatal( 'config error:\n' 'in [{}] {}: {}'.format(section[0], subsection, error)) else: for key in error: if isinstance(error[key], Exception): logger.fatal('config error:\nin {} {}: {}'.format( sectionize(section + [subsection]), key, str(error[key])) ) if abort or not results: raise InvalidSettingsError() config_checks(user_config) extras = get_extra_values(user_config) for section, value in extras: if section == (): logger.warn('unknown section "{}" in config file'.format(value)) else: section = sectionize(section) logger.warn('unknown key or subsection "{}" in ' 'section "{}"'.format(value, section)) return user_config
def read_config(configpath=None, specpath=None, checks=None, report_extra=False): """ get a (validated) config object for given config file path. :param configpath: path to config-file or a list of lines as its content :type configpath: str or list(str) :param specpath: path to spec-file :type specpath: str :param checks: custom checks to use for validator. see `validate docs <http://www.voidspace.org.uk/python/validate.html>`_ :type checks: dict str->callable, :param report_extra: log if a setting is not present in the spec file :type report_extra: boolean :raises: :class:`~alot.settings.errors.ConfigError` :rtype: `configobj.ConfigObj` """ checks = checks or {} try: config = ConfigObj(infile=configpath, configspec=specpath, file_error=True, encoding='UTF8') except ConfigObjError as e: msg = 'Error when parsing `%s`:\n%s' % (configpath, e) logging.error(msg) raise ConfigError(msg) except IOError: raise ConfigError('Could not read %s and/or %s' % (configpath, specpath)) except UnboundLocalError: # this works around a bug in configobj msg = '%s is malformed. Check for sections without parents..' raise ConfigError(msg % configpath) if specpath: validator = Validator() validator.functions.update(checks) try: results = config.validate(validator, preserve_errors=True) except ConfigObjError as e: raise ConfigError(str(e)) if results is not True: error_msg = '' for (section_list, key, res) in flatten_errors(config, results): if key is not None: if res is False: msg = 'key "%s" in section "%s" is missing.' msg = msg % (key, ', '.join(section_list)) else: msg = 'key "%s" in section "%s" failed validation: %s' msg = msg % (key, ', '.join(section_list), res) else: msg = 'section "%s" is missing' % '.'.join(section_list) error_msg += msg + '\n' raise ConfigError(error_msg) extra_values = get_extra_values(config) if report_extra else None if extra_values: msg = ['Unknown values were found in `%s`. Please check for ' 'typos if a specified setting does not seem to work:' % configpath] for sections, val in extra_values: if sections: msg.append('%s: %s' % ('->'.join(sections), val)) else: msg.append(str(val)) logging.info('\n'.join(msg)) return config
def read_config(configpath=None, specpath=None, checks=None, report_extra=False): """ get a (validated) config object for given config file path. :param configpath: path to config-file or a list of lines as its content :type configpath: str or list(str) :param specpath: path to spec-file :type specpath: str :param checks: custom checks to use for validator. see `validate docs <http://www.voidspace.org.uk/python/validate.html>`_ :type checks: dict str->callable, :param report_extra: log if a setting is not present in the spec file :type report_extra: boolean :raises: :class:`~alot.settings.errors.ConfigError` :rtype: `configobj.ConfigObj` """ checks = checks or {} try: config = ConfigObj(infile=configpath, configspec=specpath, file_error=True, encoding='UTF8') except ConfigObjError as e: msg = 'Error when parsing `%s`:\n%s' % (configpath, e) logging.error(msg) raise ConfigError(msg) except IOError: raise ConfigError('Could not read %s and/or %s' % (configpath, specpath)) except UnboundLocalError: # this works around a bug in configobj msg = '%s is malformed. Check for sections without parents..' raise ConfigError(msg % configpath) if specpath: validator = Validator() validator.functions.update(checks) try: results = config.validate(validator, preserve_errors=True) except ConfigObjError as e: raise ConfigError(str(e)) if results is not True: error_msg = '' for (section_list, key, res) in flatten_errors(config, results): if key is not None: if res is False: msg = 'key "%s" in section "%s" is missing.' msg = msg % (key, ', '.join(section_list)) else: msg = 'key "%s" in section "%s" failed validation: %s' msg = msg % (key, ', '.join(section_list), res) else: msg = 'section "%s" is missing' % '.'.join(section_list) error_msg += msg + '\n' raise ConfigError(error_msg) extra_values = get_extra_values(config) if report_extra else None if extra_values: msg = [ 'Unknown values were found in `%s`. Please check for ' 'typos if a specified setting does not seem to work:' % configpath ] for sections, val in extra_values: if sections: msg.append('%s: %s' % ('->'.join(sections), val)) else: msg.append(str(val)) logging.info('\n'.join(msg)) return config
# Validation failed failed_items = flatten_errors( cfg, test ) # Always print reason for validation failure for item in failed_items: sections, key, result = item print >> sys.stderr, ' ', for sec in sections: print >> sys.stderr, sec, ' / ', print >> sys.stderr, key if result == False: print >> sys.stderr, "ERROR, required item missing." else: print >> sys.stderr, result raise SystemExit( "ERROR gcylc.rc validation failed") extras = [] for sections, name in get_extra_values( cfg ): extra = ' ' for sec in sections: extra += sec + ' / ' extras.append( extra + name ) if len(extras) != 0: for extra in extras: print >> sys.stderr, ' ERROR, illegal entry:', extra raise SystemExit( "ERROR illegal gcylc.rc entry(s) found" ) # combine user config into default config self.inherit( dcfg, ucfg ) # theme inheritance if force_theme: # override
def __init__(self, path=None): if path is None: path = os.getcwd() res_path = resource_path() self.path = path self.resource_path = res_path settings_path = os.path.join(path, 'settings.ini') default_settings_path = os.path.join(res_path, 'settings.ini.default') settings = ConfigObj(settings_path, configspec=default_settings_path, encoding='utf-8') settings.filename = settings_path class Boolean: def __init__(self, value): self._str = value self._bool = is_boolean(value) def __bool__(self): return self._bool def __str__(self): return self._str validator = Validator(dict(boolean=lambda x: Boolean(x))) result = settings.validate(validator, preserve_errors=True, copy=True) for sections, key in get_extra_values(settings): section = settings for section_name in sections: section = section[section_name] del section[key] for sections, key, result in flatten_errors(settings, result): section = settings for section_name in sections: section = section[section_name] del section[key] assert settings.validate(validator, preserve_errors=False, copy=True) default_settings = ConfigObj(default_settings_path, configspec=default_settings_path, encoding='utf-8') default_settings.merge(settings) settings = default_settings settings.filename = settings_path settings.write() self.settings = settings self.black_flash = settings["Don't Blind Me!"]['black_flash'] self.black_smoke = settings["Don't Blind Me!"]['black_smoke'] self.ignore_temperature = False self.host = settings['Game State Integration']['host'] self.port = settings['Game State Integration'].as_int('port') gamestate_integration_cfg_template_path = os.path.join( res_path, 'gamestate_integration_dont_blind_me.cfg.template') with open(gamestate_integration_cfg_template_path) as f: gamestate_integration_cfg_template = f.read() gamestate_integration_cfg_path = os.path.join( path, 'gamestate_integration_dont_blind_me.cfg') with open(gamestate_integration_cfg_path, mode='w') as f: f.write( gamestate_integration_cfg_template.format(host=self.host, port=self.port)) self.mat_monitorgamma = settings['Video Settings'].as_float( 'mat_monitorgamma') self.mat_monitorgamma_tv_enabled = settings['Video Settings'].as_bool( 'mat_monitorgamma_tv_enabled') self.temperature = [None, 6500] self.round_phase = [None, None] self.player_alive = None self.player_flashed = [None, 0] self.player_smoked = [None, 0] self.context = Context.open()
def __init__(self, config='miners.conf'): """Аргументы: config: путь к файлу конфигурации """ # Шаблон файла конфигурации template = """ [Script] Template = string(default=miners.template.html) Refresh = integer(min=10, max=3600, default=30) Url = string(default=cgi-bin/miners.py/) Colorize = boolean(default=False) Debug = boolean(default=False) [Zabbix] Server = ip_addr(default=127.0.0.1) Port = integer(default=10051) Log = string(default=False) Send = boolean(default=False) [__many__] Host = ip_addr Port = integer(min=1, max=65535) Miner = string Request = force_list Timeout= integer(min=1, max=60, default=5) Description = string """ # Загружаем конфигурацию try: self.__config = ConfigObj( config, configspec=template.split('\n'), file_error=True, ) except (ConfigObjError, IOError) as e: # Ошибка загрузки конфигурации raise IOError( "could not read config file'{file}': {error}".format( file=config, error=e, ), ) from None # Проверяем файл конфигурации validator = Validator() result = self.__config.validate(validator, preserve_errors=True) # Если возникли ошибки if result is not True: for (sections, key, result) in flatten_errors(self.__config, result): if key is not None: if key != 'Description': raise ValueError( "the key '{key}' in the section '{section}' " "of config file failed validation: " "{result}".format( key=key, section=', '.join(sections), result=result if result else "missing key", ), ) else: raise ValueError( "the following section in " "config file was missing: '{section}'".format( section=', '.join(sections), ), ) # Проверяем на наличие дополнительных параметров for section, key in get_extra_values(self.__config): raise ValueError( "extra section or key '{key}' " "in section '{section}' of config file".format( key=key, section=', '.join(section) if section else None, ), )
def read_options_file(KOLIBRI_HOME, ini_filename="options.ini"): logger = _get_logger(KOLIBRI_HOME) ini_path = os.path.join(KOLIBRI_HOME, ini_filename) conf = ConfigObj(ini_path, configspec=get_configspec()) # validate once up front to ensure section structure is in place conf.validate(_get_validator()) # keep track of which options were overridden using environment variables, to support error reporting using_env_vars = {} # override any values from their environment variables (if set) for section, opts in option_spec.items(): for optname, attrs in opts.items(): for envvar in attrs.get("envvars", []): if os.environ.get(envvar): logger.info( "Option {optname} in section [{section}] being overridden by environment variable {envvar}".format( optname=optname, section=section, envvar=envvar ) ) conf[section][optname] = os.environ[envvar] using_env_vars[optname] = envvar break conf = clean_conf(conf) validation = conf.validate(_get_validator(), preserve_errors=True) # loop over and display any errors with config values, and then bail if validation is not True: for section_list, optname, error in flatten_errors(conf, validation): section = section_list[0] if optname in using_env_vars: logger.error( "Error processing environment variable option {envvar}: {error}".format( envvar=using_env_vars[optname], error=error ) ) else: logger.error( "Error processing {file} under section [{section}] for option {option}: {error}".format( file=ini_path, section=section, option=optname, error=error ) ) logger.critical( "Aborting: Could not process options config (see errors above for more details)" ) raise SystemExit(1) # loop over any extraneous options and warn the user that we're ignoring them for sections, name in get_extra_values(conf): # this code gets the extra values themselves the_section = conf for section in sections: the_section = the_section[section] # the_value may be a section or a value the_value = the_section.pop(name) # determine whether the extra item is a section (dict) or value kind = "section" if isinstance(the_value, dict) else "option" logger.warn( "Ignoring unknown {kind} in options file {file} under {section}: {name}.".format( kind=kind, file=ini_path, section=sections[0] if sections else "top level", name=name, ) ) # run validation once again to fill in any default values for options we deleted due to issues conf.validate(_get_validator()) # ensure all arguments under section "Paths" are fully resolved and expanded, relative to KOLIBRI_HOME _expand_paths(KOLIBRI_HOME, conf.get("Paths", {})) return conf