def collect_validation_messages(conf, error_list): validator = validate.Validator() conf.validate(validator, preserve_errors=True) message = [] cm_extras = { 'hadoop_hdfs_home': [('hadoop', 'hdfs_clusters', 'default')], 'hadoop_bin': [('hadoop', 'hdfs_clusters', 'default'), ('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_mapred_home': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_conf_dir': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'ssl_cacerts': [('beeswax', 'ssl'), ('impala', 'ssl')], 'remote_data_dir': [('liboozie', )], 'shell': [()] } whitelist_extras = ((sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or (name in cm_extras.keys() and sections in cm_extras[name]))) for sections, name in whitelist_extras: the_section = conf hierarchy_sections_string = '' try: parent = conf for section in sections: the_section = parent[section] hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " " parent = the_section except KeyError, ex: LOG.warn("Section %s not found: %s" % (section, str(ex))) the_value = '' try: # the_value may be a section or a value the_value = the_section[name] except KeyError, ex: LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
def cobj_check(settings, exception=None, copy=False): """Check for errors in config file""" if not exception: exception = Exception validator = validate.Validator() def numpy_array(val): """Define float list""" float_list = validator.functions["float_list"](val) return numpy.array(float_list) validator.functions["numpy_array"] = numpy_array results = settings.validate(validator, copy=copy, preserve_errors=True) if results is not True: output = "{0}: \n".format(settings.filename if settings. filename is not None else "configobj") for (section_list, key, error) in configobj.flatten_errors(settings, results): if key is not None: val = settings for section in section_list: val = val[section] val = val[key] if key in val else "<EMPTY>" output += " [{sections}], {key}='{val}' ({error})\n".format( sections=', '.join(section_list), key=key, val=val, error=error) else: output += "Missing section: {0}\n".format( ", ".join(section_list)) raise exception(output)
def load_prefs(self): cfgpath = os.path.expanduser(DL_RC) self.cfg = configobj.ConfigObj(cfgpath) v = validate.Validator() self.prefs = Preferences() self.prefs.email = v.check('string', self.cfg.get('email', '')) self.prefs.service.url = v.check('string', self.cfg.get('url', '')) self.prefs.service.username = v.check('string', self.cfg.get('user', '')) self.prefs.service.password = v.check('string', self.cfg.get('pass', '')) self.prefs.service.verify = v.check('boolean', self.cfg.get('verify', True)) self.prefs.service.agent = DL_AGENT self.prefs.ticket_params.permanent = v.check( 'boolean', self.cfg.get('perm', False)) self.prefs.ticket_params.total = v.check( 'integer', self.cfg.get('total_days', 365)) * 3600 * 24 self.prefs.ticket_params.lastdl = v.check( 'integer', self.cfg.get('days_after_dl', 30)) * 3600 * 24 self.prefs.ticket_params.downloads = v.check( 'integer', self.cfg.get('downloads', 0))
def loadAll(self): """Load the user prefs and the application data """ self._validator = validate.Validator() # note: self.paths['userPrefsDir'] gets set in loadSitePrefs() self.paths['appDataFile'] = join(self.paths['userPrefsDir'], 'appData.cfg') self.paths['userPrefsFile'] = join(self.paths['userPrefsDir'], 'userPrefs.cfg') self.userPrefsCfg = self.loadUserPrefs() self.appDataCfg = self.loadAppData() self.validate() #simplify namespace self.general = self.userPrefsCfg['general'] self.app = self.userPrefsCfg['app'] self.coder = self.userPrefsCfg['coder'] self.builder = self.userPrefsCfg['builder'] self.connections = self.userPrefsCfg['connections'] self.keys = self.userPrefsCfg['keyBindings'] self.appData = self.appDataCfg # keybindings: #self.keyDict = self.keysPrefsCfg['keybindings'] # == dict, with items in u'___' format #self.keys = self.convertKeyDict() # no longer a dict, no longer u'___' format self.keys = self.userPrefsCfg['keyBindings'] # connections: if self.connections['autoProxy']: self.connections['proxy'] = self.getAutoProxy()
def main(): datadir = os.path.dirname(__file__) specfile = os.path.join(datadir, 'spec.txt') conffile = os.path.join(datadir, 'config.txt') spec = configobj.ConfigObj(specfile, list_values=False) config = configobj.ConfigObj(conffile, configspec=spec) # Instead of creating a ConfigWindow we spawn an external process (will block) app = QtGui.QApplication(sys.argv) validator = validate.Validator() wnd = configobj_gui.ConfigWindow( config, spec, when_apply=configobj_gui.ConfigWindow.APPLY_OK, type_mapping={ 'mytype': (configobj_gui.create_widget_string, validate.is_integer) }) wnd.show() def printChange(option): print('%s in %s changed to %s' % (option.name, option.section.name, option.get())) def printSectionAdded(section): print('Added section %s' % (section.name)) def printSectionRemoved(section): print('Removed section %s' % (section.name)) wnd.optionChanged.connect(printChange) wnd.sectionAdded.connect(printSectionAdded) wnd.sectionRemoved.connect(printSectionRemoved) app.exec_() print(config)
def createConfig(path): spec = cfg.split("\n") config = configobj.ConfigObj(path, configspec=spec) validator = validate.Validator() config.validate(validator, copy=True) config.filename = path config.write()
def parse(self, config, explain=False, defaults=None): """Parse the conf :param config: Configuration to parse :type config: str, list or File :param explain: Whether to explain the parsing errors or not :type explain: bool :param defaults: Default options :type defaults: dict """ self.conf = {} self.conffile = config self.section = None self.defaults = defaults self.validator = validate.Validator() try: self.conf = configobj.ConfigObj(config, encoding='utf-8') self.mtime = os.path.getmtime(self.conffile) except configobj.ConfigObjError as exp: # We were unable to parse the config self.logger.critical('Unable to convert configuration') if explain: self._explain(exp) else: raise exp
def loadAll(self): """Load the user prefs and the application data """ self._validator=validate.Validator() # note: self.paths['userPrefsDir'] gets set in loadSitePrefs() self.paths['appDataFile'] = join(self.paths['userPrefsDir'], 'appData.cfg') self.paths['userPrefsFile'] = join(self.paths['userPrefsDir'], 'userPrefs.cfg') # If PsychoPy is tucked away by Py2exe in library.zip, the preferences file # cannot be found. This hack is an attempt to fix this. if "\\library.zip\\psychopy\\preferences\\" in self.paths["prefsSpecFile"]: self.paths["prefsSpecFile"] = self.paths["prefsSpecFile"].replace("\\library.zip\\psychopy\\preferences\\", "\\resources\\") self.userPrefsCfg = self.loadUserPrefs() self.appDataCfg = self.loadAppData() self.validate() #simplify namespace self.general=self.userPrefsCfg['general'] self.app = self.userPrefsCfg['app'] self.coder=self.userPrefsCfg['coder'] self.builder=self.userPrefsCfg['builder'] self.connections=self.userPrefsCfg['connections'] self.keys=self.userPrefsCfg['keyBindings'] self.appData = self.appDataCfg # keybindings: self.keys = self.userPrefsCfg['keyBindings']
def __init__(self): self.settings = configobj.ConfigObj( infile=SETTINGS_FILE, create_empty=True, indent_type=" ", configspec=SETTINGS_SPEC ) self.validator = validate.Validator() valid = self.settings.validate(self.validator) # We cannot use "if not valid" here, since validate() returns a dict # if validation fails! # See: # http://www.voidspace.org.uk/python/articles/configobj.shtml#validation if valid is not True: # What to do here? # We could only get to this point if: # 1. The user config file existed # 2. It was invalid # One option is to copy it to a different file and recreate it... log.warning("User configuration not valid. Backing up and recreating.") self.backup_and_rewrite_config()
def _check_genic_config(config): """Check that the MP-GenIC config file is sensible for running CLASS on.""" vtor = validate.Validator() config.validate(vtor) filekeys = [ 'FileWithInputSpectrum', ] if config['DifferentTransferFunctions'] == 1.: filekeys += [ 'FileWithTransferFunction', ] for ff in filekeys: if config[ff] == '': raise IOError("No savefile specified for ", ff) if os.path.exists(config[ff]): raise IOError("Refusing to write to existing file: ", config[ff]) #Check unsupported configurations if config['MWDM_Therm'] > 0: raise ValueError( "Warm dark matter power spectrum cutoff not yet supported.") if config['DifferentTransferFunctions'] == 1.: if config['InputPowerRedshift'] >= 0: raise ValueError( "Rescaling with different transfer functions not supported.")
def main(): parser = argparse.ArgumentParser(description="Upload a file to DL", epilog=DL_AGENT) parser.add_argument('-r', metavar="file", dest="rc", default="~/.dl.rc", help="Use alternate RC file") group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-g', metavar="email", dest="grant", help="Generate a grant with notification sent to 'email'") group.add_argument('file', nargs='?', help="File to upload") args = parser.parse_args() cfgpath = os.path.expanduser(args.rc) cfg = configobj.ConfigObj(cfgpath) v = validate.Validator() for param in ['user', 'pass', 'url']: if param not in cfg: die("missing \"{0}\" in configuration file".format(param)) cfg[param] = v.check('string', cfg[param]) cfg['verify'] = v.check('boolean', cfg.get('verify', True)) service = Service(cfg['url'], cfg['user'], cfg['pass'], cfg['verify'], DL_AGENT) dl = DL(service) try: if args.file: fun = progress if sys.stdout.isatty() else None answ = dl.new_ticket(args.file, progress_fn=fun) else: answ = dl.new_grant(args.grant) print(answ['url']) except KeyboardInterrupt: pass except DLError as e: die(str(e))
def load_config(): """ Load and validate configuration. :return: configuration :rtype: configobj.ConfigObj :raise IOError: if there is an issue reading the configuration file """ # todo arbitrary config files # http://www.voidspace.org.uk/python/configobj.html # http://www.voidspace.org.uk/python/validate.html if os.path.exists('bot.ini'): config = configobj.ConfigObj('bot.ini', configspec='confspec.ini') log.info("Using config file bot.ini") elif os.path.exists('default.ini'): config = configobj.ConfigObj('default.ini', configspec='confspec.ini') log.info("Using config file default.ini") else: raise IOError("Could not find config file for bot") # validate config now val = validate.Validator() results = config.validate(val, preserve_errors=True) if not results: for (section_list, key, reason) in configobj.flatten_errors(config, results): if key: msg = "CONFIG ERROR: key '%s' in section '%s' failed validation" % (key, ', '.join(section_list)) if reason: msg += " - %s" % (reason) log.error(msg) else: log.error("CONFIG ERROR: missing section '%s'" % ', '.join(section_list)) raise IOError("Errors in bot config file") return config
def load_test_config(): config = configobj.ConfigObj(BASE_CONFIG.splitlines(), configspec=CONFIG_SPEC.splitlines()) validator = validate.Validator() result = config.validate(validator, preserve_errors=True) assert result == True, "Invalid test config" return config
def load_default_conf(): """Load the default configuration""" default_conf = configobj.ConfigObj(configspec=configspec) validator = validate.Validator() default_conf.validate(validator, copy=True) default_conf = post_process_conf(default_conf) return default_conf
def validate_config_file( config: configobj.ConfigObj, logger: logging.Logger ) -> configobj.ConfigObj: """Validate config file, i.e. check that everything is set properly. This also sets all default values.""" logger.debug("Validating config file.") # 'copy' parameter of config.validate: Also copy all comments and default # values from the configspecs to the config file (to be written out later)? # Note: The config object might be completely empty, because only the first # Run of this methods sets them from the defaults if the user did not # specify a config file. # However, this method gets called 2 times (one time before and one time # after the handling of the --parameter options), # Unfortunately, if you do copy one time, an additional copy=False won't # bring it back. So we start with copy=False and if r.config.copy_all # is set to true, copy it the second time this function is called. try: copy_all = config["r"]["config"]["copy_all"] except KeyError: copy_all = False valid = config.validate( validate.Validator(), preserve_errors=True, copy=copy_all ) # adapted from https://stackoverflow.com/questions/14345879/ # answer from user sgt_pats 2017 # todo: This might need some better handling. for entry in configobj.flatten_errors(config, valid): [path, key, error] = entry if not error: msg = "The parameter {} was not in the config file\n".format(key) msg += ( "Please check to make sure this parameter is present and " "there are no mis-spellings." ) logger.critical(msg) raise ValueError(msg) if key is not None: if isinstance(error, validate.VdtValueError): optionString = config.configspec[key] msg = ( "The parameter {} was set to {} which is not one of " "the allowed values\n".format(key, config[key]) ) msg += "Please set the value to be in {}".format(optionString) logger.critical(msg) raise ValueError(msg) elif error: msg = "Validation error (section='{}', key={}): {}".format( "/".join(path), key, error ) logger.error(msg) raise ValueError(msg) return config
def collect_validation_messages(conf, error_list): validator = validate.Validator() conf.validate(validator, preserve_errors=True) message = [] cm_extras = { 'hadoop_hdfs_home': [('hadoop', 'hdfs_clusters', 'default')], 'hadoop_bin': [('hadoop', 'hdfs_clusters', 'default'), ('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_mapred_home': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'hadoop_conf_dir': [('hadoop', 'yarn_clusters', 'default'), ('hadoop', 'yarn_clusters', 'ha')], 'ssl_cacerts': [('beeswax', 'ssl'), ('impala', 'ssl')], 'remote_data_dir': [('liboozie', )], 'shell': [()], } whitelist_extras = ( (sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or ( name in list(cm_extras.keys()) and sections in cm_extras[name]))) for sections, name in whitelist_extras: the_section = conf hierarchy_sections_string = '' try: parent = conf for section in sections: the_section = parent[section] hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " " parent = the_section except KeyError as ex: LOG.warning("Section %s not found: %s" % (section, str(ex))) the_value = '' try: # the_value may be a section or a value the_value = the_section[name] except KeyError as ex: LOG.warning("Error in accessing Section or Value %s: %s" % (name, str(ex))) section_or_value = 'keyvalue' if isinstance(the_value, dict): # Sections are subclasses of dict section_or_value = 'section' section_string = hierarchy_sections_string or "top level" message.append('Extra %s, %s in the section: %s' % (section_or_value, name, section_string)) if message: error = { 'name': 'ini configuration', 'message': ', '.join(message), } error_list.append(error)
def get_config(): global _config if not _config: path = os.path.join(globalVars.appArgs.configPath, CONFIG_FILE_NAME) _config = configobj.ConfigObj(path, configspec=configspec) val = validate.Validator() _config.validate(val, copy=True) return _config
def getConfig(): global _config if not _config: path = os.path.join(config.getUserDefaultConfigPath(), "ocr.ini") _config = configobj.ConfigObj(path, configspec=configspec) val = validate.Validator() _config.validate(val) return _config
def __init__(self, cfg): """Constructor reading and validating config against given config spec @param cfg: optional parameter containing config file @type cfg: string""" self.config = configobj.ConfigObj(cfg, file_error=True, configspec=self.__CONFIG_SPEC) validator = validate.Validator() res = self.config.validate(validator, preserve_errors=True) for section_list, key, error in configobj.flatten_errors(self.config, res): # pylint: disable=W0612 raise PynatconndConfigException("Failed to validate section %s key %s in config file %s" % (", ".join(section_list), key, cfg))
def __init__(self): """ Initializes config for the tip of the day addon. """ path = os.path.join(config.getUserDefaultConfigPath(), "tip_of_day.ini") self._config = configobj.ConfigObj(path, configspec=configspec, indent_type='\t') val = validate.Validator() self._config.validate(val)
def restoreBadPrefs(self, cfg, resultOfValidate): if resultOfValidate == True: return vtor = validate.Validator() for (section_list, key, _junk) in configobj.flatten_errors(cfg, resultOfValidate): if key is not None: cfg[', '.join(section_list)][key] = vtor.get_default_value(cfg.configspec[', '.join(section_list)][key]) else: print("Section [%s] was missing in file '%s'" % (', '.join(section_list), cfg.filename))
def parse_config(path, spec=None): if spec: if isinstance(spec, str): spec = spec.split("\n") elif isinstance(spec, dict): spec = convert_dict_to_spec(spec) config = configobj.ConfigObj(path, configspec=spec) if spec: config.validate(validate.Validator(), copy=True) return config.dict()
def load_conf(conf_file): try: config = configobj.ConfigObj(conf_file, configspec=spec) except: traceback.print_exc() exit('Error parsing the ' + conf_file + ' please check it out.') validator = validate.Validator() if config.validate(validator) is not True: #), copy=True) exit(CONF_FILE + ' file has problems...please check.') return config
def load_config(package, config_file): """Load and validate package configuration file. Parameters ---------- package : str The gtecs subpackage name (e.g. 'control', 'obs', 'alert') config_file : str, or list of str The name for the package config file (e.g. '`.gtecs.conf') """ # Load package configspec file spec = pkg_resources.read_text(f'gtecs.{package}.data', 'configspec.ini').split('\n') # Create empty spec for default parameters config = configobj.ConfigObj({}, configspec=spec) # Try to find the config file, look in the home directory and # anywhere specified by GTECS_CONF environment variable if isinstance(config_file, str): filenames = [config_file] else: filenames = config_file home = os.path.expanduser('~') paths = [home, os.path.join(home, 'gtecs'), os.path.join(home, '.gtecs')] if 'GTECS_CONF' in os.environ: paths.append(os.environ['GTECS_CONF']) config_file = None config_path = None for path in paths: for file in filenames: try: with open(os.path.join(path, file)) as source: config = configobj.ConfigObj(source, configspec=spec) config_file = file config_path = path break except IOError: pass if config_file is not None and config_path is not None: loc = os.path.join(config_path, config_file) else: loc = None # Validate ConfigObj, filling defaults from configspec if missing from config file validator = validate.Validator() result = config.validate(validator) if result is not True: print('Config file validation failed') print([k for k in result if not result[k]]) raise ValueError(f'{config_file} config file validation failed') return config, spec, loc
def check_config(config_dir): try: os.makedirs(config_dir) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(config_dir): pass else: raise spec = """client_ID = string\nclient_secret = string\nrefresh_token = string""" config_spec = configobj.ConfigObj(StringIO(spec)) config_path = os.path.join(config_dir, "settings.ini") config = configobj.ConfigObj(config_path, configspec=config_spec) valid = config.validate(validate.Validator()) if valid is True: try: with open(os.path.join(config_dir, "filters.json")) as filters_file: filters_json = filters_file.read() try: filters = demjson.decode(filters_json, encoding="utf8", strict=True, allow_comments=True) except demjson.JSONDecodeError as e: filters = (False, e.pretty_description()) return config, filters except IOError as e: if e.errno == errno.ENOENT: f = open(os.path.join(config_dir, "filters.json"), "w") f.write( '{\n' ' // comment\n' ' "feed name": ["filter regexp", "another filter regexp"]\n' '}\n') f.close() else: raise elif valid is False: config["client_ID"] = raw_input("Google OAuth Client ID\n: ") config["client_secret"] = raw_input("Google OAuth Client Secret\n: ") GR = GoogleReader(config["client_ID"], config["client_secret"], None) config["refresh_token"] = GR.get_refresh_token() try: config.write() os.chmod(config_path, stat.S_IRUSR | stat.S_IWUSR) # mode -rw------- print "Config written successfully." exit(1) except Exception as e: print "{}\nConfig file was not written.".format(e) exit(2)
def make_empty_configuration_file(dest_path, spec_path): cfg = configobj.ConfigObj(create_empty=True, configspec=spec_path, default_encoding='utf-8') # Make sure the default values are copied cfg.validate(validate.Validator(), copy=True) mainlog.info("Writing configuration file {}".format(dest_path)) with open(dest_path, mode="wb") as out_file: cfg.write(out_file)
def createConfig(path): """ Create a config file using a configspec and validate it against a Validator object """ spec = cfg.split("\n") config = configobj.ConfigObj(path, configspec=spec) validator = validate.Validator() config.validate(validator, copy=True) config.filename = path config.write()
def _load_config(filename): with open(filename, 'r') as f: config = configobj.ConfigObj(f, configspec=CONFIG_SPEC.splitlines()) validator = validate.Validator() result = config.validate(validator, preserve_errors=True) if not result: print('Invalid LDAP Proxy configuration at {!r}: {!r}'.format( filename, result)) sys.exit(1) return config
def __init__(self, path): self.site = dict() path_file = Path(path) if path_file.exists(): self._config_file = ConfigObj(str(path), configspec=DEFAULT_CONF.split("\n")) else: raise FileNotFoundError() validator = validate.Validator() self._config_file.validate(validator) self.name = self._config_file["name"] self.site["name"] = self.name self.__username = None self.__password = None if "url" in self._config_file: self.site["url"] = self._config_file["url"] else: self.site["url"] = None self.site["search_field_selector"] = ActionTag( self._config_file["search_field_selector"], timeout=DEFAULT_TIMEOUT) self.site["search_button_selector"] = ActionTag( self._config_file["search_button_selector"], timeout=DEFAULT_TIMEOUT) self.site["search_response_selector"] = ActionTag( self._config_file["search_response_selector"], timeout=DEFAULT_TIMEOUT) self.site["download_link_selector"] = self._config_file[ "download_link_selector"] self.site["going_login"] = self._create_action_tag( self._config_file["login"]) self.site["going_login"] = self.add_goto_if_need( self.site["going_login"]) self.site["going_search"] = self._create_action_tag( self._config_file["going_search"]) self.site["going_search"] = self.add_goto_if_need( self.site["going_search"]) self.site["going_download"] = self._create_action_tag( self._config_file["going_download"]) self.site["lock"] = asyncio.Lock() self.site["semaphore"] = asyncio.Semaphore( self._config_file["max_request"]) self.site["logged"] = False
def load_rc_file(rc_file): """ Load settings for the qutip RC file, by default .qutiprc in the user's home directory. """ global auto_tidyup, auto_herm, auto_tidyup_atol, num_cpus, debug, atol global log_handler, colorblind_safe # Try to pull in configobj to do nicer handling of # config files instead of doing manual parsing. # We pull it in here to avoid throwing an error if configobj is missing. try: import configobj as _cobj import pkg_resources import validate except ImportError: # Don't bother warning unless the rc_file exists. import os if os.path.exists(rc_file): _logger.warn("configobj missing, not loading rc_file.", exc_info=1) return # Try to find the specification for the config file, then # use it to load the actual config. config = _cobj.ConfigObj( rc_file, configspec=pkg_resources.resource_filename('qutip', 'configspec.ini'), # doesn't throw an error if qutiprc is missing. file_error=False) # Next, validate the loaded config file against the specs. validator = validate.Validator() result = config.validate(validator) # configobj's validator returns the literal True if everything # worked, and returns a dictionary of which keys fails otherwise. # This motivates a very un-Pythonic way of checking for results, # but it's the configobj idiom. if result is not True: # OK, find which keys are bad. bad_keys = {key for key, val in result.iteritems() if not val} _logger.warn('Invalid configuration options in {}: {}'.format( rc_file, bad_keys)) else: bad_keys = {} # Now that everything's been validated, we apply the config # file to the global settings. for config_key in ('auto_tidyup', 'auto_herm', 'atol', 'auto_tidyup_atol', 'num_cpus', 'debug', 'log_handler', 'colorblind_safe'): if config_key in config and config_key not in bad_keys: _logger.debug("Applying configuration setting {} = {}.".format( config_key, config[config_key])) globals()[config_key] = config[config_key]