def load(f): p = SafeConfigParser() p.read(f) if not p.has_section('oauth2'): p.add_section('oauth2') if not p.has_section('oauth2-state'): p.add_section('oauth2-state') return p
class Config(object): def __init__(self): self.config = SafeConfigParser() self.name = '' def parse(self, fname, override): if override: override = [x for x in csv.reader( ' '.join(override).split(','), delimiter='.')] logger.info('Reading configuration file: {}'.format(fname)) if not os.path.isfile(fname): logger.interrupt('File doesn\'t exist: {}'.format(fname)) self.config.optionxform = str self.config.read(fname) for section, option, value in override: if not self.config.has_section(section): self.config.add_section(section) self.config.set(section, option, value) basename = os.path.basename(fname) self.name = os.path.splitext(basename)[0] @safe def _get_options_as_dict(self, section): if section in self.config.sections(): return {p: v for p, v in self.config.items(section)} else: return {}
def getManifest(fp, format, defaults=None): """Read the manifest from the given open file pointer according to the given ManifestFormat. Pass a dict as ``defaults`` to override the defaults from the manifest format. """ if defaults is None: defaults = format.defaults parser = SafeConfigParser() parser.readfp(fp) results = {} for key in format.keys: if parser.has_option(format.resourceType, key): results[key] = parser.get(format.resourceType, key) else: results[key] = defaults.get(key, None) for key in format.parameterSections: sectionName = "%s:%s" % (format.resourceType, key,) if parser.has_section(sectionName): results[key] = dict(parser.items(sectionName)) else: results[key] = {} return results
def set_namespace_options(namespace, options, remove=None): """ Set options in the local namespace configuration file. Can have nasty effects, be careful, only use in test code. :param namespace: the namespace to work with :param options: a dictionary with options to set :param remove: an iterable of options to remove :returns: a dictionary with all options of the namespace """ parser = SafeConfigParser({}) potential_confs = list() actual_confs = list() for p, _ in _config_paths(): potential_confs.append(p) fdone = parser.read((p,)) actual_confs.extend(fdone) if not actual_confs: raise ValueError( "Could not read configuration from any of %s" % potential_confs) if not parser.has_section(namespace): print('Namespace %s was not found in %s' % (namespace, actual_confs)) parser.add_section(namespace) for key, val in options.items(): parser.set(namespace, key, str(val)) if remove: for key in remove: parser.remove_option(namespace, key) with open(actual_confs[-1], 'w') as outfile: parser.write(outfile) return dict(parser.items(namespace))
def getManifest(fp, format, defaults=None): """Read the manifest from the given open file pointer according to the given ManifestFormat. Pass a dict as ``defaults`` to override the defaults from the manifest format. """ if defaults is None: defaults = format.defaults parser = SafeConfigParser() if six.PY2: parser.readfp(fp) else: data = fp.read() if isinstance(data, six.binary_type): data = data.decode() parser.read_string(data) results = {} for key in format.keys: if parser.has_option(format.resourceType, key): results[key] = parser.get(format.resourceType, key) else: results[key] = defaults.get(key, None) for key in format.parameterSections: sectionName = "%s:%s" % (format.resourceType, key,) if parser.has_section(sectionName): results[key] = dict(parser.items(sectionName)) else: results[key] = {} return results
def logging_config(self, name): if name != 'main': raise KeyError parser = SafeConfigParser() parser.read(self.path) for section_name in ('loggers', 'handlers', 'formatters'): if not parser.has_section(section_name): raise KeyError loggers = convert_loggers(parser) handlers = convert_handlers(parser) formatters = convert_formatters(parser) return combine(loggers, handlers, formatters)
def read_conf(conf_path, section_name=None, defaults=None, use_yaml=False): if use_yaml: return parse_config(conf_path) if defaults is None: defaults = {} parser = SafeConfigParser(defaults) success = parser.read(conf_path) if not success: print("Unable to read config from %s" % conf_path) sys.exit(1) if section_name: if parser.has_section(section_name): # if log_format is set, extract it from the parser # to prevent to expand variables which can, in case of # log_format throw a ConfigParser.InterpolationMissingOptionError log_format = None if parser.has_option(section_name, 'log_format'): log_format = parser.get(section_name, 'log_format', raw=True) # don't use remove_options because it can fail without reason parser.set(section_name, 'log_format', '') conf = dict(parser.items(section_name)) # Add log_format again, after parsing if log_format: conf['log_format'] = log_format else: print('Unable to find section %s in config %s' % (section_name, conf_path)) exit(1) else: conf = {} for section in parser.sections(): # if log_format is set, extract it from the parser # to prevent to expand variables which can, in case of # log_format throw a ConfigParser.InterpolationMissingOptionError log_format = None if parser.has_option(section, 'log_format'): log_format = parser.get(section, 'log_format', raw=True) # don't use remove_options because it can fail without reason parser.set(section, 'log_format', '') conf.update({section: dict(parser.items(section))}) # Add log_format again, after parsing if log_format: conf[section]['log_format'] = log_format return conf
def load_namespace_conf(namespace, failsafe=False, fresh=False): """ Load configuration for the namespace from the local configuration files. :param namespace: name of the namespace. :param failsafe: in case of error, return a dummy configuration. :param fresh: if True, reload configuration from files, do not use the cache. :returns: a dictionary with local namespace configuration. """ if not fresh and namespace in NS_CONF_CACHE: return NS_CONF_CACHE[namespace] parser = SafeConfigParser({}) success = False # Do not load a non-overriding file (local) if any file has # already been loaded before. loaded_files = list() for p, override in _config_paths(): if override or not loaded_files: flist = parser.read((p,)) loaded_files.extend(flist) conf = NamespaceConfiguration(loaded_files, namespace=namespace) if not loaded_files: print('Unable to read namespace config. We tried %s' % ( [x for x in _config_paths()])) else: import logging logging.info("Configuration loaded from %s", repr(loaded_files)) if not parser.has_section(namespace): print('Unable to find [%s] section in any of %s' % ( namespace, loaded_files)) else: conf.update(parser.items(namespace)) proxy = conf.get('proxy') if not proxy: print("Missing field proxy in namespace config") else: success = True NS_CONF_CACHE[namespace] = conf if not (success or failsafe): sys.exit(1) return conf
def configure(filename=None): """This function gives to the user application a chance to define where configuration file should live. Subsequent calls to this function will have no effect, unless you call :func:`reconfigure`. :param str filename: Full path to configuration file. """ global retry if getattr(configure, '_configured', False): return filename = filename or DEFAULT_CONFIG_FILENAME _ensure_directory(filename) parser = SafeConfigParser() if os.path.isfile(filename): with open(filename, 'rt') as fp: parser.readfp(fp) if not parser.has_section(RETRY_SECTION): parser.add_section(RETRY_SECTION) parser.set(RETRY_SECTION, 'max_tries', str(constants.BACKOFF_DEFAULT_MAXTRIES)) parser.set(RETRY_SECTION, 'delay', str(constants.BACKOFF_DEFAULT_DELAY)) parser.set(RETRY_SECTION, 'factor', str(constants.BACKOFF_DEFAULT_FACTOR)) with open(filename, 'wt') as fp: parser.write(fp) retry = RetrySettings(max_tries=parser.getint(RETRY_SECTION, 'max_tries'), delay=parser.getint(RETRY_SECTION, 'delay'), factor=parser.getint(RETRY_SECTION, 'factor')) setattr(configure, '_configured', True) setattr(configure, '_configured_filename', filename)
def load_namespace_conf(namespace): def places(): yield '/etc/oio/sds.conf' for f in glob('/etc/oio/sds.conf.d/*'): yield f yield path.expanduser('~/.oio/sds.conf') c = SafeConfigParser({}) success = c.read(places()) if not success: print('Unable to read namespace config') exit(1) if c.has_section(namespace): conf = dict(c.items(namespace)) else: print('Unable to find [%s] section config' % namespace) exit(1) proxy = conf.get('proxy') if not proxy: print("Missing field proxy in namespace config") exit(1) return conf
def read_conf(conf_path, section_name=None, defaults=None, use_yaml=False): if use_yaml: return parse_config(conf_path) if defaults is None: defaults = {} parser = SafeConfigParser(defaults) success = parser.read(conf_path) if not success: print("Unable to read config from %s" % conf_path) sys.exit(1) if section_name: if parser.has_section(section_name): conf = dict(parser.items(section_name)) else: print('Unable to find section %s in config %s' % (section_name, conf_path)) exit(1) else: conf = {} for section in parser.sections(): conf.update({section: dict(parser.items(section))}) return conf
def _update_settings_from_file(section, settings): tries = 0 current_directory = os.path.normpath(os.getcwd()) config_file = None while current_directory and tries < MAX_CONFIG_SEARCH_DEPTH: potential_path = os.path.join(current_directory, 'setup.cfg') if os.path.exists(potential_path): config_file = potential_path break new_directory = os.path.split(current_directory)[0] if current_directory == new_directory: break current_directory = new_directory tries += 1 if config_file and os.path.exists(config_file): with open(config_file, 'rU') as fp: config = SafeConfigParser() config.readfp(fp) if config.has_section('tool:multilint'): settings.update(sanitize(config.items('tool:multilint')))
def get_scrapycfg_targets(cfgfiles=None): cfg = SafeConfigParser() cfg.read(cfgfiles or []) baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {} targets = {} targets['default'] = baset for x in cfg.sections(): if x.startswith('deploy:'): t = baset.copy() t.update(cfg.items(x)) targets[x[7:]] = t for tname, t in list(targets.items()): try: int(t.get('project', 0)) except ValueError: # Don't import non-numeric project IDs, and also throw away the # URL and credentials associated with these projects (since the # project ID does not belong to SH, neither do the endpoint or the # auth information) del targets[tname] if t.get('url', "").endswith('scrapyd/'): t['url'] = t['url'][:-8] targets.setdefault('default', {}) return targets
def load_conf(): conf_path = os.path.expanduser(config_path) if not os.path.exists(conf_path): logger.warn("{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!".format(conf=to_printable_str(conf_path))) raise IOError else: logger.debug('{conf} is found'.format(conf=to_printable_str(conf_path))) try: with open(conf_path, 'r') as f: cp = SafeConfigParser() cp.readfp(fp=f) if not cp.has_section('common'): raise Exception("[common] section could't be found, please check your config file.") if cp.has_option('common', 'part_size'): part_size = cp.getint('common', 'part_size') else: part_size = 1 if cp.has_option('common', 'max_thread'): max_thread = cp.getint('common', 'max_thread') else: max_thread = 5 try: secret_id = cp.get('common', 'secret_id') except Exception: secret_id = cp.get('common', 'access_id') try: appid = cp.get('common', 'appid') bucket = cp.get('common', 'bucket') if bucket.endswith("-"+str(appid)): bucket = bucket.rstrip(appid) bucket = bucket[:-1] except Exception: try: bucket = cp.get('common', 'bucket') appid = bucket.split('-')[-1] bucket = bucket.rstrip(appid) bucket = bucket[:-1] except Exception: # check if user use -b bucket if (pre_bucket == ""): logger.error("The configuration file is wrong. Please reconfirm") try: schema = cp.get('common', 'schema') except Exception: schema = 'https' try: verify = cp.get('common', 'verify') except Exception: verify = 'md5' try: token = cp.get('common', 'token') except Exception: token = None try: error = cp.get('common', 'error') except Exception: error = "cos.err.log" try: success = cp.get('common', 'success') except Exception: success = "cos.suc.log" try: anonymous = cp.get('common', 'anonymous') if anonymous == 'True' or anonymous == 'true': anonymous = True else: anonymous = False except Exception: anonymous = False try: retry = int(cp.get('common', 'retry')) except Exception: retry = 2 try: timeout = int(cp.get('common', 'timeout')) except Exception: timeout = 60 region, endpoint = None, None if cp.has_option('common', 'region'): region = cp.get('common', 'region') if cp.has_option('common', 'endpoint'): endpoint = cp.get('common', 'endpoint') if pre_appid != "": appid = pre_appid if pre_bucket != "": bucket = pre_bucket if pre_region != "": region = pre_region conf = CoscmdConfig( appid=appid, secret_id=secret_id, secret_key=cp.get('common', 'secret_key'), token=token, region=compatible(region), endpoint=endpoint, bucket=bucket, part_size=part_size, max_thread=max_thread, schema=schema, anonymous=anonymous, verify=verify, retry=retry, timeout=timeout, error=error, success=success ) return conf except Exception as e: raise(e)
class PackageConfigHandler(object): """ Manager class for packages files for tracking installation of modules """ def __init__(self): # noinspection PyUnresolvedReferences from six.moves.configparser import SafeConfigParser self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages') if not os.path.isfile(self.package_cfg): print('Creating package file') with open(self.package_cfg, 'w') as outs: outs.close() self.parser = SafeConfigParser() self.parser.read(self.package_cfg) def save(self): with open(self.package_cfg, 'w') as outs: self.parser.write(outs) def add_module(self, pkg_info): """ :param pkg_info: A dict that has name, url, version, summary :return: """ if not self.parser.has_section(pkg_info['name']): self.parser.add_section(pkg_info['name']) self.parser.set(pkg_info['name'], 'url', pkg_info['url']) self.parser.set(pkg_info['name'], 'version', pkg_info['version']) self.parser.set(pkg_info['name'], 'summary', pkg_info['summary']) self.parser.set(pkg_info['name'], 'files', pkg_info['files']) self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency']) self.save() def list_modules(self): return [module for module in self.parser.sections()] def module_exists(self, name): return self.parser.has_section(name) def get_info(self, name): if self.parser.has_section(name): tbl = {} for opt, value in self.parser.items(name): tbl[opt] = value return tbl def remove_module(self, name): self.parser.remove_section(name) self.save() def get_files_installed(self, section_name): if self.parser.has_option(section_name, 'files'): files = self.parser.get(section_name, 'files').strip() return files.split(',') else: return None def get_dependencies(self, section_name): if self.parser.has_option(section_name, 'dependency'): dependencies = self.parser.get(section_name, 'dependency').strip() return set(dependencies.split(',')) if dependencies != '' else set() else: return None def get_all_dependencies(self, exclude_module=()): all_dependencies = set() for section_name in self.parser.sections(): if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'): dependencies = self.parser.get(section_name, 'dependency').strip() if dependencies != '': for dep in dependencies.split(','): all_dependencies.add(dep) return all_dependencies
class Config(object): """ Manages the configuration file """ def __init__(self): """ DEFAULT VALUES """ self._basescript = None self.recentvaults = [] self.pwlength = 10 self.search_notes = False self.search_passwd = False self.use_pwgen = False # self.alphabet = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_" self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_" self.avoid_bigrams = "cl mn nm nn rn vv VV" self._fname = self.get_config_filename() print("Using config %s" % self._fname) self._parser = SafeConfigParser() if os.path.exists(self._fname): self._parser.read(self._fname) if not self._parser.has_section("base"): self._parser.add_section("base") for num in range(10): if (not self._parser.has_option("base", "recentvaults" + str(num))): break self.recentvaults.append( self._parser.get("base", "recentvaults" + str(num))) if self._parser.has_option("base", "pwlength"): self.pwlength = int(self._parser.get("base", "pwlength")) if self._parser.has_option("base", "search_notes"): if self._parser.get("base", "search_notes") == "True": self.search_notes = True if self._parser.has_option("base", "search_passwd"): if self._parser.get("base", "search_passwd") == "True": self.search_passwd = True if self._parser.has_option("base", "use_pwgen"): if self._parser.get("base", "use_pwgen") == "True": self.use_pwgen = True if self._parser.has_option("base", "alphabet"): self.alphabet = self._parser.get("base", "alphabet") if self._parser.has_option("base", "avoid_bigrams"): self.avoid_bigrams = self._parser.get("base", "avoid_bigrams") if not os.path.exists(self._fname): self.save() def set_basescript(self, basescript): self._basescript = basescript def get_basescript(self): return self._basescript def save(self): if (not os.path.exists(os.path.dirname(self._fname))): os.mkdir(os.path.dirname(self._fname)) # remove duplicates and trim to 10 items _saved_recentvaults = [] for item in self.recentvaults: if item in _saved_recentvaults: continue self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item) _saved_recentvaults.append(item) if (len(_saved_recentvaults) >= 10): break self._parser.set("base", "pwlength", str(self.pwlength)) self._parser.set("base", "search_notes", str(self.search_notes)) self._parser.set("base", "search_passwd", str(self.search_passwd)) self._parser.set("base", "use_pwgen", str(self.use_pwgen)) self._parser.set("base", "alphabet", str(self.alphabet)) self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams)) filehandle = open(self._fname, 'w') self._parser.write(filehandle) filehandle.close() @staticmethod def get_config_filename(): """ Returns the full filename of the config file """ base_fname = "loxodo" # On Mac OS X, config files go to ~/Library/Application Support/foo/ if platform.system() == "Darwin": base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # On Microsoft Windows, config files go to $APPDATA/foo/ if platform.system() in ("Windows", "Microsoft"): if ("APPDATA" in os.environ): base_path = os.environ["APPDATA"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Allow config directory override as per freedesktop.org XDG Base Directory Specification if ("XDG_CONFIG_HOME" in os.environ): base_path = os.environ["XDG_CONFIG_HOME"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Default configuration path is ~/.config/foo/ base_path = os.path.join(os.path.expanduser("~"), ".config") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") else: return os.path.join(os.path.expanduser("~"), "." + base_fname + ".ini")
def onDiscovery(self, theme, settings, dependenciesSettings): res = queryResourceDirectory(THEME_RESOURCE_NAME, theme) if res is None: return directoryName = 'views' if 'directory' in settings: directoryName = settings['directory'] if res.isDirectory(directoryName): viewsDir = res[directoryName] layer = getattr(schemata, theme, None) if 'layer' in settings: layerName = settings['layer'] try: layer = resolve(layerName) except (ImportError, AttributeError): logger.warn("Could not import %s" % layerName) return viewConfig = SafeConfigParser() if viewsDir.isFile(VIEW_CONFIG_FILENAME): fp = viewsDir.openFile(VIEW_CONFIG_FILENAME) try: viewConfig.readfp(fp) finally: try: fp.close() except AttributeError: pass views = [] configurationMachine = ConfigurationMachine() path = viewsDir.directory for filename in os.listdir(path): if not filename.lower().endswith(EXTENSION): continue name = viewName = filename[:-3] permission = 'zope2.View' for_ = Interface class_ = None template = os.path.join(path, filename) menu = {} # Read override options from views.cfg if applicable if viewConfig.has_section(name): if viewConfig.has_option(name, 'name'): viewName = viewConfig.get(name, 'name') if viewConfig.has_option(name, 'permission'): permission = viewConfig.get(name, 'permission') if viewConfig.has_option(name, 'for'): forStr = viewConfig.get(name, 'for') if forStr != "*": for_ = resolve(forStr) if viewConfig.has_option(name, 'class'): class_ = resolve(viewConfig.get(name, 'class')) if viewConfig.has_option(name, 'menu'): menu = dict( title=viewConfig.get(name, 'menu'), menu=getattr( zope.browsermenu.metaconfigure.menus, "plone_displayviews", ), ) Products.Five.browser.metaconfigure.page( configurationMachine, name=viewName, permission=permission, for_=for_, layer=layer, template=template, class_=class_, **menu ) views.append(name) if len(views) > 0: configurationMachine.execute_actions() self.registered[theme] = views
class Config(MutableMapping): """Read and write to config file. A config object is essentially a string key-value store that can be treated like a dictionary:: c = Config() c['foo'] = 'bar' print c['foo'] The file location may be specified:: c = Config('~/matt/anotherconfig.cfg') c['where'] = 'in a different file' If no location is specified, the environment variable CHEMDATAEXTRACTOR_CONFIG is checked and used if available. Otherwise, a standard config location is used, which varies depending on the operating system. You can check the location using the ``path`` property. For more information see https://github.com/ActiveState/appdirs It is possible to edit the file by hand with a text editor. Warning: multiple instances of Config() pointing to the same file will not see each others' changes, and will overwrite the entire file when any key is changed. """ #: These values will be present in a config object unless they are explicitly defined otherwise in the config file default_values = {} def __init__(self, path=None): """ :param string path: (Optional) Path to config file location. """ self._path = path # Use CHEMDATAEXTRACTOR_CONFIG environment variable if set if not self._path: self._path = os.environ.get('CHEMDATAEXTRACTOR_CONFIG') # Use OS-dependent config directory given by appdirs if not self._path: self._path = os.path.join( appdirs.user_config_dir('ChemDataExtractor'), 'config.cfg') self._parser = SafeConfigParser() if os.path.isfile(self.path): with open(self.path) as f: self._parser.readfp(f) if not self._parser.has_section('cde'): self._parser.add_section('cde') for k, v in six.iteritems(self.default_values): if not self._parser.has_option('cde', k.encode('utf-8')): self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8')) @property def path(self): """The path to the config file.""" return self._path def _flush(self): """Save the contents of data to the file on disk. You should not need to call this manually.""" d = os.path.dirname(self.path) if not os.path.isdir(d): os.makedirs(d) with open(self.path, 'w') as f: self._parser.write(f) def __contains__(self, k): return self._parser.has_option('cde', k.encode('utf-8')) def __getitem__(self, k): try: return self._parser.get('cde', k.encode('utf-8')) except NoOptionError: raise KeyError(k) def __setitem__(self, k, v): self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8')) self._flush() def __delitem__(self, k): try: self._parser.remove_option('cde', k.encode('utf-8')) self._flush() except NoOptionError: raise KeyError(k) def __iter__(self): return (k for k, v in self._parser.items('cde')) def __len__(self): return len(self._parser.items('cde')) def __repr__(self): return '<Config: %s>' % self.path def clear(self): """Clear all values from config.""" self._parser.remove_section('cde') self._parser.add_section('cde') self._flush()
def _parse_args(args_str): keystone_auth_parser = SafeConfigParser() conf_file = keystone_auth_parser.read( '/etc/contrail/contrail-keystone-auth.conf') default_keystone_vals = { "username": "******", "tenant_name": "admin", "domain_name": "Default" } get_vars = (lambda x: keystone_auth_parser.get('KEYSTONE', x) if keystone_auth_parser.has_option('KEYSTONE', x) else None) if conf_file: if keystone_auth_parser.has_section('KEYSTONE'): username = get_vars('admin_user') if username: default_keystone_vals['username'] = username password = get_vars('admin_password') if password: default_keystone_vals['password'] = password tenant_name = get_vars('admin_tenant_name') if tenant_name: default_keystone_vals['tenant_name'] = tenant_name domain_name = get_vars('user_domain_name') if domain_name: default_keystone_vals['domain_name'] = domain_name parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description='') parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__) parser.add_argument("--debug", help="Run in debug mode, default False", action='store_true', default=False) parser.add_argument("--to-json", help="File to dump json to", default=None) parser.add_argument("--username", help="Username used to login to API Server") parser.add_argument("--password", help="Password used to login to API Server") parser.add_argument("--api-server-host", help="IP of API Server") parser.add_argument("--tenant-name", help="Name of Tenant") parser.add_argument("--domain-name", help="Domain name") parser.add_argument("--read-timeout", dest='timeout', help="Timeout for Data reading operations", default=300) parser.add_argument( "--connection-timeout", help="Timeout set for VNC API to connect to API Server", default=120) ts = calendar.timegm(time.gmtime()) if os.path.isdir("/var/log/contrail"): default_log = "/var/log/contrail/fabric_validation-{0}.log".format(ts) else: import tempfile default_log = '{0}/fabric_validation-{1}.log'.format( tempfile.gettempdir(), ts) parser.add_argument("--log_file", help="Log file to save output, default '%(default)s'", default=default_log) parser.set_defaults(**default_keystone_vals) args_obj, _ = parser.parse_known_args(args_str.split()) _args = args_obj return _args
def parse_config_file(self, config_file): config = SafeConfigParser(self.DEFAULT_CONFIG) config.readfp(config_file) blessconfig = { 'CLIENT_CONFIG': { 'domain_regex': config.get('CLIENT', 'domain_regex'), 'cache_dir': config.get('CLIENT', 'cache_dir'), 'cache_file': config.get('CLIENT', 'cache_file'), 'mfa_cache_dir': config.get('CLIENT', 'mfa_cache_dir'), 'mfa_cache_file': config.get('CLIENT', 'mfa_cache_file'), 'ip_urls': [s.strip() for s in config.get('CLIENT', 'ip_urls').split(",")], 'update_script': config.get('CLIENT', 'update_script'), 'user_session_length': int(config.get('CLIENT', 'user_session_length')), 'usebless_role_session_length': int(config.get('CLIENT', 'usebless_role_session_length')), 'update_sshagent': config.getboolean('CLIENT', 'update_sshagent'), 'use_env_creds': config.getboolean('CLIENT', 'use_env_creds'), }, 'BLESS_CONFIG': { 'ca_backend': config.get('MAIN', 'ca_backend'), 'userrole': config.get('LAMBDA', 'user_role'), 'accountid': config.get('LAMBDA', 'account_id'), 'functionname': config.get('LAMBDA', 'functionname'), 'functionversion': config.get('LAMBDA', 'functionversion'), 'certlifetime': config.getint('LAMBDA', 'certlifetime'), 'ipcachelifetime': config.getint('LAMBDA', 'ipcachelifetime'), 'timeoutconfig': { 'connect': config.getint('LAMBDA', 'timeout_connect'), 'read': config.getint('LAMBDA', 'timeout_read') } }, 'AWS_CONFIG': { 'remote_user': config.get('MAIN', 'remote_user') }, 'REGION_ALIAS': {} } if config.has_option('MAIN', 'bastion_ips'): blessconfig['AWS_CONFIG']['bastion_ips'] = config.get('MAIN', 'bastion_ips') if blessconfig['BLESS_CONFIG']['ca_backend'].lower() == 'hashicorp-vault': blessconfig['VAULT_CONFIG'] = { 'vault_addr': config.get('VAULT', 'vault_addr'), 'auth_mount': config.get('VAULT', 'auth_mount'), 'ssh_backend_mount': config.get('VAULT', 'ssh_backend_mount'), 'ssh_backend_role': config.get('VAULT', 'ssh_backend_role'), } regions = config.get('MAIN', 'region_aliases').split(",") regions = [region.strip() for region in regions] for region in regions: region = region.upper() kms_region_key = 'KMSAUTH_CONFIG_{}'.format(region) blessconfig.update({kms_region_key: self._get_region_kms_config(region, config)}) blessconfig['REGION_ALIAS'].update({region: blessconfig[kms_region_key]['awsregion']}) if config.has_section('HOUSEKEEPER'): hk_region_key = 'HOUSEKEEPER_CONFIG_{}'.format(region) blessconfig.update({hk_region_key: self._get_region_housekeeper_config(region, config)}) blessconfig['REGION_ALIAS'].update({region: blessconfig[hk_region_key]['awsregion']}) return blessconfig
def createThemeFromTemplate(title, description, baseOn='template'): """Create a new theme from the given title and description based on another theme resource directory """ source = queryResourceDirectory(THEME_RESOURCE_NAME, baseOn) if source is None: raise KeyError("Theme {0:s} not found".format(baseOn)) themeName = getUtility(IURLNormalizer).normalize(title) if isinstance(themeName, six.text_type): themeName = themeName.encode('utf-8') resources = getOrCreatePersistentResourceDirectory() resources.makeDirectory(themeName) target = resources[themeName] cloneResourceDirectory(source, target) manifest = SafeConfigParser() if MANIFEST_FILENAME in target: fp = target.openFile(MANIFEST_FILENAME) try: manifest.readfp(fp) finally: fp.close() if not manifest.has_section('theme'): manifest.add_section('theme') if not isinstance(title, str): title = title.encode('utf-8') if not isinstance(description, str): description = description.encode('utf-8') manifest.set('theme', 'title', title) manifest.set('theme', 'description', description) if manifest.has_option('theme', 'prefix'): prefix = u"/++%s++%s" % (THEME_RESOURCE_NAME, themeName) manifest.set('theme', 'prefix', prefix) if manifest.has_option('theme', 'rules'): rule = manifest.get('theme', 'rules') rule_file_name = rule.split('/')[-1] # extract real rules file name rules = u"/++%s++%s/%s" % (THEME_RESOURCE_NAME, themeName, rule_file_name) manifest.set('theme', 'rules', rules) paths_to_fix = [ 'development-css', 'production-css', 'tinymce-content-css', 'development-js', 'production-js' ] for var_path in paths_to_fix: if not manifest.has_option('theme', var_path): continue val = manifest.get('theme', var_path) if not val: continue template_prefix = '++%s++%s/' % (THEME_RESOURCE_NAME, baseOn) if template_prefix in val: # okay, fix val = val.replace(template_prefix, '++%s++%s/' % (THEME_RESOURCE_NAME, themeName)) manifest.set('theme', var_path, val) manifestContents = StringIO() manifest.write(manifestContents) target.writeFile(MANIFEST_FILENAME, manifestContents) return themeName
def parseOptions(self): """Parse the command line options.""" log = LOGGER.getChild("Options") conf_parser = argparse.ArgumentParser( "Dirac Release Notes", formatter_class=argparse.RawTextHelpFormatter, add_help=False, ) conf_parser.add_argument("-c", "--configFile", help="Specify config file", metavar="FILE", dest="configFile") conf_parser.add_argument("-t", "--token", help="API token to use", default="") conf_parser.add_argument("-d", "--debug", action="count", dest="debug", help="d, dd, ddd", default=0) args, _remaining_argv = conf_parser.parse_known_args() self.token = args.token self.printLevel = _parsePrintLevel(args.debug) LOGGER.setLevel(self.printLevel) log.debug("Parsing options") defaults = {} if args.configFile: log.debug("Parsing configFile: %r", args.configFile) config = ConfigParser() config.optionxform = str config.read([args.configFile]) defaults.update(dict(config.items("ReleaseNotes"))) log.info("Settings from config file: %r", defaults) if config.has_section("ReleaseNotes.Categories"): items = config.items("ReleaseNotes.Categories") log.info("Found Categories: %r", items) for short, system in items: self.names[short] = system parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, # Inherit options from config_parser parents=[conf_parser], ) parser.add_argument( "--branches", action="store", default=self.branches, dest="branches", nargs="+", help="branches to get release notes for", ) parser.add_argument( "--date", action="store", default=self.startDate, dest="date", help="date and optionally time after which PRs are checked (ISO 8601),\ accepting 2020-01-08 or 2018-05-20T15:23:45Z,\ default (two weeks ago): %s" % self.startDate, ) parser.add_argument( "--sinceLatestTag", action="store_true", dest="sinceLatestTag", default=self.sinceLatestTag, help="get release notes since latest tag (incompatible with --date)", ) parser.add_argument( "--headerMessage", action="store", default=self.headerMessage, dest="headerMessage", help="Header message to add between the release name and the list of PR. If it is a path,\ read the content of the file", ) parser.add_argument( "--footerMessage", action="store", default=self.footerMessage, dest="footerMessage", help="Footer message to add after the list of PR. If it is a path,\ read the content of the file", ) parser.add_argument( "--openPRs", action="store_true", dest="openPRs", default=self.openPRs, help="get release notes for open (unmerged) PRs, for testing purposes", ) parser.add_argument( "-r", "--repo", action="store", dest="repo", help="Github repository to check: [Group/]Repo", default="DiracGrid/Dirac", ) parser.add_argument( "-g", "--gitlab", action="store_true", dest="gitlab", help="Using gitlab instance", default=False ) parser.add_argument( "-u", "--gitlabUrl", action="store", dest="gitlabUrl", help="URL of the gitlab instance", default="https://gitlab.cern.ch", ) parser.add_argument( "-i", "--gitlabProjectID", action="store", dest="gitlabProjectID", help="ID of the project in Gitlab", default="0", ) parser.add_argument( "--deployRelease", action="store_true", dest="deployRelease", help="Convert an existing tag into a github/gitlab release. Requires --releaseNotes and --tagName", default=self.deployRelease, ) parser.add_argument( "--tagName", action="store", dest="tagName", help="Name of the tag to release (with --deployRelease)", default=self.tagName, ) parser.add_argument( "--releaseNotes", action="store", dest="releaseNotes", help="Path to the file containing release notes for this version (with --deployRelease)", default=self.releaseNotes, ) parser.set_defaults(**defaults) parsed = parser.parse_args() for var, val in sorted(vars(parsed).items()): log.info("Parsed options: %s = %s", var, pformat(val)) self.branches = listify(parsed.branches) log.info("Getting PRs for: %s", self.branches) # If the date parsed does not correspond to the default, # and latestTag is asked, we throw an error if (parsed.date != self.startDate) and parsed.sinceLatestTag: raise RuntimeError("--sinceLatestTag incompatible with --date") self.sinceLatestTag = parsed.sinceLatestTag if self.sinceLatestTag: log.info("Starting from the latest tag") self.startDate = None del parsed.date else: if not isinstance(parsed.date, datetime.date): parsed.date = dateutil.parser.isoparse(parsed.date) if parsed.date.tzinfo is None or parsed.date.tzinfo.utcoffset(parsed.date) is None: self.startDate = pytz.utc.localize(parsed.date) else: self.startDate = parsed.date log.info("Starting from: %s", self.startDate) self.openPRs = parsed.openPRs log.info("Also including openPRs?: %s", self.openPRs) self.headerMessage = parsed.headerMessage if self.headerMessage: log.info("Using header message: %s", self.headerMessage) self.footerMessage = parsed.footerMessage if self.footerMessage: log.info("Using footer message: %s", self.footerMessage) self.useGitlab = parsed.gitlab if isinstance(parsed.gitlab, bool) else parsed.gitlab.lower() == "true" self.useGithub = not self.useGitlab self.gitlabUrl = parsed.gitlabUrl self.glProjectID = int(parsed.gitlabProjectID) self.deployRelease = parsed.deployRelease self.releaseNotes = parsed.releaseNotes self.tagName = parsed.tagName if self.deployRelease: if not (self.releaseNotes and self.tagName): raise RuntimeError("--deployRelease requires --releaseNotes and --tagName") if not os.path.isfile(self.releaseNotes): raise RuntimeError("--releaseNotes should point to an existing file") repo = parsed.repo repos = repo.split("/") if len(repos) == 1: self.repo = repo elif len(repos) == 2: self.owner = repos[0] self.repo = repos[1] else: raise RuntimeError("Cannot parse repo option: %s" % repo) for var, val in sorted(vars(parsed).items()): log.info("Using options: %s = %s", var, pformat(val))
class Config(object): """ Manages the configuration file """ def __init__(self): """ DEFAULT VALUES """ self._basescript = None self.recentvaults = [] self.pwlength = 10 self.search_notes = False self.search_passwd = False self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_" self.avoid_bigrams = "cl mn nm nn rn vv VV" self._fname = self.get_config_filename() self._parser = SafeConfigParser() if os.path.exists(self._fname): self._parser.read(self._fname) if not self._parser.has_section("base"): self._parser.add_section("base") for num in range(10): if (not self._parser.has_option("base", "recentvaults" + str(num))): break self.recentvaults.append(self._parser.get("base", "recentvaults" + str(num))) if self._parser.has_option("base", "pwlength"): self.pwlength = int(self._parser.get("base", "pwlength")) if self._parser.has_option("base", "search_notes"): if self._parser.get("base", "search_notes") == "True": self.search_notes = True if self._parser.has_option("base", "search_passwd"): if self._parser.get("base", "search_passwd") == "True": self.search_passwd = True if self._parser.has_option("base", "alphabet"): self.alphabet = self._parser.get("base", "alphabet") if self._parser.has_option("base", "avoid_bigrams"): self.avoid_bigrams = self._parser.get("base", "avoid_bigrams") if not os.path.exists(self._fname): self.save() def set_basescript(self, basescript): self._basescript = basescript def get_basescript(self): return self._basescript def save(self): if (not os.path.exists(os.path.dirname(self._fname))): os.mkdir(os.path.dirname(self._fname)) # remove duplicates and trim to 10 items _saved_recentvaults = [] for item in self.recentvaults: if item in _saved_recentvaults: continue self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item) _saved_recentvaults.append(item) if (len(_saved_recentvaults) >= 10): break self._parser.set("base", "pwlength", str(self.pwlength)) self._parser.set("base", "search_notes", str(self.search_notes)) self._parser.set("base", "search_passwd", str(self.search_passwd)) self._parser.set("base", "alphabet", str(self.alphabet)) self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams)) filehandle = open(self._fname, 'w') self._parser.write(filehandle) filehandle.close() @staticmethod def get_config_filename(): """ Returns the full filename of the config file """ base_fname = "loxodo" # On Mac OS X, config files go to ~/Library/Application Support/foo/ if platform.system() == "Darwin": base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # On Microsoft Windows, config files go to $APPDATA/foo/ if platform.system() in ("Windows", "Microsoft"): if ("APPDATA" in os.environ): base_path = os.environ["APPDATA"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Allow config directory override as per freedesktop.org XDG Base Directory Specification if ("XDG_CONFIG_HOME" in os.environ): base_path = os.environ["XDG_CONFIG_HOME"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Default configuration path is ~/.config/foo/ base_path = os.path.join(os.path.expanduser("~"), ".config") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") else: return os.path.join(os.path.expanduser("~"),"."+ base_fname + ".ini")
class IniConfigLoader(object): """This config loader transforms a traditional INI file into a Montague Standard Format dictionary. It is compatible with most but not all PasteDeploy files.""" def __init__(self, path): self.path = path self._data = self._read() self._config = self._process() def _read(self): # We need to keep the parser around so the logging conversion can use it. path_defaults = { 'here': os.path.dirname(self.path), '__file__': self.path, } self._parser = SafeConfigParser() self._parser.read(self.path) self._globals = self._parser.defaults() data = {} for section in self._parser.sections(): section_data = data.setdefault(section, {}) for option in self._parser.options(section): if option in self._globals: continue try: section_data[option] = self._parser.get(section, option, vars=path_defaults) except InterpolationError: section_data[option] = self._parser.get(section, option, raw=True) return data def _process(self): orig = self._data config = {} for key in six.iterkeys(orig): if ':' in key: scheme, name = key.split(':', 1) kind_config = config.setdefault(SCHEMEMAP[scheme], {}) kind_config[name] = orig[key] else: config[key] = orig[key] config['globals'] = { 'here': os.path.dirname(self.path), '__file__': self.path, } for key, value in six.iteritems(self._globals): config['globals'][key] = value apps = config.setdefault('application', {}) filters = config.setdefault('filter', {}) generated_filter_count = 0 filter_apps = config.pop('filter-app', {}) for name, filter_app in six.iteritems(filter_apps): use = filter_app.pop('next') generated_filter_count += 1 filter_name = '_montague_filter_{0}'.format(generated_filter_count) apps[name] = {'use': use, 'filter-with': filter_name} filters[filter_name] = filter_app pipelines = config.pop('pipeline', {}) for name, pipeline in six.iteritems(pipelines): items = pipeline['pipeline'].split() pipeline_app = items[-1] pipeline_filters = items[:-1] pipeline_filters.reverse() apps[name] = {'use': pipeline_app} last_item = apps[name] for count, use_filter in enumerate(pipeline_filters, start=1): filter_name = '_montague_pipeline_{0}_filter_{1}'.format(name, count) filters[filter_name] = {'use': use_filter} last_item['filter-with'] = filter_name last_item = filters[filter_name] if all([self._parser.has_section(section_name) for section_name in LOGGING_SECTIONS]): loggers = convert_loggers(self._parser) handlers = convert_handlers(self._parser) formatters = convert_formatters(self._parser) config['logging'] = {'main': combine(loggers, handlers, formatters)} for key in MSF_KEYS: config.setdefault(key, {}) return config def config(self): return self._config def app_config(self, name): # This method isn't actually necessary, since montague can extract # the config information from the MSF dict returned by .config() # but it's a nice example of how to do it. if name in self._config['application']: constructor = LoadableConfig.app local_config = self._config['application'][name] elif name in self._config['composite']: constructor = LoadableConfig.composite local_config = self._config['composite'][name] else: raise KeyError return constructor( name=name, config=local_config, global_config=self._config['globals']) def server_config(self, name): if name in self._config['server']: constructor = LoadableConfig.server local_config = self._config['server'][name] else: raise KeyError return constructor( name=name, config=local_config, global_config=self._config['globals']) def filter_config(self, name): if name in self._config['filter']: constructor = LoadableConfig.filter local_config = self._config['filter'][name] else: raise KeyError return constructor( name=name, config=local_config, global_config=self._config['globals']) def logging_config(self, name): # This is provided by .config(), so no need to implement it here. raise NotImplementedError
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'scrapyd' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8') self.cp = SafeConfigParser() self.cp.readfp(StringIO(default_config)) self.cp.read(sources) for fp in extra_sources: self.cp.readfp(fp) else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _getsources(self): sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf'] sources += sorted(glob.glob('/etc/scrapyd/conf.d/*')) sources += ['scrapyd.conf'] sources += [expanduser('~/.scrapyd.conf')] scrapy_cfg = closest_scrapy_cfg() if scrapy_cfg: sources.append(scrapy_cfg) return sources def _getany(self, method, option, default): try: return method(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def get(self, option, default=None): return self._getany(self.cp.get, option, default) def getint(self, option, default=None): return self._getany(self.cp.getint, option, default) def getfloat(self, option, default=None): return self._getany(self.cp.getfloat, option, default) def getboolean(self, option, default=None): return self._getany(self.cp.getboolean, option, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise def has_section(self, section): """add section checking.""" return self.cp.has_section(section)