def load_from_config(config_file, args, values):
"""
Load config from user's home folder, and load into config object. If values are given during runtime that conflict
with values in config file, the config file values are overwritten.
:param config_file: Name of an existing config file
:param args: Array of values containing argument names from main
:param values: Array of values containing values from arguments from main
:return: key/value pairs of args/values
"""
# TODO: Handle args not existing in user config file and passed args
config = dict()
if config_file:
load_config = os.path.join(BASE_CONFIG_DIR, config_file)
if os.path.exists(load_config):
parser = SafeConfigParser()
parser.read(load_config)
for _var in CONFIG_VARS:
config[_var['var']] = parser.get('settings', _var['var'])
items = values.items()
for k, v in items:
if k in args and v is not None:
config[k] = v
for _var in CONFIG_VARS:
if _var['var'] in args and values[_var['var']] is not None:
config[_var['var']] = values[_var['var']]
if _var["var"] not in config:
click.echo(_var['error'])
return namedtuple('GenericDict', config.keys())(**config)
class Config(object):
def __init__(self):
self.config = SafeConfigParser()
self.name = ''
def parse(self, fname, override):
if override:
override = [x for x in csv.reader(
' '.join(override).split(','), delimiter='.')]
logger.info('Reading configuration file: {}'.format(fname))
if not os.path.isfile(fname):
logger.interrupt('File doesn\'t exist: {}'.format(fname))
self.config.optionxform = str
self.config.read(fname)
for section, option, value in override:
if not self.config.has_section(section):
self.config.add_section(section)
self.config.set(section, option, value)
basename = os.path.basename(fname)
self.name = os.path.splitext(basename)[0]
@safe
def _get_options_as_dict(self, section):
if section in self.config.sections():
return {p: v for p, v in self.config.items(section)}
else:
return {}
def __init__(self,
client_id,
client_secret,
access_token=None,
refresh_token=None):
self.client_id = client_id
self.client_secret = client_secret
self.access_token = access_token
self.refresh_token = refresh_token
self.expires_in = None
parser = SafeConfigParser()
parser.read(os.path.dirname(os.path.abspath(__file__)) + '/config.ini')
self._requests = requests.Session()
try:
self.SSL_VERSION = parser.get('config', 'ssl_version')
self._requests.mount(
'https://',
SSLAdapter(ssl_version=getattr(ssl, self.SSL_VERSION)))
except:
self._requests = requests
self.API_ROOT_URL = parser.get('config', 'api_root_url')
self.SDK_VERSION = parser.get('config', 'sdk_version')
self.AUTH_URL = parser.get('config', 'auth_url')
self.OAUTH_URL = parser.get('config', 'oauth_url')
def from_config(cls, path, **kwargs):
"""
Create a LastFM instance from a config file in the following format:
[lastfm]
api_key = myapikey
api_secret = myapisecret
username = thesquelched
password = plaintext_password
# Can be 'password' or 'hashed_password'
auth_method = password
You can also override config values with keyword arguments.
"""
config = SafeConfigParser()
config.add_section('lastfm')
config.read(os.path.expanduser(os.path.expandvars(path)))
return LastFM(
cls._getoption(config, kwargs, 'api_key'),
cls._getoption(config, kwargs, 'api_secret'),
username=cls._getoption(config, kwargs, 'username', None),
password=cls._getoption(config, kwargs, 'password', None),
auth_method=cls._getoption(config, kwargs, 'auth_method', None),
session_key=cls._getoption(config, kwargs, 'session_key', None),
url=cls._getoption(config, kwargs, 'url', None),
)
def auth_config(cls, email, password, hostname, namespace, dir_file):
parser = SafeConfigParser()
def try_section(section, email, password, filename):
if parser.has_section(section) or section == DEFAULTSECT:
# look at options which look like EMAIL@DOMAIN = PASSWORD
# We might want other per-domain options later
candidates = [(e,p) for e, p in parser.items(section)
if '@' in e and e.startswith(email)]
if len(candidates) == 1:
email, password = candidates[0]
elif len(candidates) > 1:
raise ValueError(
"Email prefix <{0}> is ambiguous in [{1}] of '{2}'".format(
email, section, filename))
return (email, password)
if not email:
# Do not provide default credentials. It's too dangerous.
return (None, None)
dir, file = dir_file
if dir:
# TODO: check that file is not world-readable
filename = os.path.join(dir, file)
parser.read(filename)
(email, password) = try_section(hostname, email, password, filename)
if not (email and password):
(email, password) = try_section(DEFAULTSECT, email, password, filename)
return (email, password)
def handle(self, *args, **options):
#
parsers = []
# Read config
config = SafeConfigParser()
for p in self.get_parsers():
config.read(os.path.join("etc", "address", "%s.defaults" % p))
config.read(os.path.join("etc", "address", "%s.conf" % p))
if config.getboolean(p, "enabled"):
m = __import__("noc.gis.parsers.address.%s" % p, {}, {}, "*")
for l in dir(m):
a = getattr(m, l)
if inspect.isclass(a) and issubclass(
a, AddressParser) and a != AddressParser:
parsers += [a]
else:
print("Parser '%s' is not enabled. Skipping.." % p)
# Initialize parsers
parsers = [p(config, options) for p in parsers]
# Download
if options["download"]:
for p in parsers:
print("Downloading", p.name)
if not p.download():
raise CommandError("Failed to download %s" % p.name)
else:
print("Skipping downloads")
# Sync
try:
for p in parsers:
print("Syncing", p.name)
p.sync()
except Exception:
error_report()
def read(cfgfile):
if not os.path.exists(cfgfile):
ex = IOError if six.PY2 else FileNotFoundError
raise ex('File {name} does not exist.'.format(name=cfgfile))
data = SafeConfigParser()
data.read(cfgfile)
return data
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapydartx'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__,
'default_scrapyd.conf').decode('utf8')
self.cp = SafeConfigParser()
self.cp.readfp(StringIO(default_config))
self.cp.read(sources)
for fp in extra_sources:
self.cp.readfp(fp)
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = [
'/etc/scrapydartx/scrapydartx.conf', r'c:\scrapyd\scrapyd.conf'
]
sources += sorted(glob.glob('/etc/scrapydartx/conf.d/*'))
sources += ['scrapydartx.conf']
sources += [expanduser('~/.scrapydartx.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def load(f):
p = SafeConfigParser()
p.read(f)
if not p.has_section('oauth2'):
p.add_section('oauth2')
if not p.has_section('oauth2-state'):
p.add_section('oauth2-state')
return p
def load(f):
p = SafeConfigParser()
p.read(f)
if not p.has_section('oauth2'):
p.add_section('oauth2')
if not p.has_section('oauth2-state'):
p.add_section('oauth2-state')
return p
def get_config(use_closest=True):
"""Get Scrapy config file as a SafeConfigParser"""
# 获取资源
sources = get_sources(use_closest)
# 解析配置
cfg = SafeConfigParser()
cfg.read(sources)
return cfg
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapyd'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8')
self.cp = SafeConfigParser()
self.cp.readfp(StringIO(default_config))
self.cp.read(sources)
for fp in extra_sources:
self.cp.readfp(fp)
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf']
sources += sorted(glob.glob('/etc/scrapyd/conf.d/*'))
sources += ['scrapyd.conf']
sources += [expanduser('~/.scrapyd.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def parse_config(variable=None):
"""
Parse the Trident local configuration file. This function is called
whenever Trident is imported, and it assures that Trident knows where
the default ion table datafiles exist. If a ``config.tri`` file doesn't
exist in ``$HOME/.trident`` or in the current working directory, then
Trident will launch the :class:`~trident.create_config` function to
try to automatically generate one for the user. For more information
on this process, see the installation documentation.
**Parameters**
:variable: string, optional
If you wish to get the value a variable is set to in the config
file, specify that variable name here. Will return the result
value of that variable. Default: None
"""
# Assure the ~/.trident directory exists, and read in the config file.
home = os.path.expanduser("~")
directory = os.path.join(home, '.trident')
config_filename = os.path.join(directory, 'config.tri')
# If config file exists in current directory, use it instead of file in
# $HOME/.trident. Stopgap for situations where user cannot access $HOME
local_filename = os.path.join(os.getcwd(), 'config.tri')
if os.path.exists(local_filename):
config_filename = local_filename
try:
parser = SafeConfigParser()
parser.read(config_filename)
ion_table_dir = parser.get('Trident', 'ion_table_dir')
ion_table_file = parser.get('Trident', 'ion_table_file')
except:
config_filename = create_config()
parser = SafeConfigParser()
parser.read(config_filename)
ion_table_dir = parser.get('Trident', 'ion_table_dir')
ion_table_file = parser.get('Trident', 'ion_table_file')
ion_table_dir = os.path.abspath(os.path.expanduser(ion_table_dir))
if not os.path.exists(os.path.join(ion_table_dir,
ion_table_file)):
print("")
print("No ion table data file found in %s" % ion_table_dir)
ion_table_file = get_datafiles(ion_table_dir)
parser.set('Trident', 'ion_table_file', ion_table_file)
with open(config_filename, 'w') as configfile:
parser.write(configfile)
# value to return depends on what was set for "variable"
if variable is None:
return ion_table_dir, ion_table_file
else:
return parser.get('Trident', variable)
def read_config_file(config_file_path):
# type: (str) -> None
parser = SafeConfigParser()
parser.read(config_file_path)
for section in parser.sections():
bots_config[section] = {
"email": parser.get(section, 'email'),
"key": parser.get(section, 'key'),
"site": parser.get(section, 'site'),
}
def read_config_file(config_file_path):
# type: (str) -> None
parser = SafeConfigParser()
parser.read(config_file_path)
for section in parser.sections():
bots_config[section] = {
"email": parser.get(section, 'email'),
"key": parser.get(section, 'key'),
"site": parser.get(section, 'site'),
}
def logging_config(self, name):
if name != 'main':
raise KeyError
parser = SafeConfigParser()
parser.read(self.path)
for section_name in ('loggers', 'handlers', 'formatters'):
if not parser.has_section(section_name):
raise KeyError
loggers = convert_loggers(parser)
handlers = convert_handlers(parser)
formatters = convert_formatters(parser)
return combine(loggers, handlers, formatters)
def _update_gridinit_rawx(self, port):
grid = SafeConfigParser()
grid.read(GRID_CONF)
section = "Service.%s-%s" % (self.ns, self.name)
val = grid.get(section, "Group").split(",")
val[3] = val[3].split(':')[0] + ':' + str(port)
grid.set(section, "Group", ",".join(val))
with open(GRID_CONF, "w") as fp:
grid.write(fp)
def load_config(config_files=(), overrides=()):
cp = SafeConfigParser()
cp.optionxform = str # make parsing case-sensitive
cp.read(FILENAMES + config_files)
for section, option, value in overrides:
if value is not None:
if isinstance(value, bool):
value = int(value)
cp.set(section, option, str(value))
return {section: dict(cp.items(section)) for section in cp.sections()}
def parse(file, raise_conflicts=False, separator="."):
"""
Reads in a config file and convert is to a dictionary where each
entry follows the pattern dict["section.key"]="value"
"""
data = SafeConfigParser()
data.read(file)
address_dict = {}
for s in data.sections():
for k, v in data.items(s):
key = '{}{}{}'.format(s, separator, k)
address_dict[key] = v
return address_dict
def get_config(use_closest=True):
'''
sources = ['/etc/scrapy.cfg',
'c:\\scrapy\\scrapy.cfg',
'C:\\Users\\Yingchao.wang/.config/scrapy.cfg',
'C:\\Users\\Yingchao.wang/.scrapy.cfg',
'C:\\Users\\Yingchao.wang\\Desktop\\product\\ScrapyTest\\scrapy.cfg']
'''
sources = get_sources(use_closest)
cfg = SafeConfigParser()
cfg.read(sources)
return cfg
def get_config(app_dir=None, platform=None):
"""
Load app configuration and return as config instance.
"""
config = ConfigParser()
path = os.environ.get("KNOWHOW_CONF")
if not path:
if not app_dir:
app_dir = get_app_dir(platform=platform)
assert app_dir
path = os.path.join(app_dir, "knowhow.ini")
if os.path.exists(path):
config.read(path)
return config
def get_config(app_dir=None, platform=None):
"""
Load app configuration and return as config instance.
"""
config = ConfigParser()
path = os.environ.get("KNOWHOW_CONF")
if not path:
if not app_dir:
app_dir = get_app_dir(platform=platform)
assert app_dir
path = os.path.join(app_dir, "knowhow.ini")
if os.path.exists(path):
config.read(path)
return config
def read_config_file(config_file_path):
# type: (str) -> None
config_file_path = os.path.abspath(os.path.expanduser(config_file_path))
if not os.path.isfile(config_file_path):
raise IOError("Could not read config file {}: File not found.".format(config_file_path))
parser = SafeConfigParser()
parser.read(config_file_path)
for section in parser.sections():
bots_config[section] = {
"email": parser.get(section, 'email'),
"key": parser.get(section, 'key'),
"site": parser.get(section, 'site'),
}
class Evaluator(object):
APP_NAME = 'expression-evaluators'
_default_dir = 'evaluators'
def __init__(self, plugin_dir=None):
self.config = SafeConfigParser()
config_path = save_config_path(self.APP_NAME)
self.config_file = os.path.join(config_path, self.APP_NAME + ".conf")
self.config.read(self.config_file)
this_dir = os.path.abspath(os.path.dirname(__file__))
self.plugin_dir = plugin_dir or os.path.join(this_dir,
self._default_dir)
places = [
self.plugin_dir,
]
[
places.append(os.path.join(path, self.APP_NAME, "evaluators"))
for path in xdg_data_dirs
]
PluginManagerSingleton.setBehaviour([
ConfigurablePluginManager,
VersionedPluginManager,
])
self.manager = PluginManagerSingleton.get()
self.manager.setConfigParser(self.config, self.write_config)
self.manager.setPluginInfoExtension("expr-plugin")
self.manager.setPluginPlaces(places)
self.manager.collectPlugins()
def _get_all_evaluators(self):
return self.manager.getAllPlugins()
def _get_evaluator(self, name):
pl = self.manager.getPluginByName(name)
if not pl:
raise Exception('No expression evaluator %s' % name)
return pl.plugin_object
def write_config(self):
f = open(self.config_file, "w")
self.config.write(f)
f.close()
def evaluate(self, lang, expression, *args, **kwargs):
pl = self._get_evaluator(lang)
return pl.evaluate(expression, *args, **kwargs)
def get_download_params(self, example_id):
"""Returns the local configuration file for this example_id"""
config_filename = self.get_local_path(example_id)
parser = SafeConfigParser()
parser.read(config_filename)
config_section = dict(
parser.items('config')) if 'config' in parser else {}
other_sections = {
key: value
for key, value in parser.items()
if key != 'config' and key != 'DEFAULT'
}
return config_section, other_sections
def config(request):
path = request.config.getoption("--config")
config = SafeConfigParser()
for section, values in CONFIG_DEFAULTS.items():
config.add_section(section)
for key, value in values.items():
config.set(section, key, value)
if path:
config.read([path])
with open(path) as f:
LOG.debug("Configuration:\n%s", f.read())
return config
def find_scrapy_project(project):
project_config_path = closest_scrapy_cfg()
if not project_config_path:
raise RuntimeError('Cannot find scrapy.cfg file')
project_config = SafeConfigParser()
project_config.read(project_config_path)
try:
project_settings = project_config.get('settings', project)
except (NoSectionError, NoOptionError) as e:
raise RuntimeError(e.message)
if not project_settings:
raise RuntimeError('Cannot find scrapy project settings')
project_location = os.path.dirname(project_config_path)
sys.path.append(project_location)
return project_settings
def get_config(use_closest=True):
"""Get Scrapy config file as a SafeConfigParser"""
## 获取 Scrapy 配置文件可能的路径列表: ['/etc/scrapy.cfg', ...]
sources = get_sources(use_closest)
## 实例化一个配置解析器
## 该模块是用来解析配置文件的，配置文件中的内容可以包含一个或多个节（section）
## 若传入的参数是多个配置文件的列表，会从左往右依次读取，后面的值会覆盖前面的值
## 每个节可以有多个参数（键=值）或（键:值）
## 使用配置文件的好处是使程序更加灵活，将变化抽离出来，单独管理
## Python 3.2 之后，该模块被更名为 ConfigParser
cfg = SafeConfigParser()
## 读取 sources 列表中的所有配置文件（以 .cfg 结尾），将文件中的配置按一定的
## 格式存储到配置解析器实例 cfg 的相关属性中
cfg.read(sources)
return cfg
def set_namespace_options(namespace, options, remove=None):
"""
Set options in the local namespace configuration file.
Can have nasty effects, be careful, only use in test code.
:param namespace: the namespace to work with
:param options: a dictionary with options to set
:param remove: an iterable of options to remove
:returns: a dictionary with all options of the namespace
"""
parser = SafeConfigParser({})
potential_confs = list()
actual_confs = list()
for p, _ in _config_paths():
potential_confs.append(p)
fdone = parser.read((p,))
actual_confs.extend(fdone)
if not actual_confs:
raise ValueError(
"Could not read configuration from any of %s" % potential_confs)
if not parser.has_section(namespace):
print('Namespace %s was not found in %s' % (namespace, actual_confs))
parser.add_section(namespace)
for key, val in options.items():
parser.set(namespace, key, str(val))
if remove:
for key in remove:
parser.remove_option(namespace, key)
with open(actual_confs[-1], 'w') as outfile:
parser.write(outfile)
return dict(parser.items(namespace))
def nodemgr_sighup_handler(self):
collector_list = list()
config = SafeConfigParser()
config.read([self.config.config_file_path])
if 'COLLECTOR' in config.sections():
try:
collector = config.get('COLLECTOR', 'server_list')
collector_list = collector.split()
except NoOptionError:
pass
collector_list.sort()
new_chksum = hashlib.md5(("".join(collector_list)).encode()).hexdigest()
if new_chksum != self.collector_chksum:
self.collector_chksum = new_chksum
self.random_collectors = random.sample(collector_list, len(collector_list))
self.sandesh_instance.reconfig_collectors(self.random_collectors)
class ConfigHelper(object):
NONE_VALUE = 'None'
DEFAULT_HOST = "127.0.0.1"
DEFAULT_PORT = 5696
DEFAULT_CERTFILE = os.path.normpath(os.path.join(
FILE_PATH, '../demos/certs/server.crt'))
DEFAULT_KEYFILE = os.path.normpath(os.path.join(
FILE_PATH, '../demos/certs/server.key'))
DEFAULT_CA_CERTS = os.path.normpath(os.path.join(
FILE_PATH, '../demos/certs/server.crt'))
DEFAULT_SSL_VERSION = 'PROTOCOL_SSLv23'
DEFAULT_USERNAME = None
DEFAULT_PASSWORD = None
def __init__(self):
self.logger = logging.getLogger(__name__)
self.conf = SafeConfigParser()
if self.conf.read(CONFIG_FILE):
self.logger.debug("Using config file at {0}".format(CONFIG_FILE))
else:
self.logger.warning(
"Config file {0} not found".format(CONFIG_FILE))
def get_valid_value(self, direct_value, config_section,
config_option_name, default_value):
"""Returns a value that can be used as a parameter in client or
server. If a direct_value is given, that value will be returned
instead of the value from the config file. If the appropriate config
file option is not found, the default_value is returned.
:param direct_value: represents a direct value that should be used.
supercedes values from config files
:param config_section: which section of the config file to use
:param config_option_name: name of config option value
:param default_value: default value to be used if other options not
found
:returns: a value that can be used as a parameter
"""
ARG_MSG = "Using given value '{0}' for {1}"
CONF_MSG = "Using value '{0}' from configuration file {1} for {2}"
DEFAULT_MSG = "Using default value '{0}' for {1}"
if direct_value:
return_value = direct_value
self.logger.debug(ARG_MSG.format(direct_value, config_option_name))
else:
try:
return_value = self.conf.get(config_section,
config_option_name)
self.logger.debug(CONF_MSG.format(return_value,
CONFIG_FILE,
config_option_name))
except:
return_value = default_value
self.logger.debug(DEFAULT_MSG.format(default_value,
config_option_name))
if return_value == self.NONE_VALUE:
return None
else:
return return_value
def __init__(self):
here = os.path.dirname(__file__)
self.filename = os.path.join(here, self._filename)
parser = SafeConfigParser()
parser.getlist = lambda s, o: parser.get(s, o).split()
parser.getlines = lambda s, o: [l.strip() for l in parser.get(s, o).splitlines() if l.strip()]
found = parser.read(self.filename)
if not found:
raise RuntimeError('failed to read app config %r' % self.filename)
getters = {}
for attr, options in self._getters.items():
getters.update(dict.fromkeys(options, getattr(parser, attr)))
def items(section):
for o in parser.options(section):
yield o, getters.get(o, parser.get)(section, o)
kwargs = [dict([('name', section)] + list(items(section)))
for section in parser.sections()]
apps = [App(**kw) for kw in kwargs]
# some consistency checks: names and ports must be unique to make it
# possible to deploy each app on each server.
names = [app.name for app in apps]
ports = [app.port for app in apps]
assert len(names) == len(set(names))
assert len(ports) == len(set(ports))
super(Config, self).__init__((app.name, app) for app in apps)
def read_parse_config(filename):
"""
Reads, validates and create a config dictionary
:param filename: The filename to parse
:return: return a dict with all config variables or None
"""
config_dict = dict()
config = SafeConfigParser()
config.read(filename)
#
# Sanity checks for sections
#
if not verify_section_exists(config, 'general'):
return None
#
# Default to port 33706
#
config_dict['http_server_port'] = 33706
#
# defaults for directories
#
config_dict['input_path'] = '/tmp/s3-syslog-push/input'
config_dict['log_path'] = '/tmp/s3-syslog-push/log'
try:
os.makedirs(config_dict['input_path'], 0o755)
except OSError as e:
if e.errno == 17: # FileExistsError
# TODO: handle this scenario. Shouldn't happen.
pass
try:
os.makedirs(config_dict['log_path'], 0o755)
except OSError as e:
if e.errno == 17: # FileExistsError
# TODO: handle this scenario. Shouldn't happen.
pass
for item in config.items('general'):
config_dict[item[0]] = item[1]
return config_dict
def _load(self):
"""Load the file list from the index file. @files will
be an empty dictionary if the file doesn't exist.
"""
root_logger.debug("Loading Index file from '%s'", self._index)
self.files = {}
p = SafeConfigParser()
p.optionxform = str
p.read(self._index)
for section in p.sections():
if section == "files":
for (key, value) in p.items(section):
self.files[key] = value
def _load(self):
"""Load the file list from the index file. @files will
be an empty dictionary if the file doesn't exist.
"""
root_logger.debug("Loading Index file from '%s'", self._index)
self.files = {}
p = SafeConfigParser()
p.optionxform = str
p.read(self._index)
for section in p.sections():
if section == "files":
for (key, value) in p.items(section):
self.files[key] = value
class Evaluator(object):
APP_NAME = 'expression-evaluators'
_default_dir = 'evaluators'
def __init__(self, plugin_dir=None):
self.config = SafeConfigParser()
config_path = save_config_path(self.APP_NAME)
self.config_file = os.path.join(config_path, self.APP_NAME + ".conf")
self.config.read(self.config_file)
this_dir = os.path.abspath(os.path.dirname(__file__))
self.plugin_dir = plugin_dir or os.path.join(
this_dir, self._default_dir)
places = [self.plugin_dir, ]
[places.append(os.path.join(path, self.APP_NAME, "evaluators")) for
path in xdg_data_dirs]
PluginManagerSingleton.setBehaviour([
ConfigurablePluginManager,
VersionedPluginManager,
])
self.manager = PluginManagerSingleton.get()
self.manager.setConfigParser(self.config, self.write_config)
self.manager.setPluginInfoExtension("expr-plugin")
self.manager.setPluginPlaces(places)
self.manager.collectPlugins()
def _get_all_evaluators(self):
return self.manager.getAllPlugins()
def _get_evaluator(self, name):
pl = self.manager.getPluginByName(name)
if not pl:
raise Exception('No expression evaluator %s' % name)
return pl.plugin_object
def write_config(self):
f = open(self.config_file, "w")
self.config.write(f)
f.close()
def evaluate(self, lang, expression, *args, **kwargs):
pl = self._get_evaluator(lang)
return pl.evaluate(expression, *args, **kwargs)
def get_host_domain_and_realm(self):
"""Return the hostname and IPA realm name.
IPA 4.4 introduced the requirement that the schema be
fetched when calling finalize(). This is really only used by
the ipa command-line tool but for now it is baked in.
So we have to get a TGT first but need the hostname and
realm. For now directly read the IPA config file which is
in INI format and pull those two values out and return as
a tuple.
"""
config = SafeConfigParser()
config.read('/etc/ipa/default.conf')
hostname = config.get('global', 'host')
realm = config.get('global', 'realm')
domain = config.get('global', 'domain')
return hostname, domain, realm
def start_master(options, plugins):
cfgparser = SafeConfigParser()
cfgparser.read(options.config)
cfg = Configuration(cfgparser)
pidfile = None
user, group, umask = None, None, None
def process_init():
system.drop_privileges(user, group, umask)
time.sleep(0.1)
master = WorkerMaster(process_callback=process_init)
cfg.configure_logging()
cfg.configure_amqp()
cfg.configure_mysql()
for plugin in plugins:
for entry_point in iter_entry_points('provoke.workers', plugin):
logger.debug('Loading plugin: name=%s', entry_point.name)
register = entry_point.load()
register(master, cfgparser)
break
else:
raise BadPlugin(plugin)
for res, limits in cfg.get_rlimits():
resource.setrlimit(res, limits)
if options.daemon:
logger.debug('Daemonizing master process')
pidfile = cfg.get_pidfile()
stdout, stderr, stdin = cfg.get_stdio_redirects()
user, group, umask = cfg.get_worker_privileges()
if stdout or stderr or stdin:
system.redirect_stdio(stdout, stderr, stdin)
system.daemonize()
with system.PidFile(pidfile):
handle_signals()
try:
master.run()
finally:
master.wait()
def start_master(options, plugins):
cfgparser = SafeConfigParser()
cfgparser.read(options.config)
cfg = Configuration(cfgparser)
pidfile = None
user, group, umask = None, None, None
def process_init():
system.drop_privileges(user, group, umask)
time.sleep(0.1)
master = WorkerMaster(process_callback=process_init)
cfg.configure_logging()
cfg.configure_amqp()
cfg.configure_mysql()
for plugin in plugins:
for entry_point in iter_entry_points('provoke.workers', plugin):
logger.debug('Loading plugin: name=%s', entry_point.name)
register = entry_point.load()
register(master, cfgparser)
break
else:
raise BadPlugin(plugin)
for res, limits in cfg.get_rlimits():
resource.setrlimit(res, limits)
if options.daemon:
logger.debug('Daemonizing master process')
pidfile = cfg.get_pidfile()
stdout, stderr, stdin = cfg.get_stdio_redirects()
user, group, umask = cfg.get_worker_privileges()
if stdout or stderr or stdin:
system.redirect_stdio(stdout, stderr, stdin)
system.daemonize()
with system.PidFile(pidfile):
handle_signals()
try:
master.run()
finally:
master.wait()
def _read_config(self, config_name):
parser = SafeConfigParser()
parser.read(config_name)
try:
self.ssl = parser.get("vars", "ssl").lower() in ["1", "true"]
except (NoSectionError, NoOptionError):
self.ssl = True
if self.ssl:
try:
self.trust_store_path = parser.get("vars", "trust_store_path")
except (NoSectionError, NoOptionError):
self.trust_store_path = '/etc/pki/vdsm'
else:
self.trust_store_path = None
try:
self.management_port = parser.get("addresses", "management_port")
except (NoSectionError, NoOptionError):
self.management_port = '54321'
def _update_gridinit_meta(self, name, port):
grid = SafeConfigParser()
grid.read(GRID_CONF)
section = "service.%s-%s" % (self.ns, name)
val = grid.get(section, "Group").split(",")
org_addr = val[3]
val[3] = val[3].split(':')[0] + ':' + str(port)
grid.set(section, "Group", ",".join(val))
cmd = grid.get(section, 'command')
if cmd and org_addr in cmd:
cmd = cmd.replace(org_addr, val[3])
grid.set(section, 'command', cmd)
with open(GRID_CONF, "w") as fp:
grid.write(fp)
def sighup_handler(self):
if self._args.conf_file:
config = SafeConfigParser()
config.read(self._args.conf_file)
if 'DEFAULTS' in config.sections():
try:
collectors = config.get('DEFAULTS', 'collectors')
if type(collectors) is str:
collectors = collectors.split()
new_chksum = hashlib.md5(
"".join(collectors)).hexdigest()
if new_chksum != self._chksum:
self._chksum = new_chksum
config.random_collectors = random.sample(
collectors, len(collectors))
# Reconnect to achieve loadbalance irrespective of list
self.logger.sandesh_reconfig_collectors(config)
except NoOptionError:
pass
def _read_config(self, config_name):
parser = SafeConfigParser()
parser.read(config_name)
try:
self.ssl = parser.get("vars", "ssl").lower() in ["1", "true"]
except (NoSectionError, NoOptionError):
self.ssl = True
if self.ssl:
try:
self.trust_store_path = parser.get("vars", "trust_store_path")
except (NoSectionError, NoOptionError):
self.trust_store_path = '/etc/pki/vdsm'
else:
self.trust_store_path = None
try:
self.management_port = parser.get("addresses", "management_port")
except (NoSectionError, NoOptionError):
self.management_port = '54321'
def _load(self):
"""Load the modules from the file @_path. @modules will
be an empty dictionary if the file doesn't exist.
"""
root_logger.debug("Loading StateFile from '%s'", self._path)
self.modules = {}
p = SafeConfigParser()
p.optionxform = str
p.read(self._path)
for module in p.sections():
self.modules[module] = {}
for (key, value) in p.items(module):
if value == str(True):
value = True
elif value == str(False):
value = False
self.modules[module][key] = value
def _load(self):
"""Load the modules from the file @_path. @modules will
be an empty dictionary if the file doesn't exist.
"""
root_logger.debug("Loading StateFile from '%s'", self._path)
self.modules = {}
p = SafeConfigParser()
p.optionxform = str
p.read(self._path)
for module in p.sections():
self.modules[module] = {}
for (key, value) in p.items(module):
if value == str(True):
value = True
elif value == str(False):
value = False
self.modules[module][key] = value
def create_database(config_file):
parser = SafeConfigParser()
parser.read(config_file)
# Determine which database connection to use.
database_connection = parser.get('app:main', 'install_database_connection')
if database_connection is None:
database_connection = parser.get('app:main', 'database_connection')
if database_connection is None:
database_connection = 'sqlite:///%s' % parser.get('app:main', 'database_file')
if database_connection is None:
print('Unable to determine correct database connection.')
exit(1)
'''Initialize the database file.'''
# Initialize the database connection.
engine = create_engine(database_connection)
MetaData(bind=engine)
install_session = scoped_session(sessionmaker(bind=engine, autoflush=False, autocommit=True))
model = mapping.init(database_connection)
return install_session, model
def read_config_file():
if not os.path.isfile(config_file_path):
download_config_file()
config = SafeConfigParser()
config.optionxform = str
list_of_successfully_parsed_files = config.read(config_file_path)
if config_file_path not in list_of_successfully_parsed_files:
raise Exception(
'Could not read {0} succesfully.'.format(config_file_path)
)
return config
def save_to_config(save_config, config_obj, verbose=False):
"""
Write all the values in the config object to a given file in the user's home folder
:param config_file: Name of config file to store in user's home folder
:param config_obj: The config object to export to config file
:param verbose: Specify if stdout should display a message
:return:
"""
parser = SafeConfigParser()
parser.read(save_config)
if not os.path.exists(save_config):
parser.add_section('settings')
for k, v in viewitems(config_obj._asdict()):
if v is not None and isinstance(v, str):
parser.set('settings', k, v)
with open(save_config, 'w') as f:
parser.write(f)
if verbose:
click.echo("Config file written to {}".format(save_config))
def __parse_config(discover_server = True):
p = SafeConfigParser()
p.read(IPA_DEFAULT_CONF)
try:
if not config.default_realm:
config.default_realm = p.get("global", "realm")
except Exception:
pass
if discover_server:
try:
s = p.get("global", "xmlrpc_uri")
server = urlsplit(s)
config.default_server.append(server.netloc)
except Exception:
pass
try:
if not config.default_domain:
config.default_domain = p.get("global", "domain")
except Exception:
pass
def get_scrapycfg_targets(cfgfiles=None):
cfg = SafeConfigParser()
cfg.read(cfgfiles or [])
baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {}
targets = {}
targets['default'] = baset
for x in cfg.sections():
if x.startswith('deploy:'):
t = baset.copy()
t.update(cfg.items(x))
targets[x[7:]] = t
for tname, t in list(targets.items()):
try:
int(t.get('project', 0))
except ValueError:
# Don't import non-numeric project IDs, and also throw away the
# URL and credentials associated with these projects (since the
# project ID does not belong to SH, neither do the endpoint or the
# auth information)
del targets[tname]
if t.get('url', "").endswith('scrapyd/'):
t['url'] = t['url'][:-8]
targets.setdefault('default', {})
return targets
def read(config_path):
config_path = os.path.abspath(config_path)
config_root = os.path.split(config_path)[0]
parser = SafeConfigParser()
success = parser.read(config_path)
assert config_path in success, success
subns = {"pwd": os.path.abspath(os.path.curdir)}
rv = OrderedDict()
for section in parser.sections():
rv[section] = ConfigDict(config_root)
for key in parser.options(section):
rv[section][key] = parser.get(section, key, False, subns)
return rv
class PackageConfigHandler(object):
"""
Manager class for packages files for tracking installation of modules
"""
def __init__(self):
# noinspection PyUnresolvedReferences
from six.moves.configparser import SafeConfigParser
self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
if not os.path.isfile(self.package_cfg):
print('Creating package file')
with open(self.package_cfg, 'w') as outs:
outs.close()
self.parser = SafeConfigParser()
self.parser.read(self.package_cfg)
def save(self):
with open(self.package_cfg, 'w') as outs:
self.parser.write(outs)
def add_module(self, pkg_info):
"""
:param pkg_info: A dict that has name, url, version, summary
:return:
"""
if not self.parser.has_section(pkg_info['name']):
self.parser.add_section(pkg_info['name'])
self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
self.save()
def list_modules(self):
return [module for module in self.parser.sections()]
def module_exists(self, name):
return self.parser.has_section(name)
def get_info(self, name):
if self.parser.has_section(name):
tbl = {}
for opt, value in self.parser.items(name):
tbl[opt] = value
return tbl
def remove_module(self, name):
self.parser.remove_section(name)
self.save()
def get_files_installed(self, section_name):
if self.parser.has_option(section_name, 'files'):
files = self.parser.get(section_name, 'files').strip()
return files.split(',')
else:
return None
def get_dependencies(self, section_name):
if self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
return set(dependencies.split(',')) if dependencies != '' else set()
else:
return None
def get_all_dependencies(self, exclude_module=()):
all_dependencies = set()
for section_name in self.parser.sections():
if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
if dependencies != '':
for dep in dependencies.split(','):
all_dependencies.add(dep)
return all_dependencies
def get_config(use_closest=True):
"""Get Scrapy config file as a SafeConfigParser"""
sources = get_sources(use_closest)
cfg = SafeConfigParser()
cfg.read(sources)
return cfg
raise RuntimeError(
"Database not responding at {} after {} tries. "
"Giving up".format(database_url, max_tries)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--workers",
default=2,
help="Number of workers to use by the gunicorn server. Defaults to 2."
)
parser.add_argument(
"-r",
"--reload",
action="store_true",
help="Should the gunicorn server automatically restart workers when "
"code changes? This option is only useful for development. "
"Defaults to False."
)
args = parser.parse_args()
config = SafeConfigParser()
config.read(os.getenv("PYCSW_CONFIG"))
try:
level = config.get("server", "loglevel").upper()
except NoOptionError:
level = "WARNING"
logging.basicConfig(level=getattr(logging, level))
launch_pycsw(config, workers=args.workers, reload=args.reload)
def _parse_args(self, args=None):
self.context.original_begin = time.time()
self.context.original_args = args if args is not None else sys.argv[1:]
self.option_parser.disable_interspersed_args()
try:
options, args = self.option_parser.parse_args(
self.context.original_args)
except UnboundLocalError:
# Happens sometimes with an error handler that doesn't raise its
# own exception. We'll catch the error below with
# error_encountered.
pass
self.option_parser.enable_interspersed_args()
if self.option_parser.error_encountered:
return None, None
if options.version:
self.option_parser.print_version()
return None, None
if not args or options.help:
self.option_parser.print_help()
return None, None
self.context.original_main_args = self.context.original_args[
:-len(args)]
self.context.conf = {}
if options.conf is None:
options.conf = os.environ.get('SWIFTLY_CONF', '~/.swiftly.conf')
try:
conf_parser = SafeConfigParser()
conf_parser.read(os.path.expanduser(options.conf))
for section in conf_parser.sections():
self.context.conf[section] = dict(conf_parser.items(section))
except ConfigParserError:
pass
for option_name in (
'auth_url', 'auth_user', 'auth_key', 'auth_tenant',
'auth_methods', 'region', 'direct', 'local', 'proxy', 'snet',
'no_snet', 'retries', 'cache_auth', 'no_cache_auth', 'cdn',
'no_cdn', 'concurrency', 'eventlet', 'no_eventlet', 'verbose',
'no_verbose', 'direct_object_ring'):
self._resolve_option(options, option_name, 'swiftly')
for option_name in (
'snet', 'no_snet', 'cache_auth', 'no_cache_auth', 'cdn',
'no_cdn', 'eventlet', 'no_eventlet', 'verbose', 'no_verbose'):
if isinstance(getattr(options, option_name), six.string_types):
setattr(
options, option_name,
getattr(options, option_name).lower() in TRUE_VALUES)
for option_name in ('retries', 'concurrency'):
if isinstance(getattr(options, option_name), six.string_types):
setattr(
options, option_name, int(getattr(options, option_name)))
if options.snet is None:
options.snet = False
if options.no_snet is None:
options.no_snet = False
if options.retries is None:
options.retries = 4
if options.cache_auth is None:
options.cache_auth = False
if options.no_cache_auth is None:
options.no_cache_auth = False
if options.cdn is None:
options.cdn = False
if options.no_cdn is None:
options.no_cdn = False
if options.concurrency is None:
options.concurrency = 1
if options.eventlet is None:
options.eventlet = False
if options.no_eventlet is None:
options.no_eventlet = False
if options.verbose is None:
options.verbose = False
if options.no_verbose is None:
options.no_verbose = False
self.context.eventlet = None
if options.eventlet:
self.context.eventlet = True
if options.no_eventlet:
self.context.eventlet = False
if self.context.eventlet is None:
self.context.eventlet = False
try:
import eventlet
# Eventlet 0.11.0 fixed the CPU bug
if eventlet.__version__ >= '0.11.0':
self.context.eventlet = True
except ImportError:
pass
subprocess_module = None
if self.context.eventlet:
try:
import eventlet.green.subprocess
subprocess_module = eventlet.green.subprocess
except ImportError:
pass
if subprocess_module is None:
import subprocess
subprocess_module = subprocess
self.context.io_manager.subprocess_module = subprocess_module
if options.verbose:
self.context.verbosity = 1
self.context.verbose = self._verbose
self.context.io_manager.verbose = functools.partial(
self._verbose, skip_sub_command=True)
options.retries = int(options.retries)
if args and args[0] == 'help':
return options, args
elif options.local:
self.context.client_manager = ClientManager(
LocalClient, local_path=options.local, verbose=self._verbose)
elif options.direct:
self.context.client_manager = ClientManager(
DirectClient, swift_proxy_storage_path=options.direct,
attempts=options.retries + 1, eventlet=self.context.eventlet,
verbose=self._verbose,
direct_object_ring=options.direct_object_ring)
else:
auth_cache_path = None
if options.cache_auth:
auth_cache_path = os.path.join(
tempfile.gettempdir(),
'%s.swiftly' % os.environ.get('USER', 'user'))
if not options.auth_url:
with self.context.io_manager.with_stderr() as fp:
fp.write('No Auth URL has been given.\n')
fp.flush()
return None, None
self.context.client_manager = ClientManager(
StandardClient, auth_methods=options.auth_methods,
auth_url=options.auth_url, auth_tenant=options.auth_tenant,
auth_user=options.auth_user, auth_key=options.auth_key,
auth_cache_path=auth_cache_path, region=options.region,
snet=options.snet, attempts=options.retries + 1,
eventlet=self.context.eventlet, verbose=self._verbose,
http_proxy=options.proxy)
self.context.cdn = options.cdn
self.context.concurrency = int(options.concurrency)
return options, args
def load_cfg():
cfg = SafeConfigParser()
if os.path.isfile(cfg_filename):
cfg.read([cfg_filename])
return cfg
def from_filename(cls, fname):
print("Loading issue2branch config from: '{}'".format(fname))
config = SafeConfigParser()
config.read([fname])
return cls(config)
if __name__ == '__main__':
parser = OptionParser(usage="usage: %prog [options]")
parser.add_option('-k', '--insecure',
action="store_false",
dest='insecure',
default=True,
help='Allow insecure connection when using SSL')
(options, args) = parser.parse_args()
LOG.debug('Running with parameter insecure = %s',
options.insecure)
if os.path.isfile(nova_cfg):
config = SafeConfigParser()
config.read(nova_cfg)
else:
LOG.error('Nova configuration file %s does not exist', nova_cfg)
sys.exit(1)
my_host = config.get('DEFAULT', 'host')
if not my_host:
# If host isn't set nova defaults to this
my_host = socket.gethostname()
loader = loading.get_plugin_loader('password')
auth = loader.load_from_options(
auth_url=config.get('neutron',
'auth_url'),
username=config.get('neutron',
'username'),
class Config(object):
"""
Manages the configuration file
"""
def __init__(self):
"""
DEFAULT VALUES
"""
self._basescript = None
self.recentvaults = []
self.pwlength = 10
self.search_notes = False
self.search_passwd = False
self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
self.avoid_bigrams = "cl mn nm nn rn vv VV"
self._fname = self.get_config_filename()
self._parser = SafeConfigParser()
if os.path.exists(self._fname):
self._parser.read(self._fname)
if not self._parser.has_section("base"):
self._parser.add_section("base")
for num in range(10):
if (not self._parser.has_option("base", "recentvaults" + str(num))):
break
self.recentvaults.append(self._parser.get("base", "recentvaults" + str(num)))
if self._parser.has_option("base", "pwlength"):
self.pwlength = int(self._parser.get("base", "pwlength"))
if self._parser.has_option("base", "search_notes"):
if self._parser.get("base", "search_notes") == "True":
self.search_notes = True
if self._parser.has_option("base", "search_passwd"):
if self._parser.get("base", "search_passwd") == "True":
self.search_passwd = True
if self._parser.has_option("base", "alphabet"):
self.alphabet = self._parser.get("base", "alphabet")
if self._parser.has_option("base", "avoid_bigrams"):
self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")
if not os.path.exists(self._fname):
self.save()
def set_basescript(self, basescript):
self._basescript = basescript
def get_basescript(self):
return self._basescript
def save(self):
if (not os.path.exists(os.path.dirname(self._fname))):
os.mkdir(os.path.dirname(self._fname))
# remove duplicates and trim to 10 items
_saved_recentvaults = []
for item in self.recentvaults:
if item in _saved_recentvaults:
continue
self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item)
_saved_recentvaults.append(item)
if (len(_saved_recentvaults) >= 10):
break
self._parser.set("base", "pwlength", str(self.pwlength))
self._parser.set("base", "search_notes", str(self.search_notes))
self._parser.set("base", "search_passwd", str(self.search_passwd))
self._parser.set("base", "alphabet", str(self.alphabet))
self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
filehandle = open(self._fname, 'w')
self._parser.write(filehandle)
filehandle.close()
@staticmethod
def get_config_filename():
"""
Returns the full filename of the config file
"""
base_fname = "loxodo"
# On Mac OS X, config files go to ~/Library/Application Support/foo/
if platform.system() == "Darwin":
base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# On Microsoft Windows, config files go to $APPDATA/foo/
if platform.system() in ("Windows", "Microsoft"):
if ("APPDATA" in os.environ):
base_path = os.environ["APPDATA"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Allow config directory override as per freedesktop.org XDG Base Directory Specification
if ("XDG_CONFIG_HOME" in os.environ):
base_path = os.environ["XDG_CONFIG_HOME"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Default configuration path is ~/.config/foo/
base_path = os.path.join(os.path.expanduser("~"), ".config")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
else:
return os.path.join(os.path.expanduser("~"),"."+ base_fname + ".ini")
