def getSavedProfileNames():
"""
Reads in the names of the saved profiles from the getwatchlist.conf
files in default and local and returns them as a list.
"""
profileNames = []
# First the defaults
parser = ConfigParser()
parser.optionxform = str
parser.read(getDefaultConfPath())
profileNames = parser.sections()
# Now the locals
parser = ConfigParser()
parser.optionxform = str
parser.read(getLocalConfPath())
localProfileNames = parser.sections()
# Now create one list
for profile in localProfileNames:
if not profile in profileNames:
profileNames.append(profile)
return profileNames
def get_config(p):
"""Read a config file.
:return: dict of ('section.option', value) pairs.
"""
cfg = {}
parser = ConfigParser()
if hasattr(parser, 'read_file'):
parser.read_file(Path(p).open(encoding='utf8'))
else: # pragma: no cover
assert PY2
# The `read_file` method is not available on ConfigParser in py2.7!
parser.readfp(Path(p).open(encoding='utf8'))
for section in parser.sections():
getters = {
'int': partial(parser.getint, section),
'boolean': partial(parser.getboolean, section),
'float': partial(parser.getfloat, section),
'list': lambda option: parser.get(section, option).split(),
}
default = partial(parser.get, section)
for option in parser.options(section):
type_ = option.rpartition('_')[2] if '_' in option else None
value = getters.get(type_, default)(option)
cfg['{0}.{1}'.format(section, option)] = value
return cfg
def remote_profiles(self):
"""
A list of remote profiles on the device.
"""
remote_ini = self.app_ctx.remote_profiles_ini
if not self.device.is_file(remote_ini):
raise IOError("Remote file '%s' not found" % remote_ini)
local_ini = tempfile.NamedTemporaryFile()
self.device.pull(remote_ini, local_ini.name)
cfg = ConfigParser()
cfg.read(local_ini.name)
profiles = []
for section in cfg.sections():
if cfg.has_option(section, "Path"):
if cfg.has_option(section, "IsRelative") and cfg.getint(
section, "IsRelative"
):
profiles.append(
posixpath.join(
posixpath.dirname(remote_ini), cfg.get(section, "Path")
)
)
else:
profiles.append(cfg.get(section, "Path"))
return profiles
def get_config(p):
"""Read a config file.
:return: dict of ('section.option', value) pairs.
"""
if not isinstance(p, Path):
p = Path(p)
cfg = {}
parser = ConfigParser()
parser.readfp(p.open(encoding='utf8'))
for section in parser.sections():
getters = {
'int': partial(parser.getint, section),
'boolean': partial(parser.getboolean, section),
'float': partial(parser.getfloat, section),
'list': lambda option: parser.get(section, option).split(),
}
default = partial(parser.get, section)
for option in parser.options(section):
type_ = option.rpartition('_')[2] if '_' in option else None
value = getters.get(type_, default)(option)
cfg['{0}.{1}'.format(section, option)] = value
return cfg
def build_cli_examples(_):
logger = logging.getLogger('cli-examples')
clidir = os.path.join(SPHINX_DIR, 'cli')
exini = os.path.join(clidir, 'examples.ini')
exdir = os.path.join(clidir, 'examples')
if not os.path.isdir(exdir):
os.makedirs(exdir)
config = ConfigParser()
config.read(exini)
rsts = []
for sect in config.sections():
rst, cmd = _build_cli_example(config, sect, exdir, logger)
if cmd:
logger.info('[cli] running example {0!r}'.format(sect))
logger.debug('[cli] $ {0}'.format(cmd))
subprocess.check_call(cmd, shell=True)
logger.debug('[cli] wrote {0}'.format(cmd.split()[-1]))
rsts.append(rst)
with open(os.path.join(exdir, 'examples.rst'), 'w') as f:
f.write('.. toctree::\n :glob:\n\n')
for rst in rsts:
f.write(' {0}\n'.format(rst[len(SPHINX_DIR):]))
def loadKeysConfig(path=None):
"""Load keys config file.
If path is ``None``, a file named :any:`DEFAULT_KEYS_FILE` will be looked for in the config
directory.
:param path: path of the keyboard configuration file
"""
if path is None:
path = getConfigFilePath(DEFAULT_KEYS_FILE)
cfg = ConfigParser()
cfg.optionxform = str
cfg.read([path])
for category in cfg.sections():
for actionName in cfg.options(category):
keystr = cfg.get(category, actionName)
context = Qt.WidgetShortcut
if keystr.startswith('widget:'):
keystr = keystr.split(':', 1)[1]
elif keystr.startswith('window:'):
keystr = keystr.split(':', 1)[1]
context = Qt.WindowShortcut
elif keystr.startswith('children:'):
keystr = keystr.split(':', 1)[1]
context = Qt.WidgetWithChildrenShortcut
elif keystr.startswith('application:'):
keystr = keystr.split(':', 1)[1]
context = Qt.ApplicationShortcut
qks = QKeySequence(keystr)
registerActionShortcut(category, actionName, qks, context)
def get_ic_factor(self, det):
# storing ic_factor in preferences causing issues
# ic_factor stored in detectors.cfg
p = os.path.join(paths.spectrometer_dir, 'detectors.cfg')
# factors=None
ic = 1, 0
if os.path.isfile(p):
c = ConfigParser()
c.read(p)
det = det.lower()
for si in c.sections():
if si.lower() == det:
v, e = 1, 0
if c.has_option(si, 'ic_factor'):
v = c.getfloat(si, 'ic_factor')
if c.has_option(si, 'ic_factor_err'):
e = c.getfloat(si, 'ic_factor_err')
ic = v, e
break
else:
self.debug('no detector file {}. cannot retrieve ic_factor'.format(p))
r = ufloat(*ic)
return r
def reload(self):
# clear configuration
self.properties = {}
# load defaults parameters
for prop in DEFAULT_PROPERTIES:
self.set(prop.key, prop.default_value)
# load config file TANIT_CONF_DIR
config = ConfigParser()
if osp.exists(self.conf_file):
try:
config.read(self.conf_file)
except Exception as e:
raise TanitConfigurationException(
"Exception while loading configuration file %s.",
self.conf_file, e)
_logger.info("Instantiated configuration from %s.", self.conf_file)
else:
raise TanitConfigurationException("Invalid configuration file %s.",
self.conf_file)
# load site parameters
for section in config.sections():
for (key, value) in config.items(section=section):
self.set(ConfigurationKey(section, key), value)
def load_ini(self, ini_config):
"""
Read the provided ini contents arguments and merge
the data in the ini config into the config object.
ini_config is assumed to be a string of the ini file contents.
"""
parser = ConfigParser()
parser.readfp(StringIO(ini_config))
data = {
'linters': {},
'files': {},
'branches': {},
}
if parser.has_section('files'):
ignore = parser.get('files', 'ignore')
data['files']['ignore'] = newline_value(ignore)
if parser.has_section('branches'):
ignore = parser.get('branches', 'ignore')
data['branches']['ignore'] = comma_value(ignore)
linters = []
if parser.has_section('tools'):
linters = comma_value(parser.get('tools', 'linters'))
# Setup empty config sections
for linter in linters:
data['linters'][linter] = {}
for section in parser.sections():
if not section.startswith('tool_'):
continue
# Strip off tool_
linter = section[5:]
data['linters'][linter] = dict(parser.items(section))
self.update(data)
def load(path):
"""Read the Faculty configuration from a file.
Parameters
----------
path : str or pathlib.Path
The path of the file to load configuration from.
Returns
-------
Dict[str, Profile]
The profiles loaded from the file, keyed by their names.
"""
parser = ConfigParser()
parser.read(str(path))
def _get(section, option):
try:
return parser.get(section, option)
except (NoSectionError, NoOptionError):
return None
profiles = {}
for section in parser.sections():
profiles[section] = Profile(
domain=_get(section, "domain"),
protocol=_get(section, "protocol"),
client_id=_get(section, "client_id"),
client_secret=_get(section, "client_secret"),
)
return profiles
def _load_ec_as_default_policy(proxy_conf_file, swift_conf_file, **kwargs):
"""
Override swift.conf [storage-policy:0] section to use a 2+1 EC policy.
:param proxy_conf_file: Source proxy conf filename
:param swift_conf_file: Source swift conf filename
:returns: Tuple of paths to the proxy conf file and swift conf file to use
"""
_debug('Setting configuration for default EC policy')
conf = ConfigParser()
conf.read(swift_conf_file)
# remove existing policy sections that came with swift.conf-sample
for section in list(conf.sections()):
if section.startswith('storage-policy'):
conf.remove_section(section)
# add new policy 0 section for an EC policy
conf.add_section('storage-policy:0')
ec_policy_spec = {
'name': 'ec-test',
'policy_type': 'erasure_coding',
'ec_type': 'liberasurecode_rs_vand',
'ec_num_data_fragments': 2,
'ec_num_parity_fragments': 1,
'ec_object_segment_size': 1048576,
'default': True
}
for k, v in ec_policy_spec.items():
conf.set('storage-policy:0', k, str(v))
with open(swift_conf_file, 'w') as fp:
conf.write(fp)
return proxy_conf_file, swift_conf_file
def build_cli_examples(_):
logger = logging.getLogger('cli-examples')
clidir = os.path.join(SPHINX_DIR, 'cli')
exini = os.path.join(clidir, 'examples.ini')
exdir = os.path.join(clidir, 'examples')
if not os.path.isdir(exdir):
os.makedirs(exdir)
config = ConfigParser()
config.read(exini)
rsts = []
for sect in config.sections():
rst, cmd = _build_cli_example(config, sect, exdir, logger)
if cmd:
logger.info('[cli] running example {0!r}'.format(sect))
logger.debug('[cli] $ {0}'.format(cmd))
subprocess.check_call(cmd, shell=True)
logger.debug('[cli] wrote {0}'.format(cmd.split()[-1]))
rsts.append(rst)
with open(os.path.join(exdir, 'examples.rst'), 'w') as f:
f.write('.. toctree::\n :glob:\n\n')
for rst in rsts:
f.write(' {0}\n'.format(rst[len(SPHINX_DIR):]))
def _load_ec_as_default_policy(proxy_conf_file, swift_conf_file, **kwargs):
"""
Override swift.conf [storage-policy:0] section to use a 2+1 EC policy.
:param proxy_conf_file: Source proxy conf filename
:param swift_conf_file: Source swift conf filename
:returns: Tuple of paths to the proxy conf file and swift conf file to use
"""
_debug('Setting configuration for default EC policy')
conf = ConfigParser()
conf.read(swift_conf_file)
# remove existing policy sections that came with swift.conf-sample
for section in list(conf.sections()):
if section.startswith('storage-policy'):
conf.remove_section(section)
# add new policy 0 section for an EC policy
conf.add_section('storage-policy:0')
ec_policy_spec = {
'name': 'ec-test',
'policy_type': 'erasure_coding',
'ec_type': 'liberasurecode_rs_vand',
'ec_num_data_fragments': 2,
'ec_num_parity_fragments': 1,
'ec_object_segment_size': 1048576,
'default': True
}
for k, v in ec_policy_spec.items():
conf.set('storage-policy:0', k, str(v))
with open(swift_conf_file, 'w') as fp:
conf.write(fp)
return proxy_conf_file, swift_conf_file
def parse_xml_mapping(xml_mapping_filename):
with open(xml_mapping_filename, 'r') as f:
config = ConfigParser()
if six.PY3:
config.read_file(f) # pylint: disable=no-member
else:
config.read_file(f)
return {k: dict(config.items(k)) for k in config.sections()}
def read_systemini(self):
"""read group info from system.ini
this is part of the connection process
"""
self.ftpconn.connect(**self.ftpargs)
self.ftpconn.cwd(posixpath.join(self.ftphome, 'Config'))
lines = self.ftpconn.getlines('system.ini')
self.ftpconn.close()
pvtgroups = []
self.stages = OrderedDict()
self.groups = OrderedDict()
sconf = ConfigParser()
sconf.readfp(StringIO('\n'.join(lines)))
# read and populate lists of groups first
for gtype, glist in sconf.items('GROUPS'): # ].items():
if len(glist) > 0:
for gname in glist.split(','):
gname = gname.strip()
self.groups[gname] = OrderedDict()
self.groups[gname]['category'] = gtype.strip()
self.groups[gname]['positioners'] = []
if gtype.lower().startswith('multiple'):
pvtgroups.append(gname)
for section in sconf.sections():
if section in ('DEFAULT', 'GENERAL', 'GROUPS'):
continue
items = sconf.options(section)
if section in self.groups: # this is a Group Section!
poslist = sconf.get(section, 'positionerinuse')
posnames = [a.strip() for a in poslist.split(',')]
self.groups[section]['positioners'] = posnames
elif 'plugnumber' in items: # this is a stage
self.stages[section] = {
'stagetype': sconf.get(section, 'stagename')
}
if len(pvtgroups) == 1:
self.set_trajectory_group(pvtgroups[0])
for sname in self.stages:
ret = self._xps.PositionerMaximumVelocityAndAccelerationGet(
self._sid, sname)
try:
self.stages[sname]['max_velo'] = ret[1]
self.stages[sname]['max_accel'] = ret[2] / 3.0
except:
print("could not set max velo/accel for %s" % sname)
ret = self._xps.PositionerUserTravelLimitsGet(self._sid, sname)
try:
self.stages[sname]['low_limit'] = ret[1]
self.stages[sname]['high_limit'] = ret[2]
except:
print("could not set limits for %s" % sname)
return self.groups
def get_configuration_dict(configuration=None, value_types=None):
"""
Parse the configuration files
Parameters
----------
configuration : str or list, optional
A configuration file or list of configuration files to parse,
defaults to the deploy_default.conf file in the package and
deploy.conf in the current working directory.
value_types : dict, optional
Dictionary containing classes to apply to specific items
Returns
-------
dict
Configuration dictionary
"""
if not value_types: # pragma: no cover
value_types = config_types()
if configuration is None or configuration is '': # pragma: no cover
configuration = [
# Config file that is part of the package
# PACKAGE_DEFAULT_CONFIG,
# Any deploy.conf files in the current directory
'deploy.conf'
]
config = ConfigParser()
# Set the config defaults
try:
config.read_string(config_defaults())
except AttributeError:
config.readfp(io.BytesIO(config_defaults()))
logger.debug('Working with default dict: %r', config_defaults())
config.read(configuration)
result_dict = {}
for section in config.sections():
result_dict[section] = {}
for key, val in config.items(section):
result_dict[section][key] = str_format_env(val)
config_update(result_dict)
if 'locations' not in result_dict.keys():
result_dict['locations'] = {}
result_dict['locations']['package_scripts'] = package_scripts_directory()
if not result_dict['global'].get('virtualenv_dir', None):
result_dict['global']['virtualenv_dir'] = \
default_virtualenv_directory()
cast_types(result_dict)
return result_dict
def handleApps(self, **kwargs):
l10nbuilds = urlopen(
'https://raw.githubusercontent.com/Pike/master-ball/'
'master/l10n-master/l10nbuilds.ini')
cp = ConfigParser()
cp.readfp(l10nbuilds)
for section in cp.sections():
self.stdout.write(section + '\n')
self.handleSection(section, dict(cp.items(section)))
def parse_config_file(file_path):
config = ConfigParser()
config.read(file_path)
if 'batch_scoring' not in config.sections():
# We are return empty dict, because there is nothing in this file
# that related to arguments to batch scoring.
return {}
parsed_dict = dict(config.items('batch_scoring'))
return config_validator(parsed_dict)
def handleApps(self, **kwargs):
l10nbuilds = urlopen(
'https://raw.githubusercontent.com/Pike/master-ball/'
'master/l10n-master/l10nbuilds.ini')
cp = ConfigParser()
cp.readfp(l10nbuilds)
for section in cp.sections():
self.stdout.write(section + '\n')
self.handleSection(section, dict(cp.items(section)))
def get_configuration_dict(configuration=None, value_types=None):
"""
Parse the configuration files
Parameters
----------
configuration : str or list, optional
A configuration file or list of configuration files to parse,
defaults to the deploy_default.conf file in the package and
deploy.conf in the current working directory.
value_types : dict, optional
Dictionary containing classes to apply to specific items
Returns
-------
dict
Configuration dictionary
"""
if not value_types: # pragma: no cover
value_types = config_types()
if configuration is None or configuration is '': # pragma: no cover
configuration = [
# Config file that is part of the package
# PACKAGE_DEFAULT_CONFIG,
# Any deploy.conf files in the current directory
'deploy.conf'
]
config = ConfigParser()
# Set the config defaults
try:
config.read_string(config_defaults())
except AttributeError:
config.readfp(io.BytesIO(config_defaults()))
logger.debug('Working with default dict: %r', config_defaults())
config.read(configuration)
result_dict = {}
for section in config.sections():
result_dict[section] = {}
for key, val in config.items(section):
result_dict[section][key] = str_format_env(val)
config_update(result_dict)
if 'locations' not in result_dict.keys():
result_dict['locations'] = {}
result_dict['locations']['package_scripts'] = package_scripts_directory()
if not result_dict['global'].get('virtualenv_dir', None):
result_dict['global']['virtualenv_dir'] = \
default_virtualenv_directory()
cast_types(result_dict)
return result_dict
def parse_config_file(file_path):
config = ConfigParser()
config.read(file_path)
if 'batch_scoring' not in config.sections():
# We are return empty dict, because there is nothing in this file
# that related to arguments to batch scoring.
return {}
parsed_dict = dict(config.items('batch_scoring'))
return config_validator(parsed_dict)
def read_config(filename):
"""Read the config file called *filename*.
"""
cp_ = ConfigParser()
cp_.read(filename)
res = {}
for section in cp_.sections():
res[section] = dict(cp_.items(section))
res[section].setdefault("delete", False)
if res[section]["delete"] in ["", "False", "false", "0", "off"]:
res[section]["delete"] = False
res[section].setdefault("working_directory", None)
res[section].setdefault("compression", False)
res[section].setdefault("heartbeat", True)
res[section].setdefault("req_timeout", DEFAULT_REQ_TIMEOUT)
res[section].setdefault("transfer_req_timeout",
10 * DEFAULT_REQ_TIMEOUT)
if res[section]["heartbeat"] in ["", "False", "false", "0", "off"]:
res[section]["heartbeat"] = False
if "providers" not in res[section]:
LOGGER.warning("Incomplete section " + section +
": add an 'providers' item.")
LOGGER.info("Ignoring section " + section + ": incomplete.")
del res[section]
continue
else:
res[section]["providers"] = [
"tcp://" + item for item in res[section]["providers"].split()
]
if "destination" not in res[section]:
LOGGER.warning("Incomplete section " + section +
": add an 'destination' item.")
LOGGER.info("Ignoring section " + section + ": incomplete.")
del res[section]
continue
if "topic" in res[section]:
try:
res[section]["publish_port"] = int(
res[section]["publish_port"])
except (KeyError, ValueError):
res[section]["publish_port"] = 0
elif not res[section]["heartbeat"]:
# We have no topics and therefor no subscriber (if you want to
# subscribe everything, then explicit specify an empty topic).
LOGGER.warning("Incomplete section " + section +
": add an 'topic' item or enable heartbeat.")
LOGGER.info("Ignoring section " + section + ": incomplete.")
del res[section]
continue
return res
def _read_pypirc(self):
"""Reads the .pypirc file."""
rc = self._get_rc_file()
if os.path.exists(rc):
self.announce('Using PyPI login from %s' % rc)
repository = self.repository or self.DEFAULT_REPOSITORY
config = ConfigParser()
config.read(rc)
sections = config.sections()
if 'distutils' in sections:
# let's get the list of servers
index_servers = config.get('distutils', 'index-servers')
_servers = [
server.strip() for server in index_servers.split('\n')
if server.strip() != ''
]
if _servers == []:
# nothing set, let's try to get the default pypi
if 'pypi' in sections:
_servers = ['pypi']
else:
# the file is not properly defined, returning
# an empty dict
return {}
for server in _servers:
current = {'server': server}
current['username'] = config.get(server, 'username')
# optional params
for key, default in (('repository',
self.DEFAULT_REPOSITORY),
('realm', self.DEFAULT_REALM),
('password', None)):
if config.has_option(server, key):
current[key] = config.get(server, key)
else:
current[key] = default
if (current['server'] == repository
or current['repository'] == repository):
return current
elif 'server-login' in sections:
# old format
server = 'server-login'
if config.has_option(server, 'repository'):
repository = config.get(server, 'repository')
else:
repository = self.DEFAULT_REPOSITORY
return {
'username': config.get(server, 'username'),
'password': config.get(server, 'password'),
'repository': repository,
'server': server,
'realm': self.DEFAULT_REALM
}
return {}
def getDefaultSavedProfileNames():
"""
Reads in the names of the DEFAULT saved profiles from the getwatchlist.conf
files in default and local and returns them as a list.
"""
parser = ConfigParser()
parser.optionxform = str
parser.read(getDefaultConfPath())
profileNames = parser.sections()
return profileNames
def read_conf():
config_file = "conf/settings.ini"
parser = ConfigParser()
parser.optionxform = str
parser.read(config_file)
for section_name in parser.sections():
if section_name == 'MongoDBServer':
mongodb = {x: y for x, y in parser.items(section_name)}
if section_name == 'SolrServer':
solr = {x: y for x, y in parser.items(section_name)}
return mongodb, solr
def _read(self):
parser = ConfigParser()
parser.read(self.path)
self._globals = parser.defaults()
data = {}
for section in parser.sections():
section_data = data.setdefault(section, {})
for option in parser.options(section):
if option in self._globals:
continue
section_data[option] = parser.get(section, option)
return data
def read_printers(self):
"""get invalid/valid users from cups and samba config"""
# read CUPS configuration
if os.path.isfile(ShareConfiguration.CUPS_CONF):
reg_cups = re.compile(r'\s*<Printer\s+([^>]+)>')
with open("/etc/cups/printers.conf") as fd:
for line in fd.readlines():
m_cups = reg_cups.match(line)
if m_cups:
prt = Printer(m_cups.group(1).strip())
self._printers[prt.name] = prt
# samba
if not os.path.exists(ShareConfiguration.PRINTERS_UDM_DIR):
return
for filename in os.listdir(ShareConfiguration.PRINTERS_UDM_DIR):
cfg = ConfigParser()
cfg.read(
os.path.join(ShareConfiguration.PRINTERS_UDM_DIR, filename))
try:
prt_name = cfg.sections()[0]
except IndexError:
continue
prt = None
if prt_name in self._printers:
prt = self._printers[prt_name]
else:
if cfg.has_option(prt_name, 'printer name'):
cups_name = cfg.get(prt_name, 'printer name')
if cups_name in self._printers:
prt = self._printers[cups_name]
prt.smbname = prt_name
if prt is None:
continue
if cfg.has_option(prt_name, Restrictions.INVALID_USERS):
prt.invalid_users = shlex.split(
cfg.get(prt_name, Restrictions.INVALID_USERS))
if cfg.has_option(prt_name, Restrictions.VALID_USERS):
prt.valid_users = shlex.split(
cfg.get(prt_name, Restrictions.VALID_USERS))
if cfg.has_option(prt_name, Restrictions.HOSTS_DENY):
prt.hosts_deny = shlex.split(
cfg.get(prt_name, Restrictions.HOSTS_DENY))
def load(self):
schemes = [defaultScheme]
parser = ConfigParser()
parser.read(settings.DASHBOARD_CONF)
for option, default_value in defaultUIConfig.items():
if parser.has_option('ui', option):
try:
self.ui_config[option] = parser.getint('ui', option)
except ValueError:
self.ui_config[option] = parser.get('ui', option)
else:
self.ui_config[option] = default_value
if parser.has_option('ui', 'automatic_variants'):
self.ui_config['automatic_variants'] = parser.getboolean('ui', 'automatic_variants')
else:
self.ui_config['automatic_variants'] = True
self.ui_config['keyboard_shortcuts'] = defaultKeyboardShortcuts.copy()
if parser.has_section('keyboard-shortcuts'):
self.ui_config['keyboard_shortcuts'].update( parser.items('keyboard-shortcuts') )
for section in parser.sections():
if section in ('ui', 'keyboard-shortcuts'):
continue
scheme = parser.get(section, 'scheme')
fields = []
for match in fieldRegex.finditer(scheme):
field = match.group(1)
if parser.has_option(section, '%s.label' % field):
label = parser.get(section, '%s.label' % field)
else:
label = field
fields.append({
'name' : field,
'label' : label
})
schemes.append({
'name' : section,
'pattern' : scheme,
'fields' : fields,
})
self.schemes = schemes
def load(self):
schemes = [defaultScheme]
parser = ConfigParser()
parser.read(settings.DASHBOARD_CONF)
for option, default_value in defaultUIConfig.items():
if parser.has_option('ui', option):
try:
self.ui_config[option] = parser.getint('ui', option)
except ValueError:
self.ui_config[option] = parser.get('ui', option)
else:
self.ui_config[option] = default_value
if parser.has_option('ui', 'automatic_variants'):
self.ui_config['automatic_variants'] = parser.getboolean('ui', 'automatic_variants')
else:
self.ui_config['automatic_variants'] = True
self.ui_config['keyboard_shortcuts'] = defaultKeyboardShortcuts.copy()
if parser.has_section('keyboard-shortcuts'):
self.ui_config['keyboard_shortcuts'].update( parser.items('keyboard-shortcuts') )
for section in parser.sections():
if section in ('ui', 'keyboard-shortcuts'):
continue
scheme = parser.get(section, 'scheme')
fields = []
for match in fieldRegex.finditer(scheme):
field = match.group(1)
if parser.has_option(section, '%s.label' % field):
label = parser.get(section, '%s.label' % field)
else:
label = field
fields.append({
'name' : field,
'label' : label
})
schemes.append({
'name' : section,
'pattern' : scheme,
'fields' : fields,
})
self.schemes = schemes
def _get_retentions_from_storage_schemas(self, opts):
"""Parse storage-schemas.conf and returns all retentions."""
ret = []
config_parser = ConfigParser()
if not config_parser.read(opts.storage_schemas):
raise SystemExit("Error: Couldn't read config file: %s" %
opts.storage_schemas)
for section in config_parser.sections():
options = dict(config_parser.items(section))
retentions = options['retentions'].split(',')
archives = [carbon_util.parseRetentionDef(s) for s in retentions]
ret.append(bg_accessor.Retention.from_carbon(archives))
return ret
def read_test_ini(file_dir=FILE_DIR, section="FacebookAuth"):
ini_file_path = os.path.join(file_dir, "test.ini")
ret = {}
if os.path.isfile(ini_file_path):
cp = ConfigParser()
cp.read(ini_file_path)
if section not in cp.sections():
raise EnvironmentError(
"Section '{0}' not in test.ini".format(section))
for arg in cp.options(section):
ret[arg] = cp.get(section, arg)
else:
raise EnvironmentError(
"File test.ini not existing in path '{0}'".format(FILE_DIR))
return ret
def read_config():
parser = ConfigParser()
parser.read("test.cfg")
rv = {"general": {}, "products": {}}
rv["general"].update(dict(parser.items("general")))
# This only allows one product per whatever for now
for product in parser.sections():
if product != "general":
rv["products"][product] = {}
for key, value in parser.items(product):
rv["products"][product][key] = value
return rv
def _get_retentions_from_storage_schemas(self, opts):
"""Parse storage-schemas.conf and returns all retentions."""
ret = []
config_parser = ConfigParser()
if not config_parser.read(opts.storage_schemas):
raise SystemExit(
"Error: Couldn't read config file: %s" % opts.storage_schemas
)
for section in config_parser.sections():
options = dict(config_parser.items(section))
retentions = options["retentions"].split(",")
archives = [carbon_util.parseRetentionDef(s) for s in retentions]
ret.append(bg_metric.Retention.from_carbon(archives))
return ret
def load(self):
cfp = ConfigParser()
# p = os.path.join(paths.spectrometer_dir, 'config.cfg')
p = get_spectrometer_config_path()
cfp.read(p)
gs = []
for section in cfp.sections():
g = SpectrometerParametersGroup(name=section)
ps = []
for pp in cfp.options(section):
v = cfp.getfloat(section, pp)
ps.append(Parameter(name=pp, value=v))
g.parameters = ps
gs.append(g)
self.groups = gs
def load(self):
cfp = ConfigParser()
# p = os.path.join(paths.spectrometer_dir, 'config.cfg')
p = get_spectrometer_config_path()
cfp.read(p)
gs = []
for section in cfp.sections():
g = SpectrometerParametersGroup(name=section)
ps = []
for pp in cfp.options(section):
v = cfp.getfloat(section, pp)
ps.append(Parameter(name=pp, value=v))
g.parameters = ps
gs.append(g)
self.groups = gs
def parse_config_files(self, filenames=None):
from six.moves.configparser import ConfigParser
if filenames is None:
filenames = self.find_config_files()
if DEBUG:
self.announce("Distribution.parse_config_files():")
parser = ConfigParser()
for filename in filenames:
if DEBUG:
self.announce(" reading %s" % filename)
parser.read(filename)
for section in parser.sections():
options = parser.options(section)
opt_dict = self.get_option_dict(section)
for opt in options:
if opt != '__name__':
val = parser.get(section, opt)
opt = opt.replace('-', '_')
opt_dict[opt] = (filename, val)
# Make the ConfigParser forget everything (so we retain
# the original filenames that options come from)
parser.__init__()
# If there was a "global" section in the config file, use it
# to set Distribution options.
if 'global' in self.command_options:
for (opt, (src,
val)) in list(self.command_options['global'].items()):
alias = self.negative_opt.get(opt)
try:
if alias:
setattr(self, alias, not strtobool(val))
elif opt in ('verbose', 'dry_run'): # ugh!
setattr(self, opt, strtobool(val))
else:
setattr(self, opt, val)
except ValueError as msg:
raise DistutilsOptionError(msg)
def getPluginSettings(themeDirectory, plugins=None):
"""Given an IResourceDirectory for a theme, return the settings for the
given list of plugins (or all plugins, if not given) provided as a list
of (name, plugin) pairs.
Returns a dict of dicts, with the outer dict having plugin names as keys
and containing plugins settings (key/value pairs) as values.
"""
if plugins is None:
plugins = getPlugins()
# noinspection PyPep8Naming
manifestContents = {}
if themeDirectory.isFile(MANIFEST_FILENAME):
parser = ConfigParser()
fp = themeDirectory.openFile(MANIFEST_FILENAME)
try:
if six.PY2:
parser.readfp(fp)
else:
parser.read_string(fp.read().decode())
for section in parser.sections():
manifestContents[section] = {}
for name, value in parser.items(section):
manifestContents[section][name] = value
finally:
try:
fp.close()
except AttributeError:
pass
pluginSettings = {}
for name, plugin in plugins:
pluginSettings[name] = manifestContents.get(
"%s:%s" % (THEME_RESOURCE_NAME, name), {}) # noqa
return pluginSettings
def load_ini(self, ini_config):
"""
Read the provided ini contents arguments and merge
the data in the ini config into the config object.
ini_config is assumed to be a string of the ini file contents.
"""
parser = ConfigParser()
parser.readfp(StringIO(ini_config))
data = {
'linters': {},
'files': {},
'branches': {},
'fixers': {},
'review': {}
}
if parser.has_section('files'):
ignore = parser.get('files', 'ignore')
data['files']['ignore'] = newline_value(ignore)
if parser.has_section('branches'):
ignore = parser.get('branches', 'ignore')
data['branches']['ignore'] = comma_value(ignore)
linters = []
if parser.has_section('tools'):
linters = comma_value(parser.get('tools', 'linters'))
if parser.has_section('fixers'):
data['fixers'] = dict(parser.items('fixers'))
if parser.has_section('review'):
data['review'] = dict(parser.items('review'))
# Setup empty config sections
for linter in linters:
data['linters'][linter] = {}
for section in parser.sections():
if not section.startswith('tool_'):
continue
# Strip off tool_
linter = section[5:]
data['linters'][linter] = dict(parser.items(section))
self.update(data)
def get_conf_stanzas(conf_name):
'''Get stanzas of `conf_name`
:param conf_name: Config file.
:type conf_name: ``string``
:returns: Config stanzas.
:rtype: ``dict``
Usage::
>>> stanzas = get_conf_stanzas('server')
>>> return: {'serverName': 'testServer', 'sessionTimeout': '1h', ...}
'''
if conf_name.endswith('.conf'):
conf_name = conf_name[:-5]
# TODO: dynamically caculate SPLUNK_HOME
btool_cli = [
op.join(os.environ['SPLUNK_HOME'], 'bin', 'btool'), conf_name, 'list'
]
p = subprocess.Popen(btool_cli,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, _ = p.communicate()
if isinstance(out, bytes):
out = out.decode()
out = StringIO(out)
parser = ConfigParser()
parser.optionxform = str
if sys.version_info[:2] >= (3, 2):
parser.read_file(out)
else:
parser.readfp(out)
out = {}
for section in parser.sections():
out[section] = {item[0]: item[1] for item in parser.items(section)}
return out
class Namespaces(object):
r"""Helper for namespaces.
The config file would look like:
```
[carbon-relay]
pattern = carbon\.relay\.*
[carbon-cache]
pattern = carbon\.agents\.*
[carbon-aggregator]
pattern = carbon\.aggregator\.*
[prometheus]
pattern = prometheus\.*
```
"""
def __init__(self, filename=None):
"""Initializer."""
self.config = ConfigParser({}, collections.OrderedDict)
self.patterns = collections.OrderedDict()
if not filename:
self.patterns[re.compile(".*")] = "total"
self.config.add_section("total")
return
self.config.read(filename)
for section in self.config.sections():
pattern = re.compile(self.config.get(section, "pattern"))
self.patterns[pattern] = section
def lookup(self, metric_name):
"""Return the namespace corresponding to the metric."""
for pattern, section in self.patterns.items():
if pattern.match(metric_name):
return section, self.config.items(section)
return "none", None
class Namespaces(object):
r"""Helper for namespaces.
The config file would look like:
```
[carbon-relay]
pattern = carbon\.relay\.*
[carbon-cache]
pattern = carbon\.agents\.*
[carbon-aggregator]
pattern = carbon\.aggregator\.*
[prometheus]
pattern = prometheus\.*
```
"""
def __init__(self, filename=None):
"""Initializer."""
self.config = ConfigParser({}, collections.OrderedDict)
self.patterns = collections.OrderedDict()
if not filename:
self.patterns[re.compile(".*")] = "total"
self.config.add_section("total")
return
self.config.read(filename)
for section in self.config.sections():
pattern = re.compile(self.config.get(section, "pattern"))
self.patterns[pattern] = section
def lookup(self, metric_name):
"""Return the namespace corresponding to the metric."""
for pattern, section in self.patterns.items():
if pattern.match(metric_name):
return section, self.config.items(section)
return "none", None
class BehaveTestsConfig(object):
"""
Configuration parser for tests configuration
"""
STEPS_OPTION = 'Steps'
FEATURES_OPTION = "Features"
def __init__(self, conf_path):
self._config = ConfigParser()
self._config_path = conf_path
try:
self._config.read(self._config_path)[0]
except IndexError:
logger.warning("Tests configuration '%s' can not be read!",
conf_path)
else:
logger.debug("Using Tests configuration from '%s'.", conf_path)
def __getattr__(self, name):
"""
Forward all ConfigParser attributes to ConfigParser object
:param name: name of the attribute
:return: returns the selected attribute
"""
try:
return getattr(self._config, name)
except AttributeError:
return object.__getattribute__(self, name)
def get_tests(self):
return self._config.sections()
def get_test_steps(self, test_name):
return self._config.get(test_name, self.STEPS_OPTION)
def get_test_features(self, test_name):
return self._config.get(test_name, self.FEATURES_OPTION)
def read_config(filename):
"""Read the config file called *filename*.
"""
cp_ = ConfigParser()
cp_.read(filename)
res = {}
for section in cp_.sections():
res[section] = dict(cp_.items(section))
res[section].setdefault("working_directory", None)
res[section].setdefault("compression", False)
if ("origin" not in res[section]) and ('listen' not in res[section]):
LOGGER.warning("Incomplete section " + section +
": add an 'origin' or 'listen' item.")
LOGGER.info("Ignoring section " + section + ": incomplete.")
del res[section]
continue
# if "publisher_port" not in res[section]:
# LOGGER.warning("Incomplete section " + section
# + ": add an 'publisher_port' item.")
# LOGGER.info("Ignoring section " + section
# + ": incomplete.")
# del res[section]
# continue
if "topic" not in res[section]:
LOGGER.warning("Incomplete section " + section +
": add an 'topic' item.")
LOGGER.info("Ignoring section " + section + ": incomplete.")
continue
else:
try:
res[section]["publish_port"] = int(
res[section]["publish_port"])
except (KeyError, ValueError):
res[section]["publish_port"] = 0
return res
def getLayoutsFromManifest(fp, _format, directory_name):
# support multiple sections with the same name in manifest.cfg
if six.PY2:
parser = ConfigParser(None, multidict)
parser.readfp(fp)
else:
data = fp.read()
if isinstance(data, six.binary_type):
data = data.decode()
parser = ConfigParser(dict_type=multidict, strict=False)
parser.read_string(data)
layouts = {}
for section in parser.sections():
if not section.startswith(_format.resourceType) or \
':variants' in section:
continue
# id is a combination of directory name + filename
if parser.has_option(section, 'file'):
filename = parser.get(section, 'file')
else:
filename = '' # this should not happen...
_id = directory_name + '/' + filename
if _id in layouts:
# because TTW resources are created first, we consider layouts
# with same id in a TTW to be taken before other resources
continue
data = {
'directory': directory_name
}
for key in _format.keys:
if parser.has_option(section, key):
data[key] = parser.get(section, key)
else:
data[key] = _format.defaults.get(key, None)
layouts[_id] = data
return layouts
def loadSubs(self, filename):
"""Load a substitutions file.
The file must be in the Windows-style INI format (see the
standard ConfigParser module docs for information on this
format). Each section of the file is loaded into its own
substituter.
"""
inFile = file(filename)
parser = ConfigParser()
parser.readfp(inFile, filename)
inFile.close()
for s in parser.sections():
# Add a new WordSub instance for this section. If one already
# exists, delete it.
if s in self._subbers:
del(self._subbers[s])
self._subbers[s] = WordSub()
# iterate over the key,value pairs and add them to the subber
for k, v in parser.items(s):
self._subbers[s][k] = v
class BehaveTestsConfig(object):
"""
Configuration parser for tests configuration
"""
STEPS_OPTION = 'Steps'
FEATURES_OPTION = "Features"
def __init__(self, conf_path):
self._config = ConfigParser()
self._config_path = conf_path
try:
self._config.read(self._config_path)[0]
except IndexError:
logger.warning("Tests configuration '%s' can not be read!", conf_path)
else:
logger.debug("Using Tests configuration from '%s'.", conf_path)
def __getattr__(self, name):
"""
Forward all ConfigParser attributes to ConfigParser object
:param name: name of the attribute
:return: returns the selected attribute
"""
try:
return getattr(self._config, name)
except AttributeError:
return object.__getattribute__(self, name)
def get_tests(self):
return self._config.sections()
def get_test_steps(self, test_name):
return self._config.get(test_name, self.STEPS_OPTION)
def get_test_features(self, test_name):
return self._config.get(test_name, self.FEATURES_OPTION)
def main(argv):
parser = build_cli_parser(description="Create an Excel report")
parser.add_argument("-b", "--blank-tabs", action="store_true", default=False, dest="blanktabs",
help="Display blank tabs in Excel if the query returns no results.")
parser.add_argument("-f", "--configfile", action="store", dest="configfile",
help="This is the configuration file", required=True)
parser.add_argument("-o", "--outfile", action="store", default=None, dest="outfile",
help="This is the name of the spreadsheet that we'll create", required=True)
parser.add_argument("-q", "--print-query", action="store_true", default=False, dest="printquery",
help="Print the query currently being worked.")
args = parser.parse_args()
cb = get_cb_response_object(args)
config = ConfigParser()
# build the Excel workbook
#
wb = Workbook()
# read the config file in
#
config.readfp(open(args.configfile))
# loop through the config file to pull each query
#
for section_name in config.sections():
section_type = config.get(section_name, "type")
if section_type == 'binary':
query_cls = Binary
elif section_type == 'process':
query_cls = Process
elif section_type == 'alert':
query_cls = Alert
else:
print("Invalid query type {}, skipping.".format(section_type))
continue
# print the current query if in verbose mode
#
query = config.get(section_name, "query")
if (args.verbose or args.printquery):
print('Working on query: {}'.format(query))
created_tab = False
myheader = config.get(section_name, "fields").split(",")
myheader.insert(0,"Link")
if args.blanktabs:
ws = create_tab(section_name, myheader, wb)
try:
for row, result in enumerate(cb.select(query_cls).where(query)):
if not args.blanktabs and not created_tab:
ws = create_tab(section_name, myheader, wb)
created_tab = True
for col, header_title in enumerate(myheader):
if header_title == "Link":
if section_type == "alert":
link = result.proc.webui_link
else:
link = result.webui_link
ws.cell(row=row+2, column=col+1).hyperlink = link
else:
if (header_title in ip_fields and result.get(header_title)):
ws.cell(row=row+2, column=col+1).value = int2ip(result.get(header_title, "<UNKNOWN>"))
else:
ws.cell(row=row+2, column=col+1).value = convert_to_string(result.get(header_title, "<UNKNOWN>"))
except Exception as e:
print("Encountered exception while processing query from section {0}: {1}".format(section_name, e))
traceback.print_exc()
print("Continuing...")
# save the workbook
#
wb.remove_sheet(wb.get_sheet_by_name('Sheet'))
wb.save(args.outfile)
