def get_config(p):
"""Read a config file.
:return: dict of ('section.option', value) pairs.
"""
cfg = {}
parser = ConfigParser()
if hasattr(parser, 'read_file'):
parser.read_file(Path(p).open(encoding='utf8'))
else: # pragma: no cover
assert PY2
# The `read_file` method is not available on ConfigParser in py2.7!
parser.readfp(Path(p).open(encoding='utf8'))
for section in parser.sections():
getters = {
'int': partial(parser.getint, section),
'boolean': partial(parser.getboolean, section),
'float': partial(parser.getfloat, section),
'list': lambda option: parser.get(section, option).split(),
}
default = partial(parser.get, section)
for option in parser.options(section):
type_ = option.rpartition('_')[2] if '_' in option else None
value = getters.get(type_, default)(option)
cfg['{0}.{1}'.format(section, option)] = value
return cfg
def _get_config_parser(self, text):
parser = ConfigParser()
try:
parser.read_file(StringIO(text))
except AttributeError: # Python 2
parser.readfp(StringIO(text))
return parser
def _read_config_file(self):
# Read the virt-who-<id> config file)
local_path = tempfile.mkstemp(suffix=self.config_file_name)[1]
ssh.download_file(self.remote_path, local_path, hostname=self.server)
parser = ConfigParser()
with open(local_path) as local_fp:
parser.read_file(local_fp)
return parser
def parse_color_map_from_file(f):
color_map_config = ConfigParser()
if isinstance(f, string_types):
with open(f, 'r') as fp:
color_map_config.read_file(fp)
else:
color_map_config.read_file(f)
return parse_color_map_from_configparser(color_map_config)
def parse_xml_mapping(xml_mapping_filename):
with open(xml_mapping_filename, 'r') as f:
config = ConfigParser()
if six.PY3:
config.read_file(f) # pylint: disable=no-member
else:
config.read_file(f)
return {k: dict(config.items(k)) for k in config.sections()}
def _parseINI(text):
from six.moves import cStringIO
from six.moves.configparser import ConfigParser
parser = ConfigParser()
try:
parser.read_file(cStringIO(text))
except AttributeError: # Python 2
parser.readfp(cStringIO(text))
return parser
class INIReader(object):
"""ConfigParser wrapper able to cast value when reading INI options."""
# Helper casters
cast_boolean = casts.Boolean()
cast_dict = casts.Dict()
cast_list = casts.List()
cast_logging_level = casts.LoggingLevel()
cast_tuple = casts.Tuple()
cast_webdriver_desired_capabilities = casts.WebdriverDesiredCapabilities()
def __init__(self, path):
self.config_parser = ConfigParser()
with open(path) as handler:
self.config_parser.readfp(handler)
if sys.version_info[0] < 3:
# ConfigParser.readfp is deprecated on Python3, read_file
# replaces it
self.config_parser.readfp(handler)
else:
self.config_parser.read_file(handler)
def get(self, section, option, default=None, cast=None):
"""Read an option from a section of a INI file.
The default value will return if the look up option is not available.
The value will be cast using a callable if specified otherwise a string
will be returned.
:param section: Section to look for.
:param option: Option to look for.
:param default: The value that should be used if the option is not
defined.
:param cast: If provided the value will be cast using the cast
provided.
"""
try:
value = self.config_parser.get(section, option)
if cast is not None:
if cast is bool:
value = self.cast_boolean(value)
elif cast is dict:
value = self.cast_dict(value)
elif cast is list:
value = self.cast_list(value)
elif cast is tuple:
value = self.cast_tuple(value)
else:
value = cast(value)
except (NoSectionError, NoOptionError):
value = default
return value
def has_section(self, section):
"""Check if section is available."""
return self.config_parser.has_section(section)
def _parseINI(text):
from six.moves.configparser import ConfigParser
from six.moves import cStringIO
parser = ConfigParser()
try:
parser.read_file(cStringIO(text))
except AttributeError: # Python 2
parser.readfp(cStringIO(text))
return parser
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapydartx'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8')
self.cp = ConfigParser()
self.cp.read_file(StringIO(default_config))
self.cp.read(sources)
for fp in extra_sources:
self.cp.read_file(fp)
else:
self.cp = ConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = ['/etc/scrapydartx/scrapydartx.conf', r'c:\scrapyd\scrapyd.conf']
sources += sorted(glob.glob('/etc/scrapydartx/conf.d/*'))
sources += ['scrapydartx.conf']
sources += [expanduser('~/.scrapydartx.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
class INIReader(object):
"""ConfigParser wrapper able to cast value when reading INI options."""
# Helper casters
cast_boolean = casts.Boolean()
cast_dict = casts.Dict()
cast_list = casts.List()
cast_logging_level = casts.LoggingLevel()
cast_tuple = casts.Tuple()
cast_webdriver_desired_capabilities = casts.WebdriverDesiredCapabilities()
def __init__(self, path):
self.config_parser = ConfigParser()
with open(path) as handler:
self.config_parser.readfp(handler)
if sys.version_info[0] < 3:
# ConfigParser.readfp is deprecated on Python3, read_file
# replaces it
self.config_parser.readfp(handler)
else:
self.config_parser.read_file(handler)
def get(self, section, option, default=None, cast=None):
"""Read an option from a section of a INI file.
The default value will return if the look up option is not available.
The value will be cast using a callable if specified otherwise a string
will be returned.
:param section: Section to look for.
:param option: Option to look for.
:param default: The value that should be used if the option is not
defined.
:param cast: If provided the value will be cast using the cast
provided.
"""
try:
value = self.config_parser.get(section, option)
if cast is not None:
if cast is bool:
value = self.cast_boolean(value)
elif cast is dict:
value = self.cast_dict(value)
elif cast is list:
value = self.cast_list(value)
elif cast is tuple:
value = self.cast_tuple(value)
else:
value = cast(value)
except (NoSectionError, NoOptionError):
value = default
return value
def has_section(self, section):
"""Check if section is available."""
return self.config_parser.has_section(section)
def _load_config(no_cfgfile=False):
config = ConfigParser()
config.optionxform = str # make it preserve case
# defaults
if not six.PY3:
config.readfp(BytesIO(_DEFAULT_CONFIG))
else:
config.read_file(StringIO(_DEFAULT_CONFIG))
# update from config file
if not no_cfgfile:
config.read(os.path.join(_STASH_ROOT, f) for f in _STASH_CONFIG_FILES)
return config
def _load_config(no_cfgfile=False):
config = ConfigParser()
config.optionxform = str # make it preserve case
# defaults
if not six.PY3:
config.readfp(BytesIO(_DEFAULT_CONFIG))
else:
config.read_file(StringIO(_DEFAULT_CONFIG))
# update from config file
if not no_cfgfile:
config.read(os.path.join(_STASH_ROOT, f) for f in _STASH_CONFIG_FILES)
return config
def put_ini(self, text):
"""
"""
context = self.context
parser = ConfigParser()
try:
parser.read_file(cStringIO(text))
except AttributeError: # Python 2
parser.readfp(cStringIO(text))
for option, value in parser.defaults().items():
prop_type = context.getPropertyType(option)
if prop_type is None:
context._setProperty(option, value, 'string')
else:
context._updateProperty(option, value)
def put_ini(self, text):
"""
"""
context = self.context
parser = ConfigParser()
try:
parser.read_file(cStringIO(text))
except AttributeError: # Python 2
parser.readfp(cStringIO(text))
for option, value in parser.defaults().items():
prop_type = context.getPropertyType(option)
if prop_type is None:
context._setProperty(option, value, 'string')
else:
context._updateProperty(option, value)
def makelist(mcf2):
"""recursive function for MCF by reference inclusion"""
c = ConfigParser()
LOGGER.debug('reading {}'.format(mcf2))
with codecs.open(mcf2, encoding='utf-8') as fh:
if sys.version_info >= (3, 0):
c.read_file(fh)
else:
c.readfp(fh)
mcf_dict = c.__dict__['_sections']
for section in mcf_dict.keys():
if 'base_mcf' in mcf_dict[section]:
base_mcf_path = get_abspath(mcf,
mcf_dict[section]['base_mcf'])
makelist(base_mcf_path)
mcf_list.append(mcf2)
else: # leaf
mcf_list.append(mcf2)
def read_mcf(mcf):
"""
Returns a dict representation of the passed configuration file.
The dict is structured as follows (section-name -> (attribute-name -> value)).
Note that all attribute-names are converted to lower case as part of this processing.
"""
if mcf is None:
return None
mcf_list = []
def makelist(mcf2):
"""recursive function for MCF by reference inclusion"""
c = ConfigParser()
LOGGER.debug('reading {}'.format(mcf2))
with codecs.open(mcf2, encoding='utf-8') as fh:
if sys.version_info >= (3, 0):
c.read_file(fh)
else:
c.readfp(fh)
mcf_dict = c.__dict__['_sections']
for section in mcf_dict.keys():
if 'base_mcf' in mcf_dict[section]:
base_mcf_path = get_abspath(mcf,
mcf_dict[section]['base_mcf'])
makelist(base_mcf_path)
mcf_list.append(mcf2)
else: # leaf
mcf_list.append(mcf2)
makelist(mcf)
c = ConfigParser()
for mcf_file in mcf_list:
LOGGER.debug('reading {}'.format(mcf))
with codecs.open(mcf_file, encoding='utf-8') as fh:
if sys.version_info >= (3, 0):
c.read_file(fh)
else:
c.readfp(fh)
mcf_dict = c.__dict__['_sections']
return mcf_dict
def _makeInstance(self, id, portal_type, subdir, import_context):
context = self.context
subdir = '%s/%s' % (subdir, id)
properties = self.read_data_file(import_context, '.properties', subdir)
tool = getUtility(ITypesTool)
try:
tool.constructContent(portal_type, context, id)
except ValueError: # invalid type
return None
content = context._getOb(id)
if properties is not None:
if '[DEFAULT]' not in properties:
try:
adp = FolderishDAVAwareFileAdapter
adp(content).import_(import_context, subdir)
return content
except (AttributeError, MethodNotAllowed):
# Fall through to old implemenatation below
pass
lines = properties.splitlines()
stream = StringIO('\n'.join(lines))
parser = ConfigParser(defaults={
'title': '',
'description': 'NONE'
})
try:
parser.read_file(stream)
except AttributeError: # Python 2
parser.readfp(stream)
title = parser.get('DEFAULT', 'title')
description = parser.get('DEFAULT', 'description')
content.setTitle(title)
content.setDescription(description)
return content
def _makeInstance(self, id, portal_type, subdir, import_context):
context = self.context
subdir = '%s/%s' % (subdir, id)
properties = self.read_data_file(import_context, '.properties',
subdir)
tool = getUtility(ITypesTool)
try:
tool.constructContent(portal_type, context, id)
except ValueError: # invalid type
return None
content = context._getOb(id)
if properties is not None:
if '[DEFAULT]' not in properties:
try:
adp = FolderishDAVAwareFileAdapter
adp(content).import_(import_context, subdir)
return content
except (AttributeError, MethodNotAllowed):
# Fall through to old implemenatation below
pass
lines = properties.splitlines()
stream = StringIO('\n'.join(lines))
parser = ConfigParser(defaults={'title': '',
'description': 'NONE'})
try:
parser.read_file(stream)
except AttributeError: # Python 2
parser.readfp(stream)
title = parser.get('DEFAULT', 'title')
description = parser.get('DEFAULT', 'description')
content.setTitle(title)
content.setDescription(description)
return content
def get_conf_stanzas(conf_name):
'''Get stanzas of `conf_name`
:param conf_name: Config file.
:type conf_name: ``string``
:returns: Config stanzas.
:rtype: ``dict``
Usage::
>>> stanzas = get_conf_stanzas('server')
>>> return: {'serverName': 'testServer', 'sessionTimeout': '1h', ...}
'''
if conf_name.endswith('.conf'):
conf_name = conf_name[:-5]
# TODO: dynamically caculate SPLUNK_HOME
btool_cli = [
op.join(os.environ['SPLUNK_HOME'], 'bin', 'btool'), conf_name, 'list'
]
p = subprocess.Popen(btool_cli,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, _ = p.communicate()
if isinstance(out, bytes):
out = out.decode()
out = StringIO(out)
parser = ConfigParser()
parser.optionxform = str
if sys.version_info[:2] >= (3, 2):
parser.read_file(out)
else:
parser.readfp(out)
out = {}
for section in parser.sections():
out[section] = {item[0]: item[1] for item in parser.items(section)}
return out
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
config_file = os.path.join(os.environ["HOME"], ".zuliprc")
if not os.path.exists(config_file):
raise RuntimeError("No ~/.zuliprc found")
config = ConfigParser()
with open(config_file, 'r') as f:
# Apparently, six.moves.configparser.ConfigParser is not
# consistent between Python 2 and 3!
if hasattr(config, 'read_file'):
config.read_file(f, config_file)
else:
config.readfp(f, config_file)
api_key = config.get("api", "key")
email = config.get("api", "email")
try:
realm = get_realm("zulip")
user_profile = get_user(email, realm)
user_profile.api_key = api_key
user_profile.save(update_fields=["api_key"])
except UserProfile.DoesNotExist:
print("User %s does not exist; not syncing API key" % (email, ))
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
config_file = os.path.join(os.environ["HOME"], ".zuliprc")
if not os.path.exists(config_file):
raise RuntimeError("No ~/.zuliprc found")
config = ConfigParser()
with open(config_file, 'r') as f:
# Apparently, six.moves.configparser.ConfigParser is not
# consistent between Python 2 and 3!
if hasattr(config, 'read_file'):
config.read_file(f, config_file)
else:
config.readfp(f, config_file)
api_key = config.get("api", "key")
email = config.get("api", "email")
try:
realm = get_realm("zulip")
user_profile = get_user(email, realm)
user_profile.api_key = api_key
user_profile.save(update_fields=["api_key"])
except UserProfile.DoesNotExist:
print("User %s does not exist; not syncing API key" % (email,))
def load(cfg_file=None, environment=None, overrides=None):
"""
Load configuration.
A configuration file consists of sections, led by a ``[section]`` header
and followed by ``name: value`` entries. Lines beginning with ``'#'`` are
ignored and may be used to provide comments.
A configuration file can contain multiple sections. The configuration
object is populated with values from the ``global`` section and additional
sections based on the fully qualified domain name of the local host. For
example, on the host ``deployXXXX.eqiad.wmnet`` the final value for a given
setting would be the first value found in sections:
``deployXXXX.eqiad.wmnet``, ``eqiad.wmnet``, ``wmnet`` or ``global``.
Sections not present in the configuration file will be ignored.
Configuration values are loaded from a file specified by the ``-c`` or
``--conf`` command-line options or from the default locations with the
following hierarchy, sorted by override priority:
#. ``$(pwd)/scap/environments/<environment>/scap.cfg`` or
``$(pwd)/scap/scap.cfg`` (if no environment was specified)
#. ``/etc/scap.cfg``
For example, if a configuration parameter is set in
``$(pwd)/scap/scap.cfg`` and that same parameter is set in
``/etc/scap.cfg`` the value for that parameter set in
``$(pwd)/scap/scap.cfg`` will be used during execution.
:param cfg_file: Alternate configuration file
:param environment: the string path under which scap.cfg is found
:param overrides: Dict of configuration values
:returns: dict of configuration values
"""
local_cfg = os.path.join(os.getcwd(), 'scap')
parser = ConfigParser()
if cfg_file:
try:
cfg_file = open(cfg_file)
except TypeError:
# Assume that cfg_file is already an open file
pass
if hasattr(parser, 'read_file'):
parser.read_file(cfg_file)
else:
parser.readfp(cfg_file)
else:
parser.read([
'/etc/scap.cfg',
os.path.join(local_cfg, 'scap.cfg'),
utils.get_env_specific_filename(
os.path.join(local_cfg, 'scap.cfg'), environment)
])
fqdn = socket.getfqdn().split('.')
sections = ['global']
sections += ['.'.join(fqdn[x:]) for x in range(0, len(fqdn))][::-1]
config = {key: value for key, (_, value) in DEFAULT_CONFIG.items()}
for section in sections:
if parser.has_section(section):
# Do not interpolate items in the section.
# Fixes crash on deployment server:
# 'int' object has no attribute 'find'
for key, value in parser.items(section, True):
config[key] = coerce_value(key, value)
config = override_config(config, overrides)
if not environment and config.get('environment', None):
return load(cfg_file, config.get('environment'), overrides)
config['environment'] = environment
return config
def parse_color_map_from_file(f):
color_map_config = ConfigParser()
color_map_config.read_file(f)
return parse_color_map_from_configparser(color_map_config)
class ConfigLoader(object):
'''
Takes a directory and loads config files
'''
def __init__(self, path=None, cli_cfg=''):
'''
Constructor
'''
if path is None:
path = os.getcwd()
self._config = SafeConfigParser(allow_no_value=True)
self._config.optionxform = str
user_cfg = os.path.join(FIXEDPATHS.USERDIR, 'hydrant.cfg')
with open(user_cfg) as default_cfg:
if PY32:
self._config.read_file(self.readline_generator(default_cfg))
else:
self._config.readfp(default_cfg)
workflow_cfg = ''
task_cfg = ''
if os.path.exists(os.path.join(path, 'Dockerfile')):
task_cfg = os.path.join(path, 'hydrant.cfg')
workflow_cfg = os.path.join(os.path.dirname(os.path.abspath(path)),
'hydrant.cfg')
else:
workflow_cfg = os.path.join(path, 'hydrant.cfg')
self._config.read([workflow_cfg, task_cfg, cli_cfg])
# from https://docs.python.org/3/library/configparser.html#configparser.ConfigParser.readfp
@staticmethod
def readline_generator(fp):
line = fp.readline()
while line:
yield line
line = fp.readline()
@property
def config(self):
'''Build namedtuple version of config'''
all_section = self._get_section(AllSection, 'All')
firecloud_section = self._get_section(FireCloudSection, 'FireCloud')
docker_section = self._get_section(DockerSection, 'Docker')
tasks_section = self._get_subsections(TaskSubsection, 'Task')
return Config(all_section, firecloud_section, docker_section,
tasks_section)
def _get_items(self, section):
for key, value in self._config.items(section):
if value.strip() == '': # Convert empty values to None
value = None
elif ',' in value: # Convert comma-separated fields to tuples
value = tuple(field.strip() for field in value.split(','))
yield key, value
def _get_section(self, named_tuple_class, section):
items = dict(self._get_items(section))
return named_tuple_class._make(
items.get(key) for key in named_tuple_class._fields)
def _get_subsections(self, named_tuple_class, section_prefix):
subsections = [subsection for subsection in self._config.sections() \
if subsection.startswith(section_prefix)]
if len(subsections) == 0:
return None
Section = namedtuple(section_prefix + 's',
[subsection.replace(section_prefix, '').strip() \
for subsection in subsections])
return Section._make(
self._get_section(named_tuple_class, subsection)
for subsection in subsections)
@staticmethod
def array_to_cfg_str(ary):
return ','.join(ary)
def read_file(self, f, source=None):
if PY2:
return ConfigParser.readfp(self, MySQLConfigParserFilter(f),
source)
return ConfigParser.read_file(self, MySQLConfigParserFilter(f), source) # pylint: disable=no-member
def load_config():
"""Load configuration.
:returns:
Current configuration based on configuration file and environment variables.
:rtype: dict
"""
config_parser = ConfigParser(
{key: str(value)
for key, value in DEFAULT_CONFIG.items()})
config_parser.add_section("greynoise")
if os.path.isfile(CONFIG_FILE):
LOGGER.debug("Parsing configuration file: %s...",
CONFIG_FILE,
path=CONFIG_FILE)
with open(CONFIG_FILE) as config_file:
config_parser.read_file(config_file)
else:
LOGGER.debug("Configuration file not found: %s",
CONFIG_FILE,
path=CONFIG_FILE)
if "GREYNOISE_API_KEY" in os.environ:
api_key = os.environ["GREYNOISE_API_KEY"]
LOGGER.debug("API key found in environment variable: %s",
api_key,
api_key=api_key)
# Environment variable takes precedence over configuration file content
config_parser.set("greynoise", "api_key", api_key)
if "GREYNOISE_API_SERVER" in os.environ:
api_server = os.environ["GREYNOISE_API_SERVER"]
LOGGER.debug(
"API server found in environment variable: %s",
api_server,
api_server=api_server,
)
# Environment variable takes precedence over configuration file content
config_parser.set("greynoise", "api_server", api_server)
if "GREYNOISE_TIMEOUT" in os.environ:
timeout = os.environ["GREYNOISE_TIMEOUT"]
try:
int(timeout)
except ValueError:
LOGGER.error(
"GREYNOISE_TIMEOUT environment variable "
"cannot be converted to an integer: %r",
timeout,
timeout=timeout,
)
else:
LOGGER.debug("Timeout found in environment variable: %s",
timeout,
timeout=timeout)
# Environment variable takes precedence over configuration file content
config_parser.set("greynoise", "timeout", timeout)
if "GREYNOISE_PROXY" in os.environ:
proxy = os.environ["GREYNOISE_PROXY"]
LOGGER.debug(
"Proxy found in environment variable: %s",
proxy,
proxy=proxy,
)
# Environment variable takes precedence over configuration file content
config_parser.set("greynoise", "proxy", proxy)
if "GREYNOISE_OFFERING" in os.environ:
offering = os.environ["GREYNOISE_OFFERING"]
LOGGER.debug(
"Offering found in environment variable: %s",
offering,
offering=offering,
)
# Environment variable takes precedence over configuration file content
config_parser.set("greynoise", "offering", offering)
return {
"api_key": config_parser.get("greynoise", "api_key"),
"api_server": config_parser.get("greynoise", "api_server"),
"timeout": config_parser.getint("greynoise", "timeout"),
"proxy": config_parser.get("greynoise", "proxy"),
"offering": config_parser.get("greynoise", "offering"),
}
