def __createExpCfg(self, expCfgName):
"""
Set up the experiment config section (expCfg)
"""
print "Loading experiment config file: " + expCfgName
# This is where the parser is called.
expCfg = ConfigObj(expCfgName, configspec='expCfgSpec.ini', raise_errors = True, file_error = True)
validator = Validator()
expCfgOK = expCfg.validate(validator)
if expCfgOK == True:
print "Experiment config file parsed correctly"
else:
print 'Experiment config file validation failed!'
res = expCfg.validate(validator, preserve_errors=True)
for entry in flatten_errors(expCfg, res):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error == False:
error = 'Missing value or section.'
print section_string, ' = ', error
sys.exit(1)
if expCfg.has_key('_LOAD_'):
for ld in expCfg['_LOAD_']['loadList']:
print 'Loading: ' + ld + ' as ' + expCfg['_LOAD_'][ld]['cfgFile']
curCfg = ConfigObj(expCfg['_LOAD_'][ld]['cfgFile'], configspec = expCfg['_LOAD_'][ld]['cfgSpec'], raise_errors = True, file_error = True)
validator = Validator()
expCfgOK = curCfg.validate(validator)
if expCfgOK == True:
print "Experiment config file parsed correctly"
else:
print 'Experiment config file validation failed!'
res = curCfg.validate(validator, preserve_errors=True)
for entry in flatten_errors(curCfg, res):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error == False:
error = 'Missing value or section.'
print section_string, ' = ', error
sys.exit(1)
expCfg.merge(curCfg)
self.expCfg = expCfg
def load_config():
settings = ConfigParser.RawConfigParser()
spec_file = pkg_resources.resource_filename(__name__,"/config/configspec.cfg")
settings = ConfigObj(configspec=spec_file, encoding='UTF8')
# User's FCD settings
user_config_file = os.path.join(fcd_user_dir, "firewallconfigdecryptor.cfg")
settings.merge(ConfigObj(user_config_file))
# FCD settings in current directory
settings.merge(ConfigObj("firewallconfigdecryptor.cfg"))
# FCD settings specified by environment variable
try:
fcdcfg = os.environ['firewallconfigdecryptor_CFG']
settings.merge(ConfigObj(fcdcfg))
except KeyError:
pass
results = settings.validate(validator)
if results != True:
for (section_list, key, _) in flatten_errors(settings, results):
if key is not None:
error_msg=('Invalid key "%s" in section "%s"' % (key, ', '.join(section_list)))
print("Error loading configuration file: %s"%error_msg)
raise SystemExit
else:
# ignore missing sections - use defaults
pass
return settings
def _setup(path, spec, create=False):
# validate the config
config = ConfigObj(path, configspec=spec.split("\n"))
validator = Validator()
# create config file with defaults if necessary
if create and not os.path.exists(path):
config.validate(validator, copy=True)
config.filename = path
config.write()
else:
result = config.validate(validator, preserve_errors=True)
# show config errors if there are any
if type(result) is dict:
for entry in flatten_errors(config, result):
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append("[missing section]")
section_string = " -> ".join(section_list)
if error == False:
error = "Missing value or section."
print section_string, " = ", error
raise Exception("Errors in config file")
return config
def load_config():
settings = ConfigParser.RawConfigParser()
spec_file = pkg_resources.resource_filename(__name__,"/config/configspec.cfg")
settings = ConfigObj(configspec=spec_file, encoding='UTF8')
# User's ANK settings
user_config_file = os.path.join(ank_user_dir, "autonetkit.cfg")
settings.merge(ConfigObj(user_config_file))
# ANK settings in current directory
settings.merge(ConfigObj("autonetkit.cfg"))
# ANK settings specified by environment variable
try:
ankcfg = os.environ['AUTONETKIT_CFG']
settings.merge(ConfigObj(ankcfg))
except KeyError:
pass
results = settings.validate(validator)
if results != True:
for (section_list, key, _) in flatten_errors(settings, results):
if key is not None:
print "Error loading configuration file:"
print 'Invalid key "%s" in section "%s"' % (key, ', '.join(section_list))
raise SystemExit
else:
# ignore missing sections - use defaults
#print 'The following section was missing:%s ' % ', '.join(section_list)
pass
return settings
def parse(self):
"""
Parse configuration file and validate it if a configuration specification was supplied
"""
if self.configspec and self.configfile:
cfgspec = ConfigObj(self.configspec.split('\n'),
encoding='UTF8',
interpolation=False,
list_values=False)
conf = ConfigObj(self.configfile, configspec=cfgspec)
validation = Validator()
res = conf.validate(validation, preserve_errors=True)
for entry in flatten_errors(conf, res):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
if error == False:
error = 'Missing value or section.'
raise JanuaConfigError(
'%s parameter in section %s: %s' %
(section_list[1], section_list[0], error))
return DictToObj(conf)
def setup_config(path='config.ini', create=False):
# validate the config
config = ConfigObj(path, configspec=PYDR_CONFIG_SPEC.split("\n"))
validator = Validator()
# create config file with defaults if necessary
if create and not os.path.exists(path):
config.validate(validator, copy=True)
config.filename = path
config.write()
else:
result = config.validate(validator, preserve_errors=True)
# autosubmit_interval should be less than job walltime
if config['manager']['mobile'] and config['manager']['autosubmit'] and config['manager']['autosubmit_interval'] > (config['job']['walltime']-3600):
raise Exception('Autosubmit interval is greater than job walltime, mobile server will not work!')
# snapshottime should be less than job walltime
if config['manager']['snapshottime'] > config['job']['walltime']:
raise Exception('Snapshot time is greater than job walltime!')
# replica_walltime should be less than job walltime
if config['job']['replica_walltime'] >= config['job']['walltime']:
raise Exception('Replica walltime should be less than job walltime!')
# show config errors if there are any
if type(result) is dict:
for entry in flatten_errors(config, result):
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ' -> '.join(section_list)
if error == False:
error = 'Missing value or section.'
print section_string, ' = ', error
raise Exception('Errors in config file')
return config
def load_config():
settings = ConfigParser.RawConfigParser()
spec_file = pkg_resources.resource_filename(__name__,
"/config/configspec.cfg")
settings = ConfigObj(configspec=spec_file, encoding='UTF8')
# User's FFW settings
user_config_file = os.path.join(ffw_user_dir, "muddy.cfg")
settings.merge(ConfigObj(user_config_file))
# FFW settings in current directory
settings.merge(ConfigObj("muddy.cfg"))
# FFW settings specified by environment variable
try:
ffwcfg = os.environ['muddy_CFG']
settings.merge(ConfigObj(ffwcfg))
except KeyError:
pass
results = settings.validate(validator)
if results != True:
for (section_list, key, _) in flatten_errors(settings, results):
if key is not None:
error_msg = ('Invalid key "%s" in section "%s"' %
(key, ', '.join(section_list)))
print("Error loading configuration file: %s" % error_msg)
raise SystemExit
else:
# ignore missing sections - use defaults
pass
return settings
def configuration(self):
"""Returns a dictionary for the session configuration if it is valid.
Otherwise, print the erroneous keys.
Returns
-------
configuration: dictionary
Dictionary containing the information provided in pcigale.ini.
"""
self.complete_redshifts()
self.complete_analysed_parameters()
vdt = validate.Validator(validation.functions)
validity = self.config.validate(vdt, preserve_errors=True)
if validity is not True:
print("The following issues have been found in pcigale.ini:")
for module, param, message in configobj.flatten_errors(self.config,
validity):
if len(module) > 0:
print("Module {}, parameter {}: {}".format('/'.join(module),
param, message))
else:
print("Parameter {}: {}".format(param, message))
print("Run the same command after having fixed pcigale.ini.")
return None
return self.config.dict()
def load_settings(self):
self.logger.info("Looking for %s in %s, %s.ini" % (self,
'config.ini', self))
base = ConfigObj()
for c in ["config.ini", "%s.ini" % self]:
if not os.path.exists(c):
continue
parser = ConfigObj(c, configspec='config-spec.ini')
validator = Validator()
results = parser.validate(validator)
if results:
if str(self) in parser:
base.merge(parser[str(self)])
continue
# Print an error message for sections with errors
for section_list, key, _ in flatten_errors(parser, results):
if key is not None:
self.logger.error('Failed to validate: "%s" section "%s"' \
% (key, ', '.join(section_list)))
else:
self.logger.error('Section(s) missing: %s' \
% (', '.join(section_list)))
self.settings = base
def __createSysCfg(self):
"""
Set up the system config section (sysCfg)
"""
sysCfgName = platform.node()+".cfg"
if not(os.path.isfile(sysCfgName)):
sysCfgName = "defaultSys.cfg"
print "Loading system config file: " + sysCfgName
sysCfg = ConfigObj(sysCfgName, configspec='sysCfgSpec.ini', raise_errors = True)
validator = Validator()
sysCfgOK = sysCfg.validate(validator)
if sysCfgOK == True:
print "System config file parsed correctly"
else:
print 'System config file validation failed!'
res = sysCfg.validate(validator, preserve_errors=True)
for entry in flatten_errors(sysCfg, res):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error == False:
error = 'Missing value or section.'
print section_string, ' = ', error
sys.exit(1)
self.sysCfg = sysCfg
def config_error(config, results):
"""
Parse the results of a ConfigObj validation and print the errors.
Throws a RuntimeError exception upon completion if any errors or
missing sections are encountered.
Args:
config (ConfigObj): The ConfigObj instance representing the
parsed config.
results (dict): The dictionary returned by the validation of
the 'config' arguments.
Returns:
Nothing: Nothing
Raises:
RuntimeError: Should always raise this exception.
"""
errs = 0
for (section_list, key, _) in flatten_errors(config, results):
if key is not None:
print('The "%s" key in the section "%s" failed validation'
% (key, ', '.join(section_list)))
errs += 1
else:
print('The following section was missing:%s '
% ', '.join(section_list))
errs += 1
if errs:
raise RuntimeError('There %s %d %s in configuration.'
% ('was' if errs == 1 else 'were', errs,
'error' if errs == 1 else 'errors'))
def generate_validation_report(config, validation_result):
"""
Generate a report if necessary of problems while validating.
Returns:
Either a string describing for a user the problems validating
this config or None if there are no problems.
"""
report = []
# Organize the report
for entry in flatten_errors(config, validation_result):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append(u'[missing section]')
section_string = u':'.join(section_list)
if error == False:
# We don't care about missing values for now.
continue
report.append(u"%s = %s" % (section_string, error))
if report:
return REPORT_HEADER + u"\n".join(report)
else:
return None
def check_config(config):
cameras = dict(
regions='force_list(default=list())',
sample='integer(default=2)',
max_prediction_delay='float(default=6)',
memory_decay='float(default=300)',
mmc='boolean(default=no)',
image_format=
'string(default="$(camera)_screenshots/%y-%m-%d/%H-%M-%S.%f.jpg")')
camera = dict(active='boolean(default=yes)',
url='string',
name='string',
csv_file='string(default=None)',
webhook_target='string(default=None)',
webhook_header='string(default=None)',
webhook_image='boolean(default=None)')
spec = ConfigObj()
spec['timezone'] = 'string(default="UTC")'
spec['cameras'] = dict(__many__=camera, **cameras)
config = ConfigObj(config.split('\n'), configspec=spec, raise_errors=True)
config.newlines = '\r\n' # For Windows
result = config.validate(Validator(), preserve_errors=True)
errors = flatten_errors(config, result)
if errors:
error_message = 'Config errors:'
for section_list, param, message in errors:
section_string = '/'.join(section_list)
if message is False:
message = f'param {param} is missing.'
error = f'{section_string}, param: {param}, message: {message}'
error_message += f'\n{error}'
return False, error_message
return True, None
def load_config(module_name, cfg_file=None):
'''
Loads a config file and returns a ConfigObj. A default configuration
file is also loaded from the directory containing the module given
by module_name.
'''
if module_name == None:
module_name = __name__
base_config = {}
else:
base_config = load_config(None, cfg_file)
configspec = resource_stream(module_name, 'defaults.ini')
config = ConfigObj(cfg_file, configspec=configspec)
validator = Validator()
valid = config.validate(validator)
if not valid:
for (section_list, key, _) in flatten_errors(config, valid):
if key is not None:
print 'The "%s" key in the section "%s" failed validation' % \
(key, ', '.join(section_list))
else:
print 'The following section was missing:%s ' % \
', '.join(section_list)
for sec, cfg in config.items():
if sec in base_config:
base_config[sec].update(cfg)
else:
base_config[sec] = cfg
return base_config
def _validate(self, config):
"""
Validate the config against the specification using a default
validator. The validated config values are returned if success,
otherwise, the ``ConfigError`` raised with details.
"""
validator = Validator()
try:
# NOTE:
# Use the "copy" mode, which will copy both the default values
# and all the comments.
results = config.validate(validator, preserve_errors=True,
copy=True)
except ConfigObjError as e:
raise ConfigError(e)
if results is not True:
error_msg = ""
for (section_list, key, res) in flatten_errors(config, results):
if key is not None:
if res is False:
msg = 'key "%s" in section "%s" is missing.'
msg = msg % (key, ", ".join(section_list))
else:
msg = 'key "%s" in section "%s" failed validation: %s'
msg = msg % (key, ", ".join(section_list), res)
else:
msg = 'section "%s" is missing' % ".".join(section_list)
error_msg += msg + "\n"
raise ConfigError(error_msg)
return config
def load_config():
path = find_config()
specpath = os.path.join(os.path.dirname(__file__), 'confspec.ini')
validator = Validator({
'expand_path': expand_path,
'cache_path': validate_cache_path,
'date_format': validate_date_format,
'time_format': validate_time_format,
})
config = ConfigObj(path, configspec=specpath, file_error=True)
validation = config.validate(validator, preserve_errors=True)
for section, key, error in flatten_errors(config, validation):
if not error:
raise ConfigurationException(
('{} is missing from the {} section of the configuration ' +
'file').format(key, section)
)
if isinstance(error, VdtValueError):
raise ConfigurationException(
'Bad {} setting, {}'.format(key, error.args[0])
)
return config
def generate_validation_report(config, validation_result):
"""
Generate a report if necessary of problems while validating.
Returns:
Either a string describing for a user the problems validating
this config or None if there are no problems.
"""
report = []
# Organize the report
for entry in flatten_errors(config, validation_result):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append(u'[missing section]')
section_string = u':'.join(section_list)
if error == False:
# We don't care about missing values for now.
continue
report.append(u"%s = %s" % (section_string, error))
if report:
return REPORT_HEADER + u"\n".join(report)
else:
return None
def _validate(self):
# set all str values that are 'None' to None
def set_str_to_none(d):
for k, v in d.items():
if type(v)==Section:
set_str_to_none(v)
else:
if type(v)==str and v=='None':
d[k]=None
set_str_to_none(self.data)
validator = Validator()
results = self.data.validate(validator, copy=True)
if results != True:
logger.error('Config file validation failed!')
for (section_list, key, _) in flatten_errors(self.data, results):
if key is not None:
logger.error('The "%s" key in the section "%s" failed '
'validation' % (key, ', '.join(section_list)))
else:
logger.error('The following section was missing:%s ' %
', '.join(section_list))
raise ValidationError
self.data.filename = self.filename
self._init_data_dir()
self._init_plugins_dir()
def __init__(self, filename):
"""Initializes a config file if does not exist. If exists, uses
it to validate the file, and setup default initial parameters"""
self.cfg_spec = ConfigObj(config_spec_text.splitlines(),
list_values=False)
self.cfg_filename = filename
valid = Validator()
if not os.path.exists(self.cfg_filename):
#no configuration file found
logger.info(
"File %s not found, so creating one from you from defaults" %
self.cfg_filename)
cfg = ConfigObj(configspec=self.cfg_spec,
stringify=True,
list_values=True)
cfg.filename = self.cfg_filename
test = cfg.validate(valid, copy=True)
cfg.write()
self.cfg = ConfigObj(self.cfg_filename, configspec=self.cfg_spec)
rtn = self.cfg.validate(valid, preserve_errors=True)
if type(rtn) == types.BooleanType and rtn:
logger.info("Config file validated")
self.tested = True
else:
self.tested = False
res = flatten_errors(self.cfg, rtn)
self.errortxt = ''
for row in res:
self.errortxt += 'In Section %s, key %s has error: %s' % (
row[0], row[1], row[2])
logger.error(self.errortxt)
def parse_config_file(config_file, config_spec_file):
assert config_file and isinstance(
config_file, str), 'config_file must be a non-empty str'
assert config_spec_file and isinstance(
config_spec_file, str), 'config_spec_file must be a non empty str'
try:
config = ConfigObj(config_file,
configspec=config_spec_file,
interpolation='template',
file_error=True)
except IOError as e:
raise PanoptesConfigurationParsingError('Error reading file: %s' %
str(e))
except ConfigObjError as e:
raise PanoptesConfigurationParsingError(
'Error parsing config file "%s": %s' % (config_file, str(e)))
validator = Validator()
result = config.validate(validator, preserve_errors=True)
if result is not True:
errors = ''
for (section_list, key, error) in flatten_errors(config, result):
if key is None:
errors += 'Section(s) ' + ','.join(
section_list) + ' are missing\n'
else:
errors += 'The "' + key + '" key in section "' + ','.join(
section_list) + '" failed validation\n'
raise PanoptesConfigurationParsingError(
'Error parsing the configuration file: %s' % errors)
return config
def start(fqdn, dynamo_region, dynamo_host, table_root, log_config, verbosity, environment, server_config):
"""
Starts an APScheduler job to periodically reload HAProxy config as well as run the API to register/deregister
new services, target groups and backends.
:param fqdn: The fully qualified domain name of Flyby - requests coming here will go to the API endpoints
:param dynamo_region: The AWS region of the DynamoDB tables Flyby stores and reads config in
:param dynamo_host: The hostname and port to use for DynamoDB connections. Useful for local testing with
moto or DynamoDB Local.
:param table_root: The root that will be used for table names in DynamboDB. This will be prefixed to all tables
created.
:param log_config: Location of python yaml config file.
:param verbosity: Logging verbosity, defaults to INFO.
:return:
"""
logging.getLogger().setLevel(level=getattr(logging, verbosity))
dynamo_manager = DynamoTableManagement()
config = ConfigObj(infile=server_config, configspec='flyby/configspec.ini', stringify=True)
res = config.validate(Validator(), preserve_errors=True)
if res is not True:
for section, key, msg in flatten_errors(config, res):
click.echo("{}: {} in {}".format(key, msg, section))
raise click.ClickException('bad server config')
# Create the DynamoDB tables if missing, update the DynamoDB read/write capacity if required
dynamo_manager.update_capacity(dynamo_host, dynamo_region, table_root, logger, config)
if log_config:
with open(log_config, 'r') as conf:
logging.config.dictConfig(yaml.load(conf))
scheduler = BackgroundScheduler(timezone=utc)
scheduler.add_job(update, 'interval', seconds=10, args=(fqdn,))
scheduler.start()
if environment == "development":
app.run(host='0.0.0.0')
else:
serve(app, listen='*:5000')
def validate_config(self, configspec, suppress_warnings=False):
"""
Validate this config with the given configspec
"""
self._handle_configspec(configspec)
errors = self.validate(validator, preserve_errors=True)
for entry in flatten_errors(self, errors):
section_list, key, error = entry
if not error:
LOGGER.error("Missing parameter %s",
'.'.join(section_list + [key]))
else:
LOGGER.error("Configuration error %s: %s",
'.'.join(section_list + [key]), error)
# warn about any unknown parameters before we potentially abort on
# validation errors
if not suppress_warnings:
for sections, name in get_extra_values(self):
LOGGER.warn("Unknown parameter '%s' in section '%s'", name,
".".join(sections))
if errors is not True:
raise ConfigError(
"Configuration errors were encountered while validating %r" %
self.filename)
return errors
def get_config(config_file):
"""
Parse the config file, validate it and convert types. Return a dictionary
with all settings.
"""
specs = ConfigObj(StringIO(CONFIG_SPEC), list_values=False)
config = ConfigObj(config_file, configspec=specs)
# Create validator
validator = Validator({'lowercase_string_list': lowercase_string_list})
# Convert types and validate file
result = config.validate(validator, preserve_errors=True, copy=True)
logger.debug("Config file version %d", config["version"])
# Raise exceptions for errors
for section_list, key, message in flatten_errors(config, result):
if key is not None:
raise ValueError(
"The '%s' key in the section '%s' failed validation: %s" % (
key, ", ".join(section_list), message))
else:
raise ValueError(
"The following section was missing: %s." % (
", ".join(section_list)))
# For now, no automatic update support.
if config["version"] != CONFIG_VERSION:
logger.warning(
"Config file version is %d, while expected version is %d. Please "
"check for inconsistencies and update manually.",
config["version"], CONFIG_VERSION)
return config
def get_config(config_path=None):
"""reads the config file, validates it and return a config dict
:param config_path: path to a custom config file, if none is given the
default locations will be searched
:type config_path: str
:returns: configuration
:rtype: dict
"""
if config_path is None:
config_path = _find_configuration_file()
logger.debug('using the config file at {}'.format(config_path))
config = ConfigObj(DEFAULTSPATH, interpolation=False)
try:
user_config = ConfigObj(config_path,
configspec=SPECPATH,
interpolation=False,
file_error=True,
)
except ConfigObjError as error:
logger.fatal('parsing the config file file with the following error: '
'{}'.format(error))
logger.fatal('if you recently updated khal, the config file format '
'might have changed, in that case please consult the '
'CHANGELOG or other documentation')
sys.exit(1)
fdict = {'timezone': is_timezone,
'expand_path': expand_path,
}
validator = Validator(fdict)
results = user_config.validate(validator, preserve_errors=True)
if not results:
for entry in flatten_errors(config, results):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error is False:
error = 'Missing value or section.'
print(section_string, ' = ', error)
raise ValueError # TODO own error class
config.merge(user_config)
config_checks(config)
extras = get_extra_values(user_config)
for section, value in extras:
if section == ():
logger.warn('unknown section "{}" in config file'.format(value))
else:
section = sectionize(section)
logger.warn('unknown key or subsection "{}" in '
'section "{}"'.format(value, section))
return config
def load_config():
settings = ConfigParser.RawConfigParser()
# load defaults
spec_file = pkg_resources.resource_filename(__name__,"/lib/configspec.cfg")
settings = ConfigObj(configspec=spec_file, encoding='UTF8')
# Try in ~/.autonetkit/autonetkit.cfg
user_config_file = os.path.join(ank_user_dir, "autonetkit.cfg")
settings.merge(ConfigObj(user_config_file))
#TODO: look at using configspec validation
# also try from current directory
settings.merge(ConfigObj("autonetkit.cfg"))
results = settings.validate(validator)
if results != True:
for (section_list, key, _) in flatten_errors(settings, results):
if key is not None:
print "Error loading configuration file:"
print 'Invalid key "%s" in section "%s"' % (key, ', '.join(section_list))
#TODO: throw exception here?
sys.exit(0)
else:
# ignore missing sections - use defaults
#print 'The following section was missing:%s ' % ', '.join(section_list)
pass
return settings
def config_error(config, results):
"""
Parse the results of a ConfigObj validation and log the errors.
Throws a RuntimeError exception upon completion if any errors or
missing sections are encountered.
Args:
config (ConfigObj): The ConfigObj instance representing the
parsed config.
results (dict): The dictionary returned by the validation of
the 'config' arguments.
Returns:
Nothing: Nothing
Raises:
RuntimeError: Should always raise this exception.
"""
errs = 0
for (section_list, key, _) in flatten_errors(config, results):
if key is not None:
logging.error(
'The "%s" key in the section "%s" failed validation' %
(key, ', '.join(section_list)))
errs += 1
else:
logging.error('The following section was missing:%s ' %
', '.join(section_list))
errs += 1
if errs:
raise RuntimeError('There %s %d %s in configuration.' %
('was' if errs == 1 else 'were', errs,
'error' if errs == 1 else 'errors'))
def validate(filename, specfilename):#, error_stream=sys.stderr):
conf = configobj.ConfigObj(filename, configspec=specfilename)
res = conf.validate(rich_validator, preserve_errors=True)
if not res:
flat_errs = configobj.flatten_errors(conf, res)
for _, var, ex in flat_errs:
logger.error("%s : %s\n" % (var, str(ex)))
raise v.ValidateError("Could not validate %s vs %s" % (filename, specfilename))
for k,v in conf["backends"].iteritems():
v["name"] = k
conf["backends"][k] = BackendFactory(v)
for k,v in conf["scantypes"].iteritems():
if isinstance(v, tuple):
logger.info(("exploding scanmode {0} in 2 separate scans: {0}_lon" + \
" and {0}_lat").format(k,))
conf["scantypes"].pop(k)
if ((k + "_lon" in conf["scantypes"]) or
(k + "_lat" in conf["scantypes"])):
raise ScheduleError("Cannot explode scan %s in separate subscans" % (k,))
conf["scantypes"][k + "_lon"] = v[0]
v[0].name = k + "_lon"
conf["scantypes"][k + "_lat"] = v[1]
v[1].name = k + "_lat"
else:
v.name = k
return conf
def cobj_check(settings, exception=None, copy=False):
"""Check for errors in config file"""
if not exception:
exception = Exception
validator = validate.Validator()
def numpy_array(val):
"""Define float list"""
float_list = validator.functions["float_list"](val)
return numpy.array(float_list)
validator.functions["numpy_array"] = numpy_array
results = settings.validate(validator, copy=copy, preserve_errors=True)
if results is not True:
output = "{0}: \n".format(
settings.filename if settings.filename is not None else
"configobj")
for (section_list, key, error) in configobj.flatten_errors(
settings, results):
if key is not None:
val = settings
for section in section_list:
val = val[section]
val = val[key] if key in val else "<EMPTY>"
output += " [{sections}], {key}='{val}' ({error})\n".format(
sections=', '.join(section_list),
key=key,
val=val,
error=error)
else:
output += "Missing section: {0}\n".format(
", ".join(section_list))
raise exception(output)
def validate_config(a_config, set_copy=False):
'''Validates a ConfigObj instance.
If errors are found, throws exception and reports errors.
'''
error = False
error_message_list = []
res = a_config.validate(Validator(), preserve_errors=True, copy=set_copy)
for entry in flatten_errors(a_config, res):
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if not error:
error = 'Missing value or section.'
error_message = ''.join([section_string, ' = ', str(error)])
error_message_list.append(error_message)
if error:
error_messages = '; '.join(error_message_list)
print error_messages
raise Exception(error_messages)
if not res:
error = '{0} invalid\n'.format(os.path.split(a_config.filename)[1])
raise Exception(error)
def load_config():
retval = ConfigParser.RawConfigParser()
spec_file = pkg_resources.resource_filename(__name__,
"/config/configspec.cfg")
retval = ConfigObj(configspec=spec_file, encoding='UTF8')
# User's ANK retval
user_config_file = os.path.join(ank_user_dir, "autonetkit.cfg")
retval.merge(ConfigObj(user_config_file))
# ANK retval in current directory
retval.merge(ConfigObj("autonetkit.cfg"))
# ANK retval specified by environment variable
try:
ankcfg = os.environ['AUTONETKIT_CFG']
retval.merge(ConfigObj(ankcfg))
except KeyError:
pass
results = retval.validate(validator)
if results != True:
for (section_list, key, _) in flatten_errors(retval, results):
if key is not None:
print "Error loading configuration file:"
print 'Invalid key "%s" in section "%s"' % (
key, ', '.join(section_list))
raise AutoNetkitException
else:
# ignore missing sections - use defaults
#print 'The following section was missing:%s ' % ', '.join(section_list)
pass
return retval
def _validate(self):
# set all str values that are 'None' to None
def set_str_to_none(d):
for k, v in d.items():
if type(v) == Section:
set_str_to_none(v)
else:
if type(v) == str and v == 'None':
d[k] = None
set_str_to_none(self.data)
validator = Validator()
results = self.data.validate(validator, copy=True)
if results != True:
logger.error('Config file validation failed!')
for (section_list, key, _) in flatten_errors(self.data, results):
if key is not None:
logger.error('The "%s" key in the section "%s" failed '
'validation' % (key, ', '.join(section_list)))
else:
logger.error('The following section was missing:%s ' %
', '.join(section_list))
raise ValidationError
self.data.filename = self.filename
self._init_data_dir()
self._init_plugins_dir()
def init_mailer(configfile=None):
"""Feed the mailer with the configuration file"""
global conf, log
configspec = pkg_resources.resource_filename('spamalot',
'conf/configspec.ini')
logconfig = pkg_resources.resource_filename('spamalot',
'conf/logging.ini')
conf = ConfigObj(configfile, configspec=configspec)
validator = Validator()
results = conf.validate(validator)
if results != True:
for (section_list, key, _) in flatten_errors(conf, results):
if key is not None:
print 'The "%s" key in the section "%s" failed validation' %\
(key, ', '.join(section_list))
else:
print 'The following section was missing' % ', '.join(section_list)
exit(0)
if conf['logging']['outfile']:
defaults = {
'custom_outfile': conf['logging']['outfile'],
'custom_mode': conf['logging']['mode'],
'custom_level': conf['logging']['level']
}
logging.config.fileConfig(logconfig, defaults)
# else:
# log = Dummy()
log = logging.getLogger('spamalot')
log.info('spamalot initialised')
def __init__(self, filename):
"""Initializes a config file if does not exist. If exists, uses
it to validate the file, and setup default initial parameters"""
self.cfg_spec = ConfigObj(config_spec_text.splitlines(), list_values=False)
self.cfg_filename = filename
valid = Validator()
if not os.path.exists(self.cfg_filename):
#no configuration file found
logger.info("File %s not found, so creating one from you from defaults" % self.cfg_filename)
cfg = ConfigObj(configspec=self.cfg_spec, stringify=True, list_values=True)
cfg.filename = self.cfg_filename
test = cfg.validate(valid, copy=True)
cfg.write()
self.cfg = ConfigObj(self.cfg_filename, configspec=self.cfg_spec)
rtn = self.cfg.validate(valid, preserve_errors=True)
if type(rtn) == types.BooleanType and rtn:
logger.info("Config file validated")
self.tested = True
else:
self.tested = False
res = flatten_errors(self.cfg, rtn)
self.errortxt = ''
for row in res:
self.errortxt += 'In Section %s, key %s has error: %s' % (row[0], row[1], row[2])
logger.error(self.errortxt)
def load_config():
"""
Load and validate configuration.
:return: configuration
:rtype: configobj.ConfigObj
:raise IOError: if there is an issue reading the configuration file
"""
# todo arbitrary config files
# http://www.voidspace.org.uk/python/configobj.html
# http://www.voidspace.org.uk/python/validate.html
if os.path.exists('bot.ini'):
config = configobj.ConfigObj('bot.ini', configspec='confspec.ini')
log.info("Using config file bot.ini")
elif os.path.exists('default.ini'):
config = configobj.ConfigObj('default.ini', configspec='confspec.ini')
log.info("Using config file default.ini")
else:
raise IOError("Could not find config file for bot")
# validate config now
val = validate.Validator()
results = config.validate(val, preserve_errors=True)
if not results:
for (section_list, key, reason) in configobj.flatten_errors(config, results):
if key:
msg = "CONFIG ERROR: key '%s' in section '%s' failed validation" % (key, ', '.join(section_list))
if reason:
msg += " - %s" % (reason)
log.error(msg)
else:
log.error("CONFIG ERROR: missing section '%s'" % ', '.join(section_list))
raise IOError("Errors in bot config file")
return config
def get_configuration_from_file(cls, configuration_file):
"""
Return a ConfigObj instance if configuration_file is validate
:param configuration_file: filename of configuration file
:return: ConfigObj instance
:rtype: ConfigObj
"""
configspec_file = os.path.join(
os.path.realpath(os.path.dirname(__file__)), 'spec',
'configuration.configspec')
configuration = ConfigObj(configuration_file,
configspec=configspec_file)
validator = Validator()
results = configuration.validate(validator, preserve_errors=True)
if results != True:
for (section_list, key,
_) in flatten_errors(configuration, results):
if key is not None:
print 'The "%s" key in the section "%s" failed validation' % (
key, ', '.join(section_list))
else:
print 'The following section was missing:%s ' % ', '.join(
section_list)
sys.exit(2)
return configuration
def __createSysCfg(self):
"""
Set up the system config section (sysCfg)
"""
sysCfgName = platform.node() + ".cfg"
if not (os.path.isfile(sysCfgName)):
sysCfgName = "defaultSys.cfg"
print "Loading system config file: " + sysCfgName
sysCfg = ConfigObj(sysCfgName,
configspec='sysCfgSpec.ini',
raise_errors=True)
validator = Validator()
sysCfgOK = sysCfg.validate(validator)
if sysCfgOK == True:
print "System config file parsed correctly"
else:
print 'System config file validation failed!'
res = sysCfg.validate(validator, preserve_errors=True)
for entry in flatten_errors(sysCfg, res):
# each entry is a tuple
section_list, key, error = entry
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ', '.join(section_list)
if error == False:
error = 'Missing value or section.'
print section_string, ' = ', error
sys.exit(1)
self.sysCfg = sysCfg
def get_config(config_file):
"""
Parse the config file, validate it and convert types. Return a dictionary
with all settings.
"""
specs = ConfigObj(StringIO(CONFIG_SPEC), list_values=False)
config = ConfigObj(config_file, configspec=specs)
# Create validator
validator = Validator({'lowercase_string_list': lowercase_string_list})
# Convert types and validate file
result = config.validate(validator, preserve_errors=True, copy=True)
logger.debug("Config file version %d", config["version"])
# Raise exceptions for errors
for section_list, key, message in flatten_errors(config, result):
if key is not None:
raise ValueError(
"The '%s' key in the section '%s' failed validation: %s" %
(key, ", ".join(section_list), message))
else:
raise ValueError("The following section was missing: %s." %
(", ".join(section_list)))
# For now, no automatic update support.
if config["version"] != CONFIG_VERSION:
logger.warning(
"Config file version is %d, while expected version is %d. Please "
"check for inconsistencies and update manually.",
config["version"], CONFIG_VERSION)
return config
def cobj_check(settings, exception=None, copy=False):
"""Check for errors in config file"""
if not exception:
exception = Exception
validator = validate.Validator()
def numpy_array(val):
"""Define float list"""
float_list = validator.functions["float_list"](val)
return numpy.array(float_list)
validator.functions["numpy_array"] = numpy_array
results = settings.validate(validator, copy=copy, preserve_errors=True)
if results is not True:
output = "{0}: \n".format(settings.filename if settings.
filename is not None else "configobj")
for (section_list, key,
error) in configobj.flatten_errors(settings, results):
if key is not None:
val = settings
for section in section_list:
val = val[section]
val = val[key] if key in val else "<EMPTY>"
output += " [{sections}], {key}='{val}' ({error})\n".format(
sections=', '.join(section_list),
key=key,
val=val,
error=error)
else:
output += "Missing section: {0}\n".format(
", ".join(section_list))
raise exception(output)
def __init__(self, sources, config, result):
"""
Args:
sources (:obj:`list` of :obj:`str`): list of sources of configuration values
config (:obj:`configobj.ConfigObj`): configuration
result (:obj:`dict`): dictionary of configuration errors
"""
self.sources = sources
self.config = config
self.result = result
errors = flatten_errors(config, result)
# create readable error message
messages = []
for error in errors:
section_list, key, exception = error
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
if exception == False:
message = ('.'.join(section_list)
) + ' :: ' + 'Missing value or section'
else:
message = ('.'.join(section_list)) + ' :: ' + str(exception)
messages.append(message)
self.msg = ('The following configuration sources\n {}\n\n'
'contain the following configuration errors\n {}').format(
'\n '.join(sources), '\n '.join(messages))
def test_flatten_errors(val, cfg_contents):
config = cfg_contents("""
test1=40
test2=hello
test3=3
test4=5.0
[section]
test1=40
test2=hello
test3=3
test4=5.0
[[sub section]]
test1=40
test2=hello
test3=3
test4=5.0
""")
configspec = cfg_contents("""
test1= integer(30,50)
test2= string
test3=integer
test4=float(6.0)
[section]
test1=integer(30,50)
test2=string
test3=integer
test4=float(6.0)
[[sub section]]
test1=integer(30,50)
test2=string
test3=integer
test4=float(6.0)
""")
c1 = ConfigObj(config, configspec=configspec)
res = c1.validate(val)
assert flatten_errors(c1, res) == [([], 'test4', False),
(['section'], 'test4', False),
(['section',
'sub section'], 'test4', False)]
res = c1.validate(val, preserve_errors=True)
check = flatten_errors(c1, res)
assert check[0][:2] == ([], 'test4')
assert check[1][:2] == (['section'], 'test4')
assert check[2][:2] == (['section', 'sub section'], 'test4')
for entry in check:
assert isinstance(entry[2], VdtValueTooSmallError)
assert str(entry[2]) == 'the value "5.0" is too small.'
def validate_config_file(
config: configobj.ConfigObj, logger: logging.Logger
) -> configobj.ConfigObj:
"""Validate config file, i.e. check that everything is set properly.
This also sets all default values."""
logger.debug("Validating config file.")
# 'copy' parameter of config.validate: Also copy all comments and default
# values from the configspecs to the config file (to be written out later)?
# Note: The config object might be completely empty, because only the first
# Run of this methods sets them from the defaults if the user did not
# specify a config file.
# However, this method gets called 2 times (one time before and one time
# after the handling of the --parameter options),
# Unfortunately, if you do copy one time, an additional copy=False won't
# bring it back. So we start with copy=False and if r.config.copy_all
# is set to true, copy it the second time this function is called.
try:
copy_all = config["r"]["config"]["copy_all"]
except KeyError:
copy_all = False
valid = config.validate(
validate.Validator(), preserve_errors=True, copy=copy_all
)
# adapted from https://stackoverflow.com/questions/14345879/
# answer from user sgt_pats 2017
# todo: This might need some better handling.
for entry in configobj.flatten_errors(config, valid):
[path, key, error] = entry
if not error:
msg = "The parameter {} was not in the config file\n".format(key)
msg += (
"Please check to make sure this parameter is present and "
"there are no mis-spellings."
)
logger.critical(msg)
raise ValueError(msg)
if key is not None:
if isinstance(error, validate.VdtValueError):
optionString = config.configspec[key]
msg = (
"The parameter {} was set to {} which is not one of "
"the allowed values\n".format(key, config[key])
)
msg += "Please set the value to be in {}".format(optionString)
logger.critical(msg)
raise ValueError(msg)
elif error:
msg = "Validation error (section='{}', key={}): {}".format(
"/".join(path), key, error
)
logger.error(msg)
raise ValueError(msg)
return config
def parse_config(requesting_file_path, plugin=True):
"""
parse the configuration files for a given sideboard module (or the sideboard server itself). It's
expected that this function is called from one of the files in the top-level of your module
(typically the __init__.py file)
:param requesting_file_path: the path of the file requesting a parsed config file. An example
value is:
~/sideboard/plugins/plugin_nickname/plugin_module_name/__init__.py
the containing directory (here, 'plugin_module_name') is assumed to be the module name of
the plugin that is requesting a parsed config.
:type requesting_file_path: basestring
:param plugin: if True (default) add plugin-relevant information to the returning config. Also,
treat it as if it's a plugin
:type plugin: bool
:return: the resulting configuration object
:rtype: ConfigObj
"""
module_dir, root_dir = get_dirnames(requesting_file_path)
specfile = os.path.join(module_dir, 'configspec.ini')
spec = configobj.ConfigObj(specfile, interpolation=False, list_values=False, encoding='utf-8', _inspec=True)
# to allow more/better interpolations
root_conf = [b'root = "{}"\n'.format(root_dir), b'module_root = "{}"\n'.format(module_dir)]
temp_config = configobj.ConfigObj(root_conf, interpolation=False, encoding='utf-8')
for config_path in get_config_files(requesting_file_path, plugin):
# this gracefully handles nonexistent files
temp_config.merge(configobj.ConfigObj(config_path, encoding='utf-8', interpolation=False))
# combining the merge files to one file helps configspecs with interpolation
with NamedTemporaryFile(delete=False) as config_outfile:
temp_config.write(config_outfile)
temp_name = config_outfile.name
config = configobj.ConfigObj(temp_name, encoding='utf-8', configspec=spec)
validation = config.validate(Validator(), preserve_errors=True)
unlink(temp_name)
if validation is not True:
raise ConfigurationError('configuration validation error(s) (): {!r}'.format(
configobj.flatten_errors(config, validation))
)
if plugin:
sideboard_config = globals()['config']
config['plugins'] = deepcopy(sideboard_config['plugins'])
if 'rpc_services' in config:
from sideboard.lib import register_rpc_services
register_rpc_services(config['rpc_services'])
if 'default_url' in config:
priority = config.get('default_url_priority', 0)
if priority >= sideboard_config['default_url_priority']:
sideboard_config['default_url'] = config['default_url']
return config
def __init__(self, logger, conf_file=None):
assert PanoptesValidators.valid_readable_file(
conf_file), 'conf_file must be a readable file'
self._logger = logger
logger.info('Using configuration file: ' + conf_file)
try:
config = ConfigObj(conf_file,
configspec=_CONFIG_SPEC_FILE,
interpolation='template',
file_error=True)
except (ConfigObjError, IOError):
raise
validator = Validator()
result = config.validate(validator, preserve_errors=True)
if result is not True:
errors = ''
for (section_list, key, error) in flatten_errors(config, result):
errors += 'The "' + key + '" key in section "' + ','.join(
section_list) + '" failed validation\n'
raise SyntaxError('Error parsing the configuration file: %s' %
errors)
self._setup_logging(config)
self._get_sites(config)
logger.info('Got list of sites: %s' % self._sites)
self._get_redis_urls(config)
logger.info('Got Redis URLs "%s"' % self.redis_urls)
logger.info('Got Redis URLs by group "%s"' % self.redis_urls_by_group)
logger.info('Got Redis URLs by namespace "%s"' %
self.redis_urls_by_namespace)
self._get_zookeeper_servers(config)
logger.info('Got list of ZooKeeper servers: %s' %
self._zookeeper_servers)
self._get_kafka_brokers(config)
logger.info('Got list of Kafka brokers: %s' % self._kafka_brokers)
self._get_snmp_defaults(config)
logger.info('Got SNMP defaults: %s' % self._snmp_defaults)
for plugin_type in const.PLUGIN_TYPES:
plugins_paths = config[plugin_type]['plugins_paths']
for plugins_path in plugins_paths:
if not PanoptesValidators.valid_readable_path(plugins_path):
raise PanoptesConfigurationError(
"Specified plugin path is not readable: %s" %
plugins_path)
logger.info(plugin_type + ' plugins path: ' + plugins_path)
self._config = config
def __init__(self, validator_output, config):
error_message = ''
for entry in flatten_errors(config, validator_output):
section_list, key, value = entry
error_message += f'Error in {":".join(section_list)}:{key}. This is very likely a key being ' \
f'specified with no value. Either set a value or remove the key.'
self.message = error_message
def from_file(name="config", application_name=None, path_template=None):
if not path_template:
system_config_path = "/etc/"
if platform.system() == "Windows":
system_config_path = "C:/python_app_settings/"
path_template = system_config_path + "<app_name>"
script_folder, app_name = code.get_app_name()
if not application_name:
application_name = app_name
path = path_template.replace('<app_name>',
application_name).replace('<name>', name)
config_path = os.path.join(script_folder, name + '.ini')
if not os.path.isfile(config_path):
config_path = os.path.join(path, name + '.ini')
if not os.path.isfile(config_path):
print("Settings file %s could not be loaded. Exiting." % config_path)
exit(1)
spec_path = os.path.join(script_folder, name + '.spec')
if not os.path.isfile(spec_path):
spec_path = os.path.join(path, name + '.spec')
if not os.path.isfile(spec_path):
spec_path = None
conf = {}
try:
conf = ConfigObj(config_path,
file_error=True,
unrepr=True,
raise_errors=False,
list_values=True,
configspec=spec_path)
validator = Validator()
results = conf.validate(validator, preserve_errors=True)
if not results:
#weird return value : True if validated, a dict of errors if not. So can't do a simple "if results:".
for (section_list, key, _) in flatten_errors(conf, results):
if key is not None:
print(
'The "%s" key in the section "%s" failed validation' %
(key, ', '.join(section_list)))
else:
print('The following section was missing:%s ' %
', '.join(section_list))
exit(1)
except (ConfigObjError, IOError) as e:
print('Could not read config from "%s": %s, %s' %
(config_path, e, e.__dict__))
exit(1)
config = code.AttributeDict(conf)
return config
def import_cfg(cfgfile):
"""Import settings from specified file to a dictionary.
TODO: finish this properly"""
import io
cfgspec_stringio = io.StringIO(dedent("""
[geometry]
tilt_axis = float(min=-180, max=180, default=0)
tilt_axis_shift_x = integer(default=0)
tilt_axis_shift_y = integer(default=0)
[electronbeam]
acc_voltage = float(min=0, max=1000)
energy_spread = float(min=0, max=10, default=1)
[optics]
magnification = float(min=1, max=1000000)
cs = float(min=0, max=10, default=2)
cc = float(min=0, max=10, default=2)
aperture = float(min=0, max=1000, default=50)
focal_length = float(min=0, max=100, default=2)
cond_ap_angle = float(min=0, max=10, default=0.1)
defocus_nominal = float(min=0, max=100)
"""))
cfgspec = ConfigObj(cfgspec_stringio, list_values=False, _inspec=True)
try:
cfg = ConfigObj(infile=cfgfile, list_values=False, file_error=True,
configspec=cfgspec)
except IOError:
print("Unable to read-only open file '" + cfgfile + "'.")
return None
# TODO: check error reporting
val = Validator()
validated = cfg.validate(val, preserve_errors=True)
if validated is not True:
for (section_list, key, error) in flatten_errors(cfg, validated):
if key is not None:
if error is False:
emsg = "Key '{}' in section '{}' not found.".format(
key, ': '.join(section_list))
else:
emsg = "Key '{}' in section '{}': {}.".format(
key, ': '.join(section_list), error.message)
else:
emsg = "Section '{}' missing.".format(', '.join(section_list))
raise ConfigObjError(emsg)
return cfg.dict()
def restoreBadPrefs(self, cfg, resultOfValidate):
if resultOfValidate == True:
return
vtor = validate.Validator()
for (section_list, key, _junk) in configobj.flatten_errors(cfg, resultOfValidate):
if key is not None:
cfg[', '.join(section_list)][key] = vtor.get_default_value(cfg.configspec[', '.join(section_list)][key])
else:
print("Section [%s] was missing in file '%s'" % (', '.join(section_list), cfg.filename))
def validate(self):
#def walk_nested_dict(d):
#for key1, value1 in d.items():
#if isinstance(value1, dict):
#for key2, value2 in walk_nested_dict(value1):
#yield [key1, key2], value2
#else:
#yield [key1,], value1
for key1, value1 in self.entrydict.items():
if not isinstance(value1, dict): # shouldn't happen
if key1.find('Password') == -1:
self.settings[key1] = value1.getvalue()
else:
self.settings[key1] = myutils.password_obfuscate(value1.getvalue())
else:
for key2, value2 in value1.items():
if not isinstance(value2, dict):
if key2.find('Password') == -1:
self.settings[key1][key2] = value2.getvalue()
else:
self.settings[key1][key2] = myutils.password_obfuscate(value2.getvalue())
else:
for key3, value3 in value2.items():
if not isinstance(value3, dict):
if key3.find('Password') == -1:
self.settings[key1][key2][key3] = value3.getvalue()
else:
self.settings[key1][key2][key3] = myutils.password_obfuscate(value3.getvalue())
else:
pass # shouldn't happen
errortext=["Some of your input contains errors. "
"Detailed error output below.",]
val = Validator()
val.functions['log_filename_check'] = myutils.validate_log_filename
val.functions['image_filename_check'] = myutils.validate_image_filename
valresult=self.settings.validate(val, preserve_errors=True)
if valresult != True:
for section_list, key, error in flatten_errors(self.settings,
valresult):
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ','.join(section_list)
if error == False:
error = 'Missing value or section.'
errortext.append('%s: %s' % (section_string, error))
tkMessageBox.showerror("Erroneous input. Please try again.",
'\n\n'.join(errortext), parent=self.dialog.interior())
self.settings = self.read_settings()
return False
else:
return True
def __init__(self, cfg):
"""Constructor reading and validating config against given config spec
@param cfg: optional parameter containing config file
@type cfg: string"""
self.config = configobj.ConfigObj(cfg, file_error=True, configspec=self.__CONFIG_SPEC)
validator = validate.Validator()
res = self.config.validate(validator, preserve_errors=True)
for section_list, key, error in configobj.flatten_errors(self.config, res): # pylint: disable=W0612
raise PynatconndConfigException("Failed to validate section %s key %s in config file %s" % (", ".join(section_list), key, cfg))
def niceErr(confdict,resultsdict):
for (section_list, key, _) in flatten_errors(confdict, resultsdict):
print (section_list,key)
if key is not None:
print 'The "%s" key in the section "%s" failed validation' % (key, ', '.join(section_list))
elif isinstance(confdict[key],dict):
niceErr(confdict[key],resultsdict[key])
else:
print 'The following section was missing:%s ' % key
def ConfValidation(confs, configSpec) :
vdtor = Validator()
confs.configspec = ConfigObj(configSpec, list_values=False, _inspec=True)
res = confs.validate(vdtor)
flatErrs = flatten_errors(confs, res)
_ShowErrors(flatErrs, skipMissing=True)
return confs
def print_validation_errors(config, results):
click.secho('Configuration validation failed:', fg='red', bold=True)
for (section_list, key, _) in configobj.flatten_errors(config, results):
if key is not None:
click.secho('- the "%s" key in the section "%s" failed validation'
% (key, ', '.join(section_list)), fg='red')
else:
click.secho('- the following section was missing:%s ' %
', '.join(section_list), fg='red')
def validate(self):
#def walk_nested_dict(d):
#for key1, value1 in d.items():
#if isinstance(value1, dict):
#for key2, value2 in walk_nested_dict(value1):
#yield [key1, key2], value2
#else:
#yield [key1,], value1
for key1, value1 in self.entrydict.items():
if not isinstance(value1, dict): # shouldn't happen
if key1.find('Password') == -1:
self.settings[key1] = value1.getvalue()
else:
self.settings[key1] = myutils.password_obfuscate(value1.getvalue())
else:
for key2, value2 in value1.items():
if not isinstance(value2, dict):
if key2.find('Password') == -1:
self.settings[key1][key2] = value2.getvalue()
else:
self.settings[key1][key2] = myutils.password_obfuscate(value2.getvalue())
else:
for key3, value3 in value2.items():
if not isinstance(value3, dict):
if key3.find('Password') == -1:
self.settings[key1][key2][key3] = value3.getvalue()
else:
self.settings[key1][key2][key3] = myutils.password_obfuscate(value3.getvalue())
else:
pass # shouldn't happen
errortext=["Some of your input contains errors. "
"Detailed error output below.",]
val = Validator()
val.functions['log_filename_check'] = myutils.validate_log_filename
val.functions['image_filename_check'] = myutils.validate_image_filename
valresult=self.settings.validate(val, preserve_errors=True)
if valresult != True:
for section_list, key, error in flatten_errors(self.settings,
valresult):
if key is not None:
section_list.append(key)
else:
section_list.append('[missing section]')
section_string = ','.join(section_list)
if error == False:
error = 'Missing value or section.'
errortext.append('%s: %s' % (section_string, error))
tkMessageBox.showerror("Erroneous input. Please try again.",
'\n\n'.join(errortext), parent=self.dialog.interior())
self.settings = self.read_settings()
return False
else:
return True
def restoreBadPrefs(self, cfg, resultOfValidate):
if resultOfValidate == True:
return
vtor = validate.Validator()
for (section_list, key, _) in configobj.flatten_errors(cfg, resultOfValidate):
if key is not None:
cfg[', '.join(section_list)][key] = vtor.get_default_value(cfg.configspec[', '.join(section_list)][key])
else:
print "Section [%s] was missing in file '%s'" % (', '.join(section_list), cfg.filename)
def handleConfigError(self, results):
msg = "CONFIGURATION ERROR\n"
for (section_list, key, _) in flatten_errors(self.pcssConfig, results):
if key is not None:
msg += 'The "%s" key in the section "%s" failed validation\n' % (key, ', '.join(section_list))
else:
msg += 'The following section was missing:%s ' % ', '.join(section_list)
print msg
raise pcssErrors.PcssGlobalException(msg)
def test_flatten_errors(val):
config = '''
test1=40
test2=hello
test3=3
test4=5.0
[section]
test1=40
test2=hello
test3=3
test4=5.0
[[sub section]]
test1=40
test2=hello
test3=3
test4=5.0
'''.split('\n')
configspec = '''
test1= integer(30,50)
test2= string
test3=integer
test4=float(6.0)
[section]
test1=integer(30,50)
test2=string
test3=integer
test4=float(6.0)
[[sub section]]
test1=integer(30,50)
test2=string
test3=integer
test4=float(6.0)
'''.split('\n')
c1 = ConfigObj(config, configspec=configspec)
res = c1.validate(val)
assert flatten_errors(c1, res) == [([], 'test4', False), (['section'], 'test4', False), (['section', 'sub section'], 'test4', False)]
res = c1.validate(val, preserve_errors=True)
check = flatten_errors(c1, res)
assert check[0][:2] == ([], 'test4')
assert check[1][:2] == (['section'], 'test4')
assert check[2][:2] == (['section', 'sub section'], 'test4')
for entry in check:
assert isinstance(entry[2], VdtValueTooSmallError)
assert str(entry[2]) == 'the value "5.0" is too small.'
def iter_error_strings(self, errors):
for section, key, error in flatten_errors(self, errors):
if key is None:
section, key = section[:-1], section[-1]
error = 'missing required subsection'
elif not error:
error = 'missing required value'
else:
error = str(error)
yield ('::'.join(section), key, error)
def from_file(name="config", application_name=None, path_template=None):
if not path_template:
system_config_path = "/etc/"
if platform.system() == "Windows":
system_config_path = "C:/python_app_settings/"
path_template = system_config_path + "<app_name>"
script_folder, app_name = code.get_app_name()
if not application_name:
application_name = app_name
path = path_template.replace('<app_name>', application_name).replace('<name>', name)
config_path = os.path.join(script_folder, name + '.ini')
if not os.path.isfile(config_path):
config_path = os.path.join(path, name + '.ini')
if not os.path.isfile(config_path):
print("Settings file %s could not be loaded. Exiting." % config_path)
exit(1)
spec_path = os.path.join(script_folder, name + '.spec')
if not os.path.isfile(spec_path):
spec_path = os.path.join(path, name + '.spec')
if not os.path.isfile(spec_path):
spec_path = None
conf = {}
try:
conf = ConfigObj(
config_path,
file_error=True,
unrepr=True,
raise_errors=False,
list_values=True,
configspec=spec_path
)
validator = Validator()
results = conf.validate(validator, preserve_errors=True)
if not results:
#weird return value : True if validated, a dict of errors if not. So can't do a simple "if results:".
for (section_list, key, _) in flatten_errors(conf, results):
if key is not None:
print('The "%s" key in the section "%s" failed validation' % (key, ', '.join(section_list)))
else:
print('The following section was missing:%s ' % ', '.join(section_list))
exit(1)
except (ConfigObjError, IOError) as e:
print('Could not read config from "%s": %s, %s' % (config_path, e, e.__dict__))
exit(1)
config = code.AttributeDict(conf)
return config