def get(self, fullname, default=None, raw=False, _state=[]):
if not raw and fullname in self._cache:
return self._cache[fullname]
section, name = self._normalize_name(fullname)
if section in self._cli and name in self._cli[section]:
value = self._cli[section][name]
elif section in self._env and name in self._env[section]:
value = self._env[section][name]
elif section in self._normal and name in self._normal[section]:
value = self._normal[section][name]
elif section in self._defaults and name in self._defaults[section]:
value = self._defaults[section][name]
else:
value = default
if not isinstance(value, str):
return value
if raw:
return value
tmpl = string.Template(value)
value = tmpl.safe_substitute(
ConfigurationWrapper(self, _state + [name]))
if self.schema:
param_schema = self.schema.get_parameter(name, section=section)
if param_schema:
type_validator = TypeValidatorRegistry.get_validator(
param_schema.type)
type_validation_result = type_validator.validate(value, **param_schema.type_args)
if not isinstance(type_validation_result, InvalidValueError):
value = type_validation_result
self._cache[fullname] = value
return value
def validate(self, fullname):
if not self.schema:
return None
section, name = self._normalize_name(fullname)
value = self.get(fullname, raw=True)
tmpl = string.Template(value)
value = tmpl.safe_substitute(
ConfigurationWrapper(self, [name]))
param_schema = self.schema.get_parameter(name, section=section)
if not param_schema:
return None
type_validator = TypeValidatorRegistry.get_validator(
param_schema.type)
type_validation_result = type_validator.validate(value, **param_schema.type_args)
if not isinstance(type_validation_result, InvalidValueError):
return None
return type_validation_result
def setUp(self):
super(TypeValidatorTestHelper, self).setUp()
self.validator = TypeValidatorRegistry.get_validator(self.type_name)
def generate_project_schema(project):
logger.info('Processing project %s' % project)
project_path = os.path.join(os.path.dirname(__file__), project)
files = glob.glob(os.path.join(project_path, '*.yml'))
if files == []:
logger.info("Found no YAML files in project %s. Skipping it" % project)
return
x = index(files, lambda f: f.endswith('.conf.yml'))
if x != -1:
database_file = files[x]
del files[x]
else:
database_file = os.path.join(project_path, project + '.conf.yml')
schema_records = []
if os.path.exists(database_file):
logger.debug("Processing database file %s" % database_file)
with open(database_file) as f:
schema_records.extend(yaml.load(f.read()))
schema_versions = []
for version_file in files:
logger.debug("Processing version file %s" % version_file)
with open(version_file) as f:
schema_versions.append(yaml.load(f.read()))
schema_versions = sorted(schema_versions,
key=lambda s: Version(s['version']))
parameters = OrderedDict()
for schema in schema_versions:
added = []
seen = set()
logger.debug('Processing schema version %s' % schema['version'])
for param in schema['parameters']:
# TODO(mkulkin): reduce the level of nesting
prev_param = parameters.get(param['name'], None)
if not prev_param:
logger.debug('Parameter %s does not exist yet,'
' adding it as new'
% param['name'])
added.append(param)
else:
seen.add(param['name'])
if param['type'] != prev_param['type']:
validator = TypeRegistry.get_validator(prev_param['type'])
if param['type'] == validator.base_type:
param['type'] = prev_param['type']
if param.get('default', None) is not None:
type_args = param.get('type_args', {})
value = validator.validate(param['default'], **type_args)
if not isinstance(value, Issue):
param['default'] = value
else:
logger.error("In project '%s' version %s"
" default value for parameter"
" '%s' is not valid value of"
" type %s: %s"
% (project, schema['version'],
param['name'], param['type'],
repr(param['default'])))
else:
logger.debug('Parameter %s type has'
' changed from %s to %s' %
(param['name'], prev_param['type'],
param['type']))
param['comment'] = 'Type has changed'
added.append(param)
continue
if param.get('default', None) != \
prev_param.get('default', None):
logger.debug('Parameter %s default value'
' has changed from %s to %s' %
(param['name'], prev_param['default'],
param['default']))
param['comment'] = 'Default value has changed'
added.append(param)
continue
if param.get('help', None) != prev_param.get('help', None):
param['comment'] = 'Help string has changed'
added.append(param)
removed = [name for name in parameters.keys() if name not in seen]
if len(removed) > 0:
logger.debug('Following parameters from previous'
' schema version are not present in'
' current version, marking as removed: %s'
% ','.join(removed))
# Decide either to use full schema update or incremental
changes_count = sum(map(len, [added, removed]))
logger.debug('Found %d change(s) from previous version schema'
% changes_count)
if changes_count > int(len(parameters) * DIFF_THRESHOLD):
logger.debug('Using full schema update')
new_parameters = parameters.copy()
for param in added:
new_parameters[param['name']] = param
for name in removed:
del new_parameters[name]
new_schema_record = dict(version=schema['version'],
added=new_parameters.values(),
removed=[],
checkpoint=True)
else:
logger.debug('Using incremental schema update')
new_schema_record = dict(version=schema['version'],
added=added, removed=removed)
# Place schema record either replacing existing one or appending as new
old_schema_record_idx = index(schema_records, lambda r:
str(r['version']) ==
str(new_schema_record['version']))
if old_schema_record_idx != -1:
old_schema_record = schema_records[old_schema_record_idx]
# Collect information from existing records
old_schema_parameters = {}
for param in old_schema_record.get('added', []):
old_schema_parameters[param['name']] = param
for param in added:
old_param = old_schema_parameters.get(param['name'], None)
if not old_param:
param.setdefault('comment', 'New param')
continue
extra_data = [(k, v) for k, v in old_param.items()
if k not in ['name', 'type', 'default', 'help']]
param.update(extra_data)
validator = TypeRegistry.get_validator(old_param['type'])
if param['type'] not in [old_param['type'],
validator.base_type]:
param['comment'] = 'Type has changed'
# Type has changed, enforcing old type to prevent
# accidental data loss
param['type'] = old_param['type']
if 'default' in old_param:
param['default'] = old_param['default']
if param.get('default', None) is not None:
type_args = old_param.get('type_args', {})
value = validator.validate(old_param['default'], **type_args)
if not isinstance(value, Issue):
param['default'] = value
else:
logger.error("In project '%s' version %s default value"
" for parameter '%s' is not valid value"
" of type %s: %s" %
(project, schema['version'],
param['name'], param['type'],
repr(param['default'])))
if param.get('default', None) != old_param.get('default',
None):
param['comment'] = 'Default value has changed'
continue
logger.debug('Replacing schema record %s'
% repr(new_schema_record))
schema_records[old_schema_record_idx] = new_schema_record
else:
for param in added:
param.setdefault('comment', 'New param')
logger.debug('Appending schema record %s'
% repr(new_schema_record))
schema_records.append(new_schema_record)
# Update parameter info
for param in new_schema_record.get('added', []):
parameters[param['name']] = param
for name in new_schema_record.get('removed', []):
del parameters[name]
schema_records = sorted(schema_records,
key=lambda r: Version(r['version']))
with open(database_file, 'w') as f:
f.write(yaml_dump_schema_records(schema_records))
def _parse_config_file(self,
base_mark,
config_contents,
config=Configuration(),
schema=None,
issue_reporter=None):
if issue_reporter:
def report_issue(issue):
issue_reporter.report_issue(issue)
else:
def report_issue(issue):
pass
# Parse config file
config_parser = IniConfigParser()
parsed_config = config_parser.parse('', base_mark, config_contents)
for error in parsed_config.errors:
report_issue(error)
# Validate config parameters and store them
section_name_text_f = lambda s: s.name.text
sections_by_name = groupby(sorted(parsed_config.sections,
key=section_name_text_f),
key=section_name_text_f)
for section_name, sections in sections_by_name:
sections = list(sections)
if len(sections) > 1:
report_issue(
Issue(Issue.INFO, 'Section "%s" appears multiple times' %
section_name))
seen_parameters = set()
for section in sections:
unknown_section = False
if schema:
unknown_section = not schema.has_section(section.name.text)
if unknown_section:
report_issue(
MarkedIssue(Issue.WARNING,
'Unknown section "%s"' % (section_name),
section.start_mark))
continue
for parameter in section.parameters:
parameter_schema = None
if schema:
parameter_schema = schema.get_parameter(
name=parameter.name.text,
section=section.name.text)
if not (parameter_schema or unknown_section):
report_issue(
MarkedIssue(
Issue.WARNING,
'Unknown parameter: section "%s" name "%s"'
% (section_name, parameter.name.text),
parameter.start_mark))
if parameter.name.text in seen_parameters:
report_issue(
MarkedIssue(
Issue.WARNING,
'Parameter "%s" in section "%s" redeclared' %
(parameter.name.text, section_name),
parameter.start_mark))
else:
seen_parameters.add(parameter.name.text)
parameter_fullname = parameter.name.text
if section_name != 'DEFAULT':
parameter_fullname = section_name + \
'.' + parameter_fullname
if parameter_schema:
type_validator = TypeValidatorRegistry.get_validator(
parameter_schema.type)
type_validation_result = type_validator.validate(
parameter.value.text)
if isinstance(type_validation_result, Issue):
type_validation_result.mark = parameter\
.value.start_mark.merge(
type_validation_result.mark)
type_validation_result.message = \
'Property "%s" in section "%s": %s' % (
parameter.name.text, section_name,
type_validation_result.message)
report_issue(type_validation_result)
config.set(parameter_fullname,
parameter.value.text)
else:
value = type_validation_result
config.set(parameter_fullname, value)
# if value == parameter_schema.default:
# report_issue(MarkedIssue(Issue.INFO,
# 'Explicit value equals default: section "%s"
# parameter "%s"' % (section_name,
# parameter.name.text), parameter.start_mark))
if parameter_schema.deprecation_message:
report_issue(
MarkedIssue(
Issue.WARNING,
'Deprecated parameter: section "%s" name '
'"%s". %s' %
(section_name, parameter.name.text,
parameter_schema.deprecation_message),
parameter.start_mark))
else:
config.set(parameter_fullname, parameter.value.text)
return config
def _parse_config_file(
self,
base_mark,
config_contents,
config=Configuration(),
schema=None,
issue_reporter=None):
if issue_reporter:
def report_issue(issue):
issue_reporter.report_issue(issue)
else:
def report_issue(issue):
pass
# Parse config file
config_parser = IniConfigParser()
parsed_config = config_parser.parse('', base_mark, config_contents)
for error in parsed_config.errors:
report_issue(error)
# Validate config parameters and store them
section_name_text_f = lambda s: s.name.text
sections_by_name = groupby(
sorted(
parsed_config.sections,
key=section_name_text_f),
key=section_name_text_f)
for section_name, sections in sections_by_name:
sections = list(sections)
if len(sections) > 1:
report_issue(
Issue(
Issue.INFO,
'Section "%s" appears multiple times' %
section_name))
seen_parameters = set()
for section in sections:
unknown_section = False
if schema:
unknown_section = not schema.has_section(section.name.text)
if unknown_section:
report_issue(
MarkedIssue(Issue.WARNING, 'Unknown section "%s"' %
(section_name), section.start_mark))
continue
for parameter in section.parameters:
parameter_schema = None
if schema:
parameter_schema = schema.get_parameter(
name=parameter.name.text,
section=section.name.text)
if not (parameter_schema or unknown_section):
report_issue(
MarkedIssue(
Issue.WARNING,
'Unknown parameter: section "%s" name "%s"'
% (section_name, parameter.name.text),
parameter.start_mark))
if parameter.name.text in seen_parameters:
report_issue(
MarkedIssue(
Issue.WARNING,
'Parameter "%s" in section "%s" redeclared' %
(parameter.name.text, section_name),
parameter.start_mark))
else:
seen_parameters.add(parameter.name.text)
parameter_fullname = parameter.name.text
if section_name != 'DEFAULT':
parameter_fullname = section_name + \
'.' + parameter_fullname
if parameter_schema:
type_validator = TypeValidatorRegistry.get_validator(
parameter_schema.type)
type_validation_result = type_validator.validate(
parameter.value.text)
if isinstance(type_validation_result, Issue):
type_validation_result.mark = parameter\
.value.start_mark.merge(
type_validation_result.mark)
type_validation_result.message = \
'Property "%s" in section "%s": %s' % (
parameter.name.text, section_name,
type_validation_result.message)
report_issue(type_validation_result)
config.set(
parameter_fullname,
parameter.value.text)
else:
value = type_validation_result
config.set(parameter_fullname, value)
# if value == parameter_schema.default:
# report_issue(MarkedIssue(Issue.INFO,
# 'Explicit value equals default: section "%s"
# parameter "%s"' % (section_name,
# parameter.name.text), parameter.start_mark))
if parameter_schema.deprecation_message:
report_issue(
MarkedIssue(
Issue.WARNING,
'Deprecated parameter: section "%s" name '
'"%s". %s' %
(section_name, parameter.name.text,
parameter_schema.deprecation_message),
parameter.start_mark))
else:
config.set(parameter_fullname, parameter.value.text)
return config
def setUp(self):
super(TypeValidatorTestHelper, self).setUp()
self.validator = TypeValidatorRegistry.get_validator(self.type_name)