Example #1
0
    def get(self, fullname, default=None, raw=False, _state=[]):
        if not raw and fullname in self._cache:
            return self._cache[fullname]

        section, name = self._normalize_name(fullname)

        if section in self._cli and name in self._cli[section]:
            value = self._cli[section][name]
        elif section in self._env and name in self._env[section]:
            value = self._env[section][name]
        elif section in self._normal and name in self._normal[section]:
            value = self._normal[section][name]
        elif section in self._defaults and name in self._defaults[section]:
            value = self._defaults[section][name]
        else:
            value = default

        if not isinstance(value, str):
            return value

        if raw:
            return value

        tmpl = string.Template(value)
        value = tmpl.safe_substitute(
            ConfigurationWrapper(self, _state + [name]))

        if self.schema:
            param_schema = self.schema.get_parameter(name, section=section)

            if param_schema:
                type_validator = TypeValidatorRegistry.get_validator(
                    param_schema.type)
                type_validation_result = type_validator.validate(value, **param_schema.type_args)
                if not isinstance(type_validation_result, InvalidValueError):
                    value = type_validation_result

        self._cache[fullname] = value

        return value
Example #2
0
    def validate(self, fullname):
        if not self.schema:
            return None

        section, name = self._normalize_name(fullname)

        value = self.get(fullname, raw=True)

        tmpl = string.Template(value)
        value = tmpl.safe_substitute(
            ConfigurationWrapper(self, [name]))

        param_schema = self.schema.get_parameter(name, section=section)

        if not param_schema:
            return None

        type_validator = TypeValidatorRegistry.get_validator(
            param_schema.type)
        type_validation_result = type_validator.validate(value, **param_schema.type_args)
        if not isinstance(type_validation_result, InvalidValueError):
            return None

        return type_validation_result
Example #3
0
 def setUp(self):
     super(TypeValidatorTestHelper, self).setUp()
     self.validator = TypeValidatorRegistry.get_validator(self.type_name)
Example #4
0
def generate_project_schema(project):
    logger.info('Processing project %s' % project)
    project_path = os.path.join(os.path.dirname(__file__), project)

    files = glob.glob(os.path.join(project_path, '*.yml'))
    if files == []:
        logger.info("Found no YAML files in project %s. Skipping it" % project)
        return

    x = index(files, lambda f: f.endswith('.conf.yml'))
    if x != -1:
        database_file = files[x]
        del files[x]
    else:
        database_file = os.path.join(project_path, project + '.conf.yml')

    schema_records = []
    if os.path.exists(database_file):
        logger.debug("Processing database file %s" % database_file)
        with open(database_file) as f:
            schema_records.extend(yaml.load(f.read()))

    schema_versions = []
    for version_file in files:
        logger.debug("Processing version file %s" % version_file)
        with open(version_file) as f:
            schema_versions.append(yaml.load(f.read()))

    schema_versions = sorted(schema_versions,
                             key=lambda s: Version(s['version']))

    parameters = OrderedDict()
    for schema in schema_versions:
        added = []

        seen = set()

        logger.debug('Processing schema version %s' % schema['version'])

        for param in schema['parameters']:
            # TODO(mkulkin): reduce the level of nesting
            prev_param = parameters.get(param['name'], None)

            if not prev_param:
                logger.debug('Parameter %s does not exist yet,'
                             ' adding it as new'
                             % param['name'])
                added.append(param)
            else:
                seen.add(param['name'])

                if param['type'] != prev_param['type']:
                    validator = TypeRegistry.get_validator(prev_param['type'])
                    if param['type'] == validator.base_type:
                        param['type'] = prev_param['type']

                        if param.get('default', None) is not None:
                            type_args = param.get('type_args', {})
                            value = validator.validate(param['default'], **type_args)
                            if not isinstance(value, Issue):
                                param['default'] = value
                            else:
                                logger.error("In project '%s' version %s"
                                             " default value for parameter"
                                             " '%s' is not valid value of"
                                             " type %s: %s"
                                             % (project, schema['version'],
                                                param['name'], param['type'],
                                                repr(param['default'])))
                    else:
                        logger.debug('Parameter %s type has'
                                     ' changed from %s to %s' %
                                     (param['name'], prev_param['type'],
                                      param['type']))
                        param['comment'] = 'Type has changed'
                        added.append(param)
                        continue

                if param.get('default', None) != \
                   prev_param.get('default', None):
                    logger.debug('Parameter %s default value'
                                 ' has changed from %s to %s' %
                                (param['name'], prev_param['default'],
                                 param['default']))
                    param['comment'] = 'Default value has changed'
                    added.append(param)
                    continue

                if param.get('help', None) != prev_param.get('help', None):
                    param['comment'] = 'Help string has changed'
                    added.append(param)

        removed = [name for name in parameters.keys() if name not in seen]
        if len(removed) > 0:
            logger.debug('Following parameters from previous'
                         ' schema version are not present in'
                         ' current version, marking as removed: %s'
                         % ','.join(removed))

        # Decide either to use full schema update or incremental
        changes_count = sum(map(len, [added, removed]))

        logger.debug('Found %d change(s) from previous version schema'
                     % changes_count)

        if changes_count > int(len(parameters) * DIFF_THRESHOLD):
            logger.debug('Using full schema update')

            new_parameters = parameters.copy()
            for param in added:
                new_parameters[param['name']] = param
            for name in removed:
                del new_parameters[name]

            new_schema_record = dict(version=schema['version'],
                                     added=new_parameters.values(),
                                     removed=[],
                                     checkpoint=True)
        else:
            logger.debug('Using incremental schema update')

            new_schema_record = dict(version=schema['version'],
                                     added=added, removed=removed)

        # Place schema record either replacing existing one or appending as new
        old_schema_record_idx = index(schema_records, lambda r:
                                      str(r['version']) ==
                                      str(new_schema_record['version']))

        if old_schema_record_idx != -1:
            old_schema_record = schema_records[old_schema_record_idx]
            # Collect information from existing records
            old_schema_parameters = {}
            for param in old_schema_record.get('added', []):
                old_schema_parameters[param['name']] = param

            for param in added:
                old_param = old_schema_parameters.get(param['name'], None)
                if not old_param:
                    param.setdefault('comment', 'New param')
                    continue

                extra_data = [(k, v) for k, v in old_param.items()
                              if k not in ['name', 'type', 'default', 'help']]
                param.update(extra_data)

                validator = TypeRegistry.get_validator(old_param['type'])
                if param['type'] not in [old_param['type'],
                                         validator.base_type]:
                    param['comment'] = 'Type has changed'
                    # Type has changed, enforcing old type to prevent
                    # accidental data loss
                    param['type'] = old_param['type']
                    if 'default' in old_param:
                        param['default'] = old_param['default']

                if param.get('default', None) is not None:
                    type_args = old_param.get('type_args', {})
                    value = validator.validate(old_param['default'], **type_args)
                    if not isinstance(value, Issue):
                        param['default'] = value
                    else:
                        logger.error("In project '%s' version %s default value"
                                     " for parameter '%s' is not valid value"
                                     " of type %s: %s" %
                                     (project, schema['version'],
                                      param['name'], param['type'],
                                      repr(param['default'])))

                if param.get('default', None) != old_param.get('default',
                                                               None):
                    param['comment'] = 'Default value has changed'
                    continue

            logger.debug('Replacing schema record %s'
                         % repr(new_schema_record))
            schema_records[old_schema_record_idx] = new_schema_record
        else:
            for param in added:
                param.setdefault('comment', 'New param')

            logger.debug('Appending schema record %s'
                         % repr(new_schema_record))
            schema_records.append(new_schema_record)

        # Update parameter info
        for param in new_schema_record.get('added', []):
            parameters[param['name']] = param

        for name in new_schema_record.get('removed', []):
            del parameters[name]

    schema_records = sorted(schema_records,
                            key=lambda r: Version(r['version']))

    with open(database_file, 'w') as f:
        f.write(yaml_dump_schema_records(schema_records))
Example #5
0
    def _parse_config_file(self,
                           base_mark,
                           config_contents,
                           config=Configuration(),
                           schema=None,
                           issue_reporter=None):
        if issue_reporter:

            def report_issue(issue):
                issue_reporter.report_issue(issue)
        else:

            def report_issue(issue):
                pass

        # Parse config file
        config_parser = IniConfigParser()
        parsed_config = config_parser.parse('', base_mark, config_contents)
        for error in parsed_config.errors:
            report_issue(error)

        # Validate config parameters and store them
        section_name_text_f = lambda s: s.name.text
        sections_by_name = groupby(sorted(parsed_config.sections,
                                          key=section_name_text_f),
                                   key=section_name_text_f)

        for section_name, sections in sections_by_name:
            sections = list(sections)

            if len(sections) > 1:
                report_issue(
                    Issue(Issue.INFO, 'Section "%s" appears multiple times' %
                          section_name))

            seen_parameters = set()

            for section in sections:
                unknown_section = False
                if schema:
                    unknown_section = not schema.has_section(section.name.text)

                if unknown_section:
                    report_issue(
                        MarkedIssue(Issue.WARNING,
                                    'Unknown section "%s"' % (section_name),
                                    section.start_mark))
                    continue

                for parameter in section.parameters:
                    parameter_schema = None
                    if schema:
                        parameter_schema = schema.get_parameter(
                            name=parameter.name.text,
                            section=section.name.text)
                        if not (parameter_schema or unknown_section):
                            report_issue(
                                MarkedIssue(
                                    Issue.WARNING,
                                    'Unknown parameter: section "%s" name "%s"'
                                    % (section_name, parameter.name.text),
                                    parameter.start_mark))

                    if parameter.name.text in seen_parameters:
                        report_issue(
                            MarkedIssue(
                                Issue.WARNING,
                                'Parameter "%s" in section "%s" redeclared' %
                                (parameter.name.text, section_name),
                                parameter.start_mark))
                    else:
                        seen_parameters.add(parameter.name.text)

                    parameter_fullname = parameter.name.text
                    if section_name != 'DEFAULT':
                        parameter_fullname = section_name + \
                            '.' + parameter_fullname

                    if parameter_schema:
                        type_validator = TypeValidatorRegistry.get_validator(
                            parameter_schema.type)
                        type_validation_result = type_validator.validate(
                            parameter.value.text)
                        if isinstance(type_validation_result, Issue):
                            type_validation_result.mark = parameter\
                                .value.start_mark.merge(
                                    type_validation_result.mark)
                            type_validation_result.message = \
                                'Property "%s" in section "%s": %s' % (
                                    parameter.name.text, section_name,
                                    type_validation_result.message)
                            report_issue(type_validation_result)

                            config.set(parameter_fullname,
                                       parameter.value.text)
                        else:
                            value = type_validation_result

                            config.set(parameter_fullname, value)

                            # if value == parameter_schema.default:
                            #   report_issue(MarkedIssue(Issue.INFO,
                            # 'Explicit value equals default: section "%s"
                            # parameter "%s"' % (section_name,
                            # parameter.name.text), parameter.start_mark))
                        if parameter_schema.deprecation_message:
                            report_issue(
                                MarkedIssue(
                                    Issue.WARNING,
                                    'Deprecated parameter: section "%s" name '
                                    '"%s". %s' %
                                    (section_name, parameter.name.text,
                                     parameter_schema.deprecation_message),
                                    parameter.start_mark))
                    else:
                        config.set(parameter_fullname, parameter.value.text)

        return config
Example #6
0
    def _parse_config_file(
        self,
        base_mark,
        config_contents,
        config=Configuration(),
        schema=None,
            issue_reporter=None):
        if issue_reporter:
            def report_issue(issue):
                issue_reporter.report_issue(issue)
        else:
            def report_issue(issue):
                pass

        # Parse config file
        config_parser = IniConfigParser()
        parsed_config = config_parser.parse('', base_mark, config_contents)
        for error in parsed_config.errors:
            report_issue(error)

        # Validate config parameters and store them
        section_name_text_f = lambda s: s.name.text
        sections_by_name = groupby(
            sorted(
                parsed_config.sections,
                key=section_name_text_f),
            key=section_name_text_f)

        for section_name, sections in sections_by_name:
            sections = list(sections)

            if len(sections) > 1:
                report_issue(
                    Issue(
                        Issue.INFO,
                        'Section "%s" appears multiple times' %
                        section_name))

            seen_parameters = set()

            for section in sections:
                unknown_section = False
                if schema:
                    unknown_section = not schema.has_section(section.name.text)

                if unknown_section:
                    report_issue(
                        MarkedIssue(Issue.WARNING, 'Unknown section "%s"' %
                                    (section_name), section.start_mark))
                    continue

                for parameter in section.parameters:
                    parameter_schema = None
                    if schema:
                        parameter_schema = schema.get_parameter(
                            name=parameter.name.text,
                            section=section.name.text)
                        if not (parameter_schema or unknown_section):
                            report_issue(
                                MarkedIssue(
                                    Issue.WARNING,
                                    'Unknown parameter: section "%s" name "%s"'
                                    % (section_name, parameter.name.text),
                                    parameter.start_mark))

                    if parameter.name.text in seen_parameters:
                        report_issue(
                            MarkedIssue(
                                Issue.WARNING,
                                'Parameter "%s" in section "%s" redeclared' %
                                (parameter.name.text, section_name),
                                parameter.start_mark))
                    else:
                        seen_parameters.add(parameter.name.text)

                    parameter_fullname = parameter.name.text
                    if section_name != 'DEFAULT':
                        parameter_fullname = section_name + \
                            '.' + parameter_fullname

                    if parameter_schema:
                        type_validator = TypeValidatorRegistry.get_validator(
                            parameter_schema.type)
                        type_validation_result = type_validator.validate(
                            parameter.value.text)
                        if isinstance(type_validation_result, Issue):
                            type_validation_result.mark = parameter\
                                .value.start_mark.merge(
                                    type_validation_result.mark)
                            type_validation_result.message = \
                                'Property "%s" in section "%s": %s' % (
                                    parameter.name.text, section_name,
                                    type_validation_result.message)
                            report_issue(type_validation_result)

                            config.set(
                                parameter_fullname,
                                parameter.value.text)
                        else:
                            value = type_validation_result

                            config.set(parameter_fullname, value)

                            # if value == parameter_schema.default:
                            #   report_issue(MarkedIssue(Issue.INFO,
                            # 'Explicit value equals default: section "%s"
                            # parameter "%s"' % (section_name,
                            # parameter.name.text), parameter.start_mark))
                        if parameter_schema.deprecation_message:
                            report_issue(
                                MarkedIssue(
                                    Issue.WARNING,
                                    'Deprecated parameter: section "%s" name '
                                    '"%s". %s' %
                                    (section_name, parameter.name.text,
                                     parameter_schema.deprecation_message),
                                    parameter.start_mark))
                    else:
                        config.set(parameter_fullname, parameter.value.text)

        return config
Example #7
0
 def setUp(self):
     super(TypeValidatorTestHelper, self).setUp()
     self.validator = TypeValidatorRegistry.get_validator(self.type_name)