def _validate(conf): conf.register_opts(_validator_opts) if conf.namespace: groups = generator._get_groups(generator._list_opts(conf.namespace)) opt_data = generator._generate_machine_readable_data(groups, conf) elif conf.opt_data: with open(conf.opt_data) as f: opt_data = yaml.safe_load(f) else: # TODO(bnemec): Implement this logic with group? raise RuntimeError('Neither namespace or opt-data provided.') sections = {} parser = cfg.ConfigParser(conf.input_file, sections) parser.parse() warnings = False errors = False for section, options in sections.items(): for option in options: if _validate_deprecated_opt(section, option, opt_data): logging.warn('Deprecated opt %s/%s found', section, option) warnings = True elif not _validate_opt(section, option, opt_data): logging.error('%s/%s not found', section, option) errors = True if errors or (warnings and conf.fatal_warnings): return 1 return 0
def _validate(conf): conf.register_opts(_validator_opts) if conf.namespace: groups = generator._get_groups(generator._list_opts(conf.namespace)) opt_data = generator._generate_machine_readable_data(groups, conf) elif conf.opt_data: opt_data = load_opt_data(conf) else: # TODO(bnemec): Implement this logic with group? raise RuntimeError('Neither namespace nor opt-data provided.') sections = {} parser = cfg.ConfigParser(conf.input_file, sections) parser.parse() warnings = False errors = False for section, options in sections.items(): if section in conf.exclude_group: continue for option in options: if _validate_deprecated_opt(section, option, opt_data): logging.warn('Deprecated opt %s/%s found', section, option) warnings = True elif not _validate_opt(section, option, opt_data): if section in KNOWN_BAD_GROUPS: logging.info( 'Ignoring missing option "%s" from group ' '"%s" because the group is known to have ' 'incomplete sample config data and thus ' 'cannot be validated properly.', option, section) continue logging.error('%s/%s not found', section, option) errors = True if errors or (warnings and conf.fatal_warnings): return 1 return 0
def get_conf(conf_file=None, config_file=None): """Get CONF object for specific project. """ conf = cfg.ConfigOpts() gn.register_cli_opts(conf) oslo_args = ['--config-file', conf_file] conf(oslo_args) groups = gn._get_groups(gn._list_opts(conf.namespace)) # Make new CONF new_conf = cfg.ConfigOpts() project_args = ['--config-file', config_file] new_conf(project_args) all_namespaces = [] for k, v in groups.items(): group = cfg.OptGroup(k) try: namespaces = v.get('namespaces', []) except Exception: namespaces = v list_opts = [] for namespace in namespaces: all_namespaces.append(namespace[0]) list_opts.extend(namespace[1]) new_conf.register_group(group) if k == 'DEFAULT': LOG.info("Force register DEFAULT group to easier access " "configuration values in DEFAULT group.") try: new_conf.register_opts(list_opts) except cfg.DuplicateOptError: pass LOG.info("Register group %s into new_conf.", group.name) new_conf.register_opts(list_opts, group=group) projects = [] for namespace in all_namespaces: sp = namespace.split('.') LOG.info("Get project name from namespaces.") if 'oslo' in namespace: LOG.info("Support getting project name for Oslo projects.") projects.append(".".join(sp[:2])) else: projects.append(sp[0]) for project in projects: LOG.info("Autoload all dynamic section.") try: get_name_module = DYNAMIC_SECTION_PROJECTS[project] except KeyError: LOG.info("%s project does not have dynamic section or " "not is not supported now.") continue dynamic = importlib.import_module(get_name_module) dynamic.register_dynamic_section(new_conf) return new_conf, set(projects)
def _format_option_help(app, namespaces, split_namespaces): """Generate a series of lines of restructuredtext. Format the option help as restructuredtext and return it as a list of lines. """ opts = generator._list_opts(namespaces) if split_namespaces: for namespace, opt_list in opts: for group, opts in opt_list: if isinstance(group, cfg.OptGroup): group_name = group.name else: group_name = group group = None if group_name is None: group_name = 'DEFAULT' lines = _format_group( app=app, namespace=namespace, group_name=group_name, group_obj=group, opt_list=opts, ) for line in lines: yield line else: # Merge the options from different namespaces that belong to # the same group together and format them without the # namespace. by_section = {} group_objs = {} for ignore, opt_list in opts: for group, group_opts in opt_list: if isinstance(group, cfg.OptGroup): group_name = group.name else: group_name = group group = None if group_name is None: group_name = 'DEFAULT' group_objs.setdefault(group_name, group) by_section.setdefault(group_name, []).extend(group_opts) for group_name, group_opts in sorted(by_section.items()): lines = _format_group( app=app, namespace=None, group_name=group_name, group_obj=group_objs.get(group_name), opt_list=group_opts, ) for line in lines: yield line
def test_no_modifiers_registered(self, raw_opts_loaders, get_updaters): orig_opt = cfg.StrOpt('foo', default='bar') raw_opts_loaders.return_value = [ ('namespace', lambda: [(None, [orig_opt])]), ] get_updaters.return_value = [] opts = generator._list_opts(['namespace']) # NOTE(dhellmann): Who designed this data structure? the_opt = opts[0][1][0][1][0] self.assertEqual('bar', the_opt.default) self.assertIs(orig_opt, the_opt)
def test_change_default(self, raw_opts_loaders, get_updaters): orig_opt = cfg.StrOpt('foo', default='bar') raw_opts_loaders.return_value = [ ('namespace', lambda: [(None, [orig_opt])]), ] def updater(): cfg.set_defaults([orig_opt], foo='blah') get_updaters.return_value = [updater] opts = generator._list_opts(['namespace']) # NOTE(dhellmann): Who designed this data structure? the_opt = opts[0][1][0][1][0] self.assertEqual('blah', the_opt.default) self.assertIs(orig_opt, the_opt)
def test_list_ignores_doubles(self, named_mgr): config_opts = [cfg.StrOpt("foo"), cfg.StrOpt("bar")] mock_ep1 = mock.Mock() mock_ep1.configure_mock(name="namespace", obj=[("group", config_opts)]) mock_ep2 = mock.Mock() mock_ep2.configure_mock(name="namespace", obj=[("group", config_opts)]) # These are the very same config options, but read twice. # This is possible if one misconfigures the entry point for the # sample config generator. mock_eps = [mock_ep1, mock_ep2] named_mgr.return_value = mock_eps slurped_opts = 0 for _, listing in generator._list_opts(None): for _, opts in listing: slurped_opts += len(opts) self.assertEqual(len(config_opts), slurped_opts)
def test_list_ignores_doubles(self, raw_opts_loaders): config_opts = [ (None, [cfg.StrOpt('foo'), cfg.StrOpt('bar')]), ] # These are the very same config options, but read twice. # This is possible if one misconfigures the entry point for the # sample config generator. raw_opts_loaders.return_value = [ ('namespace', lambda: config_opts), ('namespace', lambda: config_opts), ] slurped_opts = 0 for _, listing in generator._list_opts(['namespace']): for _, opts in listing: slurped_opts += len(opts) self.assertEqual(2, slurped_opts)
def test_list_ignores_doubles(self, raw_opts_loaders): config_opts = [ (None, [cfg.StrOpt('foo'), cfg.StrOpt('bar')]), ] # These are the very same config options, but read twice. # This is possible if one misconfigures the entry point for the # sample config generator. raw_opts_loaders.return_value = [ ('namespace', lambda: config_opts), ('namespace', lambda: config_opts), ] slurped_opts = 0 for _, listing in generator._list_opts(None): for _, opts in listing: slurped_opts += len(opts) self.assertEqual(2, slurped_opts)
def test_list_ignores_doubles(self, named_mgr): config_opts = [ cfg.StrOpt('foo'), cfg.StrOpt('bar'), ] mock_ep1 = mock.Mock() mock_ep1.configure_mock(name="namespace", obj=[("group", config_opts)]) mock_ep2 = mock.Mock() mock_ep2.configure_mock(name="namespace", obj=[("group", config_opts)]) # These are the very same config options, but read twice. # This is possible if one misconfigures the entry point for the # sample config generator. mock_eps = [mock_ep1, mock_ep2] named_mgr.return_value = mock_eps slurped_opts = 0 for _, listing in generator._list_opts(None): for _, opts in listing: slurped_opts += len(opts) self.assertEqual(len(config_opts), slurped_opts)
def generate_ns_data(namespace): """Generate a json string containing the namespace""" groups = generator._get_groups(generator._list_opts([namespace])) return OptionJsonEncoder(sort_keys=True).encode(groups)
def generate(conf): """Generate a sample config file. List all of the options available via the namespaces specified in the given configuration and write a description of them to the specified output file. :param conf: a ConfigOpts instance containing the generator's configuration """ conf.register_opts(_generator_opts) output_file = (open(conf.output_file, 'w') if conf.output_file else sys.stdout) output_file.write(''' # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. {{ include "%s.conf.%s_values_skeleton" .Values.conf.%s | trunc 0 }} {{ include "%s.conf.%s" .Values.conf.%s }} \n''' % (conf.helm_chart, conf.helm_namespace, conf.helm_namespace, conf.helm_chart, conf.helm_namespace, conf.helm_namespace)) output_file.write(''' {{- define "%s.conf.%s_values_skeleton" -}} \n''' % (conf.helm_chart, conf.helm_namespace)) ### values skeleton formatter = _ValuesSkeletonFormatter(output_file=output_file, wrap_width=conf.wrap_width) groups = _get_groups(_list_opts(conf.namespace)) # Output the "DEFAULT" section as the very first section _output_opts_null(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal, conf.summarize) # output all other config sections with groups in alphabetical order for group, group_data in sorted(groups.items()): _output_opts_null(formatter, group, group_data, conf.minimal, conf.summarize) output_file.write(''' {{- end -}} \n''') output_file.write(''' {{- define "%s.conf.%s" -}} \n''' % (conf.helm_chart, conf.helm_namespace)) ### helm options formatter = _HelmOptFormatter(output_file=output_file, wrap_width=conf.wrap_width) groups = _get_groups(_list_opts(conf.namespace)) # Output the "DEFAULT" section as the very first section _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal, conf.summarize) # output all other config sections with groups in alphabetical order for group, group_data in sorted(groups.items()): formatter.write('\n\n') _output_opts(formatter, group, group_data, conf.minimal, conf.summarize) output_file.write(''' {{- end -}} \n''')
def run(self): env = self.state.document.settings.env app = env.app namespace = ' '.join(self.content) opts = generator._list_opts([namespace]) result = ViewList() source_name = '<' + __name__ + '>' def _add(text): "Append some text to the output result view to be parsed." result.append(text, source_name) def _add_indented(text): """Append some text, indented by a couple of spaces. Indent everything under the option name, to format it as a definition list. """ _add(_indent(text)) by_section = {} for ignore, opt_list in opts: for group_name, opts in opt_list: by_section.setdefault(group_name, []).extend(opts) for group_name, opt_list in sorted(by_section.items()): group_name = group_name or 'DEFAULT' app.info('[oslo.config] %s %s' % (namespace, group_name)) _add('.. oslo.config:group:: %s' % group_name) _add('') for opt in opt_list: opt_type = self._TYPE_DESCRIPTIONS.get(type(opt), 'unknown type') _add('.. oslo.config:option:: %s' % opt.dest) _add('') _add_indented(':Type: %s' % opt_type) for default in generator._format_defaults(opt): if default: default = '``' + default + '``' _add_indented(':Default: %s' % default) if getattr(opt.type, 'min', None): _add_indented(':Minimum Value: %s' % opt.type.min) if getattr(opt.type, 'max', None): _add_indented(':Maximum Value: %s' % opt.type.max) if getattr(opt.type, 'choices', None): choices_text = ', '.join([self._get_choice_text(choice) for choice in opt.type.choices]) _add_indented(':Valid Values: %s' % choices_text) _add('') try: help_text = opt.help % {'default': 'the value above'} except TypeError: # There is no mention of the default in the help string. help_text = opt.help _add_indented(help_text) _add('') if opt.deprecated_opts: _list_table( _add_indented, ['Group', 'Name'], ((d.group or 'DEFAULT', d.name or opt.dest or 'UNSET') for d in opt.deprecated_opts), title='Deprecated Variations', ) if opt.deprecated_for_removal: _add_indented('.. warning::') _add_indented(' This option is deprecated for removal.') _add_indented(' Its value may be silently ignored ') _add_indented(' in the future.') if opt.deprecated_reason: _add_indented(' Reason: ' + opt.deprecated_reason) _add('') _add('') node = nodes.section() node.document = self.state.document nested_parse_with_titles(self.state, result, node) return node.children
def run(self): env = self.state.document.settings.env app = env.app namespace = ' '.join(self.content) opts = generator._list_opts([namespace]) result = ViewList() source_name = '<' + __name__ + '>' def _add(text): "Append some text to the output result view to be parsed." result.append(text, source_name) def _add_indented(text): """Append some text, indented by a couple of spaces. Indent everything under the option name, to format it as a definition list. """ _add(_indent(text)) by_section = {} for ignore, opt_list in opts: for group_name, opts in opt_list: by_section.setdefault(group_name, []).extend(opts) for group_name, opt_list in sorted(by_section.items()): group_name = group_name or 'DEFAULT' app.info('[oslo.config] %s %s' % (namespace, group_name)) _add(group_name) _add('=' * len(group_name)) _add('') for opt in opt_list: opt_type = self._TYPE_DESCRIPTIONS.get(type(opt), 'unknown type') _add('``%s``' % opt.dest) _add('') _add_indented(':Type: %s' % opt_type) for default in generator._format_defaults(opt): if default: default = '``' + default + '``' _add_indented(':Default: %s' % default) if getattr(opt.type, 'min', None): _add_indented(':Minimum Value: %s' % opt.type.min) if getattr(opt.type, 'max', None): _add_indented(':Maximum Value: %s' % opt.type.max) if getattr(opt.type, 'choices', None): choices_text = ', '.join([ self._get_choice_text(choice) for choice in opt.type.choices ]) _add_indented(':Valid Values: %s' % choices_text) _add('') _add_indented(opt.help) _add('') if opt.deprecated_opts: _list_table( _add_indented, ['Group', 'Name'], ((d.group or 'DEFAULT', d.name or opt.dest or 'UNSET') for d in opt.deprecated_opts), title='Deprecated Variations', ) if opt.deprecated_for_removal: _add_indented('.. warning:') _add_indented(' This option is deprecated for removal.') _add_indented(' Its value may be silently ignored ') _add_indented(' in the future.') _add('') _add('') node = nodes.section() node.document = self.state.document nested_parse_with_titles(self.state, result, node) return node.children