def _validate(conf):
conf.register_opts(_validator_opts)
if conf.namespace:
groups = generator._get_groups(generator._list_opts(conf.namespace))
opt_data = generator._generate_machine_readable_data(groups, conf)
elif conf.opt_data:
with open(conf.opt_data) as f:
opt_data = yaml.safe_load(f)
else:
# TODO(bnemec): Implement this logic with group?
raise RuntimeError('Neither namespace or opt-data provided.')
sections = {}
parser = cfg.ConfigParser(conf.input_file, sections)
parser.parse()
warnings = False
errors = False
for section, options in sections.items():
for option in options:
if _validate_deprecated_opt(section, option, opt_data):
logging.warn('Deprecated opt %s/%s found', section, option)
warnings = True
elif not _validate_opt(section, option, opt_data):
logging.error('%s/%s not found', section, option)
errors = True
if errors or (warnings and conf.fatal_warnings):
return 1
return 0
def _validate(conf):
conf.register_opts(_validator_opts)
if conf.namespace:
groups = generator._get_groups(generator._list_opts(conf.namespace))
opt_data = generator._generate_machine_readable_data(groups, conf)
elif conf.opt_data:
opt_data = load_opt_data(conf)
else:
# TODO(bnemec): Implement this logic with group?
raise RuntimeError('Neither namespace nor opt-data provided.')
sections = {}
parser = cfg.ConfigParser(conf.input_file, sections)
parser.parse()
warnings = False
errors = False
for section, options in sections.items():
if section in conf.exclude_group:
continue
for option in options:
if _validate_deprecated_opt(section, option, opt_data):
logging.warn('Deprecated opt %s/%s found', section, option)
warnings = True
elif not _validate_opt(section, option, opt_data):
if section in KNOWN_BAD_GROUPS:
logging.info(
'Ignoring missing option "%s" from group '
'"%s" because the group is known to have '
'incomplete sample config data and thus '
'cannot be validated properly.', option, section)
continue
logging.error('%s/%s not found', section, option)
errors = True
if errors or (warnings and conf.fatal_warnings):
return 1
return 0
def get_conf(conf_file=None, config_file=None):
"""Get CONF object for specific project.
"""
conf = cfg.ConfigOpts()
gn.register_cli_opts(conf)
oslo_args = ['--config-file', conf_file]
conf(oslo_args)
groups = gn._get_groups(gn._list_opts(conf.namespace))
# Make new CONF
new_conf = cfg.ConfigOpts()
project_args = ['--config-file', config_file]
new_conf(project_args)
all_namespaces = []
for k, v in groups.items():
group = cfg.OptGroup(k)
try:
namespaces = v.get('namespaces', [])
except Exception:
namespaces = v
list_opts = []
for namespace in namespaces:
all_namespaces.append(namespace[0])
list_opts.extend(namespace[1])
new_conf.register_group(group)
if k == 'DEFAULT':
LOG.info("Force register DEFAULT group to easier access "
"configuration values in DEFAULT group.")
try:
new_conf.register_opts(list_opts)
except cfg.DuplicateOptError:
pass
LOG.info("Register group %s into new_conf.", group.name)
new_conf.register_opts(list_opts, group=group)
projects = []
for namespace in all_namespaces:
sp = namespace.split('.')
LOG.info("Get project name from namespaces.")
if 'oslo' in namespace:
LOG.info("Support getting project name for Oslo projects.")
projects.append(".".join(sp[:2]))
else:
projects.append(sp[0])
for project in projects:
LOG.info("Autoload all dynamic section.")
try:
get_name_module = DYNAMIC_SECTION_PROJECTS[project]
except KeyError:
LOG.info("%s project does not have dynamic section or "
"not is not supported now.")
continue
dynamic = importlib.import_module(get_name_module)
dynamic.register_dynamic_section(new_conf)
return new_conf, set(projects)
def _format_option_help(app, namespaces, split_namespaces):
"""Generate a series of lines of restructuredtext.
Format the option help as restructuredtext and return it as a list
of lines.
"""
opts = generator._list_opts(namespaces)
if split_namespaces:
for namespace, opt_list in opts:
for group, opts in opt_list:
if isinstance(group, cfg.OptGroup):
group_name = group.name
else:
group_name = group
group = None
if group_name is None:
group_name = 'DEFAULT'
lines = _format_group(
app=app,
namespace=namespace,
group_name=group_name,
group_obj=group,
opt_list=opts,
)
for line in lines:
yield line
else:
# Merge the options from different namespaces that belong to
# the same group together and format them without the
# namespace.
by_section = {}
group_objs = {}
for ignore, opt_list in opts:
for group, group_opts in opt_list:
if isinstance(group, cfg.OptGroup):
group_name = group.name
else:
group_name = group
group = None
if group_name is None:
group_name = 'DEFAULT'
group_objs.setdefault(group_name, group)
by_section.setdefault(group_name, []).extend(group_opts)
for group_name, group_opts in sorted(by_section.items()):
lines = _format_group(
app=app,
namespace=None,
group_name=group_name,
group_obj=group_objs.get(group_name),
opt_list=group_opts,
)
for line in lines:
yield line
def _format_option_help(app, namespaces, split_namespaces):
"""Generate a series of lines of restructuredtext.
Format the option help as restructuredtext and return it as a list
of lines.
"""
opts = generator._list_opts(namespaces)
if split_namespaces:
for namespace, opt_list in opts:
for group, opts in opt_list:
if isinstance(group, cfg.OptGroup):
group_name = group.name
else:
group_name = group
group = None
if group_name is None:
group_name = 'DEFAULT'
lines = _format_group(
app=app,
namespace=namespace,
group_name=group_name,
group_obj=group,
opt_list=opts,
)
for line in lines:
yield line
else:
# Merge the options from different namespaces that belong to
# the same group together and format them without the
# namespace.
by_section = {}
group_objs = {}
for ignore, opt_list in opts:
for group, group_opts in opt_list:
if isinstance(group, cfg.OptGroup):
group_name = group.name
else:
group_name = group
group = None
if group_name is None:
group_name = 'DEFAULT'
group_objs.setdefault(group_name, group)
by_section.setdefault(group_name, []).extend(group_opts)
for group_name, group_opts in sorted(by_section.items()):
lines = _format_group(
app=app,
namespace=None,
group_name=group_name,
group_obj=group_objs.get(group_name),
opt_list=group_opts,
)
for line in lines:
yield line
def test_no_modifiers_registered(self, raw_opts_loaders, get_updaters):
orig_opt = cfg.StrOpt('foo', default='bar')
raw_opts_loaders.return_value = [
('namespace', lambda: [(None, [orig_opt])]),
]
get_updaters.return_value = []
opts = generator._list_opts(['namespace'])
# NOTE(dhellmann): Who designed this data structure?
the_opt = opts[0][1][0][1][0]
self.assertEqual('bar', the_opt.default)
self.assertIs(orig_opt, the_opt)
def test_no_modifiers_registered(self, raw_opts_loaders, get_updaters):
orig_opt = cfg.StrOpt('foo', default='bar')
raw_opts_loaders.return_value = [
('namespace', lambda: [(None, [orig_opt])]),
]
get_updaters.return_value = []
opts = generator._list_opts(['namespace'])
# NOTE(dhellmann): Who designed this data structure?
the_opt = opts[0][1][0][1][0]
self.assertEqual('bar', the_opt.default)
self.assertIs(orig_opt, the_opt)
def test_change_default(self, raw_opts_loaders, get_updaters):
orig_opt = cfg.StrOpt('foo', default='bar')
raw_opts_loaders.return_value = [
('namespace', lambda: [(None, [orig_opt])]),
]
def updater():
cfg.set_defaults([orig_opt], foo='blah')
get_updaters.return_value = [updater]
opts = generator._list_opts(['namespace'])
# NOTE(dhellmann): Who designed this data structure?
the_opt = opts[0][1][0][1][0]
self.assertEqual('blah', the_opt.default)
self.assertIs(orig_opt, the_opt)
def test_change_default(self, raw_opts_loaders, get_updaters):
orig_opt = cfg.StrOpt('foo', default='bar')
raw_opts_loaders.return_value = [
('namespace', lambda: [(None, [orig_opt])]),
]
def updater():
cfg.set_defaults([orig_opt], foo='blah')
get_updaters.return_value = [updater]
opts = generator._list_opts(['namespace'])
# NOTE(dhellmann): Who designed this data structure?
the_opt = opts[0][1][0][1][0]
self.assertEqual('blah', the_opt.default)
self.assertIs(orig_opt, the_opt)
def test_list_ignores_doubles(self, named_mgr):
config_opts = [cfg.StrOpt("foo"), cfg.StrOpt("bar")]
mock_ep1 = mock.Mock()
mock_ep1.configure_mock(name="namespace", obj=[("group", config_opts)])
mock_ep2 = mock.Mock()
mock_ep2.configure_mock(name="namespace", obj=[("group", config_opts)])
# These are the very same config options, but read twice.
# This is possible if one misconfigures the entry point for the
# sample config generator.
mock_eps = [mock_ep1, mock_ep2]
named_mgr.return_value = mock_eps
slurped_opts = 0
for _, listing in generator._list_opts(None):
for _, opts in listing:
slurped_opts += len(opts)
self.assertEqual(len(config_opts), slurped_opts)
def test_list_ignores_doubles(self, raw_opts_loaders):
config_opts = [
(None, [cfg.StrOpt('foo'), cfg.StrOpt('bar')]),
]
# These are the very same config options, but read twice.
# This is possible if one misconfigures the entry point for the
# sample config generator.
raw_opts_loaders.return_value = [
('namespace', lambda: config_opts),
('namespace', lambda: config_opts),
]
slurped_opts = 0
for _, listing in generator._list_opts(['namespace']):
for _, opts in listing:
slurped_opts += len(opts)
self.assertEqual(2, slurped_opts)
def test_list_ignores_doubles(self, raw_opts_loaders):
config_opts = [
(None, [cfg.StrOpt('foo'), cfg.StrOpt('bar')]),
]
# These are the very same config options, but read twice.
# This is possible if one misconfigures the entry point for the
# sample config generator.
raw_opts_loaders.return_value = [
('namespace', lambda: config_opts),
('namespace', lambda: config_opts),
]
slurped_opts = 0
for _, listing in generator._list_opts(None):
for _, opts in listing:
slurped_opts += len(opts)
self.assertEqual(2, slurped_opts)
def test_list_ignores_doubles(self, named_mgr):
config_opts = [
cfg.StrOpt('foo'),
cfg.StrOpt('bar'),
]
mock_ep1 = mock.Mock()
mock_ep1.configure_mock(name="namespace", obj=[("group", config_opts)])
mock_ep2 = mock.Mock()
mock_ep2.configure_mock(name="namespace", obj=[("group", config_opts)])
# These are the very same config options, but read twice.
# This is possible if one misconfigures the entry point for the
# sample config generator.
mock_eps = [mock_ep1, mock_ep2]
named_mgr.return_value = mock_eps
slurped_opts = 0
for _, listing in generator._list_opts(None):
for _, opts in listing:
slurped_opts += len(opts)
self.assertEqual(len(config_opts), slurped_opts)
def generate_ns_data(namespace):
"""Generate a json string containing the namespace"""
groups = generator._get_groups(generator._list_opts([namespace]))
return OptionJsonEncoder(sort_keys=True).encode(groups)
def generate(conf):
"""Generate a sample config file.
List all of the options available via the namespaces specified in the given
configuration and write a description of them to the specified output file.
:param conf: a ConfigOpts instance containing the generator's configuration
"""
conf.register_opts(_generator_opts)
output_file = (open(conf.output_file, 'w')
if conf.output_file else sys.stdout)
output_file.write('''
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{ include "%s.conf.%s_values_skeleton" .Values.conf.%s | trunc 0 }}
{{ include "%s.conf.%s" .Values.conf.%s }}
\n''' % (conf.helm_chart, conf.helm_namespace, conf.helm_namespace, conf.helm_chart, conf.helm_namespace, conf.helm_namespace))
output_file.write('''
{{- define "%s.conf.%s_values_skeleton" -}}
\n''' % (conf.helm_chart, conf.helm_namespace))
### values skeleton
formatter = _ValuesSkeletonFormatter(output_file=output_file,
wrap_width=conf.wrap_width)
groups = _get_groups(_list_opts(conf.namespace))
# Output the "DEFAULT" section as the very first section
_output_opts_null(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
conf.summarize)
# output all other config sections with groups in alphabetical order
for group, group_data in sorted(groups.items()):
_output_opts_null(formatter, group, group_data, conf.minimal,
conf.summarize)
output_file.write('''
{{- end -}}
\n''')
output_file.write('''
{{- define "%s.conf.%s" -}}
\n''' % (conf.helm_chart, conf.helm_namespace))
### helm options
formatter = _HelmOptFormatter(output_file=output_file,
wrap_width=conf.wrap_width)
groups = _get_groups(_list_opts(conf.namespace))
# Output the "DEFAULT" section as the very first section
_output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
conf.summarize)
# output all other config sections with groups in alphabetical order
for group, group_data in sorted(groups.items()):
formatter.write('\n\n')
_output_opts(formatter, group, group_data, conf.minimal,
conf.summarize)
output_file.write('''
{{- end -}}
\n''')
def run(self):
env = self.state.document.settings.env
app = env.app
namespace = ' '.join(self.content)
opts = generator._list_opts([namespace])
result = ViewList()
source_name = '<' + __name__ + '>'
def _add(text):
"Append some text to the output result view to be parsed."
result.append(text, source_name)
def _add_indented(text):
"""Append some text, indented by a couple of spaces.
Indent everything under the option name,
to format it as a definition list.
"""
_add(_indent(text))
by_section = {}
for ignore, opt_list in opts:
for group_name, opts in opt_list:
by_section.setdefault(group_name, []).extend(opts)
for group_name, opt_list in sorted(by_section.items()):
group_name = group_name or 'DEFAULT'
app.info('[oslo.config] %s %s' % (namespace, group_name))
_add('.. oslo.config:group:: %s' % group_name)
_add('')
for opt in opt_list:
opt_type = self._TYPE_DESCRIPTIONS.get(type(opt),
'unknown type')
_add('.. oslo.config:option:: %s' % opt.dest)
_add('')
_add_indented(':Type: %s' % opt_type)
for default in generator._format_defaults(opt):
if default:
default = '``' + default + '``'
_add_indented(':Default: %s' % default)
if getattr(opt.type, 'min', None):
_add_indented(':Minimum Value: %s' % opt.type.min)
if getattr(opt.type, 'max', None):
_add_indented(':Maximum Value: %s' % opt.type.max)
if getattr(opt.type, 'choices', None):
choices_text = ', '.join([self._get_choice_text(choice)
for choice in opt.type.choices])
_add_indented(':Valid Values: %s' % choices_text)
_add('')
try:
help_text = opt.help % {'default': 'the value above'}
except TypeError:
# There is no mention of the default in the help string.
help_text = opt.help
_add_indented(help_text)
_add('')
if opt.deprecated_opts:
_list_table(
_add_indented,
['Group', 'Name'],
((d.group or 'DEFAULT',
d.name or opt.dest or 'UNSET')
for d in opt.deprecated_opts),
title='Deprecated Variations',
)
if opt.deprecated_for_removal:
_add_indented('.. warning::')
_add_indented(' This option is deprecated for removal.')
_add_indented(' Its value may be silently ignored ')
_add_indented(' in the future.')
if opt.deprecated_reason:
_add_indented(' Reason: ' + opt.deprecated_reason)
_add('')
_add('')
node = nodes.section()
node.document = self.state.document
nested_parse_with_titles(self.state, result, node)
return node.children
def run(self):
env = self.state.document.settings.env
app = env.app
namespace = ' '.join(self.content)
opts = generator._list_opts([namespace])
result = ViewList()
source_name = '<' + __name__ + '>'
def _add(text):
"Append some text to the output result view to be parsed."
result.append(text, source_name)
def _add_indented(text):
"""Append some text, indented by a couple of spaces.
Indent everything under the option name,
to format it as a definition list.
"""
_add(_indent(text))
by_section = {}
for ignore, opt_list in opts:
for group_name, opts in opt_list:
by_section.setdefault(group_name, []).extend(opts)
for group_name, opt_list in sorted(by_section.items()):
group_name = group_name or 'DEFAULT'
app.info('[oslo.config] %s %s' % (namespace, group_name))
_add(group_name)
_add('=' * len(group_name))
_add('')
for opt in opt_list:
opt_type = self._TYPE_DESCRIPTIONS.get(type(opt),
'unknown type')
_add('``%s``' % opt.dest)
_add('')
_add_indented(':Type: %s' % opt_type)
for default in generator._format_defaults(opt):
if default:
default = '``' + default + '``'
_add_indented(':Default: %s' % default)
if getattr(opt.type, 'min', None):
_add_indented(':Minimum Value: %s' % opt.type.min)
if getattr(opt.type, 'max', None):
_add_indented(':Maximum Value: %s' % opt.type.max)
if getattr(opt.type, 'choices', None):
choices_text = ', '.join([
self._get_choice_text(choice)
for choice in opt.type.choices
])
_add_indented(':Valid Values: %s' % choices_text)
_add('')
_add_indented(opt.help)
_add('')
if opt.deprecated_opts:
_list_table(
_add_indented,
['Group', 'Name'],
((d.group or 'DEFAULT', d.name or opt.dest or 'UNSET')
for d in opt.deprecated_opts),
title='Deprecated Variations',
)
if opt.deprecated_for_removal:
_add_indented('.. warning:')
_add_indented(' This option is deprecated for removal.')
_add_indented(' Its value may be silently ignored ')
_add_indented(' in the future.')
_add('')
_add('')
node = nodes.section()
node.document = self.state.document
nested_parse_with_titles(self.state, result, node)
return node.children
def generate_ns_data(namespace):
"""Generate a json string containing the namespace"""
groups = generator._get_groups(generator._list_opts([namespace]))
return OptionJsonEncoder(sort_keys=True).encode(groups)
