def from_plugin(cls, subparser, plugin, base_groups):
"""Reads spec & vars from plugin and constructs the parser instance
:param subparser: argparse.subparser to extend
:param plugin: InfraredPlugin object
:param base_groups: dict, included groups
:return: SpecParser object based on given plugin spec & vars
"""
spec_dict = base_groups or {}
with open(plugin.spec) as stream:
spec = yaml.safe_load(stream) or {}
dict_utils.dict_merge(
base_groups,
spec,
dict_utils.ConflictResolver.unique_append_list_resolver)
# The "try-excpet" block here is for adding spec file path if it
# includes an unsupported option type
try:
return SpecParser(subparser, spec_dict, plugin.vars_dir,
plugin.defaults_dir, plugin.path)
except exceptions.IRUnsupportedSpecOptionType as ex:
ex.message += ' in file: {}'.format(plugin.spec)
raise ex
def get_answers_file_args(self, cli_args):
"""Resolve arguments' values from answers INI file. """
file_result = {}
args_to_remove = []
for (parser_name, parser_dict, arg_name, arg_value,
option_spec) in self._iterate_received_arguments(cli_args):
file_result[parser_name] = file_result.get(parser_name, {})
if option_spec and option_spec.get('action', '') == 'read-answers':
# we have config option. saving it.
self._convert_non_cli_args(parser_name, parser_dict[arg_name])
dict_utils.dict_merge(file_result[parser_name],
parser_dict[arg_name])
# remove from cli args
args_to_remove.append((parser_name, arg_name))
# remove parser dict outside loop to avoid iteration dict modification
for parser_name, arg_name in args_to_remove:
for spec_parser in self.spec_helper.iterate_parsers():
if spec_parser['name'] in cli_args and spec_parser[
'name'] == parser_name:
parser_dict = cli_args[spec_parser['name']]
parser_dict.pop(arg_name)
break
return file_result
def from_plugin(cls, subparser, plugin, base_groups):
"""Reads spec & vars from plugin and constructs the parser instance
:param subparser: argparse.subparser to extend
:param plugin: InfraredPlugin object
:param base_groups: dict, included groups
:return: SpecParser object based on given plugin spec & vars
"""
spec_dict = base_groups or {}
with open(plugin.spec) as stream:
spec = yaml.load(stream) or {}
dict_utils.dict_merge(
base_groups,
spec,
dict_utils.ConflictResolver.unique_append_list_resolver)
# The "try-excpet" block here is for adding spec file path if it
# includes an unsupported option type
try:
return SpecParser(subparser, spec_dict, plugin.vars_dir,
plugin.defaults_dir, plugin.path)
except exceptions.IRUnsupportedSpecOptionType as ex:
ex.message += ' in file: {}'.format(plugin.spec)
raise ex
def test_dict_merge_none_resolver(first, second, expected):
from infrared.core.utils.dict_utils import dict_merge, ConflictResolver
first, second, expected = read_yaml(first), read_yaml(second), read_yaml(
expected)
dict_merge(first,
second,
conflict_resolver=ConflictResolver.greedy_resolver)
assert first == expected
def test_dict_merge():
from infrared.core.utils.dict_utils import dict_merge
first_dict = {'a': 1, 'b': 2, 'c': {'d': 'foo1', 'e': 'bar', 'list1': [
'a', 'b', 'c']}, 'list2': [1, 2, 3]}
second_dict = {'a': 2, 'c': {'d': 'foo2', 'f': 5, 'list1': [3, 4, 5]},
'g': 'bla', 5: 'yy', 'list3': ['a', 2]}
expected_result = {'a': 2, 'b': 2, 'c': {'d': 'foo2', 'e': 'bar', 'f': 5,
'list1': [3, 4, 5]}, 'g': 'bla',
5: 'yy', 'list2': [1, 2, 3], 'list3': ['a', 2]}
dict_merge(first_dict, second_dict)
assert first_dict == expected_result
def get_answers_file_args(self, cli_args):
"""Resolve arguments' values from answers INI file. """
file_result = {}
for (parser_name, parser_dict, arg_name, arg_value,
option_spec) in self._iterate_received_arguments(cli_args):
file_result[parser_name] = file_result.get(parser_name, {})
if option_spec and option_spec.get('action', '') == 'read-answers':
# we have config option. saving it.
self._convert_non_cli_args(parser_name, parser_dict[arg_name])
dict_utils.dict_merge(file_result[parser_name],
parser_dict[arg_name])
# remove from cli args
parser_dict.pop(arg_name)
return file_result
def parse_args(self, arg_parser, args=None):
"""Parses all the arguments (cli, answers file)
:return: None, if ``--generate-answers-file`` in arg_arg_parser
:return: (dict, dict):
* command arguments dict (arguments to control the IR logic)
* nested arguments dict (arguments to pass to the playbooks)
"""
spec_defaults = self.get_spec_defaults()
cli_args = CliParser.parse_cli_input(arg_parser, args)
file_args = self.get_answers_file_args(cli_args)
# generate answers file and exit
if self.generate_answers_file(cli_args, spec_defaults):
LOG.warning("Answers file generated. Exiting.")
# print warnings when something was overridden from non-cli source.
self.validate_arg_sources(cli_args, file_args,
spec_defaults)
# print warnings for deprecated
self.validate_arg_deprecation(cli_args, file_args)
# now filter defaults to have only parser defined in cli
defaults = dict((key, spec_defaults[key])
for key in cli_args.keys() if
key in spec_defaults)
# copy cli args with the same name to all parser groups
self._merge_duplicated_cli_args(cli_args)
self._merge_duplicated_cli_args(file_args)
dict_utils.dict_merge(defaults, file_args)
dict_utils.dict_merge(defaults, cli_args)
self.validate_requires_args(defaults)
self.validate_length_args(defaults)
self.validate_choices_args(defaults)
self.validate_min_max_args(defaults)
# now resolve complex types.
self.resolve_custom_types(defaults)
nested, control, custom = \
self.get_nested_custom_and_control_args(defaults)
return nested, control, custom
def parse_args(self, arg_parser, args=None):
"""Parses all the arguments (cli, answers file)
:return: None, if ``--generate-answers-file`` in arg_arg_parser
:return: (dict, dict):
* command arguments dict (arguments to control the IR logic)
* nested arguments dict (arguments to pass to the playbooks)
"""
spec_defaults = self.get_spec_defaults()
cli_args = CliParser.parse_cli_input(arg_parser, args)
file_args = self.get_answers_file_args(cli_args)
# generate answers file and exit
if self.generate_answers_file(cli_args, spec_defaults):
LOG.warning("Answers file generated. Exiting.")
# print warnings when something was overridden from non-cli source.
self.validate_arg_sources(cli_args, file_args,
spec_defaults)
# print warnings for deprecated
self.validate_arg_deprecation(cli_args, file_args)
# now filter defaults to have only parser defined in cli
defaults = dict((key, spec_defaults[key])
for key in cli_args.keys() if
key in spec_defaults)
# copy cli args with the same name to all parser groups
self._merge_duplicated_cli_args(cli_args)
self._merge_duplicated_cli_args(file_args)
dict_utils.dict_merge(defaults, file_args)
dict_utils.dict_merge(defaults, cli_args)
self.validate_requires_args(defaults)
self.validate_length_args(defaults)
self.validate_choices_args(defaults)
self.validate_min_max_args(defaults)
# now resolve complex types.
self.resolve_custom_types(defaults)
nested, control = self.get_nested_and_control_args(defaults)
return nested, control
def get_answers_file_args(self, cli_args):
"""Resolve arguments' values from answers INI file. """
file_result = {}
custom_file_result = {}
args_to_remove = []
for (parser_name, parser_dict, arg_name, arg_value,
option_spec) in self._iterate_received_arguments(cli_args):
file_result[parser_name] = file_result.get(parser_name, {})
if option_spec and option_spec.get('action', '') == 'read-answers':
# we have config option. saving it.
self._convert_non_cli_args(parser_name, parser_dict[arg_name])
dict_utils.dict_merge(file_result[parser_name],
parser_dict[arg_name])
# remove from cli args
args_to_remove.append((parser_name, arg_name))
# Attempt to iterate over plugin arguments to determine if they should be treated as custom
try:
for plugin_parser, plugin_option in self.spec_helper.iterate_option_specs(
):
for plugin_arg in arg_value:
if plugin_arg == plugin_option['name']:
if plugin_option.get('ansible_variable'):
custom_arg = plugin_option.get(
'ansible_variable')
custom_file_result[custom_arg] = arg_value[
plugin_arg]
# remove custom arg from file_result
del file_result[parser_name][plugin_arg]
except TypeError:
pass
# remove parser dict outside loop to avoid iteration dict modification
for parser_name, arg_name in args_to_remove:
for spec_parser in self.spec_helper.iterate_parsers():
if spec_parser['name'] in cli_args and spec_parser[
'name'] == parser_name:
parser_dict = cli_args[spec_parser['name']]
parser_dict.pop(arg_name)
break
return file_result, custom_file_result
def get_answers_file_args(self, cli_args):
"""Resolve arguments' values from answers INI file. """
file_result = {}
for (parser_name, parser_dict, arg_name, arg_value,
option_spec) in self._iterate_received_arguments(cli_args):
file_result[parser_name] = file_result.get(parser_name, {})
if option_spec and option_spec.get(
'action', '') == 'read-answers':
# we have config option. saving it.
self._convert_non_cli_args(
parser_name, parser_dict[arg_name])
dict_utils.dict_merge(
file_result[parser_name],
parser_dict[arg_name])
# remove from cli args
parser_dict.pop(arg_name)
return file_result
def validate_requires_args(self, args, custom_parsed_cli_args):
"""Check if all the required arguments have been provided. """
silent_args = self.get_silent_args(args)
def validate_parser(parser_name, expected_options, parser_args):
"""Helper method to resolve dict_merge. """
result = collections.defaultdict(list)
condition_req_args = self._get_conditionally_required_args(
parser_name, expected_options, args)
for option in expected_options:
name = option['name']
# check required options.
if (option.get('required', False) and
name not in parser_args or
option['name'] in condition_req_args) and \
name not in silent_args:
if option.get('ansible_variable') and \
option['ansible_variable'] in custom_parsed_cli_args:
continue
result[parser_name].append(name)
return result
res = {}
for command_data in self.spec_helper.iterate_parsers():
cmd_name = command_data['name']
if cmd_name in args:
dict_utils.dict_merge(
res,
validate_parser(
cmd_name,
self.spec_helper.get_parser_option_specs(cmd_name),
args[cmd_name]))
missing_args = dict((cmd_name, args) for cmd_name, args in res.items()
if len(args) > 0)
if missing_args:
raise exceptions.IRRequiredArgsMissingException(missing_args)
def test_dict_merge():
from infrared.core.utils.dict_utils import dict_merge
first_dict = {
'a': 1,
'b': 2,
'c': {
'd': 'foo1',
'e': 'bar',
'list1': ['a', 'b', 'c']
},
'list2': [1, 2, 3]
}
second_dict = {
'a': 2,
'c': {
'd': 'foo2',
'f': 5,
'list1': [3, 4, 5]
},
'g': 'bla',
5: 'yy',
'list3': ['a', 2]
}
expected_result = {
'a': 2,
'b': 2,
'c': {
'd': 'foo2',
'e': 'bar',
'f': 5,
'list1': [3, 4, 5]
},
'g': 'bla',
5: 'yy',
'list2': [1, 2, 3],
'list3': ['a', 2]
}
dict_merge(first_dict, second_dict)
assert not cmp(first_dict, expected_result)
def get_answers_file_args(self, cli_args):
"""Resolve arguments' values from answers INI file. """
file_result = {}
args_to_remove = []
for (parser_name, parser_dict, arg_name, arg_value,
option_spec) in self._iterate_received_arguments(cli_args):
file_result[parser_name] = file_result.get(parser_name, {})
if option_spec and option_spec.get(
'action', '') == 'read-answers':
# Iterate over arguments supplied by file
for parsed_arg in parser_dict[arg_name]:
# Supplied arguments' value can be a list
if isinstance(parser_dict[arg_name][parsed_arg], list):
i = 0
# Iterrate over argument values list
for parsed_value in parser_dict[arg_name][parsed_arg]:
parser_dict[arg_name][parsed_arg][i] = \
SpecParser.parse_env_variable_from_file(parsed_value)
i += 1
else:
parser_dict[arg_name][parsed_arg] = \
SpecParser.parse_env_variable_from_file(parser_dict[arg_name][parsed_arg])
# we have config option. saving it.
self._convert_non_cli_args(
parser_name, parser_dict[arg_name])
dict_utils.dict_merge(
file_result[parser_name],
parser_dict[arg_name])
# remove from cli args
args_to_remove.append((parser_name, arg_name))
# remove parser dict outside loop to avoid iteration dict modification
for parser_name, arg_name in args_to_remove:
for spec_parser in self.spec_helper.iterate_parsers():
if spec_parser['name'] in cli_args and spec_parser['name'] == parser_name:
parser_dict = cli_args[spec_parser['name']]
parser_dict.pop(arg_name)
break
return file_result
def validate_requires_args(self, args):
"""Check if all the required arguments have been provided. """
silent_args = self.get_silent_args(args)
def validate_parser(parser_name, expected_options, parser_args):
"""Helper method to resolve dict_merge. """
result = collections.defaultdict(list)
condition_req_args = self._get_conditionally_required_args(
parser_name, expected_options, args)
for option in expected_options:
name = option['name']
# check required options.
if (option.get('required', False) and
name not in parser_args or
option['name'] in condition_req_args) and \
name not in silent_args:
result[parser_name].append(name)
return result
res = {}
for command_data in self.spec_helper.iterate_parsers():
cmd_name = command_data['name']
if cmd_name in args:
dict_utils.dict_merge(
res,
validate_parser(
cmd_name,
self.spec_helper.get_parser_option_specs(cmd_name),
args[cmd_name]))
missing_args = dict((cmd_name, args)
for cmd_name, args in res.items() if len(args) > 0)
if missing_args:
raise exceptions.IRRequiredArgsMissingException(missing_args)
def merge_extra_vars(vars_dict, extra_vars=None):
"""Extend ``vars_dict`` with ``extra-vars``
:param vars_dict: Dictionary to merge extra-vars into
:param extra_vars: List of extra-vars
"""
for extra_var in extra_vars or []:
if extra_var.startswith('@'):
with open(extra_var[1:]) as f_obj:
loaded_yml = yaml.load(f_obj)
dict_utils.dict_merge(
vars_dict,
loaded_yml,
conflict_resolver=dict_utils.ConflictResolver.
unique_append_list_resolver)
else:
if '=' not in extra_var:
raise exceptions.IRExtraVarsException(extra_var)
key, value = extra_var.split("=")
dict_utils.dict_insert(vars_dict, value, *key.split("."))
def parse_args(self, arg_parser, args=None):
"""Parses all the arguments (cli, answers file)
:return: None, if ``--generate-answers-file`` in arg_arg_parser
:return: (dict, dict):
* command arguments dict (arguments to control the IR logic)
* nested arguments dict (arguments to pass to the playbooks)
* custom_args custom arguments dict (arguments with custom ansible variables)
"""
spec_defaults, custom_spec_defaults = self.get_spec_defaults()
cli_args, custom_cli_args = CliParser.parse_cli_input(arg_parser, args)
helper_custom_cli_args = dict(custom_cli_args)
# parse custom ansible variables and their values
custom_parsed_cli_args = {}
for custom_parse in custom_cli_args.values():
custom_parsed_cli_args.update(custom_parse)
file_args, custom_file_args = self.get_answers_file_args(cli_args)
# generate answers file and exit
if self.generate_answers_file(cli_args, custom_cli_args,
spec_defaults):
LOG.warning("Answers file generated. Exiting.")
# print warnings when something was overridden from non-cli source.
self.validate_arg_sources(cli_args, file_args, spec_defaults)
# print warnings for deprecated
self.validate_arg_deprecation(cli_args, file_args)
# now filter defaults to have only parser defined in cli
defaults = dict((key, spec_defaults[key]) for key in cli_args.keys()
if key in spec_defaults)
# now filter custom ansible defaults to have values defined:
for custom_default in custom_spec_defaults:
if custom_default not in custom_parsed_cli_args:
custom_parsed_cli_args[custom_default] = custom_spec_defaults[
custom_default]
# copy cli args with the same name to all parser groups
self._merge_duplicated_cli_args(cli_args)
self._merge_duplicated_cli_args(file_args)
dict_utils.dict_merge(defaults, file_args)
# combine custom cli commands with custom file parsing if needed
if custom_file_args:
custom_file_args.update(custom_parsed_cli_args)
custom_parsed_cli_args = custom_file_args
dict_utils.dict_merge(defaults, cli_args)
self.validate_requires_args(defaults, custom_parsed_cli_args)
self.validate_length_args(defaults)
self.validate_choices_args(defaults)
self.validate_min_max_args(defaults)
# now resolve complex types.
self.resolve_custom_types(defaults, custom_cli_args, custom_file_args)
nested, control = self.get_nested_and_control_args(defaults)
# populate custom cli args with resolved complextypes when parsing from CLI
if helper_custom_cli_args:
for helper_arg in helper_custom_cli_args:
resolved_custom_value = custom_cli_args[helper_arg]
for custom_variable in helper_custom_cli_args[helper_arg]:
custom_parsed_cli_args[
custom_variable] = resolved_custom_value
return nested, control, custom_parsed_cli_args
def test_dict_merge_none_resolver(first, second, expected):
from infrared.core.utils.dict_utils import dict_merge, ConflictResolver
dict_merge(first, second, conflict_resolver=ConflictResolver.none_resolver)
assert not cmp(first, expected)
def test_dict_merge_none_resolver(first, second, expected):
from infrared.core.utils.dict_utils import dict_merge, ConflictResolver
dict_merge(first, second, conflict_resolver=ConflictResolver.none_resolver)
assert first == expected
def test_dict_merge_none_resolver(first, second, expected):
from infrared.core.utils.dict_utils import dict_merge, ConflictResolver
first, second, expected = read_yaml(first), read_yaml(second), read_yaml(expected)
dict_merge(first, second, conflict_resolver=ConflictResolver.greedy_resolver)
assert first == expected