def _validate_task_handler_action_for_role(th_action: dict) -> None:
"""Verify that the task handler action is valid for role include."""
module = th_action['__ansible_module__']
if 'name' not in th_action:
raise MatchError(message=f"Failed to find required 'name' key in {module!s}")
if not isinstance(th_action['name'], str):
raise MatchError(
message=f"Value assigned to 'name' key on '{module!s}' is not a string.",
)
def create_matcherror(self,
message: Optional[str] = None,
linenumber: int = 0,
details: str = "",
filename: Optional[Union[str, Lintable]] = None,
tag: str = "") -> MatchError:
match = MatchError(message=message,
linenumber=linenumber,
details=details,
filename=filename,
rule=copy.copy(self))
if tag:
match.tag = tag
return match
def test_matcherror_compare_with_dummy_sentinel(operation, expected_value):
"""Check that MatchError comparison runs other types fallbacks."""
dummy_obj = DummySentinelTestObject()
# NOTE: This assertion abuses the CPython property to cache short string
# NOTE: objects because the identity check is more presice and we don't
# NOTE: want extra operator protocol methods to influence the test.
assert operation(MatchError("foo"), dummy_obj) is expected_value
def _taskshandlers_children(basedir, k, v, parent_type):
results = []
for th in v:
if 'include' in th:
append_children(th['include'], basedir, k, parent_type, results)
elif 'include_tasks' in th:
append_children(th['include_tasks'], basedir, k, parent_type, results)
elif 'import_playbook' in th:
append_children(th['import_playbook'], basedir, k, parent_type, results)
elif 'import_tasks' in th:
append_children(th['import_tasks'], basedir, k, parent_type, results)
elif 'include_role' in th or 'import_role' in th:
th = normalize_task_v2(th)
module = th['action']['__ansible_module__']
if "name" not in th['action']:
raise MatchError(
"Failed to find required 'name' key in %s" % module)
if not isinstance(th['action']["name"], str):
raise RuntimeError(
"Value assigned to 'name' key on '%s' is not a string." %
module)
results.extend(_roles_children(basedir, k, [th['action'].get("name")],
parent_type,
main=th['action'].get('tasks_from', 'main')))
elif 'block' in th:
results.extend(_taskshandlers_children(basedir, k, th['block'], parent_type))
if 'rescue' in th:
results.extend(_taskshandlers_children(basedir, k, th['rescue'], parent_type))
if 'always' in th:
results.extend(_taskshandlers_children(basedir, k, th['always'], parent_type))
return results
def matchlines(self, file, text) -> List[MatchError]:
matches: List[MatchError] = []
if not self.match:
return matches
# arrays are 0-based, line numbers are 1-based
# so use prev_line_no as the counter
for (prev_line_no, line) in enumerate(text.split("\n")):
if line.lstrip().startswith('#'):
continue
rule_id_list = get_rule_skips_from_line(line)
if self.id in rule_id_list:
continue
result = self.match(file, line)
if not result:
continue
message = None
if isinstance(result, str):
message = result
m = MatchError(message=message,
linenumber=prev_line_no + 1,
details=line,
filename=file['path'],
rule=self)
matches.append(m)
return matches
def test_matcherror_invalid():
"""Ensure that MatchError requires message or rule."""
expected_err = (
r"^MatchError\(\) missing a required argument: one of 'message' or 'rule'$"
)
with pytest.raises(TypeError, match=expected_err):
MatchError()
def test_matcherror_compare_with_other_fallback(
other,
operation,
expected_value,
):
"""Check that MatchError comparison runs other types fallbacks."""
assert operation(MatchError("foo"), other) is expected_value
def test_format_coloured_string(self):
match = MatchError(message="message",
linenumber=1,
details="hello",
filename="filename.yml",
rule=self.rule)
self.formatter.format(match)
def run(
self, file: Lintable, tags: Set[str] = set(), skip_list: List[str] = []
) -> List[MatchError]:
matches: List[MatchError] = list()
if not file.path.is_dir():
try:
if file.content is not None: # loads the file content
pass
except IOError as e:
return [
MatchError(
message=str(e),
filename=file,
rule=LoadingFailureRule(),
tag=e.__class__.__name__.lower(),
)
]
for rule in self.rules:
if (
not tags
or rule.has_dynamic_tags
or not set(rule.tags).union([rule.id]).isdisjoint(tags)
):
rule_definition = set(rule.tags)
rule_definition.add(rule.id)
if set(rule_definition).isdisjoint(skip_list):
matches.extend(rule.getmatches(file))
# some rules can produce matches with tags that are inside our
# skip_list, so we need to cleanse the matches
matches = [m for m in matches if m.tag not in skip_list]
return matches
def test_matcherror_compare_with_other_fallback(
other: object,
operation: Callable[..., bool],
expected_value: bool,
) -> None:
"""Check that MatchError comparison runs other types fallbacks."""
assert operation(MatchError("foo"), other) is expected_value
def test_unicode_format_string(self):
match = MatchError(message=u'\U0001f427',
linenumber=1,
details="hello",
filename="filename.yml",
rule=self.rule)
self.formatter.format(match)
def matchyaml(self, file: Lintable) -> List[MatchError]:
matches: List[MatchError] = []
if not self.matchplay or str(file.base_kind) != 'text/yaml':
return matches
yaml = ansiblelint.utils.parse_yaml_linenumbers(file)
# yaml returned can be an AnsibleUnicode (a string) when the yaml
# file contains a single string. YAML spec allows this but we consider
# this an fatal error.
if isinstance(yaml, str):
if yaml.startswith('$ANSIBLE_VAULT'):
return []
return [MatchError(filename=str(file.path), rule=LoadingFailureRule())]
if not yaml:
return matches
if isinstance(yaml, dict):
yaml = [yaml]
yaml = ansiblelint.skip_utils.append_skipped_rules(yaml, file)
for play in yaml:
# Bug #849
if play is None:
continue
if self.id in play.get('skipped_rules', ()):
continue
matches.extend(self.matchplay(file, play))
return matches
def normalize_task_v2(task: dict) -> dict: # noqa: C901
"""Ensure tasks have an action key and strings are converted to python objects."""
result = dict()
if 'always_run' in task:
# FIXME(ssbarnea): Delayed import to avoid circular import
# See https://github.com/ansible/ansible-lint/issues/880
# noqa: # pylint:disable=cyclic-import,import-outside-toplevel
from ansiblelint.rules.AlwaysRunRule import AlwaysRunRule
raise MatchError(rule=AlwaysRunRule,
filename=task[FILENAME_KEY],
linenumber=task[LINE_NUMBER_KEY])
sanitized_task = _sanitize_task(task)
mod_arg_parser = ModuleArgsParser(sanitized_task)
try:
action, arguments, result['delegate_to'] = mod_arg_parser.parse()
except AnsibleParserError as e:
try:
task_info = "%s:%s" % (task[FILENAME_KEY], task[LINE_NUMBER_KEY])
except KeyError:
task_info = "Unknown"
pp = pprint.PrettyPrinter(indent=2)
task_pprint = pp.pformat(sanitized_task)
_logger.critical("Couldn't parse task at %s (%s)\n%s", task_info,
e.message, task_pprint)
raise SystemExit(ANSIBLE_FAILURE_RC)
# denormalize shell -> command conversion
if '_uses_shell' in arguments:
action = 'shell'
del arguments['_uses_shell']
for (k, v) in list(task.items()):
if k in ('action', 'local_action', 'args',
'delegate_to') or k == action:
# we don't want to re-assign these values, which were
# determined by the ModuleArgsParser() above
continue
else:
result[k] = v
result['action'] = dict(__ansible_module__=action)
if '_raw_params' in arguments:
result['action']['__ansible_arguments__'] = arguments[
'_raw_params'].split(' ')
del arguments['_raw_params']
else:
result['action']['__ansible_arguments__'] = list()
if 'argv' in arguments and not result['action']['__ansible_arguments__']:
result['action']['__ansible_arguments__'] = arguments['argv']
del arguments['argv']
result['action'].update(arguments)
return result
def _get_ansible_syntax_check_matches(lintable: Lintable) -> List[MatchError]:
"""Run ansible syntax check and return a list of MatchError(s)."""
if lintable.kind != 'playbook':
return []
with timed_info("Executing syntax check on %s", lintable.path):
cmd = ['ansible-playbook', '--syntax-check', str(lintable.path)]
run = subprocess.run(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False, # needed when command is a list
universal_newlines=True,
check=False
)
result = []
if run.returncode != 0:
message = None
filename = str(lintable.path)
linenumber = 0
column = None
stderr = strip_ansi_escape(run.stderr)
stdout = strip_ansi_escape(run.stdout)
if stderr:
details = stderr
if stdout:
details += "\n" + stdout
else:
details = stdout
m = _ansible_syntax_check_re.search(stderr)
if m:
message = m.groupdict()['title']
# Ansible returns absolute paths
filename = m.groupdict()['filename']
linenumber = int(m.groupdict()['line'])
column = int(m.groupdict()['column'])
if run.returncode == 4:
rule: BaseRule = AnsibleSyntaxCheckRule()
else:
rule = RuntimeErrorRule()
if not message:
message = (
f"Unexpected error code {run.returncode} from "
f"execution of: {' '.join(cmd)}")
result.append(MatchError(
message=message,
filename=filename,
linenumber=linenumber,
column=column,
rule=rule,
details=details
))
return result
def test_dict_format_line(self) -> None:
match = MatchError(
message="xyz",
linenumber=1,
details={'hello': 'world'}, # type: ignore
filename="filename.yml",
rule=self.rule,
)
self.formatter.format(match)
def matchlines(self, file, text):
matches = []
# arrays are 0-based, line numbers are 1-based
# so use prev_line_no as the counter
for (prev_line_no, line) in enumerate(text.split("\n")):
if issuspicious(line):
matche = MatchError(line, prev_line_no + 1, self.shortdesc, file['path'], self)
matches.append(matche)
return matches
def test_dict_format_line(self):
match = MatchError(
"xyz",
1,
{'hello': 'world'},
"filename.yml",
self.rule,
)
self.formatter.format(match, True)
def test_matcherror_compare_no_other_fallback(other, operation, operator_char):
"""Check that MatchError comparison with other types causes TypeError."""
expected_error = (r'^('
r'unsupported operand type\(s\) for {operator!s}:|'
r"'{operator!s}' not supported between instances of"
r") 'MatchError' and '{other_type!s}'$".format(
other_type=type(other).__name__,
operator=operator_char))
with pytest.raises(TypeError, match=expected_error):
operation(MatchError("foo"), other)
def _taskshandlers_children(
basedir: str, k: str, v: Union[None, Any], parent_type: FileType
) -> List[Lintable]:
results: List[Lintable] = []
if v is None:
raise MatchError(
message="A malformed block was encountered while loading a block.",
rule=RuntimeErrorRule(),
)
for th in v:
# ignore empty tasks, `-`
if not th:
continue
with contextlib.suppress(LookupError):
children = _get_task_handler_children_for_tasks_or_playbooks(
th,
basedir,
k,
parent_type,
)
results.append(children)
continue
if (
'include_role' in th or 'import_role' in th
): # lgtm [py/unreachable-statement]
th = normalize_task_v2(th)
_validate_task_handler_action_for_role(th['action'])
results.extend(
_roles_children(
basedir,
k,
[th['action'].get("name")],
parent_type,
main=th['action'].get('tasks_from', 'main'),
)
)
continue
if 'block' not in th:
continue
results.extend(_taskshandlers_children(basedir, k, th['block'], parent_type))
if 'rescue' in th:
results.extend(
_taskshandlers_children(basedir, k, th['rescue'], parent_type)
)
if 'always' in th:
results.extend(
_taskshandlers_children(basedir, k, th['always'], parent_type)
)
return results
def normalize_task_v2(task: Dict[str, Any]) -> Dict[str, Any]:
"""Ensure tasks have a normalized action key and strings are converted to python objects."""
result = dict()
sanitized_task = _sanitize_task(task)
mod_arg_parser = ModuleArgsParser(sanitized_task)
try:
action, arguments, result['delegate_to'] = mod_arg_parser.parse(
skip_action_validation=options.skip_action_validation)
except AnsibleParserError as e:
raise MatchError(
rule=AnsibleParserErrorRule(),
message=e.message,
filename=task.get(FILENAME_KEY, "Unknown"),
linenumber=task.get(LINE_NUMBER_KEY, 0),
)
# denormalize shell -> command conversion
if '_uses_shell' in arguments:
action = 'shell'
del arguments['_uses_shell']
for (k, v) in list(task.items()):
if k in ('action', 'local_action', 'args',
'delegate_to') or k == action:
# we don't want to re-assign these values, which were
# determined by the ModuleArgsParser() above
continue
result[k] = v
if not isinstance(action, str):
raise RuntimeError("Task actions can only be strings, got %s" % action)
action_unnormalized = action
# convert builtin fqn calls to short forms because most rules know only
# about short calls but in the future we may switch the normalization to do
# the opposite. Mainly we currently consider normalized the module listing
# used by `ansible-doc -t module -l 2>/dev/null`
action = removeprefix(action, "ansible.builtin.")
result['action'] = dict(__ansible_module__=action,
__ansible_module_original__=action_unnormalized)
if '_raw_params' in arguments:
result['action']['__ansible_arguments__'] = arguments[
'_raw_params'].split(' ')
del arguments['_raw_params']
else:
result['action']['__ansible_arguments__'] = list()
if 'argv' in arguments and not result['action']['__ansible_arguments__']:
result['action']['__ansible_arguments__'] = arguments['argv']
del arguments['argv']
result['action'].update(arguments)
return result
def create_matcherror(
self,
message: str = None,
linenumber: int = 0,
details: str = "",
filename: str = None) -> MatchError:
return MatchError(
message=message,
linenumber=linenumber,
details=details,
filename=filename,
rule=self
)
def matchyaml(self, file: Lintable) -> List[MatchError]:
matches: List[MatchError] = []
if not self.matchplay:
return matches
yaml = ansiblelint.utils.parse_yaml_linenumbers(
file.content, file.path)
# yaml returned can be an AnsibleUnicode (a string) when the yaml
# file contains a single string. YAML spec allows this but we consider
# this an fatal error.
if isinstance(yaml, str):
return [MatchError(filename=file.path, rule=LoadingFailureRule())]
if not yaml:
return matches
if isinstance(yaml, dict):
yaml = [yaml]
yaml = ansiblelint.skip_utils.append_skipped_rules(
yaml, file.content, file.kind)
for play in yaml:
# Bug #849
if play is None:
continue
if self.id in play.get('skipped_rules', ()):
continue
result = self.matchplay(file, play)
if not result:
continue
if isinstance(result, tuple):
result = [result]
if not isinstance(result, list):
raise TypeError("{} is not a list".format(result))
for section, message, *optional_linenumber in result:
linenumber = self._matchplay_linenumber(
play, optional_linenumber)
matches.append(
self.create_matcherror(message=message,
linenumber=linenumber,
details=str(section),
filename=file.path))
return matches
def setup_class(self) -> None:
"""Set up few MatchError objects."""
self.rule = AnsibleLintRule()
self.rule.id = "TCF0001"
self.rule.severity = "VERY_HIGH"
self.matches = []
self.matches.append(
MatchError(
message="message",
linenumber=1,
details="hello",
filename="filename.yml",
rule=self.rule,
))
self.matches.append(
MatchError(
message="message",
linenumber=2,
details="hello",
filename="filename.yml",
rule=self.rule,
))
self.formatter = CodeclimateJSONFormatter(pathlib.Path.cwd(),
display_relative_path=True)
def normalize_task_v2(task: dict) -> dict: # noqa: C901
"""Ensure tasks have an action key and strings are converted to python objects."""
result = dict()
sanitized_task = _sanitize_task(task)
mod_arg_parser = ModuleArgsParser(sanitized_task)
try:
action, arguments, result['delegate_to'] = mod_arg_parser.parse()
except AnsibleParserError as e:
raise MatchError(
rule=AnsibleParserErrorRule(),
message=e.message,
filename=task.get(FILENAME_KEY, "Unknown"),
linenumber=task.get(LINE_NUMBER_KEY, 0),
)
# denormalize shell -> command conversion
if '_uses_shell' in arguments:
action = 'shell'
del arguments['_uses_shell']
for (k, v) in list(task.items()):
if k in ('action', 'local_action', 'args',
'delegate_to') or k == action:
# we don't want to re-assign these values, which were
# determined by the ModuleArgsParser() above
continue
else:
result[k] = v
result['action'] = dict(__ansible_module__=action)
if '_raw_params' in arguments:
result['action']['__ansible_arguments__'] = arguments[
'_raw_params'].split(' ')
del arguments['_raw_params']
else:
result['action']['__ansible_arguments__'] = list()
if 'argv' in arguments and not result['action']['__ansible_arguments__']:
result['action']['__ansible_arguments__'] = arguments['argv']
del arguments['argv']
result['action'].update(arguments)
return result
def _emit_matches(self, files: List[Lintable]) -> Generator[MatchError, None, None]:
visited: Set[Lintable] = set()
while visited != self.lintables:
for lintable in self.lintables - visited:
try:
for child in ansiblelint.utils.find_children(lintable):
if self.is_excluded(str(child.path)):
continue
self.lintables.add(child)
files.append(child)
except MatchError as e:
e.rule = LoadingFailureRule()
yield e
except AttributeError:
yield MatchError(
filename=str(lintable.path), rule=LoadingFailureRule()
)
visited.add(lintable)
def find_children(lintable: Lintable) -> List[Lintable]: # noqa: C901
if not lintable.path.exists():
return []
playbook_dir = str(lintable.path.parent)
_set_collections_basedir(playbook_dir or os.path.abspath('.'))
add_all_plugin_dirs(playbook_dir or '.')
if lintable.kind == 'role':
playbook_ds = AnsibleMapping({'roles': [{'role': str(lintable.path)}]})
elif lintable.kind not in ("playbook", "tasks"):
return []
else:
try:
playbook_ds = parse_yaml_from_file(str(lintable.path))
except AnsibleError as e:
raise SystemExit(str(e))
results = []
basedir = os.path.dirname(str(lintable.path))
# playbook_ds can be an AnsibleUnicode string, which we consider invalid
if isinstance(playbook_ds, str):
raise MatchError(filename=str(lintable.path),
rule=LoadingFailureRule())
for item in _playbook_items(playbook_ds):
# if lintable.kind not in ["playbook"]:
# continue
for child in play_children(basedir, item, lintable.kind, playbook_dir):
# We avoid processing parametrized children
path_str = str(child.path)
if "$" in path_str or "{{" in path_str:
continue
# Repair incorrect paths obtained when old syntax was used, like:
# - include: simpletask.yml tags=nginx
valid_tokens = list()
for token in split_args(path_str):
if '=' in token:
break
valid_tokens.append(token)
path = ' '.join(valid_tokens)
if path != path_str:
child.path = Path(path)
child.name = child.path.name
results.append(child)
return results
def run(self, playbookfile, tags=set(), skip_list=frozenset()) -> List:
text = ""
matches: List = list()
error: Optional[IOError] = None
for i in range(3):
try:
with open(playbookfile['path'], mode='r',
encoding='utf-8') as f:
text = f.read()
break
except IOError as e:
_logger.warning("Couldn't open %s - %s [try:%s]",
playbookfile['path'], e.strerror, i)
error = e
sleep(1)
continue
else:
return [
MatchError(message=str(error),
filename=playbookfile['path'],
rule=LoadingFailureRule())
]
for rule in self.rules:
if not tags or not set(rule.tags).union([rule.id
]).isdisjoint(tags):
rule_definition = set(rule.tags)
rule_definition.add(rule.id)
if set(rule_definition).isdisjoint(skip_list):
matches.extend(rule.matchlines(playbookfile, text))
matches.extend(rule.matchtasks(playbookfile, text))
matches.extend(
rule.matchyaml(
Lintable(playbookfile['path'],
content=text,
kind=playbookfile['type'])))
# some rules can produce matches with tags that are inside our
# skip_list, so we need to cleanse the matches
matches = [m for m in matches if m.tag not in skip_list]
return matches
def matchtasks(self, file: str,
text: str) -> List[MatchError]: # noqa: C901
matches: List[MatchError] = []
if not self.matchtask:
return matches
if file['type'] == 'meta':
return matches
yaml = ansiblelint.utils.parse_yaml_linenumbers(text, file['path'])
if not yaml:
return matches
yaml = append_skipped_rules(yaml, text, file['type'])
try:
tasks = ansiblelint.utils.get_normalized_tasks(yaml, file)
except MatchError as e:
return [e]
for task in tasks:
if self.id in task.get('skipped_rules', ()):
continue
if 'action' not in task:
continue
result = self.matchtask(file, task)
if not result:
continue
message = None
if isinstance(result, str):
message = result
task_msg = "Task/Handler: " + ansiblelint.utils.task_to_str(task)
m = MatchError(message=message,
linenumber=task[ansiblelint.utils.LINE_NUMBER_KEY],
details=task_msg,
filename=file['path'],
rule=self)
matches.append(m)
return matches
def find_children(playbook: Tuple[str, str],
playbook_dir: str) -> List[Lintable]:
if not os.path.exists(playbook[0]):
return []
_set_collections_basedir(playbook_dir or os.path.abspath('.'))
add_all_plugin_dirs(playbook_dir or '.')
if playbook[1] == 'role':
playbook_ds = {'roles': [{'role': playbook[0]}]}
else:
try:
playbook_ds = parse_yaml_from_file(playbook[0])
except AnsibleError as e:
raise SystemExit(str(e))
results = []
basedir = os.path.dirname(playbook[0])
# playbook_ds can be an AnsibleUnicode string, which we consider invalid
if isinstance(playbook_ds, str):
raise MatchError(filename=playbook[0], rule=LoadingFailureRule)
items = _playbook_items(playbook_ds)
for item in items:
for child in play_children(basedir, item, playbook[1], playbook_dir):
# We avoid processing parametrized children
path_str = str(child.path)
if "$" in path_str or "{{" in path_str:
continue
# Repair incorrect paths obtained when old syntax was used, like:
# - include: simpletask.yml tags=nginx
valid_tokens = list()
for token in split_args(path_str):
if '=' in token:
break
valid_tokens.append(token)
path = ' '.join(valid_tokens)
if path != path_str:
child.path = Path(path)
child.name = child.path.name
results.append(child)
return results
def matchyaml(self, file: str, text: str) -> List[MatchError]:
matches: List[MatchError] = []
if not self.matchplay:
return matches
yaml = ansiblelint.utils.parse_yaml_linenumbers(text, file['path'])
if not yaml:
return matches
if isinstance(yaml, dict):
yaml = [yaml]
yaml = ansiblelint.skip_utils.append_skipped_rules(
yaml, text, file['type'])
for play in yaml:
if self.id in play.get('skipped_rules', ()):
continue
result = self.matchplay(file, play)
if not result:
continue
if isinstance(result, tuple):
result = [result]
if not isinstance(result, list):
raise TypeError("{} is not a list".format(result))
for section, message, *optional_linenumber in result:
linenumber = self._matchplay_linenumber(
play, optional_linenumber)
m = MatchError(message=message,
linenumber=linenumber,
details=str(section),
filename=file['path'],
rule=self)
matches.append(m)
return matches