def get_lintables(
options: Namespace = Namespace(), args: Optional[List[str]] = None
) -> List[Lintable]:
"""Detect files and directories that are lintable."""
lintables: List[Lintable] = []
# passing args bypass auto-detection mode
if args:
for arg in args:
lintable = Lintable(arg)
if lintable.kind in ("yaml", None):
_logger.warning(
"Overriding detected file kind '%s' with 'playbook' "
"for given positional argument: %s",
lintable.kind,
arg,
)
lintable = Lintable(arg, kind="playbook")
lintables.append(lintable)
else:
for filename in get_yaml_files(options):
p = Path(filename)
# skip exclusions
try:
for file_path in options.exclude_paths:
if str(p.resolve()).startswith(str(file_path)):
raise FileNotFoundError(
f'File {file_path} matched exclusion entry: {p}'
)
except FileNotFoundError as e:
_logger.debug('Ignored %s due to: %s', p, e)
continue
lintables.append(Lintable(p))
# stage 2: guess roles from current lintables, as there is no unique
# file that must be present in any kind of role.
for lintable in lintables:
parts = lintable.path.parent.parts
if 'roles' in parts:
role = lintable.path
while role.parent.name != "roles" and role.name:
role = role.parent
if role.exists:
lintable = Lintable(role, kind="role")
if lintable not in lintables:
_logger.debug("Added role: %s", lintable)
lintables.append(lintable)
return lintables
def pytest_collect_file(parent, path) -> Optional["Node"]:
"""Transform each found molecule.yml into a pytest test."""
# additional mappings not known by ansible-lint (yet)
extra_kinds = [
{"molecule": "**/molecule/*/molecule.yml"},
{"zuul": "**/zuul.d/*.{yaml,yml}"},
{"zuul": "**/.zuul.yaml"},
{"ansible-lint": "**/.ansiblelint"},
{"ansible-lint": "**/.config/ansiblelint.yml"},
{"json-schema": "f/*.json"},
]
# appending extra kinds at beginning of default ones
options.kinds = [*extra_kinds, *options.kinds]
if not path.fnmatch("*/examples/*") and not path.fnmatch("f/*.json"):
# We care only about f/ and examples/
return None
try:
lintable = Lintable(str(path))
except RuntimeError:
# ignore unknown file types
return None
if lintable.kind == 'json-schema':
return SchemaFile.from_parent(parent, fspath=path) # type: ignore
if not lintable.kind:
return None
return YamlFile.from_parent(parent, fspath=path) # type: ignore
def _get_task_handler_children_for_tasks_or_playbooks(
task_handler,
basedir: str,
k,
parent_type: FileType,
) -> Lintable:
"""Try to get children of taskhandler for include/import tasks/playbooks."""
child_type = k if parent_type == 'playbook' else parent_type
task_include_keys = 'include', 'include_tasks', 'import_playbook', 'import_tasks'
for task_handler_key in task_include_keys:
with contextlib.suppress(KeyError):
# ignore empty tasks
if not task_handler:
continue
return Lintable(
path_dwim(basedir, task_handler[task_handler_key]), kind=child_type
)
raise LookupError(
f'The node contains none of: {", ".join(task_include_keys)}',
)
def run(self) -> List[MatchError]:
"""Execute the linting process."""
files: List[Lintable] = list()
for playbook in self.playbooks:
if self.is_excluded(playbook[0]) or playbook[1] == 'role':
continue
files.append(
Lintable(ansiblelint.file_utils.normpath(playbook[0]),
kind=playbook[1])) # type: ignore
matches = set(self._emit_matches(files))
# remove duplicates from files list
files = [
value for n, value in enumerate(files) if value not in files[:n]
]
# remove files that have already been checked
files = [x for x in files if x['path'] not in self.checked_files]
for file in files:
_logger.debug("Examining %s of type %s",
ansiblelint.file_utils.normpath(file['path']),
file['type'])
matches = matches.union(
self.rules.run(file,
tags=set(self.tags),
skip_list=self.skip_list))
# update list of checked files
self.checked_files.update([x['path'] for x in files])
return sorted(matches)
def __init__(self,
rules: "RulesCollection",
lintable: Union[Lintable, str],
tags: FrozenSet[Any] = frozenset(),
skip_list: List[str] = [],
exclude_paths: List[str] = [],
verbosity: int = 0,
checked_files: Optional[Set[str]] = None) -> None:
"""Initialize a Runner instance."""
self.rules = rules
self.lintables: Set[Lintable] = set()
if isinstance(lintable, str):
lintable = Lintable(lintable)
self.lintables.add(lintable)
self.playbook_dir = lintable.dir
self.tags = tags
self.skip_list = skip_list
self._update_exclude_paths(exclude_paths)
self.verbosity = verbosity
if checked_files is None:
checked_files = set()
self.checked_files = checked_files
def __init__(self,
*lintables: Union[Lintable, str],
rules: "RulesCollection",
tags: FrozenSet[Any] = frozenset(),
skip_list: List[str] = [],
exclude_paths: List[str] = [],
verbosity: int = 0,
checked_files: Optional[Set[Lintable]] = None) -> None:
"""Initialize a Runner instance."""
self.rules = rules
self.lintables: Set[Lintable] = set()
# Assure consistent type
for item in lintables:
if not isinstance(item, Lintable):
item = Lintable(item)
self.lintables.add(item)
# Expand folders (roles) to their components
expand_dirs_in_lintables(self.lintables)
self.tags = tags
self.skip_list = skip_list
self._update_exclude_paths(exclude_paths)
self.verbosity = verbosity
if checked_files is None:
checked_files = set()
self.checked_files = checked_files
def lint_error_lines(test_playbook: str) -> List[int]:
"""Get VarHasSpacesRules linting results on test_playbook."""
collection = RulesCollection()
collection.register(VariableHasSpacesRule())
lintable = Lintable(test_playbook)
results = Runner(lintable, rules=collection).run()
return list(map(lambda item: item.linenumber, results))
def test_auto_detect(monkeypatch, path: str, kind: FileType) -> None:
"""Verify auto-detection logic."""
options = cli.get_config([])
def mockreturn(options):
return [path]
# assert Lintable is able to determine file type
lintable_detected = Lintable(path)
lintable_expected = Lintable(path, kind=kind)
assert lintable_detected == lintable_expected
monkeypatch.setattr(utils, 'get_yaml_files', mockreturn)
result = utils.get_lintables(options)
# get_lintable could return additional files and we only care to see
# that the given file is among the returned list.
assert lintable_expected in result
def test_included_tasks(default_rules_collection, filename, file_count,
match_count):
"""Check if number of loaded files is correct."""
lintable = Lintable(filename)
runner = Runner(default_rules_collection, lintable, [], [], [])
result = runner.run()
assert len(runner.playbooks) == file_count
assert len(result) == match_count
def test_206() -> None:
"""Verify rule."""
collection = RulesCollection()
collection.register(VariableHasSpacesRule())
lintable = Lintable("examples/playbooks/206.yml")
results = Runner(collection, lintable=lintable).run()
assert len(results) == 3
def test_runner_unicode_format(default_rules_collection, formatter_cls) -> None:
formatter = formatter_cls(os.getcwd(), display_relative_path=True)
runner = Runner(
rules=default_rules_collection,
lintable=Lintable('test/unicode.yml', "playbook"))
matches = runner.run()
formatter.format(matches[0])
def test_var_spacing() -> None:
"""Verify rule."""
collection = RulesCollection()
collection.register(VariableHasSpacesRule())
lintable = Lintable("examples/playbooks/var-spacing.yml")
results = Runner(lintable, rules=collection).run()
assert len(results) == 3
def test_auto_detect_exclude(monkeypatch: MonkeyPatch) -> None:
"""Verify that exclude option can be used to narrow down detection."""
options = cli.get_config(['--exclude', 'foo'])
def mockreturn(options: Namespace) -> List[str]:
return ['foo/playbook.yml', 'bar/playbook.yml']
monkeypatch.setattr(utils, 'discover_lintables', mockreturn)
result = utils.get_lintables(options)
assert result == [Lintable('bar/playbook.yml', kind='playbook')]
def test_default_kinds(monkeypatch: MonkeyPatch, path: str,
kind: FileType) -> None:
"""Verify auto-detection logic based on DEFAULT_KINDS."""
options = cli.get_config([])
def mockreturn(options: Namespace) -> Dict[str, Any]:
return {path: kind}
# assert Lintable is able to determine file type
lintable_detected = Lintable(path)
lintable_expected = Lintable(path, kind=kind)
assert lintable_detected == lintable_expected
monkeypatch.setattr(file_utils, 'discover_lintables', mockreturn)
result = file_utils.discover_lintables(options)
# get_lintable could return additional files and we only care to see
# that the given file is among the returned list.
assert lintable_detected.name in result
assert lintable_detected.kind == result[lintable_expected.name]
def test_auto_detect_exclude(monkeypatch):
"""Verify that exclude option can be used to narrow down detection."""
options = cli.get_config(['--exclude', 'foo'])
def mockreturn(options):
return ['foo/playbook.yml', 'bar/playbook.yml']
monkeypatch.setattr(utils, 'get_yaml_files', mockreturn)
result = utils.get_lintables(options)
assert result == [Lintable('bar/playbook.yml', kind='playbook')]
def test_auto_detect(monkeypatch, path: str, kind: FileType) -> None:
"""Verify auto-detection logic."""
options = cli.get_config([])
def mockreturn(options):
return [path]
monkeypatch.setattr(utils, 'get_yaml_files', mockreturn)
result = utils.get_lintables(options)
assert result == [Lintable(path, kind=kind)]
def test_get_ansible_syntax_check_matches() -> None:
"""Validate parsing of ansible output."""
lintable = Lintable('examples/playbooks/conflicting_action.yml', kind='playbook')
result = AnsibleSyntaxCheckRule._get_ansible_syntax_check_matches(lintable)
assert result[0].linenumber == 3
assert result[0].column == 7
assert result[0].message == "conflicting action statements: debug, command"
# We internaly convert absolute paths returned by ansible into paths
# relative to current directory.
assert result[0].filename.endswith("/conflicting_action.yml")
assert len(result) == 1
def test_extra_vars_passed_to_command(config_options) -> None:
"""Validate `extra-vars` are passed to syntax check command."""
config_options.extra_vars = {
'foo': 'bar',
'complex_variable': ':{;\t$()',
}
lintable = Lintable('examples/playbooks/extra_vars.yml', kind='playbook')
result = AnsibleSyntaxCheckRule._get_ansible_syntax_check_matches(lintable)
assert not result
def test_empty_playbook() -> None:
"""Validate detection of empty-playbook."""
lintable = Lintable('examples/playbooks/empty_playbook.yml', kind='playbook')
result = AnsibleSyntaxCheckRule._get_ansible_syntax_check_matches(lintable)
assert result[0].linenumber == 0
# We internally convert absolute paths returned by ansible into paths
# relative to current directory.
assert result[0].filename.endswith("/empty_playbook.yml")
assert result[0].tag == "empty-playbook"
assert result[0].message == "Empty playbook, nothing to do"
assert len(result) == 1
def _include_children(basedir: str, k, v, parent_type) -> List[Lintable]:
# handle special case include_tasks: name=filename.yml
if k == 'include_tasks' and isinstance(v, dict) and 'file' in v:
v = v['file']
# handle include: filename.yml tags=blah
(command, args, kwargs) = tokenize("{0}: {1}".format(k, v))
result = path_dwim(basedir, args[0])
if not os.path.exists(result):
result = path_dwim(os.path.join(os.path.dirname(basedir)), v)
return [Lintable(result, kind=parent_type)]
def test_included_tasks(
default_rules_collection: RulesCollection,
filename: str,
file_count: int,
match_count: int,
) -> None:
"""Check if number of loaded files is correct."""
lintable = Lintable(filename)
runner = Runner(lintable, rules=default_rules_collection)
result = runner.run()
assert len(runner.lintables) == file_count
assert len(result) == match_count
def _extend_with_roles(lintables: List[Lintable]) -> None:
"""Detect roles among lintables and adds them to the list."""
for lintable in lintables:
parts = lintable.path.parent.parts
if 'roles' in parts:
role = lintable.path
while role.parent.name != "roles" and role.name:
role = role.parent
if role.exists and not role.is_file():
lintable = Lintable(role, kind="role")
if lintable not in lintables:
_logger.debug("Added role: %s", lintable)
lintables.append(lintable)
def test_runner_unicode_format(
default_rules_collection: RulesCollection,
formatter_cls: Type[formatters.BaseFormatter[Any]],
) -> None:
formatter = formatter_cls(os.getcwd(), display_relative_path=True)
runner = Runner(
Lintable('examples/playbooks/unicode.yml', "playbook"),
rules=default_rules_collection,
)
matches = runner.run()
formatter.format(matches[0])
def test_rule_no_handler() -> None:
"""Verify rule."""
collection = RulesCollection()
collection.register(UseHandlerRatherThanWhenChangedRule())
lintable = Lintable('examples/playbooks/rule-no-handler.yml')
results = Runner(lintable, rules=collection).run()
assert len(results) == 3
assert results[0].filename == 'examples/playbooks/roles/a-role/tasks/main.yml'
assert results[0].linenumber == 3
assert results[1].filename == 'examples/playbooks/rule-no-handler.yml'
assert results[1].linenumber == 14
assert results[2].filename == 'examples/playbooks/rule-no-handler.yml'
assert results[2].linenumber == 18
def _emit_matches(
self, files: List[Lintable]) -> Generator[MatchError, None, None]:
visited: Set = set()
while visited != self.playbooks:
for arg in self.playbooks - visited:
try:
for child in ansiblelint.utils.find_children(
arg, self.playbook_dir):
if self.is_excluded(child['path']):
continue
self.playbooks.add((child['path'], child['type']))
files.append(
Lintable(child['path'], kind=child['type']))
except MatchError as e:
e.rule = LoadingFailureRule()
yield e
visited.add(arg)
def _look_for_role_files(basedir: str, role: str, main='main') -> List[Lintable]:
role_path = _rolepath(basedir, role)
if not role_path:
return []
results = []
for kind in ['tasks', 'meta', 'handlers']:
current_path = os.path.join(role_path, kind)
for folder, subdirs, files in os.walk(current_path):
for file in files:
file_ignorecase = file.lower()
if file_ignorecase.endswith(('.yml', '.yaml')):
thpath = os.path.join(folder, file)
# TODO(ssbarnea): Find correct way to pass kind: FileType
results.append(Lintable(thpath, kind=kind)) # type: ignore
return results
def run(self, playbookfile, tags=set(), skip_list=frozenset()) -> List:
text = ""
matches: List = list()
error: Optional[IOError] = None
for i in range(3):
try:
with open(playbookfile['path'], mode='r',
encoding='utf-8') as f:
text = f.read()
break
except IOError as e:
_logger.warning("Couldn't open %s - %s [try:%s]",
playbookfile['path'], e.strerror, i)
error = e
sleep(1)
continue
else:
return [
MatchError(message=str(error),
filename=playbookfile['path'],
rule=LoadingFailureRule())
]
for rule in self.rules:
if not tags or not set(rule.tags).union([rule.id
]).isdisjoint(tags):
rule_definition = set(rule.tags)
rule_definition.add(rule.id)
if set(rule_definition).isdisjoint(skip_list):
matches.extend(rule.matchlines(playbookfile, text))
matches.extend(rule.matchtasks(playbookfile, text))
matches.extend(
rule.matchyaml(
Lintable(playbookfile['path'],
content=text,
kind=playbookfile['type'])))
# some rules can produce matches with tags that are inside our
# skip_list, so we need to cleanse the matches
matches = [m for m in matches if m.tag not in skip_list]
return matches
def _look_for_role_files(basedir: str,
role: str,
main: Optional[str] = 'main') -> List[Lintable]:
role_path = _rolepath(basedir, role)
if not role_path:
return []
results = []
for kind in ['tasks', 'meta', 'handlers', 'vars', 'defaults']:
current_path = os.path.join(role_path, kind)
for folder, subdirs, files in os.walk(current_path):
for file in files:
file_ignorecase = file.lower()
if file_ignorecase.endswith(('.yml', '.yaml')):
thpath = os.path.join(folder, file)
results.append(Lintable(thpath))
return results
import pytest
from ansiblelint.file_utils import Lintable
PLAY_IN_THE_PLACE = Lintable(
'playbook.yml',
'''\
- hosts: all
roles:
- include_in_the_place
''',
)
PLAY_RELATIVE = Lintable(
'playbook.yml',
'''\
- hosts: all
roles:
- include_relative
''',
)
PLAY_MISS_INCLUDE = Lintable(
'playbook.yml',
'''\
- hosts: all
roles:
- include_miss
''',
)
def test_runner_with_directory(default_rules_collection,
directory_name) -> None:
runner = Runner(directory_name, rules=default_rules_collection)
expected = Lintable(name=directory_name, kind="role")
assert expected in runner.lintables
