def get_lintables( options: Namespace = Namespace(), args: Optional[List[str]] = None ) -> List[Lintable]: """Detect files and directories that are lintable.""" lintables: List[Lintable] = [] # passing args bypass auto-detection mode if args: for arg in args: lintable = Lintable(arg) if lintable.kind in ("yaml", None): _logger.warning( "Overriding detected file kind '%s' with 'playbook' " "for given positional argument: %s", lintable.kind, arg, ) lintable = Lintable(arg, kind="playbook") lintables.append(lintable) else: for filename in get_yaml_files(options): p = Path(filename) # skip exclusions try: for file_path in options.exclude_paths: if str(p.resolve()).startswith(str(file_path)): raise FileNotFoundError( f'File {file_path} matched exclusion entry: {p}' ) except FileNotFoundError as e: _logger.debug('Ignored %s due to: %s', p, e) continue lintables.append(Lintable(p)) # stage 2: guess roles from current lintables, as there is no unique # file that must be present in any kind of role. for lintable in lintables: parts = lintable.path.parent.parts if 'roles' in parts: role = lintable.path while role.parent.name != "roles" and role.name: role = role.parent if role.exists: lintable = Lintable(role, kind="role") if lintable not in lintables: _logger.debug("Added role: %s", lintable) lintables.append(lintable) return lintables
def pytest_collect_file(parent, path) -> Optional["Node"]: """Transform each found molecule.yml into a pytest test.""" # additional mappings not known by ansible-lint (yet) extra_kinds = [ {"molecule": "**/molecule/*/molecule.yml"}, {"zuul": "**/zuul.d/*.{yaml,yml}"}, {"zuul": "**/.zuul.yaml"}, {"ansible-lint": "**/.ansiblelint"}, {"ansible-lint": "**/.config/ansiblelint.yml"}, {"json-schema": "f/*.json"}, ] # appending extra kinds at beginning of default ones options.kinds = [*extra_kinds, *options.kinds] if not path.fnmatch("*/examples/*") and not path.fnmatch("f/*.json"): # We care only about f/ and examples/ return None try: lintable = Lintable(str(path)) except RuntimeError: # ignore unknown file types return None if lintable.kind == 'json-schema': return SchemaFile.from_parent(parent, fspath=path) # type: ignore if not lintable.kind: return None return YamlFile.from_parent(parent, fspath=path) # type: ignore
def _get_task_handler_children_for_tasks_or_playbooks( task_handler, basedir: str, k, parent_type: FileType, ) -> Lintable: """Try to get children of taskhandler for include/import tasks/playbooks.""" child_type = k if parent_type == 'playbook' else parent_type task_include_keys = 'include', 'include_tasks', 'import_playbook', 'import_tasks' for task_handler_key in task_include_keys: with contextlib.suppress(KeyError): # ignore empty tasks if not task_handler: continue return Lintable( path_dwim(basedir, task_handler[task_handler_key]), kind=child_type ) raise LookupError( f'The node contains none of: {", ".join(task_include_keys)}', )
def run(self) -> List[MatchError]: """Execute the linting process.""" files: List[Lintable] = list() for playbook in self.playbooks: if self.is_excluded(playbook[0]) or playbook[1] == 'role': continue files.append( Lintable(ansiblelint.file_utils.normpath(playbook[0]), kind=playbook[1])) # type: ignore matches = set(self._emit_matches(files)) # remove duplicates from files list files = [ value for n, value in enumerate(files) if value not in files[:n] ] # remove files that have already been checked files = [x for x in files if x['path'] not in self.checked_files] for file in files: _logger.debug("Examining %s of type %s", ansiblelint.file_utils.normpath(file['path']), file['type']) matches = matches.union( self.rules.run(file, tags=set(self.tags), skip_list=self.skip_list)) # update list of checked files self.checked_files.update([x['path'] for x in files]) return sorted(matches)
def __init__(self, rules: "RulesCollection", lintable: Union[Lintable, str], tags: FrozenSet[Any] = frozenset(), skip_list: List[str] = [], exclude_paths: List[str] = [], verbosity: int = 0, checked_files: Optional[Set[str]] = None) -> None: """Initialize a Runner instance.""" self.rules = rules self.lintables: Set[Lintable] = set() if isinstance(lintable, str): lintable = Lintable(lintable) self.lintables.add(lintable) self.playbook_dir = lintable.dir self.tags = tags self.skip_list = skip_list self._update_exclude_paths(exclude_paths) self.verbosity = verbosity if checked_files is None: checked_files = set() self.checked_files = checked_files
def __init__(self, *lintables: Union[Lintable, str], rules: "RulesCollection", tags: FrozenSet[Any] = frozenset(), skip_list: List[str] = [], exclude_paths: List[str] = [], verbosity: int = 0, checked_files: Optional[Set[Lintable]] = None) -> None: """Initialize a Runner instance.""" self.rules = rules self.lintables: Set[Lintable] = set() # Assure consistent type for item in lintables: if not isinstance(item, Lintable): item = Lintable(item) self.lintables.add(item) # Expand folders (roles) to their components expand_dirs_in_lintables(self.lintables) self.tags = tags self.skip_list = skip_list self._update_exclude_paths(exclude_paths) self.verbosity = verbosity if checked_files is None: checked_files = set() self.checked_files = checked_files
def lint_error_lines(test_playbook: str) -> List[int]: """Get VarHasSpacesRules linting results on test_playbook.""" collection = RulesCollection() collection.register(VariableHasSpacesRule()) lintable = Lintable(test_playbook) results = Runner(lintable, rules=collection).run() return list(map(lambda item: item.linenumber, results))
def test_auto_detect(monkeypatch, path: str, kind: FileType) -> None: """Verify auto-detection logic.""" options = cli.get_config([]) def mockreturn(options): return [path] # assert Lintable is able to determine file type lintable_detected = Lintable(path) lintable_expected = Lintable(path, kind=kind) assert lintable_detected == lintable_expected monkeypatch.setattr(utils, 'get_yaml_files', mockreturn) result = utils.get_lintables(options) # get_lintable could return additional files and we only care to see # that the given file is among the returned list. assert lintable_expected in result
def test_included_tasks(default_rules_collection, filename, file_count, match_count): """Check if number of loaded files is correct.""" lintable = Lintable(filename) runner = Runner(default_rules_collection, lintable, [], [], []) result = runner.run() assert len(runner.playbooks) == file_count assert len(result) == match_count
def test_206() -> None: """Verify rule.""" collection = RulesCollection() collection.register(VariableHasSpacesRule()) lintable = Lintable("examples/playbooks/206.yml") results = Runner(collection, lintable=lintable).run() assert len(results) == 3
def test_runner_unicode_format(default_rules_collection, formatter_cls) -> None: formatter = formatter_cls(os.getcwd(), display_relative_path=True) runner = Runner( rules=default_rules_collection, lintable=Lintable('test/unicode.yml', "playbook")) matches = runner.run() formatter.format(matches[0])
def test_var_spacing() -> None: """Verify rule.""" collection = RulesCollection() collection.register(VariableHasSpacesRule()) lintable = Lintable("examples/playbooks/var-spacing.yml") results = Runner(lintable, rules=collection).run() assert len(results) == 3
def test_auto_detect_exclude(monkeypatch: MonkeyPatch) -> None: """Verify that exclude option can be used to narrow down detection.""" options = cli.get_config(['--exclude', 'foo']) def mockreturn(options: Namespace) -> List[str]: return ['foo/playbook.yml', 'bar/playbook.yml'] monkeypatch.setattr(utils, 'discover_lintables', mockreturn) result = utils.get_lintables(options) assert result == [Lintable('bar/playbook.yml', kind='playbook')]
def test_default_kinds(monkeypatch: MonkeyPatch, path: str, kind: FileType) -> None: """Verify auto-detection logic based on DEFAULT_KINDS.""" options = cli.get_config([]) def mockreturn(options: Namespace) -> Dict[str, Any]: return {path: kind} # assert Lintable is able to determine file type lintable_detected = Lintable(path) lintable_expected = Lintable(path, kind=kind) assert lintable_detected == lintable_expected monkeypatch.setattr(file_utils, 'discover_lintables', mockreturn) result = file_utils.discover_lintables(options) # get_lintable could return additional files and we only care to see # that the given file is among the returned list. assert lintable_detected.name in result assert lintable_detected.kind == result[lintable_expected.name]
def test_auto_detect_exclude(monkeypatch): """Verify that exclude option can be used to narrow down detection.""" options = cli.get_config(['--exclude', 'foo']) def mockreturn(options): return ['foo/playbook.yml', 'bar/playbook.yml'] monkeypatch.setattr(utils, 'get_yaml_files', mockreturn) result = utils.get_lintables(options) assert result == [Lintable('bar/playbook.yml', kind='playbook')]
def test_auto_detect(monkeypatch, path: str, kind: FileType) -> None: """Verify auto-detection logic.""" options = cli.get_config([]) def mockreturn(options): return [path] monkeypatch.setattr(utils, 'get_yaml_files', mockreturn) result = utils.get_lintables(options) assert result == [Lintable(path, kind=kind)]
def test_get_ansible_syntax_check_matches() -> None: """Validate parsing of ansible output.""" lintable = Lintable('examples/playbooks/conflicting_action.yml', kind='playbook') result = AnsibleSyntaxCheckRule._get_ansible_syntax_check_matches(lintable) assert result[0].linenumber == 3 assert result[0].column == 7 assert result[0].message == "conflicting action statements: debug, command" # We internaly convert absolute paths returned by ansible into paths # relative to current directory. assert result[0].filename.endswith("/conflicting_action.yml") assert len(result) == 1
def test_extra_vars_passed_to_command(config_options) -> None: """Validate `extra-vars` are passed to syntax check command.""" config_options.extra_vars = { 'foo': 'bar', 'complex_variable': ':{;\t$()', } lintable = Lintable('examples/playbooks/extra_vars.yml', kind='playbook') result = AnsibleSyntaxCheckRule._get_ansible_syntax_check_matches(lintable) assert not result
def test_empty_playbook() -> None: """Validate detection of empty-playbook.""" lintable = Lintable('examples/playbooks/empty_playbook.yml', kind='playbook') result = AnsibleSyntaxCheckRule._get_ansible_syntax_check_matches(lintable) assert result[0].linenumber == 0 # We internally convert absolute paths returned by ansible into paths # relative to current directory. assert result[0].filename.endswith("/empty_playbook.yml") assert result[0].tag == "empty-playbook" assert result[0].message == "Empty playbook, nothing to do" assert len(result) == 1
def _include_children(basedir: str, k, v, parent_type) -> List[Lintable]: # handle special case include_tasks: name=filename.yml if k == 'include_tasks' and isinstance(v, dict) and 'file' in v: v = v['file'] # handle include: filename.yml tags=blah (command, args, kwargs) = tokenize("{0}: {1}".format(k, v)) result = path_dwim(basedir, args[0]) if not os.path.exists(result): result = path_dwim(os.path.join(os.path.dirname(basedir)), v) return [Lintable(result, kind=parent_type)]
def test_included_tasks( default_rules_collection: RulesCollection, filename: str, file_count: int, match_count: int, ) -> None: """Check if number of loaded files is correct.""" lintable = Lintable(filename) runner = Runner(lintable, rules=default_rules_collection) result = runner.run() assert len(runner.lintables) == file_count assert len(result) == match_count
def _extend_with_roles(lintables: List[Lintable]) -> None: """Detect roles among lintables and adds them to the list.""" for lintable in lintables: parts = lintable.path.parent.parts if 'roles' in parts: role = lintable.path while role.parent.name != "roles" and role.name: role = role.parent if role.exists and not role.is_file(): lintable = Lintable(role, kind="role") if lintable not in lintables: _logger.debug("Added role: %s", lintable) lintables.append(lintable)
def test_runner_unicode_format( default_rules_collection: RulesCollection, formatter_cls: Type[formatters.BaseFormatter[Any]], ) -> None: formatter = formatter_cls(os.getcwd(), display_relative_path=True) runner = Runner( Lintable('examples/playbooks/unicode.yml', "playbook"), rules=default_rules_collection, ) matches = runner.run() formatter.format(matches[0])
def test_rule_no_handler() -> None: """Verify rule.""" collection = RulesCollection() collection.register(UseHandlerRatherThanWhenChangedRule()) lintable = Lintable('examples/playbooks/rule-no-handler.yml') results = Runner(lintable, rules=collection).run() assert len(results) == 3 assert results[0].filename == 'examples/playbooks/roles/a-role/tasks/main.yml' assert results[0].linenumber == 3 assert results[1].filename == 'examples/playbooks/rule-no-handler.yml' assert results[1].linenumber == 14 assert results[2].filename == 'examples/playbooks/rule-no-handler.yml' assert results[2].linenumber == 18
def _emit_matches( self, files: List[Lintable]) -> Generator[MatchError, None, None]: visited: Set = set() while visited != self.playbooks: for arg in self.playbooks - visited: try: for child in ansiblelint.utils.find_children( arg, self.playbook_dir): if self.is_excluded(child['path']): continue self.playbooks.add((child['path'], child['type'])) files.append( Lintable(child['path'], kind=child['type'])) except MatchError as e: e.rule = LoadingFailureRule() yield e visited.add(arg)
def _look_for_role_files(basedir: str, role: str, main='main') -> List[Lintable]: role_path = _rolepath(basedir, role) if not role_path: return [] results = [] for kind in ['tasks', 'meta', 'handlers']: current_path = os.path.join(role_path, kind) for folder, subdirs, files in os.walk(current_path): for file in files: file_ignorecase = file.lower() if file_ignorecase.endswith(('.yml', '.yaml')): thpath = os.path.join(folder, file) # TODO(ssbarnea): Find correct way to pass kind: FileType results.append(Lintable(thpath, kind=kind)) # type: ignore return results
def run(self, playbookfile, tags=set(), skip_list=frozenset()) -> List: text = "" matches: List = list() error: Optional[IOError] = None for i in range(3): try: with open(playbookfile['path'], mode='r', encoding='utf-8') as f: text = f.read() break except IOError as e: _logger.warning("Couldn't open %s - %s [try:%s]", playbookfile['path'], e.strerror, i) error = e sleep(1) continue else: return [ MatchError(message=str(error), filename=playbookfile['path'], rule=LoadingFailureRule()) ] for rule in self.rules: if not tags or not set(rule.tags).union([rule.id ]).isdisjoint(tags): rule_definition = set(rule.tags) rule_definition.add(rule.id) if set(rule_definition).isdisjoint(skip_list): matches.extend(rule.matchlines(playbookfile, text)) matches.extend(rule.matchtasks(playbookfile, text)) matches.extend( rule.matchyaml( Lintable(playbookfile['path'], content=text, kind=playbookfile['type']))) # some rules can produce matches with tags that are inside our # skip_list, so we need to cleanse the matches matches = [m for m in matches if m.tag not in skip_list] return matches
def _look_for_role_files(basedir: str, role: str, main: Optional[str] = 'main') -> List[Lintable]: role_path = _rolepath(basedir, role) if not role_path: return [] results = [] for kind in ['tasks', 'meta', 'handlers', 'vars', 'defaults']: current_path = os.path.join(role_path, kind) for folder, subdirs, files in os.walk(current_path): for file in files: file_ignorecase = file.lower() if file_ignorecase.endswith(('.yml', '.yaml')): thpath = os.path.join(folder, file) results.append(Lintable(thpath)) return results
import pytest from ansiblelint.file_utils import Lintable PLAY_IN_THE_PLACE = Lintable( 'playbook.yml', '''\ - hosts: all roles: - include_in_the_place ''', ) PLAY_RELATIVE = Lintable( 'playbook.yml', '''\ - hosts: all roles: - include_relative ''', ) PLAY_MISS_INCLUDE = Lintable( 'playbook.yml', '''\ - hosts: all roles: - include_miss ''', )
def test_runner_with_directory(default_rules_collection, directory_name) -> None: runner = Runner(directory_name, rules=default_rules_collection) expected = Lintable(name=directory_name, kind="role") assert expected in runner.lintables