def filter_targets( self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test.""" return [ target for target in targets if os.path.splitext(target.path)[1] == '.py' and ( is_subdir(target.path, data_context().content.module_path) or is_subdir(target.path, data_context().content.module_utils_path)) ]
def __init__(self, path, module_path, module_prefix, base_path): """ :type path: str :type module_path: str | None :type module_prefix: str | None :type base_path: str """ super(TestTarget, self).__init__() self.name = path self.path = path self.base_path = base_path + '/' if base_path else None name, ext = os.path.splitext(os.path.basename(self.path)) if module_path and is_subdir(path, module_path) and name != '__init__' and ext in MODULE_EXTENSIONS: self.module = name[len(module_prefix or ''):].lstrip('_') self.modules = (self.module,) else: self.module = None self.modules = tuple() aliases = [self.path, self.module] parts = self.path.split('/') for i in range(1, len(parts)): alias = '%s/' % '/'.join(parts[:i]) aliases.append(alias) aliases = [a for a in aliases if a] self.aliases = tuple(sorted(aliases))
def create_content_layout( layout_providers, # type: t.List[t.Type[LayoutProvider]] source_providers, # type: t.List[t.Type[SourceProvider]] root, # type: str walk, # type: bool ): # type: (...) -> ContentLayout """Create a content layout using the given providers and root path.""" layout_provider = find_path_provider(LayoutProvider, layout_providers, root, walk) try: source_provider = find_path_provider(SourceProvider, source_providers, root, walk) except ProviderNotFoundForPath: source_provider = UnversionedSource(layout_provider.root) if source_provider.root != layout_provider.root and is_subdir( source_provider.root, layout_provider.root): raise UnexpectedSourceRoot(source_provider.root, layout_provider.root) layout = layout_provider.create( layout_provider.root, source_provider.get_paths(layout_provider.root)) return layout
def __init__(self): content_path = os.environ.get('ANSIBLE_TEST_CONTENT_ROOT') current_path = os.getcwd() self.__layout_providers = get_path_provider_classes(LayoutProvider) self.__source_providers = get_path_provider_classes(SourceProvider) self.payload_callbacks = [ ] # type: t.List[t.Callable[t.List[t.Tuple[str, str]], None]] if content_path: content = self.create_content_layout(self.__layout_providers, self.__source_providers, content_path, False) if content.is_ansible: install = content else: install = None elif is_subdir(current_path, ANSIBLE_ROOT): content = self.create_content_layout(self.__layout_providers, self.__source_providers, ANSIBLE_ROOT, False) install = InstallLayout(ANSIBLE_ROOT, content.all_files()) else: content = self.create_content_layout(self.__layout_providers, self.__source_providers, current_path, True) install = None self.__install = install # type: t.Optional[InstallLayout] self.content = content # type: ContentLayout
def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test.""" if self.no_targets: return [] if self.text is not None: if self.text: targets = [target for target in targets if not is_binary_file(target.path)] else: targets = [target for target in targets if is_binary_file(target.path)] if self.extensions: targets = [target for target in targets if os.path.splitext(target.path)[1] in self.extensions or (is_subdir(target.path, 'bin') and '.py' in self.extensions)] if self.prefixes: targets = [target for target in targets if any(target.path.startswith(pre) for pre in self.prefixes)] if self.files: targets = [target for target in targets if os.path.basename(target.path) in self.files] if self.ignore_self and data_context().content.is_ansible: relative_self_path = os.path.relpath(self.path, data_context().content.root) targets = [target for target in targets if target.path != relative_self_path] return targets
def filter_targets( self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test.""" return [ target for target in targets if os.path.splitext(target.path)[1] == '.py' or is_subdir(target.path, 'bin') ]
def filter_targets( self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test.""" yaml_targets = [ target for target in targets if os.path.splitext(target.path)[1] in ('.yml', '.yaml') ] for plugin_type, plugin_path in sorted( data_context().content.plugin_paths.items()): if plugin_type == 'module_utils': continue yaml_targets.extend([ target for target in targets if os.path.splitext(target.path)[1] == '.py' and os.path.basename(target.path) != '__init__.py' and is_subdir(target.path, plugin_path) ]) return yaml_targets
def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test.""" # This should use documentable plugins from constants instead plugin_type_blacklist = set([ # not supported by ansible-doc 'action', 'doc_fragments', 'filter', 'module_utils', 'netconf', 'terminal', 'test', ]) plugin_paths = [plugin_path for plugin_type, plugin_path in data_context().content.plugin_paths.items() if plugin_type not in plugin_type_blacklist] return [target for target in targets if os.path.splitext(target.path)[1] == '.py' and os.path.basename(target.path) != '__init__.py' and any(is_subdir(target.path, path) for path in plugin_paths) ]
def get_dependent_paths_internal(self, path): """ :type path: str :rtype: list[str] """ ext = os.path.splitext(os.path.split(path)[1])[1] if is_subdir(path, data_context().content.module_utils_path): if ext == '.py': return self.get_python_module_utils_usage(path) if ext == '.psm1': return self.get_powershell_module_utils_usage(path) if ext == '.cs': return self.get_csharp_module_utils_usage(path) if path.startswith('test/integration/targets/'): return self.get_integration_target_usage(path) return []
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ paths = [ [ i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml') ], ] for plugin_type, plugin_path in sorted( data_context().content.plugin_paths.items()): if plugin_type == 'module_utils': continue paths.append([ target.path for target in targets.include if os.path.splitext(target.path)[1] == '.py' and os.path.basename(target.path) != '__init__.py' and is_subdir(target.path, plugin_path) ]) paths = [sorted(p) for p in paths if p] if not paths: return SanitySkipped(self.name) results = [] for test_paths in paths: results += self.test_paths(args, test_paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ settings = self.load_settings(args, 'ansible-test') paths = [ i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml') ] for plugin_type, plugin_path in sorted( data_context().content.plugin_paths.items()): if plugin_type == 'module_utils': continue paths.extend([ target.path for target in targets.include if os.path.splitext(target.path)[1] == '.py' and os.path.basename(target.path) != '__init__.py' and is_subdir(target.path, plugin_path) ]) paths = settings.filter_skipped_paths(paths) if not paths: return SanitySkipped(self.name) results = self.test_paths(args, paths) results = settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning('Skipping pylint on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) plugin_dir = os.path.join(INSTALL_ROOT, 'test/sanity/pylint/plugins') plugin_names = sorted(p[0] for p in [ os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__') skip_paths = read_lines_without_comments(PYLINT_SKIP_PATH, optional=True) invalid_ignores = [] supported_versions = set(SUPPORTED_PYTHON_VERSIONS) - set(UNSUPPORTED_PYTHON_VERSIONS) supported_versions = set([v.split('.')[0] for v in supported_versions]) | supported_versions ignore_entries = read_lines_without_comments(PYLINT_IGNORE_PATH, optional=True) ignore = collections.defaultdict(dict) # type: t.Dict[str, t.Dict[str, int]] line = 0 for ignore_entry in ignore_entries: line += 1 if not ignore_entry: continue if ' ' not in ignore_entry: invalid_ignores.append((line, 'Invalid syntax')) continue path, code = ignore_entry.split(' ', 1) if not os.path.exists(path): invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path)) continue if ' ' in code: code, version = code.split(' ', 1) if version not in supported_versions: invalid_ignores.append((line, 'Invalid version: %s' % version)) continue if version not in (args.python_version, args.python_version.split('.')[0]): continue # ignore version specific entries for other versions ignore[path][code] = line skip_paths_set = set(skip_paths) paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/')) and i.path not in skip_paths_set) module_paths = [os.path.relpath(p, 'lib/ansible/modules/').split(os.path.sep) for p in paths if is_subdir(p, 'lib/ansible/modules/')] module_dirs = sorted(set([p[0] for p in module_paths if len(p) > 1])) large_module_group_threshold = 500 large_module_groups = [key for key, value in itertools.groupby(module_paths, lambda p: p[0] if len(p) > 1 else '') if len(list(value)) > large_module_group_threshold] large_module_group_paths = [os.path.relpath(p, 'lib/ansible/modules/').split(os.path.sep) for p in paths if any(is_subdir(p, os.path.join('lib/ansible/modules/', g)) for g in large_module_groups)] large_module_group_dirs = sorted(set([os.path.sep.join(p[:2]) for p in large_module_group_paths if len(p) > 2])) contexts = [] remaining_paths = set(paths) def add_context(available_paths, context_name, context_filter): """ :type available_paths: set[str] :type context_name: str :type context_filter: (str) -> bool """ filtered_paths = set(p for p in available_paths if context_filter(p)) contexts.append((context_name, sorted(filtered_paths))) available_paths -= filtered_paths def filter_path(path_filter=None): """ :type path_filter: str :rtype: (str) -> bool """ def context_filter(path_to_filter): """ :type path_to_filter: str :rtype: bool """ return is_subdir(path_to_filter, path_filter) return context_filter for large_module_group_dir in large_module_group_dirs: add_context(remaining_paths, 'modules/%s' % large_module_group_dir, filter_path('lib/ansible/modules/%s/' % large_module_group_dir)) for module_dir in module_dirs: add_context(remaining_paths, 'modules/%s' % module_dir, filter_path('lib/ansible/modules/%s/' % module_dir)) add_context(remaining_paths, 'modules', filter_path('lib/ansible/modules/')) add_context(remaining_paths, 'module_utils', filter_path('lib/ansible/module_utils/')) add_context(remaining_paths, 'units', filter_path('test/units/')) add_context(remaining_paths, 'validate-modules', filter_path('test/sanity/validate-modules/')) add_context(remaining_paths, 'sanity', filter_path('test/sanity/')) add_context(remaining_paths, 'ansible-test', filter_path('test/runner/')) add_context(remaining_paths, 'test', filter_path('test/')) add_context(remaining_paths, 'hacking', filter_path('hacking/')) add_context(remaining_paths, 'ansible', lambda p: True) messages = [] context_times = [] test_start = datetime.datetime.utcnow() for context, context_paths in sorted(contexts): if not context_paths: continue context_start = datetime.datetime.utcnow() messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names) context_end = datetime.datetime.utcnow() context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start)) test_end = datetime.datetime.utcnow() for context_time in context_times: display.info(context_time, verbosity=4) display.info('total: %d (%s)' % (len(paths), test_end - test_start), verbosity=4) errors = [SanityMessage( message=m['message'].replace('\n', ' '), path=m['path'], line=int(m['line']), column=int(m['column']), level=m['type'], code=m['symbol'], ) for m in messages] if args.explain: return SanitySuccess(self.name) line = 0 filtered = [] for error in errors: if error.code in ignore[error.path]: ignore[error.path][error.code] = 0 # error ignored, clear line number of ignore entry to track usage else: filtered.append(error) # error not ignored errors = filtered for invalid_ignore in invalid_ignores: errors.append(SanityMessage( code='A201', message=invalid_ignore[1], path=PYLINT_IGNORE_PATH, line=invalid_ignore[0], column=1, confidence=calculate_confidence(PYLINT_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, )) for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append(SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PYLINT_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence(((PYLINT_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in paths: if path not in ignore: continue for code in ignore[path]: line = ignore[path][code] if not line: continue errors.append(SanityMessage( code='A102', message='Remove since "%s" passes "%s" pylint test' % (path, code), path=PYLINT_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence(((PYLINT_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def context_filter(path_to_filter): """ :type path_to_filter: str :rtype: bool """ return is_subdir(path_to_filter, path_filter)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning('Skipping pylint on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) plugin_dir = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/plugins') plugin_names = sorted(p[0] for p in [ os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__') settings = self.load_settings(args, 'ansible-test') paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/')) paths = settings.filter_skipped_paths(paths) module_paths = [os.path.relpath(p, data_context().content.module_path).split(os.path.sep) for p in paths if is_subdir(p, data_context().content.module_path)] module_dirs = sorted(set([p[0] for p in module_paths if len(p) > 1])) large_module_group_threshold = 500 large_module_groups = [key for key, value in itertools.groupby(module_paths, lambda p: p[0] if len(p) > 1 else '') if len(list(value)) > large_module_group_threshold] large_module_group_paths = [os.path.relpath(p, data_context().content.module_path).split(os.path.sep) for p in paths if any(is_subdir(p, os.path.join(data_context().content.module_path, g)) for g in large_module_groups)] large_module_group_dirs = sorted(set([os.path.sep.join(p[:2]) for p in large_module_group_paths if len(p) > 2])) contexts = [] remaining_paths = set(paths) def add_context(available_paths, context_name, context_filter): """ :type available_paths: set[str] :type context_name: str :type context_filter: (str) -> bool """ filtered_paths = set(p for p in available_paths if context_filter(p)) contexts.append((context_name, sorted(filtered_paths))) available_paths -= filtered_paths def filter_path(path_filter=None): """ :type path_filter: str :rtype: (str) -> bool """ def context_filter(path_to_filter): """ :type path_to_filter: str :rtype: bool """ return is_subdir(path_to_filter, path_filter) return context_filter for large_module_dir in large_module_group_dirs: add_context(remaining_paths, 'modules/%s' % large_module_dir, filter_path(os.path.join(data_context().content.module_path, large_module_dir))) for module_dir in module_dirs: add_context(remaining_paths, 'modules/%s' % module_dir, filter_path(os.path.join(data_context().content.module_path, module_dir))) add_context(remaining_paths, 'modules', filter_path(data_context().content.module_path)) add_context(remaining_paths, 'module_utils', filter_path(data_context().content.module_utils_path)) add_context(remaining_paths, 'units', filter_path(data_context().content.unit_path)) if data_context().content.collection: add_context(remaining_paths, 'collection', lambda p: True) else: add_context(remaining_paths, 'validate-modules', filter_path('test/sanity/validate-modules/')) add_context(remaining_paths, 'sanity', filter_path('test/sanity/')) add_context(remaining_paths, 'ansible-test', filter_path('test/runner/')) add_context(remaining_paths, 'test', filter_path('test/')) add_context(remaining_paths, 'hacking', filter_path('hacking/')) add_context(remaining_paths, 'ansible', lambda p: True) messages = [] context_times = [] test_start = datetime.datetime.utcnow() for context, context_paths in sorted(contexts): if not context_paths: continue context_start = datetime.datetime.utcnow() messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names) context_end = datetime.datetime.utcnow() context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start)) test_end = datetime.datetime.utcnow() for context_time in context_times: display.info(context_time, verbosity=4) display.info('total: %d (%s)' % (len(paths), test_end - test_start), verbosity=4) errors = [SanityMessage( message=m['message'].replace('\n', ' '), path=m['path'], line=int(m['line']), column=int(m['column']), level=m['type'], code=m['symbol'], ) for m in messages] if args.explain: return SanitySuccess(self.name) errors = settings.process_errors(errors, paths) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ settings = self.load_processor(args) paths = [target.path for target in targets.include] doc_targets = collections.defaultdict(list) target_paths = collections.defaultdict(dict) remap_types = dict( modules='module', ) for plugin_type, plugin_path in data_context().content.plugin_paths.items(): plugin_type = remap_types.get(plugin_type, plugin_type) for plugin_file_path in [target.name for target in targets.include if is_subdir(target.path, plugin_path)]: plugin_name = os.path.splitext(os.path.basename(plugin_file_path))[0] if plugin_name.startswith('_'): plugin_name = plugin_name[1:] doc_targets[plugin_type].append(data_context().content.prefix + plugin_name) target_paths[plugin_type][data_context().content.prefix + plugin_name] = plugin_file_path env = ansible_environment(args, color=False) error_messages = [] for doc_type in sorted(doc_targets): cmd = ['ansible-doc', '-t', doc_type] + sorted(doc_targets[doc_type]) try: with coverage_context(args): stdout, stderr = intercept_command(args, cmd, target_name='ansible-doc', env=env, capture=True, python_version=python_version) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: errors = stderr.strip().splitlines() messages = [self.parse_error(e, target_paths) for e in errors] if messages and all(messages): error_messages += messages continue if status: summary = u'%s' % SubprocessError(cmd=cmd, status=status, stderr=stderr) return SanityFailure(self.name, summary=summary) if stdout: display.info(stdout.strip(), verbosity=3) if stderr: summary = u'Output on stderr from ansible-doc is considered an error.\n\n%s' % SubprocessError(cmd, stderr=stderr) return SanityFailure(self.name, summary=summary) error_messages = settings.process_errors(error_messages, paths) if error_messages: return SanityFailure(self.name, messages=error_messages) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ settings = self.load_settings(args, None, python_version) paths = sorted( i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' and (is_subdir(i.path, data_context().content.module_path) or is_subdir(i.path, data_context().content.module_utils_path)) ) paths = settings.filter_skipped_paths(paths) if not paths: return SanitySkipped(self.name, python_version=python_version) env = ansible_environment(args, color=False) # create a clean virtual environment to minimize the available imports beyond the python standard library virtual_environment_path = os.path.abspath('test/runner/.tox/minimal-py%s' % python_version.replace('.', '')) virtual_environment_bin = os.path.join(virtual_environment_path, 'bin') remove_tree(virtual_environment_path) python = find_python(python_version) cmd = [python, '-m', 'virtualenv', virtual_environment_path, '--python', python, '--no-setuptools', '--no-wheel'] if not args.coverage: cmd.append('--no-pip') run_command(args, cmd, capture=True) # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') if not args.explain: os.symlink(os.path.abspath(os.path.join(ANSIBLE_ROOT, 'test/sanity/import/importer.py')), importer_path) # create a minimal python library python_path = os.path.abspath('test/runner/.tox/import/lib') ansible_path = os.path.join(python_path, 'ansible') ansible_init = os.path.join(ansible_path, '__init__.py') ansible_link = os.path.join(ansible_path, 'module_utils') if not args.explain: remove_tree(ansible_path) make_dirs(ansible_path) with open(ansible_init, 'w'): pass os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/module_utils'), ansible_link) # activate the virtual environment env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH']) env['PYTHONPATH'] = python_path # make sure coverage is available in the virtual environment if needed if args.coverage: run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env) run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env) run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env) run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env) cmd = ['importer.py'] data = '\n'.join(paths) display.info(data, verbosity=4) results = [] virtualenv_python = os.path.join(virtual_environment_bin, 'python') try: with coverage_context(args): stdout, stderr = intercept_command(args, cmd, self.name, env, capture=True, data=data, python_version=python_version, virtualenv=virtualenv_python) if stdout or stderr: raise SubprocessError(cmd, stdout=stdout, stderr=stderr) except SubprocessError as ex: if ex.status != 10 or ex.stderr or not ex.stdout: raise pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' results = parse_to_list_of_dict(pattern, ex.stdout) results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), ) for r in results] results = settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)