def filter_targets(
self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget]
"""Return the given list of test targets, filtered to include only those relevant for the test."""
return [
target for target in targets
if os.path.splitext(target.path)[1] == '.py' and (
is_subdir(target.path,
data_context().content.module_path)
or is_subdir(target.path,
data_context().content.module_utils_path))
]
def __init__(self, path, module_path, module_prefix, base_path):
"""
:type path: str
:type module_path: str | None
:type module_prefix: str | None
:type base_path: str
"""
super(TestTarget, self).__init__()
self.name = path
self.path = path
self.base_path = base_path + '/' if base_path else None
name, ext = os.path.splitext(os.path.basename(self.path))
if module_path and is_subdir(path, module_path) and name != '__init__' and ext in MODULE_EXTENSIONS:
self.module = name[len(module_prefix or ''):].lstrip('_')
self.modules = (self.module,)
else:
self.module = None
self.modules = tuple()
aliases = [self.path, self.module]
parts = self.path.split('/')
for i in range(1, len(parts)):
alias = '%s/' % '/'.join(parts[:i])
aliases.append(alias)
aliases = [a for a in aliases if a]
self.aliases = tuple(sorted(aliases))
def create_content_layout(
layout_providers, # type: t.List[t.Type[LayoutProvider]]
source_providers, # type: t.List[t.Type[SourceProvider]]
root, # type: str
walk, # type: bool
): # type: (...) -> ContentLayout
"""Create a content layout using the given providers and root path."""
layout_provider = find_path_provider(LayoutProvider, layout_providers,
root, walk)
try:
source_provider = find_path_provider(SourceProvider,
source_providers, root, walk)
except ProviderNotFoundForPath:
source_provider = UnversionedSource(layout_provider.root)
if source_provider.root != layout_provider.root and is_subdir(
source_provider.root, layout_provider.root):
raise UnexpectedSourceRoot(source_provider.root,
layout_provider.root)
layout = layout_provider.create(
layout_provider.root,
source_provider.get_paths(layout_provider.root))
return layout
def __init__(self):
content_path = os.environ.get('ANSIBLE_TEST_CONTENT_ROOT')
current_path = os.getcwd()
self.__layout_providers = get_path_provider_classes(LayoutProvider)
self.__source_providers = get_path_provider_classes(SourceProvider)
self.payload_callbacks = [
] # type: t.List[t.Callable[t.List[t.Tuple[str, str]], None]]
if content_path:
content = self.create_content_layout(self.__layout_providers,
self.__source_providers,
content_path, False)
if content.is_ansible:
install = content
else:
install = None
elif is_subdir(current_path, ANSIBLE_ROOT):
content = self.create_content_layout(self.__layout_providers,
self.__source_providers,
ANSIBLE_ROOT, False)
install = InstallLayout(ANSIBLE_ROOT, content.all_files())
else:
content = self.create_content_layout(self.__layout_providers,
self.__source_providers,
current_path, True)
install = None
self.__install = install # type: t.Optional[InstallLayout]
self.content = content # type: ContentLayout
def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget]
"""Return the given list of test targets, filtered to include only those relevant for the test."""
if self.no_targets:
return []
if self.text is not None:
if self.text:
targets = [target for target in targets if not is_binary_file(target.path)]
else:
targets = [target for target in targets if is_binary_file(target.path)]
if self.extensions:
targets = [target for target in targets if os.path.splitext(target.path)[1] in self.extensions
or (is_subdir(target.path, 'bin') and '.py' in self.extensions)]
if self.prefixes:
targets = [target for target in targets if any(target.path.startswith(pre) for pre in self.prefixes)]
if self.files:
targets = [target for target in targets if os.path.basename(target.path) in self.files]
if self.ignore_self and data_context().content.is_ansible:
relative_self_path = os.path.relpath(self.path, data_context().content.root)
targets = [target for target in targets if target.path != relative_self_path]
return targets
def filter_targets(
self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget]
"""Return the given list of test targets, filtered to include only those relevant for the test."""
return [
target for target in targets
if os.path.splitext(target.path)[1] == '.py'
or is_subdir(target.path, 'bin')
]
def filter_targets(
self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget]
"""Return the given list of test targets, filtered to include only those relevant for the test."""
yaml_targets = [
target for target in targets
if os.path.splitext(target.path)[1] in ('.yml', '.yaml')
]
for plugin_type, plugin_path in sorted(
data_context().content.plugin_paths.items()):
if plugin_type == 'module_utils':
continue
yaml_targets.extend([
target for target in targets
if os.path.splitext(target.path)[1] == '.py'
and os.path.basename(target.path) != '__init__.py'
and is_subdir(target.path, plugin_path)
])
return yaml_targets
def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget]
"""Return the given list of test targets, filtered to include only those relevant for the test."""
# This should use documentable plugins from constants instead
plugin_type_blacklist = set([
# not supported by ansible-doc
'action',
'doc_fragments',
'filter',
'module_utils',
'netconf',
'terminal',
'test',
])
plugin_paths = [plugin_path for plugin_type, plugin_path in data_context().content.plugin_paths.items() if plugin_type not in plugin_type_blacklist]
return [target for target in targets
if os.path.splitext(target.path)[1] == '.py'
and os.path.basename(target.path) != '__init__.py'
and any(is_subdir(target.path, path) for path in plugin_paths)
]
def get_dependent_paths_internal(self, path):
"""
:type path: str
:rtype: list[str]
"""
ext = os.path.splitext(os.path.split(path)[1])[1]
if is_subdir(path, data_context().content.module_utils_path):
if ext == '.py':
return self.get_python_module_utils_usage(path)
if ext == '.psm1':
return self.get_powershell_module_utils_usage(path)
if ext == '.cs':
return self.get_csharp_module_utils_usage(path)
if path.startswith('test/integration/targets/'):
return self.get_integration_target_usage(path)
return []
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
paths = [
[
i.path for i in targets.include
if os.path.splitext(i.path)[1] in ('.yml', '.yaml')
],
]
for plugin_type, plugin_path in sorted(
data_context().content.plugin_paths.items()):
if plugin_type == 'module_utils':
continue
paths.append([
target.path for target in targets.include
if os.path.splitext(target.path)[1] == '.py'
and os.path.basename(target.path) != '__init__.py'
and is_subdir(target.path, plugin_path)
])
paths = [sorted(p) for p in paths if p]
if not paths:
return SanitySkipped(self.name)
results = []
for test_paths in paths:
results += self.test_paths(args, test_paths)
if results:
return SanityFailure(self.name, messages=results)
return SanitySuccess(self.name)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
settings = self.load_settings(args, 'ansible-test')
paths = [
i.path for i in targets.include
if os.path.splitext(i.path)[1] in ('.yml', '.yaml')
]
for plugin_type, plugin_path in sorted(
data_context().content.plugin_paths.items()):
if plugin_type == 'module_utils':
continue
paths.extend([
target.path for target in targets.include
if os.path.splitext(target.path)[1] == '.py'
and os.path.basename(target.path) != '__init__.py'
and is_subdir(target.path, plugin_path)
])
paths = settings.filter_skipped_paths(paths)
if not paths:
return SanitySkipped(self.name)
results = self.test_paths(args, paths)
results = settings.process_errors(results, paths)
if results:
return SanityFailure(self.name, messages=results)
return SanitySuccess(self.name)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
display.warning('Skipping pylint on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
plugin_dir = os.path.join(INSTALL_ROOT, 'test/sanity/pylint/plugins')
plugin_names = sorted(p[0] for p in [
os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__')
skip_paths = read_lines_without_comments(PYLINT_SKIP_PATH, optional=True)
invalid_ignores = []
supported_versions = set(SUPPORTED_PYTHON_VERSIONS) - set(UNSUPPORTED_PYTHON_VERSIONS)
supported_versions = set([v.split('.')[0] for v in supported_versions]) | supported_versions
ignore_entries = read_lines_without_comments(PYLINT_IGNORE_PATH, optional=True)
ignore = collections.defaultdict(dict) # type: t.Dict[str, t.Dict[str, int]]
line = 0
for ignore_entry in ignore_entries:
line += 1
if not ignore_entry:
continue
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
path, code = ignore_entry.split(' ', 1)
if not os.path.exists(path):
invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path))
continue
if ' ' in code:
code, version = code.split(' ', 1)
if version not in supported_versions:
invalid_ignores.append((line, 'Invalid version: %s' % version))
continue
if version not in (args.python_version, args.python_version.split('.')[0]):
continue # ignore version specific entries for other versions
ignore[path][code] = line
skip_paths_set = set(skip_paths)
paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/')) and i.path not in skip_paths_set)
module_paths = [os.path.relpath(p, 'lib/ansible/modules/').split(os.path.sep) for p in paths if is_subdir(p, 'lib/ansible/modules/')]
module_dirs = sorted(set([p[0] for p in module_paths if len(p) > 1]))
large_module_group_threshold = 500
large_module_groups = [key for key, value in
itertools.groupby(module_paths, lambda p: p[0] if len(p) > 1 else '') if len(list(value)) > large_module_group_threshold]
large_module_group_paths = [os.path.relpath(p, 'lib/ansible/modules/').split(os.path.sep) for p in paths
if any(is_subdir(p, os.path.join('lib/ansible/modules/', g)) for g in large_module_groups)]
large_module_group_dirs = sorted(set([os.path.sep.join(p[:2]) for p in large_module_group_paths if len(p) > 2]))
contexts = []
remaining_paths = set(paths)
def add_context(available_paths, context_name, context_filter):
"""
:type available_paths: set[str]
:type context_name: str
:type context_filter: (str) -> bool
"""
filtered_paths = set(p for p in available_paths if context_filter(p))
contexts.append((context_name, sorted(filtered_paths)))
available_paths -= filtered_paths
def filter_path(path_filter=None):
"""
:type path_filter: str
:rtype: (str) -> bool
"""
def context_filter(path_to_filter):
"""
:type path_to_filter: str
:rtype: bool
"""
return is_subdir(path_to_filter, path_filter)
return context_filter
for large_module_group_dir in large_module_group_dirs:
add_context(remaining_paths, 'modules/%s' % large_module_group_dir, filter_path('lib/ansible/modules/%s/' % large_module_group_dir))
for module_dir in module_dirs:
add_context(remaining_paths, 'modules/%s' % module_dir, filter_path('lib/ansible/modules/%s/' % module_dir))
add_context(remaining_paths, 'modules', filter_path('lib/ansible/modules/'))
add_context(remaining_paths, 'module_utils', filter_path('lib/ansible/module_utils/'))
add_context(remaining_paths, 'units', filter_path('test/units/'))
add_context(remaining_paths, 'validate-modules', filter_path('test/sanity/validate-modules/'))
add_context(remaining_paths, 'sanity', filter_path('test/sanity/'))
add_context(remaining_paths, 'ansible-test', filter_path('test/runner/'))
add_context(remaining_paths, 'test', filter_path('test/'))
add_context(remaining_paths, 'hacking', filter_path('hacking/'))
add_context(remaining_paths, 'ansible', lambda p: True)
messages = []
context_times = []
test_start = datetime.datetime.utcnow()
for context, context_paths in sorted(contexts):
if not context_paths:
continue
context_start = datetime.datetime.utcnow()
messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names)
context_end = datetime.datetime.utcnow()
context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start))
test_end = datetime.datetime.utcnow()
for context_time in context_times:
display.info(context_time, verbosity=4)
display.info('total: %d (%s)' % (len(paths), test_end - test_start), verbosity=4)
errors = [SanityMessage(
message=m['message'].replace('\n', ' '),
path=m['path'],
line=int(m['line']),
column=int(m['column']),
level=m['type'],
code=m['symbol'],
) for m in messages]
if args.explain:
return SanitySuccess(self.name)
line = 0
filtered = []
for error in errors:
if error.code in ignore[error.path]:
ignore[error.path][error.code] = 0 # error ignored, clear line number of ignore entry to track usage
else:
filtered.append(error) # error not ignored
errors = filtered
for invalid_ignore in invalid_ignores:
errors.append(SanityMessage(
code='A201',
message=invalid_ignore[1],
path=PYLINT_IGNORE_PATH,
line=invalid_ignore[0],
column=1,
confidence=calculate_confidence(PYLINT_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None,
))
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(
code='A101',
message='Remove "%s" since it does not exist' % path,
path=PYLINT_SKIP_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(((PYLINT_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None,
))
for path in paths:
if path not in ignore:
continue
for code in ignore[path]:
line = ignore[path][code]
if not line:
continue
errors.append(SanityMessage(
code='A102',
message='Remove since "%s" passes "%s" pylint test' % (path, code),
path=PYLINT_IGNORE_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(((PYLINT_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None,
))
if errors:
return SanityFailure(self.name, messages=errors)
return SanitySuccess(self.name)
def context_filter(path_to_filter):
"""
:type path_to_filter: str
:rtype: bool
"""
return is_subdir(path_to_filter, path_filter)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
display.warning('Skipping pylint on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
plugin_dir = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/plugins')
plugin_names = sorted(p[0] for p in [
os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__')
settings = self.load_settings(args, 'ansible-test')
paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/'))
paths = settings.filter_skipped_paths(paths)
module_paths = [os.path.relpath(p, data_context().content.module_path).split(os.path.sep) for p in
paths if is_subdir(p, data_context().content.module_path)]
module_dirs = sorted(set([p[0] for p in module_paths if len(p) > 1]))
large_module_group_threshold = 500
large_module_groups = [key for key, value in
itertools.groupby(module_paths, lambda p: p[0] if len(p) > 1 else '') if len(list(value)) > large_module_group_threshold]
large_module_group_paths = [os.path.relpath(p, data_context().content.module_path).split(os.path.sep) for p in paths
if any(is_subdir(p, os.path.join(data_context().content.module_path, g)) for g in large_module_groups)]
large_module_group_dirs = sorted(set([os.path.sep.join(p[:2]) for p in large_module_group_paths if len(p) > 2]))
contexts = []
remaining_paths = set(paths)
def add_context(available_paths, context_name, context_filter):
"""
:type available_paths: set[str]
:type context_name: str
:type context_filter: (str) -> bool
"""
filtered_paths = set(p for p in available_paths if context_filter(p))
contexts.append((context_name, sorted(filtered_paths)))
available_paths -= filtered_paths
def filter_path(path_filter=None):
"""
:type path_filter: str
:rtype: (str) -> bool
"""
def context_filter(path_to_filter):
"""
:type path_to_filter: str
:rtype: bool
"""
return is_subdir(path_to_filter, path_filter)
return context_filter
for large_module_dir in large_module_group_dirs:
add_context(remaining_paths, 'modules/%s' % large_module_dir, filter_path(os.path.join(data_context().content.module_path, large_module_dir)))
for module_dir in module_dirs:
add_context(remaining_paths, 'modules/%s' % module_dir, filter_path(os.path.join(data_context().content.module_path, module_dir)))
add_context(remaining_paths, 'modules', filter_path(data_context().content.module_path))
add_context(remaining_paths, 'module_utils', filter_path(data_context().content.module_utils_path))
add_context(remaining_paths, 'units', filter_path(data_context().content.unit_path))
if data_context().content.collection:
add_context(remaining_paths, 'collection', lambda p: True)
else:
add_context(remaining_paths, 'validate-modules', filter_path('test/sanity/validate-modules/'))
add_context(remaining_paths, 'sanity', filter_path('test/sanity/'))
add_context(remaining_paths, 'ansible-test', filter_path('test/runner/'))
add_context(remaining_paths, 'test', filter_path('test/'))
add_context(remaining_paths, 'hacking', filter_path('hacking/'))
add_context(remaining_paths, 'ansible', lambda p: True)
messages = []
context_times = []
test_start = datetime.datetime.utcnow()
for context, context_paths in sorted(contexts):
if not context_paths:
continue
context_start = datetime.datetime.utcnow()
messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names)
context_end = datetime.datetime.utcnow()
context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start))
test_end = datetime.datetime.utcnow()
for context_time in context_times:
display.info(context_time, verbosity=4)
display.info('total: %d (%s)' % (len(paths), test_end - test_start), verbosity=4)
errors = [SanityMessage(
message=m['message'].replace('\n', ' '),
path=m['path'],
line=int(m['line']),
column=int(m['column']),
level=m['type'],
code=m['symbol'],
) for m in messages]
if args.explain:
return SanitySuccess(self.name)
errors = settings.process_errors(errors, paths)
if errors:
return SanityFailure(self.name, messages=errors)
return SanitySuccess(self.name)
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
settings = self.load_processor(args)
paths = [target.path for target in targets.include]
doc_targets = collections.defaultdict(list)
target_paths = collections.defaultdict(dict)
remap_types = dict(
modules='module',
)
for plugin_type, plugin_path in data_context().content.plugin_paths.items():
plugin_type = remap_types.get(plugin_type, plugin_type)
for plugin_file_path in [target.name for target in targets.include if is_subdir(target.path, plugin_path)]:
plugin_name = os.path.splitext(os.path.basename(plugin_file_path))[0]
if plugin_name.startswith('_'):
plugin_name = plugin_name[1:]
doc_targets[plugin_type].append(data_context().content.prefix + plugin_name)
target_paths[plugin_type][data_context().content.prefix + plugin_name] = plugin_file_path
env = ansible_environment(args, color=False)
error_messages = []
for doc_type in sorted(doc_targets):
cmd = ['ansible-doc', '-t', doc_type] + sorted(doc_targets[doc_type])
try:
with coverage_context(args):
stdout, stderr = intercept_command(args, cmd, target_name='ansible-doc', env=env, capture=True, python_version=python_version)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
errors = stderr.strip().splitlines()
messages = [self.parse_error(e, target_paths) for e in errors]
if messages and all(messages):
error_messages += messages
continue
if status:
summary = u'%s' % SubprocessError(cmd=cmd, status=status, stderr=stderr)
return SanityFailure(self.name, summary=summary)
if stdout:
display.info(stdout.strip(), verbosity=3)
if stderr:
summary = u'Output on stderr from ansible-doc is considered an error.\n\n%s' % SubprocessError(cmd, stderr=stderr)
return SanityFailure(self.name, summary=summary)
error_messages = settings.process_errors(error_messages, paths)
if error_messages:
return SanityFailure(self.name, messages=error_messages)
return SanitySuccess(self.name)
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
settings = self.load_settings(args, None, python_version)
paths = sorted(
i.path
for i in targets.include
if os.path.splitext(i.path)[1] == '.py' and
(is_subdir(i.path, data_context().content.module_path) or is_subdir(i.path, data_context().content.module_utils_path))
)
paths = settings.filter_skipped_paths(paths)
if not paths:
return SanitySkipped(self.name, python_version=python_version)
env = ansible_environment(args, color=False)
# create a clean virtual environment to minimize the available imports beyond the python standard library
virtual_environment_path = os.path.abspath('test/runner/.tox/minimal-py%s' % python_version.replace('.', ''))
virtual_environment_bin = os.path.join(virtual_environment_path, 'bin')
remove_tree(virtual_environment_path)
python = find_python(python_version)
cmd = [python, '-m', 'virtualenv', virtual_environment_path, '--python', python, '--no-setuptools', '--no-wheel']
if not args.coverage:
cmd.append('--no-pip')
run_command(args, cmd, capture=True)
# add the importer to our virtual environment so it can be accessed through the coverage injector
importer_path = os.path.join(virtual_environment_bin, 'importer.py')
if not args.explain:
os.symlink(os.path.abspath(os.path.join(ANSIBLE_ROOT, 'test/sanity/import/importer.py')), importer_path)
# create a minimal python library
python_path = os.path.abspath('test/runner/.tox/import/lib')
ansible_path = os.path.join(python_path, 'ansible')
ansible_init = os.path.join(ansible_path, '__init__.py')
ansible_link = os.path.join(ansible_path, 'module_utils')
if not args.explain:
remove_tree(ansible_path)
make_dirs(ansible_path)
with open(ansible_init, 'w'):
pass
os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/module_utils'), ansible_link)
# activate the virtual environment
env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH'])
env['PYTHONPATH'] = python_path
# make sure coverage is available in the virtual environment if needed
if args.coverage:
run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env)
run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env)
run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env)
run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env)
cmd = ['importer.py']
data = '\n'.join(paths)
display.info(data, verbosity=4)
results = []
virtualenv_python = os.path.join(virtual_environment_bin, 'python')
try:
with coverage_context(args):
stdout, stderr = intercept_command(args, cmd, self.name, env, capture=True, data=data, python_version=python_version,
virtualenv=virtualenv_python)
if stdout or stderr:
raise SubprocessError(cmd, stdout=stdout, stderr=stderr)
except SubprocessError as ex:
if ex.status != 10 or ex.stderr or not ex.stdout:
raise
pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$'
results = parse_to_list_of_dict(pattern, ex.stdout)
results = [SanityMessage(
message=r['message'],
path=r['path'],
line=int(r['line']),
column=int(r['column']),
) for r in results]
results = settings.process_errors(results, paths)
if results:
return SanityFailure(self.name, messages=results, python_version=python_version)
return SanitySuccess(self.name, python_version=python_version)
