def list_valid_collection_paths(search_paths=None, warn=False):
"""
Filter out non existing or invalid search_paths for collections
:param search_paths: list of text-string paths, if none load default config
:param warn: display warning if search_path does not exist
:return: subset of original list
"""
if search_paths is None:
search_paths = []
search_paths.extend(AnsibleCollectionLoader().n_collection_paths)
for path in search_paths:
b_path = to_bytes(path)
if not os.path.exists(b_path):
# warn for missing, but not if default
if warn:
display.warning(
"The configured collection path {0} does not exist.".
format(path))
continue
if not os.path.isdir(b_path):
if warn:
display.warning(
"The configured collection path {0}, exists, but it is not a directory."
.format(path))
continue
yield path
def pytest_configure():
from ansible.utils.collection_loader import AnsibleCollectionLoader
# allow unit tests to import code from collections
sys.meta_path.insert(0, AnsibleCollectionLoader())
import py._path.local
# force collections unit tests to be loaded with the ansible_collections namespace
# original idea from https://stackoverflow.com/questions/50174130/how-do-i-pytest-a-project-using-pep-420-namespace-packages/50175552#50175552
py._path.local.LocalPath.pypkgpath = collection_pypkgpath
def post_validate(self, templar):
'''
Override of base class post_validate, to also do final validation on
the block and task include (if any) to which this task belongs.
'''
if self._parent:
self._parent.post_validate(templar)
if AnsibleCollectionLoader().default_collection:
pass
super(Task, self).post_validate(templar)
def pytest_configure():
"""Configure this pytest plugin."""
from ansible.utils.collection_loader import AnsibleCollectionLoader
# allow unit tests to import code from collections
sys.meta_path.insert(0, AnsibleCollectionLoader())
# noinspection PyProtectedMember
import py._path.local
# force collections unit tests to be loaded with the ansible_collections namespace
# original idea from https://stackoverflow.com/questions/50174130/how-do-i-pytest-a-project-using-pep-420-namespace-packages/50175552#50175552
# noinspection PyProtectedMember
py._path.local.LocalPath.pypkgpath = collection_pypkgpath # pylint: disable=protected-access
def main():
"""Main program function."""
base_dir = os.getcwd()
messages = set()
if AnsibleCollectionLoader:
# allow importing code from collections
sys.meta_path.insert(0, AnsibleCollectionLoader())
for path in sys.argv[1:] or sys.stdin.read().splitlines():
test_python_module(path, base_dir, messages, False)
test_python_module(path, base_dir, messages, True)
if messages:
exit(10)
def _ensure_default_collection(collection_list=None):
default_collection = AnsibleCollectionLoader().default_collection
# Will be None when used as the default
if collection_list is None:
collection_list = []
# FIXME: exclude role tasks?
if default_collection and default_collection not in collection_list:
collection_list.insert(0, default_collection)
# if there's something in the list, ensure that builtin or legacy is always there too
if collection_list and 'ansible.builtin' not in collection_list and 'ansible.legacy' not in collection_list:
collection_list.append('ansible.legacy')
return collection_list
def _ensure_default_collection(collection_list=None):
default_collection = AnsibleCollectionLoader().default_collection
if collection_list is None:
collection_list = []
if default_collection: # FIXME: exclude role tasks?
if isinstance(collection_list, string_types):
collection_list = [collection_list]
if default_collection not in collection_list:
collection_list.insert(0, default_collection)
# if there's something in the list, ensure that builtin or legacy is always there too
if collection_list and 'ansible.builtin' not in collection_list and 'ansible.legacy' not in collection_list:
collection_list.append('ansible.legacy')
return collection_list
def _play_prereqs():
options = context.CLIARGS
# all needs loader
loader = DataLoader()
basedir = options.get('basedir', False)
if basedir:
loader.set_basedir(basedir)
add_all_plugin_dirs(basedir)
set_collection_playbook_paths(basedir)
default_collection = get_collection_name_from_path(basedir)
if default_collection:
display.warning(u'running with default collection {0}'.format(
default_collection))
AnsibleCollectionLoader().set_default_collection(
default_collection)
vault_ids = list(options['vault_ids'])
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
vault_secrets = CLI.setup_vault_secrets(
loader,
vault_ids=vault_ids,
vault_password_files=list(options['vault_password_files']),
ask_vault_pass=options['ask_vault_pass'],
auto_prompt=False)
loader.set_vault_secrets(vault_secrets)
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader,
sources=options['inventory'])
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager(
loader=loader,
inventory=inventory,
version_info=CLI.version_info(gitinfo=False))
return loader, inventory, variable_manager
# Copyright (c) IBM Corporation 2020
# Apache License, Version 2.0 (see https://opensource.org/licenses/Apache-2.0)
import re
import sys
from ansible_doc_extractor.cli import main
from ansible.utils.collection_loader import AnsibleCollectionLoader
if __name__ == '__main__':
# allow doc-extractor to import code from collections so doc_fragment plugins work
sys.meta_path.insert(0, AnsibleCollectionLoader())
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
def preprocess_data(self, ds):
'''
tasks are especially complex arguments so need pre-processing.
keep it short.
'''
if not isinstance(ds, dict):
raise AnsibleAssertionError('ds (%s) should be a dict but was a %s' % (ds, type(ds)))
# the new, cleaned datastructure, which will have legacy
# items reduced to a standard structure suitable for the
# attributes of the task class
new_ds = AnsibleMapping()
if isinstance(ds, AnsibleBaseYAMLObject):
new_ds.ansible_pos = ds.ansible_pos
# since this affects the task action parsing, we have to resolve in preprocess instead of in typical validator
default_collection = AnsibleCollectionLoader().default_collection
# use the parent value if our ds doesn't define it
collections_list = ds.get('collections', self.collections)
if collections_list is None:
collections_list = []
if isinstance(collections_list, string_types):
collections_list = [collections_list]
if default_collection and not self._role: # FIXME: and not a collections role
if collections_list:
if default_collection not in collections_list:
collections_list.insert(0, default_collection)
else:
collections_list = [default_collection]
if collections_list and 'ansible.builtin' not in collections_list and 'ansible.legacy' not in collections_list:
collections_list.append('ansible.legacy')
if collections_list:
ds['collections'] = collections_list
# use the args parsing class to determine the action, args,
# and the delegate_to value from the various possible forms
# supported as legacy
args_parser = ModuleArgsParser(task_ds=ds, collection_list=collections_list)
try:
(action, args, delegate_to) = args_parser.parse()
except AnsibleParserError as e:
# if the raises exception was created with obj=ds args, then it includes the detail
# so we dont need to add it so we can just re raise.
if e._obj:
raise
# But if it wasn't, we can add the yaml object now to get more detail
raise AnsibleParserError(to_native(e), obj=ds, orig_exc=e)
# the command/shell/script modules used to support the `cmd` arg,
# which corresponds to what we now call _raw_params, so move that
# value over to _raw_params (assuming it is empty)
if action in ('command', 'shell', 'script'):
if 'cmd' in args:
if args.get('_raw_params', '') != '':
raise AnsibleError("The 'cmd' argument cannot be used when other raw parameters are specified."
" Please put everything in one or the other place.", obj=ds)
args['_raw_params'] = args.pop('cmd')
new_ds['action'] = action
new_ds['args'] = args
new_ds['delegate_to'] = delegate_to
# we handle any 'vars' specified in the ds here, as we may
# be adding things to them below (special handling for includes).
# When that deprecated feature is removed, this can be too.
if 'vars' in ds:
# _load_vars is defined in Base, and is used to load a dictionary
# or list of dictionaries in a standard way
new_ds['vars'] = self._load_vars(None, ds.get('vars'))
else:
new_ds['vars'] = dict()
for (k, v) in iteritems(ds):
if k in ('action', 'local_action', 'args', 'delegate_to') or k == action or k == 'shell':
# we don't want to re-assign these values, which were determined by the ModuleArgsParser() above
continue
elif k.startswith('with_') and k.replace("with_", "") in lookup_loader:
# transform into loop property
self._preprocess_with_loop(ds, new_ds, k, v)
else:
# pre-2.0 syntax allowed variables for include statements at the top level of the task,
# so we move those into the 'vars' dictionary here, and show a deprecation message
# as we will remove this at some point in the future.
if action in ('include',) and k not in self._valid_attrs and k not in self.DEPRECATED_ATTRIBUTES:
display.deprecated("Specifying include variables at the top-level of the task is deprecated."
" Please see:\nhttps://docs.ansible.com/ansible/playbooks_roles.html#task-include-files-and-encouraging-reuse\n\n"
" for currently supported syntax regarding included files and variables", version="2.12")
new_ds['vars'][k] = v
elif C.INVALID_TASK_ATTRIBUTE_FAILED or k in self._valid_attrs:
new_ds[k] = v
else:
display.warning("Ignoring invalid attribute: %s" % k)
return super(Task, self).preprocess_data(new_ds)
def _load_role_data(self, role_include, parent_role=None):
self._role_name = role_include.role
self._role_path = role_include.get_role_path()
self._role_collection = role_include._role_collection
self._role_params = role_include.get_role_params()
self._variable_manager = role_include.get_variable_manager()
self._loader = role_include.get_loader()
if parent_role:
self.add_parent(parent_role)
# copy over all field attributes from the RoleInclude
# update self._attributes directly, to avoid squashing
for (attr_name, _) in iteritems(self._valid_attrs):
if attr_name in ('when', 'tags'):
self._attributes[attr_name] = self._extend_value(
self._attributes[attr_name],
role_include._attributes[attr_name],
)
else:
self._attributes[attr_name] = role_include._attributes[
attr_name]
# vars and default vars are regular dictionaries
self._role_vars = self._load_role_yaml(
'vars', main=self._from_files.get('vars'), allow_dir=True)
if self._role_vars is None:
self._role_vars = dict()
elif not isinstance(self._role_vars, dict):
raise AnsibleParserError(
"The vars/main.yml file for role '%s' must contain a dictionary of variables"
% self._role_name)
self._default_vars = self._load_role_yaml(
'defaults', main=self._from_files.get('defaults'), allow_dir=True)
if self._default_vars is None:
self._default_vars = dict()
elif not isinstance(self._default_vars, dict):
raise AnsibleParserError(
"The defaults/main.yml file for role '%s' must contain a dictionary of variables"
% self._role_name)
# load the role's other files, if they exist
metadata = self._load_role_yaml('meta')
if metadata:
self._metadata = RoleMetadata.load(
metadata,
owner=self,
variable_manager=self._variable_manager,
loader=self._loader)
self._dependencies = self._load_dependencies()
else:
self._metadata = RoleMetadata()
# reset collections list; roles do not inherit collections from parents, just use the defaults
# FUTURE: use a private config default for this so we can allow it to be overridden later
self.collections = []
# configure plugin/collection loading; either prepend the current role's collection or configure legacy plugin loading
# FIXME: need exception for explicit ansible.legacy?
if self._role_collection: # this is a collection-hosted role
self.collections.insert(0, self._role_collection)
else: # this is a legacy role, but set the default collection if there is one
default_collection = AnsibleCollectionLoader().default_collection
if default_collection:
self.collections.insert(0, default_collection)
# legacy role, ensure all plugin dirs under the role are added to plugin search path
add_all_plugin_dirs(self._role_path)
# collections can be specified in metadata for legacy or collection-hosted roles
if self._metadata.collections:
self.collections.extend((c for c in self._metadata.collections
if c not in self.collections))
# if any collections were specified, ensure that core or legacy synthetic collections are always included
if self.collections:
# default append collection is core for collection-hosted roles, legacy for others
default_append_collection = 'ansible.builtin' if self._role_collection else 'ansible.legacy'
if 'ansible.builtin' not in self.collections and 'ansible.legacy' not in self.collections:
self.collections.append(default_append_collection)
task_data = self._load_role_yaml('tasks',
main=self._from_files.get('tasks'))
if task_data:
try:
self._task_blocks = load_list_of_blocks(
task_data,
play=self._play,
role=self,
loader=self._loader,
variable_manager=self._variable_manager)
except AssertionError as e:
raise AnsibleParserError(
"The tasks/main.yml file for role '%s' must contain a list of tasks"
% self._role_name,
obj=task_data,
orig_exc=e)
handler_data = self._load_role_yaml(
'handlers', main=self._from_files.get('handlers'))
if handler_data:
try:
self._handler_blocks = load_list_of_blocks(
handler_data,
play=self._play,
role=self,
use_handlers=True,
loader=self._loader,
variable_manager=self._variable_manager)
except AssertionError as e:
raise AnsibleParserError(
"The handlers/main.yml file for role '%s' must contain a list of tasks"
% self._role_name,
obj=handler_data,
orig_exc=e)
def _configure_collection_loader():
if not any(
(isinstance(l, AnsibleCollectionLoader) for l in sys.meta_path)):
sys.meta_path.insert(0, AnsibleCollectionLoader())
def main():
"""
Main program function used to isolate globals from imported code.
Changes to globals in imported modules on Python 2.x will overwrite our own globals.
"""
import contextlib
import os
import re
import runpy
import sys
import traceback
import types
import warnings
ansible_path = os.environ['PYTHONPATH']
temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep
collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME')
try:
# noinspection PyCompatibility
from importlib import import_module
except ImportError:
def import_module(name):
__import__(name)
return sys.modules[name]
try:
# noinspection PyCompatibility
from StringIO import StringIO
except ImportError:
from io import StringIO
# pre-load an empty ansible package to prevent unwanted code in __init__.py from loading
# without this the ansible.release import there would pull in many Python modules which Ansible modules should not have access to
ansible_module = types.ModuleType('ansible')
ansible_module.__file__ = os.path.join(os.environ['PYTHONPATH'], 'ansible', '__init__.py')
ansible_module.__path__ = [os.path.dirname(ansible_module.__file__)]
ansible_module.__package__ = 'ansible'
sys.modules['ansible'] = ansible_module
if collection_full_name:
# allow importing code from collections when testing a collection
from ansible.utils.collection_loader import AnsibleCollectionLoader
from ansible.module_utils._text import to_bytes
def get_source(self, fullname):
mod = sys.modules.get(fullname)
if not mod:
mod = self.load_module(fullname)
with open(to_bytes(mod.__file__), 'rb') as mod_file:
source = mod_file.read()
return source
def get_code(self, fullname):
return compile(source=self.get_source(fullname), filename=self.get_filename(fullname), mode='exec', flags=0, dont_inherit=True)
def is_package(self, fullname):
return self.get_filename(fullname).endswith('__init__.py')
def get_filename(self, fullname):
mod = sys.modules.get(fullname) or self.load_module(fullname)
return mod.__file__
# monkeypatch collection loader to work with runpy
# remove this (and the associated code above) once implemented natively in the collection loader
AnsibleCollectionLoader.get_source = get_source
AnsibleCollectionLoader.get_code = get_code
AnsibleCollectionLoader.is_package = is_package
AnsibleCollectionLoader.get_filename = get_filename
collection_loader = AnsibleCollectionLoader()
# noinspection PyCallingNonCallable
sys.meta_path.insert(0, collection_loader)
else:
# do not support collection loading when not testing a collection
collection_loader = None
class ImporterAnsibleModuleException(Exception):
"""Exception thrown during initialization of ImporterAnsibleModule."""
class ImporterAnsibleModule:
"""Replacement for AnsibleModule to support import testing."""
def __init__(self, *args, **kwargs):
raise ImporterAnsibleModuleException()
class ImportBlacklist:
"""Blacklist inappropriate imports."""
def __init__(self, path, name):
self.path = path
self.name = name
self.loaded_modules = set()
def find_module(self, fullname, path=None):
"""Return self if the given fullname is blacklisted, otherwise return None.
:param fullname: str
:param path: str
:return: ImportBlacklist | None
"""
if fullname in self.loaded_modules:
return None # ignore modules that are already being loaded
if is_name_in_namepace(fullname, ['ansible']):
if fullname in ('ansible.module_utils.basic', 'ansible.module_utils.common.removed'):
return self # intercept loading so we can modify the result
if is_name_in_namepace(fullname, ['ansible.module_utils', self.name]):
return None # module_utils and module under test are always allowed
if os.path.exists(convert_ansible_name_to_absolute_path(fullname)):
return self # blacklist ansible files that exist
return None # ansible file does not exist, do not blacklist
if is_name_in_namepace(fullname, ['ansible_collections']):
if not collection_loader:
return self # blacklist collections when we are not testing a collection
if is_name_in_namepace(fullname, ['ansible_collections...plugins.module_utils', self.name]):
return None # module_utils and module under test are always allowed
if collection_loader.find_module(fullname, path):
return self # blacklist collection files that exist
return None # collection file does not exist, do not blacklist
# not a namespace we care about
return None
def load_module(self, fullname):
"""Raise an ImportError.
:type fullname: str
"""
if fullname == 'ansible.module_utils.basic':
module = self.__load_module(fullname)
# stop Ansible module execution during AnsibleModule instantiation
module.AnsibleModule = ImporterAnsibleModule
# no-op for _load_params since it may be called before instantiating AnsibleModule
module._load_params = lambda *args, **kwargs: {} # pylint: disable=protected-access
return module
if fullname == 'ansible.module_utils.common.removed':
module = self.__load_module(fullname)
# no-op for removed_module since it is called in place of AnsibleModule instantiation
module.removed_module = lambda *args, **kwargs: None
return module
raise ImportError('import of "%s" is not allowed in this context' % fullname)
def __load_module(self, fullname):
"""Load the requested module while avoiding infinite recursion.
:type fullname: str
:rtype: module
"""
self.loaded_modules.add(fullname)
return import_module(fullname)
def run():
"""Main program function."""
base_dir = os.getcwd()
messages = set()
for path in sys.argv[1:] or sys.stdin.read().splitlines():
name = convert_relative_path_to_name(path)
test_python_module(path, name, base_dir, messages)
if messages:
exit(10)
def test_python_module(path, name, base_dir, messages):
"""Test the given python module by importing it.
:type path: str
:type name: str
:type base_dir: str
:type messages: set[str]
"""
if name in sys.modules:
return # cannot be tested because it has already been loaded
is_ansible_module = (path.startswith('lib/ansible/modules/') or path.startswith('plugins/modules/')) and os.path.basename(path) != '__init__.py'
run_main = is_ansible_module
if path == 'lib/ansible/modules/utilities/logic/async_wrapper.py':
# async_wrapper is a non-standard Ansible module (does not use AnsibleModule) so we cannot test the main function
run_main = False
capture_normal = Capture()
capture_main = Capture()
try:
with monitor_sys_modules(path, messages):
with blacklist_imports(path, name, messages):
with capture_output(capture_normal):
import_module(name)
if run_main:
with monitor_sys_modules(path, messages):
with blacklist_imports(path, name, messages):
with capture_output(capture_main):
runpy.run_module(name, run_name='__main__')
except ImporterAnsibleModuleException:
# module instantiated AnsibleModule without raising an exception
pass
except BaseException as ex: # pylint: disable=locally-disabled, broad-except
# intentionally catch all exceptions, including calls to sys.exit
exc_type, _exc, exc_tb = sys.exc_info()
message = str(ex)
results = list(reversed(traceback.extract_tb(exc_tb)))
line = 0
offset = 0
full_path = os.path.join(base_dir, path)
base_path = base_dir + os.path.sep
source = None
# avoid line wraps in messages
message = re.sub(r'\n *', ': ', message)
for result in results:
if result[0] == full_path:
# save the line number for the file under test
line = result[1] or 0
if not source and result[0].startswith(base_path) and not result[0].startswith(temp_path):
# save the first path and line number in the traceback which is in our source tree
source = (os.path.relpath(result[0], base_path), result[1] or 0, 0)
if isinstance(ex, SyntaxError):
# SyntaxError has better information than the traceback
if ex.filename == full_path: # pylint: disable=locally-disabled, no-member
# syntax error was reported in the file under test
line = ex.lineno or 0 # pylint: disable=locally-disabled, no-member
offset = ex.offset or 0 # pylint: disable=locally-disabled, no-member
elif ex.filename.startswith(base_path) and not ex.filename.startswith(temp_path): # pylint: disable=locally-disabled, no-member
# syntax error was reported in our source tree
source = (os.path.relpath(ex.filename, base_path), ex.lineno or 0, ex.offset or 0) # pylint: disable=locally-disabled, no-member
# remove the filename and line number from the message
# either it was extracted above, or it's not really useful information
message = re.sub(r' \(.*?, line [0-9]+\)$', '', message)
if source and source[0] != path:
message += ' (at %s:%d:%d)' % (source[0], source[1], source[2])
report_message(path, line, offset, 'traceback', '%s: %s' % (exc_type.__name__, message), messages)
finally:
capture_report(path, capture_normal, messages)
capture_report(path, capture_main, messages)
def is_name_in_namepace(name, namespaces):
"""Returns True if the given name is one of the given namespaces, otherwise returns False."""
name_parts = name.split('.')
for namespace in namespaces:
namespace_parts = namespace.split('.')
length = min(len(name_parts), len(namespace_parts))
truncated_name = name_parts[0:length]
truncated_namespace = namespace_parts[0:length]
# empty parts in the namespace are treated as wildcards
# to simplify the comparison, use those empty parts to indicate the positions in the name to be empty as well
for idx, part in enumerate(truncated_namespace):
if not part:
truncated_name[idx] = part
# example: name=ansible, allowed_name=ansible.module_utils
# example: name=ansible.module_utils.system.ping, allowed_name=ansible.module_utils
if truncated_name == truncated_namespace:
return True
return False
def check_sys_modules(path, before, messages):
"""Check for unwanted changes to sys.modules.
:type path: str
:type before: dict[str, module]
:type messages: set[str]
"""
after = sys.modules
removed = set(before.keys()) - set(after.keys())
changed = set(key for key, value in before.items() if key in after and value != after[key])
# additions are checked by our custom PEP 302 loader, so we don't need to check them again here
for module in sorted(removed):
report_message(path, 0, 0, 'unload', 'unloading of "%s" in sys.modules is not supported' % module, messages)
for module in sorted(changed):
report_message(path, 0, 0, 'reload', 'reloading of "%s" in sys.modules is not supported' % module, messages)
def convert_ansible_name_to_absolute_path(name):
"""Calculate the module path from the given name.
:type name: str
:rtype: str
"""
return os.path.join(ansible_path, name.replace('.', os.path.sep))
def convert_relative_path_to_name(path):
"""Calculate the module name from the given path.
:type path: str
:rtype: str
"""
if path.endswith('/__init__.py'):
clean_path = os.path.dirname(path)
else:
clean_path = path
clean_path = os.path.splitext(clean_path)[0]
name = clean_path.replace(os.path.sep, '.')
if collection_loader:
# when testing collections the relative paths (and names) being tested are within the collection under test
name = 'ansible_collections.%s.%s' % (collection_full_name, name)
else:
# when testing ansible all files being imported reside under the lib directory
name = name[len('lib/'):]
return name
class Capture:
"""Captured output and/or exception."""
def __init__(self):
self.stdout = StringIO()
self.stderr = StringIO()
def capture_report(path, capture, messages):
"""Report on captured output.
:type path: str
:type capture: Capture
:type messages: set[str]
"""
if capture.stdout.getvalue():
first = capture.stdout.getvalue().strip().splitlines()[0].strip()
report_message(path, 0, 0, 'stdout', first, messages)
if capture.stderr.getvalue():
first = capture.stderr.getvalue().strip().splitlines()[0].strip()
report_message(path, 0, 0, 'stderr', first, messages)
def report_message(path, line, column, code, message, messages):
"""Report message if not already reported.
:type path: str
:type line: int
:type column: int
:type code: str
:type message: str
:type messages: set[str]
"""
message = '%s:%d:%d: %s: %s' % (path, line, column, code, message)
if message not in messages:
messages.add(message)
print(message)
@contextlib.contextmanager
def blacklist_imports(path, name, messages):
"""Blacklist imports.
:type path: str
:type name: str
:type messages: set[str]
"""
blacklist = ImportBlacklist(path, name)
sys.meta_path.insert(0, blacklist)
try:
yield
finally:
if sys.meta_path[0] != blacklist:
report_message(path, 0, 0, 'metapath', 'changes to sys.meta_path[0] are not permitted', messages)
while blacklist in sys.meta_path:
sys.meta_path.remove(blacklist)
@contextlib.contextmanager
def monitor_sys_modules(path, messages):
"""Monitor sys.modules for unwanted changes, reverting any additions made to our own namespaces."""
snapshot = sys.modules.copy()
try:
yield
finally:
check_sys_modules(path, snapshot, messages)
for key in set(sys.modules.keys()) - set(snapshot.keys()):
if is_name_in_namepace(key, ('ansible', 'ansible_collections')):
del sys.modules[key] # only unload our own code since we know it's native Python
@contextlib.contextmanager
def capture_output(capture):
"""Capture sys.stdout and sys.stderr.
:type capture: Capture
"""
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = capture.stdout
sys.stderr = capture.stderr
# clear all warnings registries to make all warnings available
for module in sys.modules.values():
try:
module.__warningregistry__.clear()
except AttributeError:
pass
with warnings.catch_warnings():
warnings.simplefilter('error')
try:
yield
finally:
sys.stdout = old_stdout
sys.stderr = old_stderr
run()
def run(self):
super(PlaybookCLI, self).run()
# Note: slightly wrong, this is written so that implicit localhost
# manages passwords
sshpass = None
becomepass = None
passwords = {}
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
b_playbook_dirs = []
for playbook in context.CLIARGS['args']:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" %
playbook)
if not (os.path.isfile(playbook)
or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError(
"the playbook: %s does not appear to be a file" % playbook)
b_playbook_dir = os.path.dirname(
os.path.abspath(
to_bytes(playbook, errors='surrogate_or_strict')))
# load plugins from all playbooks in case they add callbacks/inventory/etc
add_all_plugin_dirs(b_playbook_dir)
b_playbook_dirs.append(b_playbook_dir)
set_collection_playbook_paths(b_playbook_dirs)
playbook_collection = get_collection_name_from_path(b_playbook_dirs[0])
if playbook_collection:
display.warning("running playbook inside collection {0}".format(
playbook_collection))
AnsibleCollectionLoader().set_default_collection(
playbook_collection)
# don't deal with privilege escalation or passwords when we don't need to
if not (context.CLIARGS['listhosts'] or context.CLIARGS['listtasks']
or context.CLIARGS['listtags'] or context.CLIARGS['syntax']):
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
# create base objects
loader, inventory, variable_manager = self._play_prereqs()
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
CLI.get_host_list(inventory, context.CLIARGS['subset'])
# flush fact cache if requested
if context.CLIARGS['flush_cache']:
self._flush_cache(inventory, variable_manager)
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(playbooks=context.CLIARGS['args'],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords)
results = pbex.run()
if isinstance(results, list):
for p in results:
display.display('\nplaybook: %s' % p['playbook'])
for idx, play in enumerate(p['plays']):
if play._included_path is not None:
loader.set_basedir(play._included_path)
else:
pb_dir = os.path.realpath(
os.path.dirname(p['playbook']))
loader.set_basedir(pb_dir)
msg = "\n play #%d (%s): %s" % (idx + 1, ','.join(
play.hosts), play.name)
mytags = set(play.tags)
msg += '\tTAGS: [%s]' % (','.join(mytags))
if context.CLIARGS['listhosts']:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (
play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
display.display(msg)
all_tags = set()
if context.CLIARGS['listtags'] or context.CLIARGS[
'listtasks']:
taskmsg = ''
if context.CLIARGS['listtasks']:
taskmsg = ' tasks:\n'
def _process_block(b):
taskmsg = ''
for task in b.block:
if isinstance(task, Block):
taskmsg += _process_block(task)
else:
if task.action == 'meta':
continue
all_tags.update(task.tags)
if context.CLIARGS['listtasks']:
cur_tags = list(
mytags.union(set(task.tags)))
cur_tags.sort()
if task.name:
taskmsg += " %s" % task.get_name(
)
else:
taskmsg += " %s" % task.action
taskmsg += "\tTAGS: [%s]\n" % ', '.join(
cur_tags)
return taskmsg
all_vars = variable_manager.get_vars(play=play)
for block in play.compile():
block = block.filter_tagged_tasks(all_vars)
if not block.has_tasks():
continue
taskmsg += _process_block(block)
if context.CLIARGS['listtags']:
cur_tags = list(mytags.union(all_tags))
cur_tags.sort()
taskmsg += " TASK TAGS: [%s]\n" % ', '.join(
cur_tags)
display.display(taskmsg)
return 0
else:
return results
def test_import_from_collection(monkeypatch):
collection_root = os.path.join(os.path.dirname(__file__), 'fixtures',
'collections')
collection_path = os.path.join(
collection_root,
'ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py'
)
# the trace we're expecting to be generated when running the code below:
# answer = question()
expected_trace_log = [
(collection_path, 5, 'call'),
(collection_path, 6, 'line'),
(collection_path, 6, 'return'),
]
# define the collection root before any ansible code has been loaded
# otherwise config will have already been loaded and changing the environment will have no effect
monkeypatch.setenv('ANSIBLE_COLLECTIONS_PATHS', collection_root)
from ansible.utils.collection_loader import AnsibleCollectionLoader
# zap the singleton collection loader instance if it exists
AnsibleCollectionLoader._Singleton__instance = None
for index in [
idx for idx, obj in enumerate(sys.meta_path)
if isinstance(obj, AnsibleCollectionLoader)
]:
# replace any existing collection loaders that may exist
# since these were loaded during unit test collection
# they will not have the correct configuration
sys.meta_path[index] = AnsibleCollectionLoader()
# make sure the collection loader is installed
# this will be a no-op if the collection loader is already installed
# which will depend on whether or not any tests being run imported ansible.plugins.loader during unit test collection
from ansible.plugins.loader import _configure_collection_loader
_configure_collection_loader(
) # currently redundant, the import above already calls this
from ansible_collections.my_namespace.my_collection.plugins.module_utils.my_util import question
original_trace_function = sys.gettrace()
trace_log = []
if original_trace_function:
# enable tracing while preserving the existing trace function (coverage)
def my_trace_function(frame, event, arg):
trace_log.append((frame.f_code.co_filename, frame.f_lineno, event))
# the original trace function expects to have itself set as the trace function
sys.settrace(original_trace_function)
# call the original trace function
original_trace_function(frame, event, arg)
# restore our trace function
sys.settrace(my_trace_function)
return my_trace_function
else:
# no existing trace function, so our trace function is much simpler
def my_trace_function(frame, event, arg):
trace_log.append((frame.f_code.co_filename, frame.f_lineno, event))
return my_trace_function
sys.settrace(my_trace_function)
try:
# run a minimal amount of code while the trace is running
# adding more code here, including use of a context manager, will add more to our trace
answer = question()
finally:
sys.settrace(original_trace_function)
# make sure 'import ... as ...' works on builtin synthetic collections
# the following import is not supported (it tries to find module_utils in ansible.plugins)
# import ansible_collections.ansible.builtin.plugins.module_utils as c1
import ansible_collections.ansible.builtin.plugins.action as c2
import ansible_collections.ansible.builtin.plugins as c3
import ansible_collections.ansible.builtin as c4
import ansible_collections.ansible as c5
import ansible_collections as c6
# make sure 'import ...' works on builtin synthetic collections
import ansible_collections.ansible.builtin.plugins.module_utils
import ansible_collections.ansible.builtin.plugins.action
assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2
import ansible_collections.ansible.builtin.plugins
assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3
import ansible_collections.ansible.builtin
assert ansible_collections.ansible.builtin == c5.builtin == c4
import ansible_collections.ansible
assert ansible_collections.ansible == c6.ansible == c5
import ansible_collections
assert ansible_collections == c6
# make sure 'from ... import ...' works on builtin synthetic collections
from ansible_collections.ansible import builtin
from ansible_collections.ansible.builtin import plugins
assert builtin.plugins == plugins
from ansible_collections.ansible.builtin.plugins import action
from ansible_collections.ansible.builtin.plugins.action import command
assert action.command == command
from ansible_collections.ansible.builtin.plugins.module_utils import basic
from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule
assert basic.AnsibleModule == AnsibleModule
# make sure relative imports work from collections code
# these require __package__ to be set correctly
import ansible_collections.my_namespace.my_collection.plugins.module_utils.my_other_util
import ansible_collections.my_namespace.my_collection.plugins.action.my_action
# verify that code loaded from a collection does not inherit __future__ statements from the collection loader
if sys.version_info[0] == 2:
# if the collection code inherits the division future feature from the collection loader this will fail
assert answer == 1
else:
assert answer == 1.5
# verify that the filename and line number reported by the trace is correct
# this makes sure that collection loading preserves file paths and line numbers
assert trace_log == expected_trace_log