def list_valid_collection_paths(search_paths=None, warn=False): """ Filter out non existing or invalid search_paths for collections :param search_paths: list of text-string paths, if none load default config :param warn: display warning if search_path does not exist :return: subset of original list """ if search_paths is None: search_paths = [] search_paths.extend(AnsibleCollectionLoader().n_collection_paths) for path in search_paths: b_path = to_bytes(path) if not os.path.exists(b_path): # warn for missing, but not if default if warn: display.warning( "The configured collection path {0} does not exist.". format(path)) continue if not os.path.isdir(b_path): if warn: display.warning( "The configured collection path {0}, exists, but it is not a directory." .format(path)) continue yield path
def pytest_configure(): from ansible.utils.collection_loader import AnsibleCollectionLoader # allow unit tests to import code from collections sys.meta_path.insert(0, AnsibleCollectionLoader()) import py._path.local # force collections unit tests to be loaded with the ansible_collections namespace # original idea from https://stackoverflow.com/questions/50174130/how-do-i-pytest-a-project-using-pep-420-namespace-packages/50175552#50175552 py._path.local.LocalPath.pypkgpath = collection_pypkgpath
def post_validate(self, templar): ''' Override of base class post_validate, to also do final validation on the block and task include (if any) to which this task belongs. ''' if self._parent: self._parent.post_validate(templar) if AnsibleCollectionLoader().default_collection: pass super(Task, self).post_validate(templar)
def pytest_configure(): """Configure this pytest plugin.""" from ansible.utils.collection_loader import AnsibleCollectionLoader # allow unit tests to import code from collections sys.meta_path.insert(0, AnsibleCollectionLoader()) # noinspection PyProtectedMember import py._path.local # force collections unit tests to be loaded with the ansible_collections namespace # original idea from https://stackoverflow.com/questions/50174130/how-do-i-pytest-a-project-using-pep-420-namespace-packages/50175552#50175552 # noinspection PyProtectedMember py._path.local.LocalPath.pypkgpath = collection_pypkgpath # pylint: disable=protected-access
def main(): """Main program function.""" base_dir = os.getcwd() messages = set() if AnsibleCollectionLoader: # allow importing code from collections sys.meta_path.insert(0, AnsibleCollectionLoader()) for path in sys.argv[1:] or sys.stdin.read().splitlines(): test_python_module(path, base_dir, messages, False) test_python_module(path, base_dir, messages, True) if messages: exit(10)
def _ensure_default_collection(collection_list=None): default_collection = AnsibleCollectionLoader().default_collection # Will be None when used as the default if collection_list is None: collection_list = [] # FIXME: exclude role tasks? if default_collection and default_collection not in collection_list: collection_list.insert(0, default_collection) # if there's something in the list, ensure that builtin or legacy is always there too if collection_list and 'ansible.builtin' not in collection_list and 'ansible.legacy' not in collection_list: collection_list.append('ansible.legacy') return collection_list
def _ensure_default_collection(collection_list=None): default_collection = AnsibleCollectionLoader().default_collection if collection_list is None: collection_list = [] if default_collection: # FIXME: exclude role tasks? if isinstance(collection_list, string_types): collection_list = [collection_list] if default_collection not in collection_list: collection_list.insert(0, default_collection) # if there's something in the list, ensure that builtin or legacy is always there too if collection_list and 'ansible.builtin' not in collection_list and 'ansible.legacy' not in collection_list: collection_list.append('ansible.legacy') return collection_list
def _play_prereqs(): options = context.CLIARGS # all needs loader loader = DataLoader() basedir = options.get('basedir', False) if basedir: loader.set_basedir(basedir) add_all_plugin_dirs(basedir) set_collection_playbook_paths(basedir) default_collection = get_collection_name_from_path(basedir) if default_collection: display.warning(u'running with default collection {0}'.format( default_collection)) AnsibleCollectionLoader().set_default_collection( default_collection) vault_ids = list(options['vault_ids']) default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST vault_ids = default_vault_ids + vault_ids vault_secrets = CLI.setup_vault_secrets( loader, vault_ids=vault_ids, vault_password_files=list(options['vault_password_files']), ask_vault_pass=options['ask_vault_pass'], auto_prompt=False) loader.set_vault_secrets(vault_secrets) # create the inventory, and filter it based on the subset specified (if any) inventory = InventoryManager(loader=loader, sources=options['inventory']) # create the variable manager, which will be shared throughout # the code, ensuring a consistent view of global variables variable_manager = VariableManager( loader=loader, inventory=inventory, version_info=CLI.version_info(gitinfo=False)) return loader, inventory, variable_manager
# Copyright (c) IBM Corporation 2020 # Apache License, Version 2.0 (see https://opensource.org/licenses/Apache-2.0) import re import sys from ansible_doc_extractor.cli import main from ansible.utils.collection_loader import AnsibleCollectionLoader if __name__ == '__main__': # allow doc-extractor to import code from collections so doc_fragment plugins work sys.meta_path.insert(0, AnsibleCollectionLoader()) sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) sys.exit(main())
def preprocess_data(self, ds): ''' tasks are especially complex arguments so need pre-processing. keep it short. ''' if not isinstance(ds, dict): raise AnsibleAssertionError('ds (%s) should be a dict but was a %s' % (ds, type(ds))) # the new, cleaned datastructure, which will have legacy # items reduced to a standard structure suitable for the # attributes of the task class new_ds = AnsibleMapping() if isinstance(ds, AnsibleBaseYAMLObject): new_ds.ansible_pos = ds.ansible_pos # since this affects the task action parsing, we have to resolve in preprocess instead of in typical validator default_collection = AnsibleCollectionLoader().default_collection # use the parent value if our ds doesn't define it collections_list = ds.get('collections', self.collections) if collections_list is None: collections_list = [] if isinstance(collections_list, string_types): collections_list = [collections_list] if default_collection and not self._role: # FIXME: and not a collections role if collections_list: if default_collection not in collections_list: collections_list.insert(0, default_collection) else: collections_list = [default_collection] if collections_list and 'ansible.builtin' not in collections_list and 'ansible.legacy' not in collections_list: collections_list.append('ansible.legacy') if collections_list: ds['collections'] = collections_list # use the args parsing class to determine the action, args, # and the delegate_to value from the various possible forms # supported as legacy args_parser = ModuleArgsParser(task_ds=ds, collection_list=collections_list) try: (action, args, delegate_to) = args_parser.parse() except AnsibleParserError as e: # if the raises exception was created with obj=ds args, then it includes the detail # so we dont need to add it so we can just re raise. if e._obj: raise # But if it wasn't, we can add the yaml object now to get more detail raise AnsibleParserError(to_native(e), obj=ds, orig_exc=e) # the command/shell/script modules used to support the `cmd` arg, # which corresponds to what we now call _raw_params, so move that # value over to _raw_params (assuming it is empty) if action in ('command', 'shell', 'script'): if 'cmd' in args: if args.get('_raw_params', '') != '': raise AnsibleError("The 'cmd' argument cannot be used when other raw parameters are specified." " Please put everything in one or the other place.", obj=ds) args['_raw_params'] = args.pop('cmd') new_ds['action'] = action new_ds['args'] = args new_ds['delegate_to'] = delegate_to # we handle any 'vars' specified in the ds here, as we may # be adding things to them below (special handling for includes). # When that deprecated feature is removed, this can be too. if 'vars' in ds: # _load_vars is defined in Base, and is used to load a dictionary # or list of dictionaries in a standard way new_ds['vars'] = self._load_vars(None, ds.get('vars')) else: new_ds['vars'] = dict() for (k, v) in iteritems(ds): if k in ('action', 'local_action', 'args', 'delegate_to') or k == action or k == 'shell': # we don't want to re-assign these values, which were determined by the ModuleArgsParser() above continue elif k.startswith('with_') and k.replace("with_", "") in lookup_loader: # transform into loop property self._preprocess_with_loop(ds, new_ds, k, v) else: # pre-2.0 syntax allowed variables for include statements at the top level of the task, # so we move those into the 'vars' dictionary here, and show a deprecation message # as we will remove this at some point in the future. if action in ('include',) and k not in self._valid_attrs and k not in self.DEPRECATED_ATTRIBUTES: display.deprecated("Specifying include variables at the top-level of the task is deprecated." " Please see:\nhttps://docs.ansible.com/ansible/playbooks_roles.html#task-include-files-and-encouraging-reuse\n\n" " for currently supported syntax regarding included files and variables", version="2.12") new_ds['vars'][k] = v elif C.INVALID_TASK_ATTRIBUTE_FAILED or k in self._valid_attrs: new_ds[k] = v else: display.warning("Ignoring invalid attribute: %s" % k) return super(Task, self).preprocess_data(new_ds)
def _load_role_data(self, role_include, parent_role=None): self._role_name = role_include.role self._role_path = role_include.get_role_path() self._role_collection = role_include._role_collection self._role_params = role_include.get_role_params() self._variable_manager = role_include.get_variable_manager() self._loader = role_include.get_loader() if parent_role: self.add_parent(parent_role) # copy over all field attributes from the RoleInclude # update self._attributes directly, to avoid squashing for (attr_name, _) in iteritems(self._valid_attrs): if attr_name in ('when', 'tags'): self._attributes[attr_name] = self._extend_value( self._attributes[attr_name], role_include._attributes[attr_name], ) else: self._attributes[attr_name] = role_include._attributes[ attr_name] # vars and default vars are regular dictionaries self._role_vars = self._load_role_yaml( 'vars', main=self._from_files.get('vars'), allow_dir=True) if self._role_vars is None: self._role_vars = dict() elif not isinstance(self._role_vars, dict): raise AnsibleParserError( "The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name) self._default_vars = self._load_role_yaml( 'defaults', main=self._from_files.get('defaults'), allow_dir=True) if self._default_vars is None: self._default_vars = dict() elif not isinstance(self._default_vars, dict): raise AnsibleParserError( "The defaults/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name) # load the role's other files, if they exist metadata = self._load_role_yaml('meta') if metadata: self._metadata = RoleMetadata.load( metadata, owner=self, variable_manager=self._variable_manager, loader=self._loader) self._dependencies = self._load_dependencies() else: self._metadata = RoleMetadata() # reset collections list; roles do not inherit collections from parents, just use the defaults # FUTURE: use a private config default for this so we can allow it to be overridden later self.collections = [] # configure plugin/collection loading; either prepend the current role's collection or configure legacy plugin loading # FIXME: need exception for explicit ansible.legacy? if self._role_collection: # this is a collection-hosted role self.collections.insert(0, self._role_collection) else: # this is a legacy role, but set the default collection if there is one default_collection = AnsibleCollectionLoader().default_collection if default_collection: self.collections.insert(0, default_collection) # legacy role, ensure all plugin dirs under the role are added to plugin search path add_all_plugin_dirs(self._role_path) # collections can be specified in metadata for legacy or collection-hosted roles if self._metadata.collections: self.collections.extend((c for c in self._metadata.collections if c not in self.collections)) # if any collections were specified, ensure that core or legacy synthetic collections are always included if self.collections: # default append collection is core for collection-hosted roles, legacy for others default_append_collection = 'ansible.builtin' if self._role_collection else 'ansible.legacy' if 'ansible.builtin' not in self.collections and 'ansible.legacy' not in self.collections: self.collections.append(default_append_collection) task_data = self._load_role_yaml('tasks', main=self._from_files.get('tasks')) if task_data: try: self._task_blocks = load_list_of_blocks( task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager) except AssertionError as e: raise AnsibleParserError( "The tasks/main.yml file for role '%s' must contain a list of tasks" % self._role_name, obj=task_data, orig_exc=e) handler_data = self._load_role_yaml( 'handlers', main=self._from_files.get('handlers')) if handler_data: try: self._handler_blocks = load_list_of_blocks( handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader, variable_manager=self._variable_manager) except AssertionError as e: raise AnsibleParserError( "The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name, obj=handler_data, orig_exc=e)
def _configure_collection_loader(): if not any( (isinstance(l, AnsibleCollectionLoader) for l in sys.meta_path)): sys.meta_path.insert(0, AnsibleCollectionLoader())
def main(): """ Main program function used to isolate globals from imported code. Changes to globals in imported modules on Python 2.x will overwrite our own globals. """ import contextlib import os import re import runpy import sys import traceback import types import warnings ansible_path = os.environ['PYTHONPATH'] temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME') try: # noinspection PyCompatibility from importlib import import_module except ImportError: def import_module(name): __import__(name) return sys.modules[name] try: # noinspection PyCompatibility from StringIO import StringIO except ImportError: from io import StringIO # pre-load an empty ansible package to prevent unwanted code in __init__.py from loading # without this the ansible.release import there would pull in many Python modules which Ansible modules should not have access to ansible_module = types.ModuleType('ansible') ansible_module.__file__ = os.path.join(os.environ['PYTHONPATH'], 'ansible', '__init__.py') ansible_module.__path__ = [os.path.dirname(ansible_module.__file__)] ansible_module.__package__ = 'ansible' sys.modules['ansible'] = ansible_module if collection_full_name: # allow importing code from collections when testing a collection from ansible.utils.collection_loader import AnsibleCollectionLoader from ansible.module_utils._text import to_bytes def get_source(self, fullname): mod = sys.modules.get(fullname) if not mod: mod = self.load_module(fullname) with open(to_bytes(mod.__file__), 'rb') as mod_file: source = mod_file.read() return source def get_code(self, fullname): return compile(source=self.get_source(fullname), filename=self.get_filename(fullname), mode='exec', flags=0, dont_inherit=True) def is_package(self, fullname): return self.get_filename(fullname).endswith('__init__.py') def get_filename(self, fullname): mod = sys.modules.get(fullname) or self.load_module(fullname) return mod.__file__ # monkeypatch collection loader to work with runpy # remove this (and the associated code above) once implemented natively in the collection loader AnsibleCollectionLoader.get_source = get_source AnsibleCollectionLoader.get_code = get_code AnsibleCollectionLoader.is_package = is_package AnsibleCollectionLoader.get_filename = get_filename collection_loader = AnsibleCollectionLoader() # noinspection PyCallingNonCallable sys.meta_path.insert(0, collection_loader) else: # do not support collection loading when not testing a collection collection_loader = None class ImporterAnsibleModuleException(Exception): """Exception thrown during initialization of ImporterAnsibleModule.""" class ImporterAnsibleModule: """Replacement for AnsibleModule to support import testing.""" def __init__(self, *args, **kwargs): raise ImporterAnsibleModuleException() class ImportBlacklist: """Blacklist inappropriate imports.""" def __init__(self, path, name): self.path = path self.name = name self.loaded_modules = set() def find_module(self, fullname, path=None): """Return self if the given fullname is blacklisted, otherwise return None. :param fullname: str :param path: str :return: ImportBlacklist | None """ if fullname in self.loaded_modules: return None # ignore modules that are already being loaded if is_name_in_namepace(fullname, ['ansible']): if fullname in ('ansible.module_utils.basic', 'ansible.module_utils.common.removed'): return self # intercept loading so we can modify the result if is_name_in_namepace(fullname, ['ansible.module_utils', self.name]): return None # module_utils and module under test are always allowed if os.path.exists(convert_ansible_name_to_absolute_path(fullname)): return self # blacklist ansible files that exist return None # ansible file does not exist, do not blacklist if is_name_in_namepace(fullname, ['ansible_collections']): if not collection_loader: return self # blacklist collections when we are not testing a collection if is_name_in_namepace(fullname, ['ansible_collections...plugins.module_utils', self.name]): return None # module_utils and module under test are always allowed if collection_loader.find_module(fullname, path): return self # blacklist collection files that exist return None # collection file does not exist, do not blacklist # not a namespace we care about return None def load_module(self, fullname): """Raise an ImportError. :type fullname: str """ if fullname == 'ansible.module_utils.basic': module = self.__load_module(fullname) # stop Ansible module execution during AnsibleModule instantiation module.AnsibleModule = ImporterAnsibleModule # no-op for _load_params since it may be called before instantiating AnsibleModule module._load_params = lambda *args, **kwargs: {} # pylint: disable=protected-access return module if fullname == 'ansible.module_utils.common.removed': module = self.__load_module(fullname) # no-op for removed_module since it is called in place of AnsibleModule instantiation module.removed_module = lambda *args, **kwargs: None return module raise ImportError('import of "%s" is not allowed in this context' % fullname) def __load_module(self, fullname): """Load the requested module while avoiding infinite recursion. :type fullname: str :rtype: module """ self.loaded_modules.add(fullname) return import_module(fullname) def run(): """Main program function.""" base_dir = os.getcwd() messages = set() for path in sys.argv[1:] or sys.stdin.read().splitlines(): name = convert_relative_path_to_name(path) test_python_module(path, name, base_dir, messages) if messages: exit(10) def test_python_module(path, name, base_dir, messages): """Test the given python module by importing it. :type path: str :type name: str :type base_dir: str :type messages: set[str] """ if name in sys.modules: return # cannot be tested because it has already been loaded is_ansible_module = (path.startswith('lib/ansible/modules/') or path.startswith('plugins/modules/')) and os.path.basename(path) != '__init__.py' run_main = is_ansible_module if path == 'lib/ansible/modules/utilities/logic/async_wrapper.py': # async_wrapper is a non-standard Ansible module (does not use AnsibleModule) so we cannot test the main function run_main = False capture_normal = Capture() capture_main = Capture() try: with monitor_sys_modules(path, messages): with blacklist_imports(path, name, messages): with capture_output(capture_normal): import_module(name) if run_main: with monitor_sys_modules(path, messages): with blacklist_imports(path, name, messages): with capture_output(capture_main): runpy.run_module(name, run_name='__main__') except ImporterAnsibleModuleException: # module instantiated AnsibleModule without raising an exception pass except BaseException as ex: # pylint: disable=locally-disabled, broad-except # intentionally catch all exceptions, including calls to sys.exit exc_type, _exc, exc_tb = sys.exc_info() message = str(ex) results = list(reversed(traceback.extract_tb(exc_tb))) line = 0 offset = 0 full_path = os.path.join(base_dir, path) base_path = base_dir + os.path.sep source = None # avoid line wraps in messages message = re.sub(r'\n *', ': ', message) for result in results: if result[0] == full_path: # save the line number for the file under test line = result[1] or 0 if not source and result[0].startswith(base_path) and not result[0].startswith(temp_path): # save the first path and line number in the traceback which is in our source tree source = (os.path.relpath(result[0], base_path), result[1] or 0, 0) if isinstance(ex, SyntaxError): # SyntaxError has better information than the traceback if ex.filename == full_path: # pylint: disable=locally-disabled, no-member # syntax error was reported in the file under test line = ex.lineno or 0 # pylint: disable=locally-disabled, no-member offset = ex.offset or 0 # pylint: disable=locally-disabled, no-member elif ex.filename.startswith(base_path) and not ex.filename.startswith(temp_path): # pylint: disable=locally-disabled, no-member # syntax error was reported in our source tree source = (os.path.relpath(ex.filename, base_path), ex.lineno or 0, ex.offset or 0) # pylint: disable=locally-disabled, no-member # remove the filename and line number from the message # either it was extracted above, or it's not really useful information message = re.sub(r' \(.*?, line [0-9]+\)$', '', message) if source and source[0] != path: message += ' (at %s:%d:%d)' % (source[0], source[1], source[2]) report_message(path, line, offset, 'traceback', '%s: %s' % (exc_type.__name__, message), messages) finally: capture_report(path, capture_normal, messages) capture_report(path, capture_main, messages) def is_name_in_namepace(name, namespaces): """Returns True if the given name is one of the given namespaces, otherwise returns False.""" name_parts = name.split('.') for namespace in namespaces: namespace_parts = namespace.split('.') length = min(len(name_parts), len(namespace_parts)) truncated_name = name_parts[0:length] truncated_namespace = namespace_parts[0:length] # empty parts in the namespace are treated as wildcards # to simplify the comparison, use those empty parts to indicate the positions in the name to be empty as well for idx, part in enumerate(truncated_namespace): if not part: truncated_name[idx] = part # example: name=ansible, allowed_name=ansible.module_utils # example: name=ansible.module_utils.system.ping, allowed_name=ansible.module_utils if truncated_name == truncated_namespace: return True return False def check_sys_modules(path, before, messages): """Check for unwanted changes to sys.modules. :type path: str :type before: dict[str, module] :type messages: set[str] """ after = sys.modules removed = set(before.keys()) - set(after.keys()) changed = set(key for key, value in before.items() if key in after and value != after[key]) # additions are checked by our custom PEP 302 loader, so we don't need to check them again here for module in sorted(removed): report_message(path, 0, 0, 'unload', 'unloading of "%s" in sys.modules is not supported' % module, messages) for module in sorted(changed): report_message(path, 0, 0, 'reload', 'reloading of "%s" in sys.modules is not supported' % module, messages) def convert_ansible_name_to_absolute_path(name): """Calculate the module path from the given name. :type name: str :rtype: str """ return os.path.join(ansible_path, name.replace('.', os.path.sep)) def convert_relative_path_to_name(path): """Calculate the module name from the given path. :type path: str :rtype: str """ if path.endswith('/__init__.py'): clean_path = os.path.dirname(path) else: clean_path = path clean_path = os.path.splitext(clean_path)[0] name = clean_path.replace(os.path.sep, '.') if collection_loader: # when testing collections the relative paths (and names) being tested are within the collection under test name = 'ansible_collections.%s.%s' % (collection_full_name, name) else: # when testing ansible all files being imported reside under the lib directory name = name[len('lib/'):] return name class Capture: """Captured output and/or exception.""" def __init__(self): self.stdout = StringIO() self.stderr = StringIO() def capture_report(path, capture, messages): """Report on captured output. :type path: str :type capture: Capture :type messages: set[str] """ if capture.stdout.getvalue(): first = capture.stdout.getvalue().strip().splitlines()[0].strip() report_message(path, 0, 0, 'stdout', first, messages) if capture.stderr.getvalue(): first = capture.stderr.getvalue().strip().splitlines()[0].strip() report_message(path, 0, 0, 'stderr', first, messages) def report_message(path, line, column, code, message, messages): """Report message if not already reported. :type path: str :type line: int :type column: int :type code: str :type message: str :type messages: set[str] """ message = '%s:%d:%d: %s: %s' % (path, line, column, code, message) if message not in messages: messages.add(message) print(message) @contextlib.contextmanager def blacklist_imports(path, name, messages): """Blacklist imports. :type path: str :type name: str :type messages: set[str] """ blacklist = ImportBlacklist(path, name) sys.meta_path.insert(0, blacklist) try: yield finally: if sys.meta_path[0] != blacklist: report_message(path, 0, 0, 'metapath', 'changes to sys.meta_path[0] are not permitted', messages) while blacklist in sys.meta_path: sys.meta_path.remove(blacklist) @contextlib.contextmanager def monitor_sys_modules(path, messages): """Monitor sys.modules for unwanted changes, reverting any additions made to our own namespaces.""" snapshot = sys.modules.copy() try: yield finally: check_sys_modules(path, snapshot, messages) for key in set(sys.modules.keys()) - set(snapshot.keys()): if is_name_in_namepace(key, ('ansible', 'ansible_collections')): del sys.modules[key] # only unload our own code since we know it's native Python @contextlib.contextmanager def capture_output(capture): """Capture sys.stdout and sys.stderr. :type capture: Capture """ old_stdout = sys.stdout old_stderr = sys.stderr sys.stdout = capture.stdout sys.stderr = capture.stderr # clear all warnings registries to make all warnings available for module in sys.modules.values(): try: module.__warningregistry__.clear() except AttributeError: pass with warnings.catch_warnings(): warnings.simplefilter('error') try: yield finally: sys.stdout = old_stdout sys.stderr = old_stderr run()
def run(self): super(PlaybookCLI, self).run() # Note: slightly wrong, this is written so that implicit localhost # manages passwords sshpass = None becomepass = None passwords = {} # initial error check, to make sure all specified playbooks are accessible # before we start running anything through the playbook executor b_playbook_dirs = [] for playbook in context.CLIARGS['args']: if not os.path.exists(playbook): raise AnsibleError("the playbook: %s could not be found" % playbook) if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise AnsibleError( "the playbook: %s does not appear to be a file" % playbook) b_playbook_dir = os.path.dirname( os.path.abspath( to_bytes(playbook, errors='surrogate_or_strict'))) # load plugins from all playbooks in case they add callbacks/inventory/etc add_all_plugin_dirs(b_playbook_dir) b_playbook_dirs.append(b_playbook_dir) set_collection_playbook_paths(b_playbook_dirs) playbook_collection = get_collection_name_from_path(b_playbook_dirs[0]) if playbook_collection: display.warning("running playbook inside collection {0}".format( playbook_collection)) AnsibleCollectionLoader().set_default_collection( playbook_collection) # don't deal with privilege escalation or passwords when we don't need to if not (context.CLIARGS['listhosts'] or context.CLIARGS['listtasks'] or context.CLIARGS['listtags'] or context.CLIARGS['syntax']): (sshpass, becomepass) = self.ask_passwords() passwords = {'conn_pass': sshpass, 'become_pass': becomepass} # create base objects loader, inventory, variable_manager = self._play_prereqs() # (which is not returned in list_hosts()) is taken into account for # warning if inventory is empty. But it can't be taken into account for # checking if limit doesn't match any hosts. Instead we don't worry about # limit if only implicit localhost was in inventory to start with. # # Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts()) CLI.get_host_list(inventory, context.CLIARGS['subset']) # flush fact cache if requested if context.CLIARGS['flush_cache']: self._flush_cache(inventory, variable_manager) # create the playbook executor, which manages running the plays via a task queue manager pbex = PlaybookExecutor(playbooks=context.CLIARGS['args'], inventory=inventory, variable_manager=variable_manager, loader=loader, passwords=passwords) results = pbex.run() if isinstance(results, list): for p in results: display.display('\nplaybook: %s' % p['playbook']) for idx, play in enumerate(p['plays']): if play._included_path is not None: loader.set_basedir(play._included_path) else: pb_dir = os.path.realpath( os.path.dirname(p['playbook'])) loader.set_basedir(pb_dir) msg = "\n play #%d (%s): %s" % (idx + 1, ','.join( play.hosts), play.name) mytags = set(play.tags) msg += '\tTAGS: [%s]' % (','.join(mytags)) if context.CLIARGS['listhosts']: playhosts = set(inventory.get_hosts(play.hosts)) msg += "\n pattern: %s\n hosts (%d):" % ( play.hosts, len(playhosts)) for host in playhosts: msg += "\n %s" % host display.display(msg) all_tags = set() if context.CLIARGS['listtags'] or context.CLIARGS[ 'listtasks']: taskmsg = '' if context.CLIARGS['listtasks']: taskmsg = ' tasks:\n' def _process_block(b): taskmsg = '' for task in b.block: if isinstance(task, Block): taskmsg += _process_block(task) else: if task.action == 'meta': continue all_tags.update(task.tags) if context.CLIARGS['listtasks']: cur_tags = list( mytags.union(set(task.tags))) cur_tags.sort() if task.name: taskmsg += " %s" % task.get_name( ) else: taskmsg += " %s" % task.action taskmsg += "\tTAGS: [%s]\n" % ', '.join( cur_tags) return taskmsg all_vars = variable_manager.get_vars(play=play) for block in play.compile(): block = block.filter_tagged_tasks(all_vars) if not block.has_tasks(): continue taskmsg += _process_block(block) if context.CLIARGS['listtags']: cur_tags = list(mytags.union(all_tags)) cur_tags.sort() taskmsg += " TASK TAGS: [%s]\n" % ', '.join( cur_tags) display.display(taskmsg) return 0 else: return results
def test_import_from_collection(monkeypatch): collection_root = os.path.join(os.path.dirname(__file__), 'fixtures', 'collections') collection_path = os.path.join( collection_root, 'ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py' ) # the trace we're expecting to be generated when running the code below: # answer = question() expected_trace_log = [ (collection_path, 5, 'call'), (collection_path, 6, 'line'), (collection_path, 6, 'return'), ] # define the collection root before any ansible code has been loaded # otherwise config will have already been loaded and changing the environment will have no effect monkeypatch.setenv('ANSIBLE_COLLECTIONS_PATHS', collection_root) from ansible.utils.collection_loader import AnsibleCollectionLoader # zap the singleton collection loader instance if it exists AnsibleCollectionLoader._Singleton__instance = None for index in [ idx for idx, obj in enumerate(sys.meta_path) if isinstance(obj, AnsibleCollectionLoader) ]: # replace any existing collection loaders that may exist # since these were loaded during unit test collection # they will not have the correct configuration sys.meta_path[index] = AnsibleCollectionLoader() # make sure the collection loader is installed # this will be a no-op if the collection loader is already installed # which will depend on whether or not any tests being run imported ansible.plugins.loader during unit test collection from ansible.plugins.loader import _configure_collection_loader _configure_collection_loader( ) # currently redundant, the import above already calls this from ansible_collections.my_namespace.my_collection.plugins.module_utils.my_util import question original_trace_function = sys.gettrace() trace_log = [] if original_trace_function: # enable tracing while preserving the existing trace function (coverage) def my_trace_function(frame, event, arg): trace_log.append((frame.f_code.co_filename, frame.f_lineno, event)) # the original trace function expects to have itself set as the trace function sys.settrace(original_trace_function) # call the original trace function original_trace_function(frame, event, arg) # restore our trace function sys.settrace(my_trace_function) return my_trace_function else: # no existing trace function, so our trace function is much simpler def my_trace_function(frame, event, arg): trace_log.append((frame.f_code.co_filename, frame.f_lineno, event)) return my_trace_function sys.settrace(my_trace_function) try: # run a minimal amount of code while the trace is running # adding more code here, including use of a context manager, will add more to our trace answer = question() finally: sys.settrace(original_trace_function) # make sure 'import ... as ...' works on builtin synthetic collections # the following import is not supported (it tries to find module_utils in ansible.plugins) # import ansible_collections.ansible.builtin.plugins.module_utils as c1 import ansible_collections.ansible.builtin.plugins.action as c2 import ansible_collections.ansible.builtin.plugins as c3 import ansible_collections.ansible.builtin as c4 import ansible_collections.ansible as c5 import ansible_collections as c6 # make sure 'import ...' works on builtin synthetic collections import ansible_collections.ansible.builtin.plugins.module_utils import ansible_collections.ansible.builtin.plugins.action assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2 import ansible_collections.ansible.builtin.plugins assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3 import ansible_collections.ansible.builtin assert ansible_collections.ansible.builtin == c5.builtin == c4 import ansible_collections.ansible assert ansible_collections.ansible == c6.ansible == c5 import ansible_collections assert ansible_collections == c6 # make sure 'from ... import ...' works on builtin synthetic collections from ansible_collections.ansible import builtin from ansible_collections.ansible.builtin import plugins assert builtin.plugins == plugins from ansible_collections.ansible.builtin.plugins import action from ansible_collections.ansible.builtin.plugins.action import command assert action.command == command from ansible_collections.ansible.builtin.plugins.module_utils import basic from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule assert basic.AnsibleModule == AnsibleModule # make sure relative imports work from collections code # these require __package__ to be set correctly import ansible_collections.my_namespace.my_collection.plugins.module_utils.my_other_util import ansible_collections.my_namespace.my_collection.plugins.action.my_action # verify that code loaded from a collection does not inherit __future__ statements from the collection loader if sys.version_info[0] == 2: # if the collection code inherits the division future feature from the collection loader this will fail assert answer == 1 else: assert answer == 1.5 # verify that the filename and line number reported by the trace is correct # this makes sure that collection loading preserves file paths and line numbers assert trace_log == expected_trace_log