def test_bogus_imports():
finder = get_default_finder()
reset_collections_loader_state(finder)
# ensure ImportError on known-bogus imports
bogus_imports = ['bogus_toplevel', 'ansible_collections.bogusns', 'ansible_collections.testns.boguscoll',
'ansible_collections.testns.testcoll.bogussub', 'ansible_collections.ansible.builtin.bogussub']
for bogus_import in bogus_imports:
with pytest.raises(ImportError):
import_module(bogus_import)
def run(self, terms, variables=None, **kwargs):
result = []
self.set_options(var_options=variables, direct=kwargs)
not_found = self.get_option('result_not_found')
no_version = self.get_option('result_no_version')
for term in terms:
if not FQCN_RE.match(term):
raise AnsibleLookupError(
'"{term}" is not a FQCN'.format(term=term))
try:
collection_pkg = import_module(
'ansible_collections.{fqcn}'.format(fqcn=term))
except ImportError:
# Collection not found
result.append(not_found)
continue
try:
data = load_collection_meta(collection_pkg,
no_version=no_version)
except Exception as exc:
raise AnsibleLookupError(
'Error while loading metadata for {fqcn}: {error}'.format(
fqcn=term, error=exc))
result.append(data.get('version', no_version))
return result
def get_collection_path(self, collection_name):
if not AnsibleCollectionRef.is_valid_collection_name(collection_name):
raise ValueError('{0} is not a valid collection name'.format(to_native(collection_name)))
m = import_module('ansible_collections.{0}'.format(collection_name))
return m.__file__
def get_collection_role_path(role_name, collection_list=None):
acr = AnsibleCollectionRef.try_parse_fqcr(role_name, 'role')
if acr:
# looks like a valid qualified collection ref; skip the collection_list
collection_list = [acr.collection]
subdirs = acr.subdirs
resource = acr.resource
elif not collection_list:
return None # not a FQ role and no collection search list spec'd, nothing to do
else:
resource = role_name # treat as unqualified, loop through the collection search list to try and resolve
subdirs = ''
for collection_name in collection_list:
try:
acr = AnsibleCollectionRef(collection_name=collection_name, subdirs=subdirs, resource=resource, ref_type='role')
# FIXME: error handling/logging; need to catch any import failures and move along
# FIXME: this line shouldn't be necessary, but py2 pkgutil.get_data is delegating back to built-in loader when it shouldn't
pkg = import_module(acr.n_python_package_name)
if pkg is not None:
# the package is now loaded, get the collection's package and ask where it lives
path = os.path.dirname(to_bytes(sys.modules[acr.n_python_package_name].__file__, errors='surrogate_or_strict'))
return resource, to_text(path, errors='surrogate_or_strict'), collection_name
except IOError:
continue
except Exception as ex:
# FIXME: pick out typical import errors first, then error logging
continue
return None
def _add_module(self, name, wrapper=False):
m, ext = name
m = to_text(m)
if m.startswith("Ansible."):
# Builtin util, use plugin loader to get the data
mu_path = ps_module_utils_loader.find_plugin(m, ext)
if not mu_path:
raise AnsibleError(
'Could not find imported module support code '
'for \'%s\'' % m)
module_util_data = to_bytes(_slurp(mu_path))
else:
# Collection util, load the package data based on the util import.
submodules = tuple(m.split("."))
n_package_name = to_native('.'.join(submodules[:-1]),
errors='surrogate_or_strict')
n_resource_name = to_native(submodules[-1] + ext,
errors='surrogate_or_strict')
try:
module_util = import_module(to_native(n_package_name))
module_util_data = to_bytes(pkgutil.get_data(
n_package_name, n_resource_name),
errors='surrogate_or_strict')
# Get the path of the util which is required for coverage collection.
resource_paths = list(module_util.__path__)
if len(resource_paths) != 1:
# This should never happen with a collection but we are just being defensive about it.
raise AnsibleError(
"Internal error: Referenced module_util package '%s' contains 0 or multiple "
"import locations when we only expect 1." %
n_package_name)
mu_path = os.path.join(resource_paths[0], n_resource_name)
except OSError as err:
if err.errno == errno.ENOENT:
raise AnsibleError(
'Could not find collection imported module support code for \'%s\''
% to_native(m))
else:
raise
util_info = {
'data': module_util_data,
'path': to_text(mu_path),
}
if ext == ".psm1":
self.ps_modules[m] = util_info
else:
if wrapper:
self.cs_utils_wrapper[m] = util_info
else:
self.cs_utils_module[m] = util_info
self.scan_module(module_util_data,
wrapper=wrapper,
powershell=(ext == ".psm1"))
def _find_fq_plugin(self, fq_name, extension):
"""Search builtin paths to find a plugin. No external paths are searched,
meaning plugins inside roles inside collections will be ignored.
"""
plugin_type = AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(
self.subdir)
acr = AnsibleCollectionRef.from_fqcr(fq_name, plugin_type)
n_resource = to_native(acr.resource, errors='strict')
# we want this before the extension is added
full_name = '{0}.{1}'.format(acr.n_python_package_name, n_resource)
if extension:
n_resource += extension
pkg = sys.modules.get(acr.n_python_package_name)
if not pkg:
# FIXME: there must be cheaper/safer way to do this
pkg = import_module(acr.n_python_package_name)
# if the package is one of our flatmaps, we need to consult its loader to find the path, since the file could be
# anywhere in the tree
if hasattr(pkg, '__loader__') and isinstance(pkg.__loader__,
AnsibleFlatMapLoader):
try:
file_path = pkg.__loader__.find_file(n_resource)
return full_name, to_text(file_path)
except IOError:
# this loader already takes care of extensionless files, so if we didn't find it, just bail
return None, None
pkg_path = os.path.dirname(pkg.__file__)
n_resource_path = os.path.join(pkg_path, n_resource)
# FIXME: and is file or file link or ...
if os.path.exists(n_resource_path):
return full_name, to_text(n_resource_path)
# look for any matching extension in the package location (sans filter)
ext_blacklist = ['.pyc', '.pyo']
found_files = [
f for f in glob.iglob(os.path.join(pkg_path, n_resource) + '.*') if
os.path.isfile(f) and os.path.splitext(f)[1] not in ext_blacklist
]
if not found_files:
return None, None
if len(found_files) > 1:
# TODO: warn?
pass
return full_name, to_text(found_files[0])
def _get_keywords_docs(keys):
data = {}
descs = DocCLI._list_keywords()
for keyword in keys:
if keyword.startswith('with_'):
keyword = 'loop'
try:
# if no desc, typeerror raised ends this block
kdata = {'description': descs[keyword]}
# get playbook objects for keyword and use first to get keyword attributes
kdata['applies_to'] = []
for pobj in PB_OBJECTS:
if pobj not in PB_LOADED:
obj_class = 'ansible.playbook.%s' % pobj.lower()
loaded_class = importlib.import_module(obj_class)
PB_LOADED[pobj] = getattr(loaded_class, pobj, None)
if keyword in PB_LOADED[pobj]._valid_attrs:
kdata['applies_to'].append(pobj)
# we should only need these once
if 'type' not in kdata:
fa = getattr(PB_LOADED[pobj], '_%s' % keyword)
if getattr(fa, 'private'):
kdata = {}
raise KeyError
kdata['type'] = getattr(fa, 'isa', 'string')
if keyword.endswith('when'):
kdata['template'] = 'implicit'
elif getattr(fa, 'static'):
kdata['template'] = 'static'
else:
kdata['template'] = 'explicit'
# those that require no processing
for visible in ('alias', 'priority'):
kdata[visible] = getattr(fa, visible)
# remove None keys
for k in list(kdata.keys()):
if kdata[k] is None:
del kdata[k]
data[keyword] = kdata
except KeyError as e:
display.warning("Skipping Invalid keyword '%s' specified: %s" %
(keyword, to_native(e)))
return data
def test_on_collection_load():
finder = get_default_finder()
reset_collections_loader_state(finder)
load_handler = MagicMock()
AnsibleCollectionConfig.on_collection_load += load_handler
m = import_module('ansible_collections.testns.testcoll')
load_handler.assert_called_once_with(collection_name='testns.testcoll', collection_path=os.path.dirname(m.__file__))
_meta = _get_collection_metadata('testns.testcoll')
assert _meta
# FIXME: compare to disk
finder = get_default_finder()
reset_collections_loader_state(finder)
AnsibleCollectionConfig.on_collection_load += MagicMock(side_effect=Exception('bang'))
with pytest.raises(Exception) as ex:
import_module('ansible_collections.testns.testcoll')
assert 'bang' in str(ex.value)
def test_collpkg_loader_load_module():
reset_collections_loader_state()
with patch('ansible.utils.collection_loader.AnsibleCollectionConfig') as p:
for name in [
'ansible_collections.ansible.builtin',
'ansible_collections.testns.testcoll'
]:
parent_pkg = name.rpartition('.')[0]
module_to_load = name.rpartition('.')[2]
paths = extend_paths(default_test_collection_paths, parent_pkg)
existing_child_paths = [
p for p in extend_paths(paths, module_to_load)
if os.path.exists(p)
]
is_builtin = 'ansible.builtin' in name
if name in sys.modules:
del sys.modules[name]
loader = _AnsibleCollectionPkgLoader(name, path_list=paths)
assert repr(loader).startswith('_AnsibleCollectionPkgLoader(path=')
module = loader.load_module(name)
assert module.__name__ == name
assert isinstance(module.__loader__, _AnsibleCollectionPkgLoader)
if is_builtin:
assert module.__path__ == []
else:
assert module.__path__ == [existing_child_paths[0]]
assert module.__package__ == name
if is_builtin:
assert module.__file__ == '<ansible_synthetic_collection_package>'
else:
assert module.__file__.endswith(
'__synthetic__') and os.path.isdir(
os.path.dirname(module.__file__))
assert sys.modules.get(name) == module
assert hasattr(module, '_collection_meta') and isinstance(
module._collection_meta, dict)
# FIXME: validate _collection_meta contents match what's on disk (or not)
# if the module has metadata, try loading it with busted metadata
if module._collection_meta:
_collection_finder = import_module(
'ansible.utils.collection_loader._collection_finder')
with patch.object(_collection_finder,
'_meta_yml_to_dict',
side_effect=Exception('bang')):
with pytest.raises(Exception) as ex:
_AnsibleCollectionPkgLoader(
name, path_list=paths).load_module(name)
assert 'error parsing collection metadata' in str(ex.value)
def __init__(self, config=None):
if config:
paths = config.get_config_value('COLLECTIONS_PATHS')
else:
paths = os.environ.get('ANSIBLE_COLLECTIONS_PATHS',
'').split(os.pathsep)
if isinstance(paths, string_types):
paths = [paths]
elif paths is None:
paths = []
# expand any placeholders in configured paths
paths = [
to_native(os.path.expanduser(p), errors='surrogate_or_strict')
for p in paths
]
# Append all ``ansible_collections`` dirs from sys.path to the end
for path in sys.path:
if (path not in paths and os.path.isdir(
to_bytes(
os.path.join(path, 'ansible_collections'),
errors='surrogate_or_strict',
))):
paths.append(path)
self._n_configured_paths = paths
self._n_playbook_paths = []
self._default_collection = None
# pre-inject grafted package maps so we can force them to use the right loader instead of potentially delegating to a "normal" loader
for syn_pkg_def in (p for p in iteritems(_SYNTHETIC_PACKAGES)
if p[1].get('graft')):
pkg_name = syn_pkg_def[0]
pkg_def = syn_pkg_def[1]
newmod = ModuleType(pkg_name)
newmod.__package__ = pkg_name
newmod.__file__ = '<ansible_synthetic_collection_package>'
pkg_type = pkg_def.get('type')
# TODO: need to rethink map style so we can just delegate all the loading
if pkg_type == 'flatmap':
newmod.__loader__ = AnsibleFlatMapLoader(
import_module(pkg_def['flatmap']))
newmod.__path__ = []
sys.modules[pkg_name] = newmod
def __getitem__(self, key):
if not isinstance(key, string_types):
raise ValueError('key must be a string')
key = to_native(key)
if '.' not in key: # might be a built-in value, delegate to base dict
return self._delegatee.__getitem__(key)
func = self._collection_jinja_func_cache.get(key)
if func:
return func
acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname)
if not acr:
raise KeyError('invalid plugin name: {0}'.format(key))
try:
pkg = import_module(acr.n_python_package_name)
except ImportError:
raise KeyError()
parent_prefix = acr.collection
if acr.subdirs:
parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs)
for dummy, module_name, ispkg in pkgutil.iter_modules(
pkg.__path__, prefix=parent_prefix + '.'):
if ispkg:
continue
try:
plugin_impl = self._pluginloader.get(module_name)
except Exception as e:
raise TemplateSyntaxError(to_native(e), 0)
method_map = getattr(plugin_impl, self._method_map_name)
for f in iteritems(method_map()):
fq_name = '.'.join((parent_prefix, f[0]))
# FIXME: detect/warn on intra-collection function name collisions
self._collection_jinja_func_cache[fq_name] = f[1]
function_impl = self._collection_jinja_func_cache[key]
return function_impl
def test_collection_get_data():
finder = get_default_finder()
reset_collections_loader_state(finder)
# something that's there
d = pkgutil.get_data('ansible_collections.testns.testcoll', 'plugins/action/my_action.py')
assert b'hello from my_action.py' in d
# something that's not there
d = pkgutil.get_data('ansible_collections.testns.testcoll', 'bogus/bogus')
assert d is None
with pytest.raises(ValueError):
plugins_pkg = import_module('ansible_collections.ansible.builtin')
assert not os.path.exists(os.path.dirname(plugins_pkg.__file__))
d = pkgutil.get_data('ansible_collections.ansible.builtin', 'plugins/connection/local.py')
def _find_module(self, fullname, path, load):
# this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others
if not fullname.startswith(
'ansible_collections.') and fullname != 'ansible_collections':
return False, None
if sys.modules.get(fullname):
if not load:
return True, None
return True, sys.modules[fullname]
newmod = None
# this loader implements key functionality for Ansible collections
# * implicit distributed namespace packages for the root Ansible namespace (no pkgutil.extend_path hackery reqd)
# * implicit package support for Python 2.7 (no need for __init__.py in collections, except to use standard Py2.7 tooling)
# * preventing controller-side code injection during collection loading
# * (default loader would execute arbitrary package code from all __init__.py's)
parent_pkg_name = '.'.join(fullname.split('.')[:-1])
parent_pkg = sys.modules.get(parent_pkg_name)
if parent_pkg_name and not parent_pkg:
raise ImportError(
'parent package {0} not found'.format(parent_pkg_name))
# are we at or below the collection level? eg a.mynamespace.mycollection.something.else
# if so, we don't want distributed namespace behavior; first mynamespace.mycollection on the path is where
# we'll load everything from (ie, don't fall back to another mynamespace.mycollection lower on the path)
sub_collection = fullname.count('.') > 1
synpkg_def = _SYNTHETIC_PACKAGES.get(fullname)
synpkg_remainder = ''
if not synpkg_def:
# if the parent is a grafted package, we have some special work to do, otherwise just look for stuff on disk
parent_synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name)
if parent_synpkg_def and parent_synpkg_def.get('graft'):
synpkg_def = parent_synpkg_def
synpkg_remainder = '.' + fullname.rpartition('.')[2]
# FUTURE: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?)
if synpkg_def:
pkg_type = synpkg_def.get('type')
if not pkg_type:
raise KeyError(
'invalid synthetic package type (no package "type" specified)'
)
if pkg_type == 'map':
map_package = synpkg_def.get('map')
if not map_package:
raise KeyError(
'invalid synthetic map package definition (no target "map" defined)'
)
if not load:
return True, None
mod = import_module(map_package + synpkg_remainder)
sys.modules[fullname] = mod
return True, mod
elif pkg_type == 'flatmap':
raise NotImplementedError()
elif pkg_type == 'pkg_only':
if not load:
return True, None
newmod = ModuleType(fullname)
newmod.__package__ = fullname
newmod.__file__ = '<ansible_synthetic_collection_package>'
newmod.__loader__ = self
newmod.__path__ = []
if not synpkg_def.get('allow_external_subpackages'):
# if external subpackages are NOT allowed, we're done
sys.modules[fullname] = newmod
return True, newmod
# if external subpackages ARE allowed, check for on-disk implementations and return a normal
# package if we find one, otherwise return the one we created here
if not parent_pkg: # top-level package, look for NS subpackages on all collection paths
package_paths = [
self._extend_path_with_ns(p, fullname)
for p in self.n_collection_paths
]
else: # subpackage; search in all subpaths (we'll limit later inside a collection)
package_paths = [
self._extend_path_with_ns(p, fullname)
for p in parent_pkg.__path__
]
for candidate_child_path in package_paths:
code_object = None
is_package = True
location = None
# check for implicit sub-package first
if os.path.isdir(to_bytes(candidate_child_path)):
# Py3.x implicit namespace packages don't have a file location, so they don't support get_data
# (which assumes the parent dir or that the loader has an internal mapping); so we have to provide
# a bogus leaf file on the __file__ attribute for pkgutil.get_data to strip off
location = os.path.join(candidate_child_path, '__synthetic__')
else:
for source_path in [
os.path.join(candidate_child_path, '__init__.py'),
candidate_child_path + '.py'
]:
if not os.path.isfile(to_bytes(source_path)):
continue
if not load:
return True, None
with open(to_bytes(source_path), 'rb') as fd:
source = fd.read()
code_object = compile(source=source,
filename=source_path,
mode='exec',
flags=0,
dont_inherit=True)
location = source_path
is_package = source_path.endswith('__init__.py')
break
if not location:
continue
newmod = ModuleType(fullname)
newmod.__file__ = location
newmod.__loader__ = self
if is_package:
if sub_collection: # we never want to search multiple instances of the same collection; use first found
newmod.__path__ = [candidate_child_path]
else:
newmod.__path__ = package_paths
newmod.__package__ = fullname
else:
newmod.__package__ = parent_pkg_name
sys.modules[fullname] = newmod
if code_object:
# FIXME: decide cases where we don't actually want to exec the code?
exec(code_object, newmod.__dict__)
return True, newmod
# even if we didn't find one on disk, fall back to a synthetic package if we have one...
if newmod:
sys.modules[fullname] = newmod
return True, newmod
# FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin
return False, None
def __getitem__(self, key):
original_key = key
self._load_ansible_plugins()
try:
if not isinstance(key, string_types):
raise ValueError('key must be a string')
key = to_native(key)
if '.' not in key: # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect
func = self._delegatee.get(key)
if func:
return func
key, leaf_key = get_fqcr_and_name(key)
seen = set()
while True:
if key in seen:
raise TemplateSyntaxError(
'recursive collection redirect found for %r' %
original_key, 0)
seen.add(key)
acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname)
if not acr:
raise KeyError('invalid plugin name: {0}'.format(key))
ts = _get_collection_metadata(acr.collection)
# TODO: implement cycle detection (unified across collection redir as well)
routing_entry = ts.get('plugin_routing',
{}).get(self._dirname,
{}).get(leaf_key, {})
deprecation_entry = routing_entry.get('deprecation')
if deprecation_entry:
warning_text = deprecation_entry.get('warning_text')
removal_date = deprecation_entry.get('removal_date')
removal_version = deprecation_entry.get('removal_version')
if not warning_text:
warning_text = '{0} "{1}" is deprecated'.format(
self._dirname, key)
display.deprecated(warning_text,
version=removal_version,
date=removal_date,
collection_name=acr.collection)
tombstone_entry = routing_entry.get('tombstone')
if tombstone_entry:
warning_text = tombstone_entry.get('warning_text')
removal_date = tombstone_entry.get('removal_date')
removal_version = tombstone_entry.get('removal_version')
if not warning_text:
warning_text = '{0} "{1}" has been removed'.format(
self._dirname, key)
exc_msg = display.get_deprecation_message(
warning_text,
version=removal_version,
date=removal_date,
collection_name=acr.collection,
removed=True)
raise AnsiblePluginRemovedError(exc_msg)
redirect = routing_entry.get('redirect', None)
if redirect:
next_key, leaf_key = get_fqcr_and_name(
redirect, collection=acr.collection)
display.vvv(
'redirecting (type: {0}) {1}.{2} to {3}'.format(
self._dirname, acr.collection, acr.resource,
next_key))
key = next_key
else:
break
func = self._collection_jinja_func_cache.get(key)
if func:
return func
try:
pkg = import_module(acr.n_python_package_name)
except ImportError:
raise KeyError()
parent_prefix = acr.collection
if acr.subdirs:
parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs)
# TODO: implement collection-level redirect
for dummy, module_name, ispkg in pkgutil.iter_modules(
pkg.__path__, prefix=parent_prefix + '.'):
if ispkg:
continue
try:
plugin_impl = self._pluginloader.get(module_name)
except Exception as e:
raise TemplateSyntaxError(to_native(e), 0)
try:
method_map = getattr(plugin_impl, self._method_map_name)
func_items = method_map().items()
except Exception as e:
display.warning(
"Skipping %s plugin %s as it seems to be invalid: %r" %
(self._dirname, to_text(
plugin_impl._original_path), e), )
continue
for func_name, func in func_items:
fq_name = '.'.join((parent_prefix, func_name))
# FIXME: detect/warn on intra-collection function name collisions
if self._pluginloader.class_name == 'FilterModule':
if fq_name.startswith(('ansible.builtin.', 'ansible.legacy.')) and \
func_name in C.STRING_TYPE_FILTERS:
self._collection_jinja_func_cache[
fq_name] = _wrap_native_text(func)
else:
self._collection_jinja_func_cache[
fq_name] = _unroll_iterator(func)
else:
self._collection_jinja_func_cache[fq_name] = func
function_impl = self._collection_jinja_func_cache[key]
return function_impl
except AnsiblePluginRemovedError as apre:
raise TemplateSyntaxError(to_native(apre), 0)
except KeyError:
raise
except Exception as ex:
display.warning(
'an unexpected error occurred during Jinja2 environment setup: {0}'
.format(to_native(ex)))
display.vvv(
'exception during Jinja2 environment setup: {0}'.format(
format_exc()))
raise TemplateSyntaxError(to_native(ex), 0)
def __getitem__(self, key):
try:
if not isinstance(key, string_types):
raise ValueError('key must be a string')
key = to_native(key)
if '.' not in key: # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect
func = self._delegatee.get(key)
if func:
return func
ts = _get_collection_metadata('ansible.builtin')
# TODO: implement support for collection-backed redirect (currently only builtin)
# TODO: implement cycle detection (unified across collection redir as well)
redirect_fqcr = ts.get('plugin_routing', {}).get(self._dirname, {}).get(key, {}).get('redirect', None)
if redirect_fqcr:
acr = AnsibleCollectionRef.from_fqcr(ref=redirect_fqcr, ref_type=self._dirname)
display.vvv('redirecting {0} {1} to {2}.{3}'.format(self._dirname, key, acr.collection, acr.resource))
key = redirect_fqcr
# TODO: handle recursive forwarding (not necessary for builtin, but definitely for further collection redirs)
func = self._collection_jinja_func_cache.get(key)
if func:
return func
acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname)
if not acr:
raise KeyError('invalid plugin name: {0}'.format(key))
try:
pkg = import_module(acr.n_python_package_name)
except ImportError:
raise KeyError()
parent_prefix = acr.collection
if acr.subdirs:
parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs)
# TODO: implement collection-level redirect
for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'):
if ispkg:
continue
try:
plugin_impl = self._pluginloader.get(module_name)
except Exception as e:
raise TemplateSyntaxError(to_native(e), 0)
method_map = getattr(plugin_impl, self._method_map_name)
for f in iteritems(method_map()):
fq_name = '.'.join((parent_prefix, f[0]))
# FIXME: detect/warn on intra-collection function name collisions
self._collection_jinja_func_cache[fq_name] = f[1]
function_impl = self._collection_jinja_func_cache[key]
return function_impl
except KeyError:
raise
except Exception as ex:
display.warning('an unexpected error occurred during Jinja2 environment setup: {0}'.format(to_native(ex)))
display.vvv('exception during Jinja2 environment setup: {0}'.format(format_exc()))
raise
def __getitem__(self, key):
try:
if not isinstance(key, string_types):
raise ValueError('key must be a string')
key = to_native(key)
if '.' not in key: # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect
func = self._delegatee.get(key)
if func:
return func
# didn't find it in the pre-built Jinja env, assume it's a former builtin and follow the normal routing path
leaf_key = key
key = 'ansible.builtin.' + key
else:
leaf_key = key.split('.')[-1]
acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname)
if not acr:
raise KeyError('invalid plugin name: {0}'.format(key))
ts = _get_collection_metadata(acr.collection)
# TODO: implement support for collection-backed redirect (currently only builtin)
# TODO: implement cycle detection (unified across collection redir as well)
routing_entry = ts.get('plugin_routing', {}).get(self._dirname, {}).get(leaf_key, {})
deprecation_entry = routing_entry.get('deprecation')
if deprecation_entry:
warning_text = deprecation_entry.get('warning_text')
removal_date = deprecation_entry.get('removal_date')
removal_version = deprecation_entry.get('removal_version')
if not warning_text:
warning_text = '{0} "{1}" is deprecated'.format(self._dirname, key)
display.deprecated(warning_text, version=removal_version, date=removal_date, collection_name=acr.collection)
tombstone_entry = routing_entry.get('tombstone')
if tombstone_entry:
warning_text = tombstone_entry.get('warning_text')
removal_date = tombstone_entry.get('removal_date')
removal_version = tombstone_entry.get('removal_version')
if not warning_text:
warning_text = '{0} "{1}" has been removed'.format(self._dirname, key)
exc_msg = display.get_deprecation_message(warning_text, version=removal_version, date=removal_date,
collection_name=acr.collection, removed=True)
raise AnsiblePluginRemovedError(exc_msg)
redirect_fqcr = routing_entry.get('redirect', None)
if redirect_fqcr:
acr = AnsibleCollectionRef.from_fqcr(ref=redirect_fqcr, ref_type=self._dirname)
display.vvv('redirecting {0} {1} to {2}.{3}'.format(self._dirname, key, acr.collection, acr.resource))
key = redirect_fqcr
# TODO: handle recursive forwarding (not necessary for builtin, but definitely for further collection redirs)
func = self._collection_jinja_func_cache.get(key)
if func:
return func
try:
pkg = import_module(acr.n_python_package_name)
except ImportError:
raise KeyError()
parent_prefix = acr.collection
if acr.subdirs:
parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs)
# TODO: implement collection-level redirect
for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'):
if ispkg:
continue
try:
plugin_impl = self._pluginloader.get(module_name)
except Exception as e:
raise TemplateSyntaxError(to_native(e), 0)
method_map = getattr(plugin_impl, self._method_map_name)
for func_name, func in iteritems(method_map()):
fq_name = '.'.join((parent_prefix, func_name))
# FIXME: detect/warn on intra-collection function name collisions
if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS:
self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
else:
self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
function_impl = self._collection_jinja_func_cache[key]
return function_impl
except AnsiblePluginRemovedError as apre:
raise TemplateSyntaxError(to_native(apre), 0)
except KeyError:
raise
except Exception as ex:
display.warning('an unexpected error occurred during Jinja2 environment setup: {0}'.format(to_native(ex)))
display.vvv('exception during Jinja2 environment setup: {0}'.format(format_exc()))
raise TemplateSyntaxError(to_native(ex), 0)
def _add_module(self, name, ext, fqn, optional, wrapper=False):
m = to_text(name)
util_fqn = None
if m.startswith("Ansible."):
# Builtin util, use plugin loader to get the data
mu_path = ps_module_utils_loader.find_plugin(m, ext)
if not mu_path:
if optional:
return
raise AnsibleError(
'Could not find imported module support code '
'for \'%s\'' % m)
module_util_data = to_bytes(_slurp(mu_path))
else:
# Collection util, load the package data based on the util import.
submodules = m.split(".")
if m.startswith('.'):
fqn_submodules = fqn.split('.')
for submodule in submodules:
if submodule:
break
del fqn_submodules[-1]
submodules = fqn_submodules + [s for s in submodules if s]
n_package_name = to_native('.'.join(submodules[:-1]),
errors='surrogate_or_strict')
n_resource_name = to_native(submodules[-1] + ext,
errors='surrogate_or_strict')
try:
module_util = import_module(n_package_name)
pkg_data = pkgutil.get_data(n_package_name, n_resource_name)
if pkg_data is None:
raise ImportError("No package data found")
module_util_data = to_bytes(pkg_data,
errors='surrogate_or_strict')
util_fqn = to_text("%s.%s " % (n_package_name, submodules[-1]),
errors='surrogate_or_strict')
# Get the path of the util which is required for coverage collection.
resource_paths = list(module_util.__path__)
if len(resource_paths) != 1:
# This should never happen with a collection but we are just being defensive about it.
raise AnsibleError(
"Internal error: Referenced module_util package '%s' contains 0 or multiple "
"import locations when we only expect 1." %
n_package_name)
mu_path = os.path.join(resource_paths[0], n_resource_name)
except (ImportError, OSError) as err:
if getattr(err, "errno", errno.ENOENT) == errno.ENOENT:
if optional:
return
raise AnsibleError(
'Could not find collection imported module support code for \'%s\''
% to_native(m))
else:
raise
util_info = {
'data': module_util_data,
'path': to_text(mu_path),
}
if ext == ".psm1":
self.ps_modules[m] = util_info
else:
if wrapper:
self.cs_utils_wrapper[m] = util_info
else:
self.cs_utils_module[m] = util_info
self.scan_module(module_util_data,
fqn=util_fqn,
wrapper=wrapper,
powershell=(ext == ".psm1"))
