def minion_mods(opts, context=None, whitelist=None):
'''
Load execution modules
Returns a dictionary of execution modules appropriate for the current
system by evaluating the __virtual__() function in each module.
.. code-block:: python
import salt.config
import salt.loader
__opts__ salt.config.minion_config('/etc/salt/minion')
__salt__ = salt.loader.minion_mods(__opts__)
__salt__['test.ping']()
'''
load = _create_loader(opts, 'modules', 'module')
if context is None:
context = {}
pack = {'name': '__context__',
'value': context}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
functions = load.gen_functions(
pack,
whitelist=whitelist,
provider_overrides=True
)
# Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions)
return functions
def minion_mods(opts, context=None, whitelist=None):
'''
Load execution modules
Returns a dictionary of execution modules appropriate for the current
system by evaluating the __virtual__() function in each module.
.. code-block:: python
import salt.config
import salt.loader
__opts__ = salt.config.minion_config('/etc/salt/minion')
__salt__ = salt.loader.minion_mods(__opts__)
__salt__['test.ping']()
'''
load = _create_loader(opts, 'modules', 'module')
if context is None:
context = {}
pack = {'name': '__context__', 'value': context}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
functions = load.gen_functions(pack,
whitelist=whitelist,
provider_overrides=True)
# Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions)
return functions
def minion_mods(opts, context=None, whitelist=None):
'''
Returns the minion modules
'''
load = _create_loader(opts, 'modules', 'module')
if context is None:
context = {}
pack = {'name': '__context__',
'value': context}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
functions = load.gen_functions(
pack,
whitelist=whitelist
)
# Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions)
if opts.get('providers', False):
if isinstance(opts['providers'], dict):
for mod, provider in opts['providers'].items():
funcs = raw_mod(opts, provider, functions)
if funcs:
for func in funcs:
f_key = '{0}{1}'.format(mod, func[func.rindex('.'):])
functions[f_key] = funcs[func]
return functions
def minion_mods(opts, context=None, whitelist=None):
'''
Returns the minion modules
'''
load = _create_loader(opts, 'modules', 'module')
if context is None:
context = {}
pack = {'name': '__context__', 'value': context}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
functions = load.gen_functions(pack,
whitelist=whitelist,
provider_overrides=True)
# Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions)
return functions
def minion_mods(opts, context=None, whitelist=None):
'''
Returns the minion modules
'''
load = _create_loader(opts, 'modules', 'module')
if context is None:
context = {}
pack = {'name': '__context__',
'value': context}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
functions = load.gen_functions(
pack,
whitelist=whitelist,
provider_overrides=True
)
# Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions)
return functions
def minion_mods(opts, context=None, whitelist=None):
'''
Returns the minion modules
'''
load = _create_loader(opts, 'modules', 'module')
if context is None:
context = {}
pack = {'name': '__context__', 'value': context}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
functions = load.gen_functions(pack, whitelist=whitelist)
# Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions)
if opts.get('providers', False):
if isinstance(opts['providers'], dict):
for mod, provider in opts['providers'].items():
funcs = raw_mod(opts, provider, functions)
if funcs:
for func in funcs:
f_key = '{0}{1}'.format(mod, func[func.rindex('.'):])
functions[f_key] = funcs[func]
return functions
def _load_module(self, name):
mod = None
fpath, suffix = self.file_mapping[name]
self.loaded_files.add(name)
try:
sys.path.append(os.path.dirname(fpath))
if suffix == '.pyx':
mod = self.pyximport.load_module(name, fpath,
tempfile.gettempdir())
else:
desc = self.suffix_map[suffix]
# if it is a directory, we dont open a file
if suffix == '':
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(self.loaded_base_name,
self.mod_type_check(fpath),
self.tag, name), None, fpath,
desc)
# reload all submodules if necessary
if not self.initial_load:
self._reload_submodules(mod)
else:
with open(fpath, desc[1]) as fn_:
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(fpath), self.tag, name),
fn_, fpath, desc)
except IOError:
raise
except ImportError:
log.debug('Failed to import {0} {1}:\n'.format(self.tag, name),
exc_info=True)
return mod
except Exception as error:
log.error('Failed to import {0} {1}, this is due most likely to a '
'syntax error:\n'.format(self.tag, name),
exc_info=True)
return mod
except SystemExit:
log.error('Failed to import {0} {1} as the module called exit()\n'.
format(self.tag, name),
exc_info=True)
return mod
finally:
sys.path.pop()
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self._grains
mod.__pillar__ = self._pillar
# pack whatever other globals we were asked to
for p_name, p_value in six.iteritems(self.pack):
setattr(mod, p_name, p_value)
# Call a module's initialization method if it exists
module_init = getattr(mod, '__init__', None)
if inspect.isfunction(module_init):
try:
module_init(self.opts)
except TypeError as e:
log.error(e)
module_name = mod.__name__.rsplit('.', 1)[-1]
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
if self.virtual_enable:
(virtual_ret, module_name, virtual_err) = self.process_virtual(
mod,
module_name,
)
if virtual_err is not None:
log.debug('Error loading {0}.{1}: {2}'.format(
self.tag,
module_name,
virtual_err,
))
# if process_virtual returned a non-True value then we are
# supposed to not process this module
if virtual_ret is not True:
# If a module has information about why it could not be loaded, record it
self.missing_modules[module_name] = virtual_err
self.missing_modules[name] = virtual_err
return False
# If this is a proxy minion then MOST modules cannot work. Therefore, require that
# any module that does work with salt-proxy-minion define __proxyenabled__ as a list
# containing the names of the proxy types that the module supports.
if not hasattr(mod, 'render') and 'proxy' in self.opts:
if not hasattr(mod, '__proxyenabled__') or \
self.opts['proxy']['proxytype'] in mod.__proxyenabled__ or \
'*' in mod.__proxyenabled__:
err_string = 'not a proxy_minion enabled module'
self.missing_modules[module_name] = err_string
self.missing_modules[name] = err_string
return False
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from the '
'provided __load__ attribute'.format(module_name))
mod_dict = salt.utils.odict.OrderedDict()
for attr in getattr(mod, '__load__', dir(mod)):
if attr.startswith('_'):
# private functions are skipped
continue
func = getattr(mod, attr)
if not inspect.isfunction(func):
# Not a function!? Skip it!!!
continue
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must be a
# dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute name
# if no alias is defined.
funcname = getattr(mod, '__func_alias__', {}).get(attr, attr)
# Save many references for lookups
self._dict['{0}.{1}'.format(module_name, funcname)] = func
setattr(mod_dict, funcname, func)
mod_dict[funcname] = func
self._apply_outputter(func, mod)
# enforce depends
Depends.enforce_dependencies(self._dict, self.tag)
self.loaded_modules[module_name] = mod_dict
return True
def _load_module(self, name):
mod = None
fpath, suffix = self.file_mapping[name][:2]
# if the fpath has `.cpython-3x` in it, but the running Py version
# is 3.y, the following will cause us to return immediately and we won't try to import this .pyc.
# This is for the unusual case where several Python versions share a single
# source tree and drop their .pycs in the same __pycache__ folder.
# If we were to load a .pyc for another Py version it's not a big problem
# but the log will get spammed with "Bad Magic Number" messages that
# can be very misleading if the user is debugging another problem.
try:
(implementation_tag,
cache_tag_ver) = sys.implementation.cache_tag.split("-")
if cache_tag_ver not in fpath and implementation_tag in fpath:
log.trace(
"Trying to load %s on %s, returning False.",
fpath,
sys.implementation.cache_tag,
)
return False
except AttributeError:
# Most likely Py 2.7 or some other Python version we don't really support
pass
self.loaded_files.add(name)
fpath_dirname = os.path.dirname(fpath)
try:
self.__populate_sys_path()
sys.path.append(fpath_dirname)
if suffix == ".pyx":
mod = pyximport.load_module(name, fpath, tempfile.gettempdir())
elif suffix == ".o":
top_mod = __import__(fpath, globals(), locals(), [])
comps = fpath.split(".")
if len(comps) < 2:
mod = top_mod
else:
mod = top_mod
for subname in comps[1:]:
mod = getattr(mod, subname)
elif suffix == ".zip":
mod = zipimporter(fpath).load_module(name)
else:
desc = self.suffix_map[suffix]
# if it is a directory, we don't open a file
try:
mod_namespace = ".".join((
self.loaded_base_name,
self.mod_type_check(fpath),
self.tag,
name,
))
except TypeError:
mod_namespace = "{}.{}.{}.{}".format(
self.loaded_base_name,
self.mod_type_check(fpath),
self.tag,
name,
)
if suffix == "":
# pylint: disable=no-member
# Package directory, look for __init__
loader_details = [
(
importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES,
),
(
importlib.machinery.SourcelessFileLoader,
importlib.machinery.BYTECODE_SUFFIXES,
),
(
importlib.machinery.ExtensionFileLoader,
importlib.machinery.EXTENSION_SUFFIXES,
),
]
file_finder = importlib.machinery.FileFinder(
fpath_dirname, *loader_details)
spec = file_finder.find_spec(mod_namespace)
if spec is None:
raise ImportError()
# TODO: Get rid of load_module in favor of
# exec_module below. load_module is deprecated, but
# loading using exec_module has been causing odd things
# with the magic dunders we pack into the loaded
# modules, most notably with salt-ssh's __opts__.
mod = spec.loader.load_module()
# mod = importlib.util.module_from_spec(spec)
# spec.loader.exec_module(mod)
# pylint: enable=no-member
sys.modules[mod_namespace] = mod
# reload all submodules if necessary
if not self.initial_load:
self._reload_submodules(mod)
else:
# pylint: disable=no-member
loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath)
spec = importlib.util.spec_from_file_location(
mod_namespace, fpath, loader=loader)
if spec is None:
raise ImportError()
# TODO: Get rid of load_module in favor of
# exec_module below. load_module is deprecated, but
# loading using exec_module has been causing odd things
# with the magic dunders we pack into the loaded
# modules, most notably with salt-ssh's __opts__.
mod = self.run(spec.loader.load_module)
# mod = importlib.util.module_from_spec(spec)
# spec.loader.exec_module(mod)
# pylint: enable=no-member
sys.modules[mod_namespace] = mod
except OSError:
raise
except ImportError as exc:
if "magic number" in str(exc):
error_msg = (
"Failed to import {} {}. Bad magic number. If migrating from"
" Python2 to Python3, remove all .pyc files and try again."
.format(self.tag, name))
log.warning(error_msg)
self.missing_modules[name] = error_msg
log.debug("Failed to import %s %s:\n",
self.tag,
name,
exc_info=True)
self.missing_modules[name] = exc
return False
except Exception as error: # pylint: disable=broad-except
log.error(
"Failed to import %s %s, this is due most likely to a syntax error:\n",
self.tag,
name,
exc_info=True,
)
self.missing_modules[name] = error
return False
except SystemExit as error:
try:
fn_, _, caller, _ = traceback.extract_tb(sys.exc_info()[2])[-1]
except Exception: # pylint: disable=broad-except
pass
else:
tgt_fns = [
os.path.join("salt", "utils", "process.py"),
os.path.join("salt", "cli", "daemons.py"),
os.path.join("salt", "cli", "api.py"),
]
for tgt_fn in tgt_fns:
if fn_.endswith(tgt_fn) and "_handle_signals" in caller:
# Race conditon, SIGTERM or SIGINT received while loader
# was in process of loading a module. Call sys.exit to
# ensure that the process is killed.
sys.exit(salt.defaults.exitcodes.EX_OK)
log.error(
"Failed to import %s %s as the module called exit()\n",
self.tag,
name,
exc_info=True,
)
self.missing_modules[name] = error
return False
finally:
sys.path.remove(fpath_dirname)
self.__clean_sys_path()
loader_context = salt.loader.context.LoaderContext()
if hasattr(mod, "__salt_loader__"):
if not isinstance(mod.__salt_loader__,
salt.loader.context.LoaderContext):
log.warning("Override __salt_loader__: %s", mod)
mod.__salt_loader__ = loader_context
else:
mod.__salt_loader__ = loader_context
if hasattr(mod, "__opts__"):
if not isinstance(mod.__opts__,
salt.loader.context.NamedLoaderContext):
if not hasattr(mod, "__orig_opts__"):
mod.__orig_opts__ = copy.deepcopy(mod.__opts__)
mod.__opts__ = copy.deepcopy(mod.__orig_opts__)
mod.__opts__.update(self.opts)
else:
if not hasattr(mod, "__orig_opts__"):
mod.__orig_opts__ = {}
mod.__opts__ = copy.deepcopy(mod.__orig_opts__)
mod.__opts__.update(self.opts)
# pack whatever other globals we were asked to
for p_name, p_value in self.pack.items():
if p_name == "__opts__":
continue
mod_named_context = getattr(mod, p_name, None)
if hasattr(mod_named_context, "default"):
default = copy.deepcopy(mod_named_context.default)
else:
default = None
named_context = loader_context.named_context(p_name, default)
if mod_named_context is None:
setattr(mod, p_name, named_context)
elif named_context != mod_named_context:
log.debug("Override %s: %s", p_name, mod)
setattr(mod, p_name, named_context)
else:
setattr(mod, p_name, named_context)
if self.pack_self is not None:
mod_named_context = getattr(mod, self.pack_self, None)
if hasattr(mod_named_context, "default"):
default = copy.deepcopy(mod_named_context.default)
else:
default = None
named_context = loader_context.named_context(
self.pack_self, default)
if mod_named_context is None:
setattr(mod, self.pack_self, named_context)
elif named_context != mod_named_context:
log.debug("Override %s: %s", self.pack_self, mod)
setattr(mod, self.pack_self, named_context)
else:
setattr(mod, self.pack_self, named_context)
module_name = mod.__name__.rsplit(".", 1)[-1]
# Call a module's initialization method if it exists
module_init = getattr(mod, "__init__", None)
if inspect.isfunction(module_init):
try:
self.run(module_init, self.opts)
except TypeError as e:
log.error(e)
except Exception: # pylint: disable=broad-except
err_string = "__init__ failed"
log.debug(
"Error loading %s.%s: %s",
self.tag,
module_name,
err_string,
exc_info=True,
)
self.missing_modules[module_name] = err_string
self.missing_modules[name] = err_string
return False
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
if self.virtual_enable:
virtual_funcs_to_process = ["__virtual__"] + self.virtual_funcs
for virtual_func in virtual_funcs_to_process:
(
virtual_ret,
module_name,
virtual_err,
virtual_aliases,
) = self._process_virtual(mod, module_name, virtual_func)
if virtual_err is not None:
log.trace("Error loading %s.%s: %s", self.tag, module_name,
virtual_err)
# if _process_virtual returned a non-True value then we are
# supposed to not process this module
if virtual_ret is not True and module_name not in self.missing_modules:
# If a module has information about why it could not be loaded, record it
self.missing_modules[module_name] = virtual_err
self.missing_modules[name] = virtual_err
return False
else:
virtual_aliases = ()
# If this is a proxy minion then MOST modules cannot work. Therefore, require that
# any module that does work with salt-proxy-minion define __proxyenabled__ as a list
# containing the names of the proxy types that the module supports.
#
# Render modules and state modules are OK though
if "proxy" in self.opts:
if self.tag in ["grains", "proxy"]:
if not hasattr(mod, "__proxyenabled__") or (
self.opts["proxy"]["proxytype"]
not in mod.__proxyenabled__
and "*" not in mod.__proxyenabled__):
err_string = "not a proxy_minion enabled module"
self.missing_modules[module_name] = err_string
self.missing_modules[name] = err_string
return False
if getattr(mod, "__load__", False) is not False:
log.info(
"The functions from module '%s' are being loaded from the "
"provided __load__ attribute",
module_name,
)
# If we had another module by the same virtual name, we should put any
# new functions under the existing dictionary.
mod_names = [module_name] + list(virtual_aliases)
for attr in getattr(mod, "__load__", dir(mod)):
if attr.startswith("_"):
# private functions are skipped
continue
func = getattr(mod, attr)
if not inspect.isfunction(func) and not isinstance(
func, functools.partial):
# Not a function!? Skip it!!!
continue
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must be a
# dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute name
# if no alias is defined.
funcname = getattr(mod, "__func_alias__", {}).get(attr, attr)
for tgt_mod in mod_names:
try:
full_funcname = ".".join((tgt_mod, funcname))
except TypeError:
full_funcname = "{}.{}".format(tgt_mod, funcname)
# Save many references for lookups
# Careful not to overwrite existing (higher priority) functions
if full_funcname not in self._dict:
self._dict[full_funcname] = func
self._apply_outputter(func, mod)
self.loaded_modules.add(tgt_mod)
# enforce depends
try:
Depends.enforce_dependencies(self._dict, self.tag, name)
except RuntimeError as exc:
log.info(
"Depends.enforce_dependencies() failed for the following reason: %s",
exc,
)
return True
def _load_module(self, name):
mod = None
fpath, suffix = self.file_mapping[name]
self.loaded_files.add(name)
try:
sys.path.append(os.path.dirname(fpath))
if suffix == '.pyx':
mod = self.pyximport.load_module(name, fpath, tempfile.gettempdir())
else:
desc = self.suffix_map[suffix]
# if it is a directory, we dont open a file
if suffix == '':
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(fpath),
self.tag,
name
), None, fpath, desc)
# reload all submodules if necessary
if not self.initial_load:
self._reload_submodules(mod)
else:
with open(fpath, desc[1]) as fn_:
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(fpath),
self.tag,
name
), fn_, fpath, desc)
except IOError:
raise
except ImportError:
log.debug(
'Failed to import {0} {1}:\n'.format(
self.tag, name
),
exc_info=True
)
return mod
except Exception:
log.error(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error:\n'.format(
self.tag, name
),
exc_info=True
)
return mod
except SystemExit:
log.error(
'Failed to import {0} {1} as the module called exit()\n'.format(
self.tag, name
),
exc_info=True
)
return mod
finally:
sys.path.pop()
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self._grains
mod.__pillar__ = self._pillar
# pack whatever other globals we were asked to
for p_name, p_value in six.iteritems(self.pack):
setattr(mod, p_name, p_value)
# Call a module's initialization method if it exists
module_init = getattr(mod, '__init__', None)
if inspect.isfunction(module_init):
try:
module_init(self.opts)
except TypeError:
pass
module_name = mod.__name__.rsplit('.', 1)[-1]
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
if self.virtual_enable:
(virtual_ret, module_name, virtual_err) = self.process_virtual(
mod,
module_name,
)
if virtual_err is not None:
log.debug('Error loading {0}.{1}: {2}'.format(self.tag,
module_name,
virtual_err,
))
# if process_virtual returned a non-True value then we are
# supposed to not process this module
if virtual_ret is not True:
# If a module has information about why it could not be loaded, record it
self.missing_modules[module_name] = virtual_err
self.missing_modules[name] = virtual_err
return False
# If this is a proxy minion then MOST modules cannot work. Therefore, require that
# any module that does work with salt-proxy-minion define __proxyenabled__ as a list
# containing the names of the proxy types that the module supports.
#
# Render modules and state modules are OK though
if 'proxy' in self.opts:
if self.tag not in ['render', 'states']:
if not hasattr(mod, '__proxyenabled__') or \
(self.opts['proxy']['proxytype'] not in mod.__proxyenabled__ and
'*' not in mod.__proxyenabled__):
err_string = 'not a proxy_minion enabled module'
self.missing_modules[module_name] = err_string
self.missing_modules[name] = err_string
return False
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from the '
'provided __load__ attribute'.format(
module_name
)
)
mod_dict = salt.utils.odict.OrderedDict()
for attr in getattr(mod, '__load__', dir(mod)):
if attr.startswith('_'):
# private functions are skipped
continue
func = getattr(mod, attr)
if not inspect.isfunction(func):
# Not a function!? Skip it!!!
continue
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must be a
# dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute name
# if no alias is defined.
funcname = getattr(mod, '__func_alias__', {}).get(attr, attr)
# Save many references for lookups
self._dict['{0}.{1}'.format(module_name, funcname)] = func
setattr(mod_dict, funcname, func)
mod_dict[funcname] = func
self._apply_outputter(func, mod)
# enforce depends
try:
Depends.enforce_dependencies(self._dict, self.tag)
except RuntimeError as e:
log.info('Depends.enforce_dependencies() failed '
'for reasons: {0}'.format(e))
self.loaded_modules[module_name] = mod_dict
return True
