def minion_mods(opts, context=None, whitelist=None): ''' Load execution modules Returns a dictionary of execution modules appropriate for the current system by evaluating the __virtual__() function in each module. .. code-block:: python import salt.config import salt.loader __opts__ salt.config.minion_config('/etc/salt/minion') __salt__ = salt.loader.minion_mods(__opts__) __salt__['test.ping']() ''' load = _create_loader(opts, 'modules', 'module') if context is None: context = {} pack = {'name': '__context__', 'value': context} if not whitelist: whitelist = opts.get('whitelist_modules', None) functions = load.gen_functions( pack, whitelist=whitelist, provider_overrides=True ) # Enforce dependencies of module functions from "functions" Depends.enforce_dependencies(functions) return functions
def minion_mods(opts, context=None, whitelist=None): ''' Load execution modules Returns a dictionary of execution modules appropriate for the current system by evaluating the __virtual__() function in each module. .. code-block:: python import salt.config import salt.loader __opts__ = salt.config.minion_config('/etc/salt/minion') __salt__ = salt.loader.minion_mods(__opts__) __salt__['test.ping']() ''' load = _create_loader(opts, 'modules', 'module') if context is None: context = {} pack = {'name': '__context__', 'value': context} if not whitelist: whitelist = opts.get('whitelist_modules', None) functions = load.gen_functions(pack, whitelist=whitelist, provider_overrides=True) # Enforce dependencies of module functions from "functions" Depends.enforce_dependencies(functions) return functions
def minion_mods(opts, context=None, whitelist=None): ''' Returns the minion modules ''' load = _create_loader(opts, 'modules', 'module') if context is None: context = {} pack = {'name': '__context__', 'value': context} if not whitelist: whitelist = opts.get('whitelist_modules', None) functions = load.gen_functions( pack, whitelist=whitelist ) # Enforce dependencies of module functions from "functions" Depends.enforce_dependencies(functions) if opts.get('providers', False): if isinstance(opts['providers'], dict): for mod, provider in opts['providers'].items(): funcs = raw_mod(opts, provider, functions) if funcs: for func in funcs: f_key = '{0}{1}'.format(mod, func[func.rindex('.'):]) functions[f_key] = funcs[func] return functions
def minion_mods(opts, context=None, whitelist=None): ''' Returns the minion modules ''' load = _create_loader(opts, 'modules', 'module') if context is None: context = {} pack = {'name': '__context__', 'value': context} if not whitelist: whitelist = opts.get('whitelist_modules', None) functions = load.gen_functions(pack, whitelist=whitelist, provider_overrides=True) # Enforce dependencies of module functions from "functions" Depends.enforce_dependencies(functions) return functions
def minion_mods(opts, context=None, whitelist=None): ''' Returns the minion modules ''' load = _create_loader(opts, 'modules', 'module') if context is None: context = {} pack = {'name': '__context__', 'value': context} if not whitelist: whitelist = opts.get('whitelist_modules', None) functions = load.gen_functions( pack, whitelist=whitelist, provider_overrides=True ) # Enforce dependencies of module functions from "functions" Depends.enforce_dependencies(functions) return functions
def minion_mods(opts, context=None, whitelist=None): ''' Returns the minion modules ''' load = _create_loader(opts, 'modules', 'module') if context is None: context = {} pack = {'name': '__context__', 'value': context} if not whitelist: whitelist = opts.get('whitelist_modules', None) functions = load.gen_functions(pack, whitelist=whitelist) # Enforce dependencies of module functions from "functions" Depends.enforce_dependencies(functions) if opts.get('providers', False): if isinstance(opts['providers'], dict): for mod, provider in opts['providers'].items(): funcs = raw_mod(opts, provider, functions) if funcs: for func in funcs: f_key = '{0}{1}'.format(mod, func[func.rindex('.'):]) functions[f_key] = funcs[func] return functions
def _load_module(self, name): mod = None fpath, suffix = self.file_mapping[name] self.loaded_files.add(name) try: sys.path.append(os.path.dirname(fpath)) if suffix == '.pyx': mod = self.pyximport.load_module(name, fpath, tempfile.gettempdir()) else: desc = self.suffix_map[suffix] # if it is a directory, we dont open a file if suffix == '': mod = imp.load_module( '{0}.{1}.{2}.{3}'.format(self.loaded_base_name, self.mod_type_check(fpath), self.tag, name), None, fpath, desc) # reload all submodules if necessary if not self.initial_load: self._reload_submodules(mod) else: with open(fpath, desc[1]) as fn_: mod = imp.load_module( '{0}.{1}.{2}.{3}'.format( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name), fn_, fpath, desc) except IOError: raise except ImportError: log.debug('Failed to import {0} {1}:\n'.format(self.tag, name), exc_info=True) return mod except Exception as error: log.error('Failed to import {0} {1}, this is due most likely to a ' 'syntax error:\n'.format(self.tag, name), exc_info=True) return mod except SystemExit: log.error('Failed to import {0} {1} as the module called exit()\n'. format(self.tag, name), exc_info=True) return mod finally: sys.path.pop() if hasattr(mod, '__opts__'): mod.__opts__.update(self.opts) else: mod.__opts__ = self.opts mod.__grains__ = self._grains mod.__pillar__ = self._pillar # pack whatever other globals we were asked to for p_name, p_value in six.iteritems(self.pack): setattr(mod, p_name, p_value) # Call a module's initialization method if it exists module_init = getattr(mod, '__init__', None) if inspect.isfunction(module_init): try: module_init(self.opts) except TypeError as e: log.error(e) module_name = mod.__name__.rsplit('.', 1)[-1] # if virtual modules are enabled, we need to look for the # __virtual__() function inside that module and run it. if self.virtual_enable: (virtual_ret, module_name, virtual_err) = self.process_virtual( mod, module_name, ) if virtual_err is not None: log.debug('Error loading {0}.{1}: {2}'.format( self.tag, module_name, virtual_err, )) # if process_virtual returned a non-True value then we are # supposed to not process this module if virtual_ret is not True: # If a module has information about why it could not be loaded, record it self.missing_modules[module_name] = virtual_err self.missing_modules[name] = virtual_err return False # If this is a proxy minion then MOST modules cannot work. Therefore, require that # any module that does work with salt-proxy-minion define __proxyenabled__ as a list # containing the names of the proxy types that the module supports. if not hasattr(mod, 'render') and 'proxy' in self.opts: if not hasattr(mod, '__proxyenabled__') or \ self.opts['proxy']['proxytype'] in mod.__proxyenabled__ or \ '*' in mod.__proxyenabled__: err_string = 'not a proxy_minion enabled module' self.missing_modules[module_name] = err_string self.missing_modules[name] = err_string return False if getattr(mod, '__load__', False) is not False: log.info( 'The functions from module {0!r} are being loaded from the ' 'provided __load__ attribute'.format(module_name)) mod_dict = salt.utils.odict.OrderedDict() for attr in getattr(mod, '__load__', dir(mod)): if attr.startswith('_'): # private functions are skipped continue func = getattr(mod, attr) if not inspect.isfunction(func): # Not a function!? Skip it!!! continue # Let's get the function name. # If the module has the __func_alias__ attribute, it must be a # dictionary mapping in the form of(key -> value): # <real-func-name> -> <desired-func-name> # # It default's of course to the found callable attribute name # if no alias is defined. funcname = getattr(mod, '__func_alias__', {}).get(attr, attr) # Save many references for lookups self._dict['{0}.{1}'.format(module_name, funcname)] = func setattr(mod_dict, funcname, func) mod_dict[funcname] = func self._apply_outputter(func, mod) # enforce depends Depends.enforce_dependencies(self._dict, self.tag) self.loaded_modules[module_name] = mod_dict return True
def _load_module(self, name): mod = None fpath, suffix = self.file_mapping[name][:2] # if the fpath has `.cpython-3x` in it, but the running Py version # is 3.y, the following will cause us to return immediately and we won't try to import this .pyc. # This is for the unusual case where several Python versions share a single # source tree and drop their .pycs in the same __pycache__ folder. # If we were to load a .pyc for another Py version it's not a big problem # but the log will get spammed with "Bad Magic Number" messages that # can be very misleading if the user is debugging another problem. try: (implementation_tag, cache_tag_ver) = sys.implementation.cache_tag.split("-") if cache_tag_ver not in fpath and implementation_tag in fpath: log.trace( "Trying to load %s on %s, returning False.", fpath, sys.implementation.cache_tag, ) return False except AttributeError: # Most likely Py 2.7 or some other Python version we don't really support pass self.loaded_files.add(name) fpath_dirname = os.path.dirname(fpath) try: self.__populate_sys_path() sys.path.append(fpath_dirname) if suffix == ".pyx": mod = pyximport.load_module(name, fpath, tempfile.gettempdir()) elif suffix == ".o": top_mod = __import__(fpath, globals(), locals(), []) comps = fpath.split(".") if len(comps) < 2: mod = top_mod else: mod = top_mod for subname in comps[1:]: mod = getattr(mod, subname) elif suffix == ".zip": mod = zipimporter(fpath).load_module(name) else: desc = self.suffix_map[suffix] # if it is a directory, we don't open a file try: mod_namespace = ".".join(( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name, )) except TypeError: mod_namespace = "{}.{}.{}.{}".format( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name, ) if suffix == "": # pylint: disable=no-member # Package directory, look for __init__ loader_details = [ ( importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES, ), ( importlib.machinery.SourcelessFileLoader, importlib.machinery.BYTECODE_SUFFIXES, ), ( importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES, ), ] file_finder = importlib.machinery.FileFinder( fpath_dirname, *loader_details) spec = file_finder.find_spec(mod_namespace) if spec is None: raise ImportError() # TODO: Get rid of load_module in favor of # exec_module below. load_module is deprecated, but # loading using exec_module has been causing odd things # with the magic dunders we pack into the loaded # modules, most notably with salt-ssh's __opts__. mod = spec.loader.load_module() # mod = importlib.util.module_from_spec(spec) # spec.loader.exec_module(mod) # pylint: enable=no-member sys.modules[mod_namespace] = mod # reload all submodules if necessary if not self.initial_load: self._reload_submodules(mod) else: # pylint: disable=no-member loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath) spec = importlib.util.spec_from_file_location( mod_namespace, fpath, loader=loader) if spec is None: raise ImportError() # TODO: Get rid of load_module in favor of # exec_module below. load_module is deprecated, but # loading using exec_module has been causing odd things # with the magic dunders we pack into the loaded # modules, most notably with salt-ssh's __opts__. mod = self.run(spec.loader.load_module) # mod = importlib.util.module_from_spec(spec) # spec.loader.exec_module(mod) # pylint: enable=no-member sys.modules[mod_namespace] = mod except OSError: raise except ImportError as exc: if "magic number" in str(exc): error_msg = ( "Failed to import {} {}. Bad magic number. If migrating from" " Python2 to Python3, remove all .pyc files and try again." .format(self.tag, name)) log.warning(error_msg) self.missing_modules[name] = error_msg log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True) self.missing_modules[name] = exc return False except Exception as error: # pylint: disable=broad-except log.error( "Failed to import %s %s, this is due most likely to a syntax error:\n", self.tag, name, exc_info=True, ) self.missing_modules[name] = error return False except SystemExit as error: try: fn_, _, caller, _ = traceback.extract_tb(sys.exc_info()[2])[-1] except Exception: # pylint: disable=broad-except pass else: tgt_fns = [ os.path.join("salt", "utils", "process.py"), os.path.join("salt", "cli", "daemons.py"), os.path.join("salt", "cli", "api.py"), ] for tgt_fn in tgt_fns: if fn_.endswith(tgt_fn) and "_handle_signals" in caller: # Race conditon, SIGTERM or SIGINT received while loader # was in process of loading a module. Call sys.exit to # ensure that the process is killed. sys.exit(salt.defaults.exitcodes.EX_OK) log.error( "Failed to import %s %s as the module called exit()\n", self.tag, name, exc_info=True, ) self.missing_modules[name] = error return False finally: sys.path.remove(fpath_dirname) self.__clean_sys_path() loader_context = salt.loader.context.LoaderContext() if hasattr(mod, "__salt_loader__"): if not isinstance(mod.__salt_loader__, salt.loader.context.LoaderContext): log.warning("Override __salt_loader__: %s", mod) mod.__salt_loader__ = loader_context else: mod.__salt_loader__ = loader_context if hasattr(mod, "__opts__"): if not isinstance(mod.__opts__, salt.loader.context.NamedLoaderContext): if not hasattr(mod, "__orig_opts__"): mod.__orig_opts__ = copy.deepcopy(mod.__opts__) mod.__opts__ = copy.deepcopy(mod.__orig_opts__) mod.__opts__.update(self.opts) else: if not hasattr(mod, "__orig_opts__"): mod.__orig_opts__ = {} mod.__opts__ = copy.deepcopy(mod.__orig_opts__) mod.__opts__.update(self.opts) # pack whatever other globals we were asked to for p_name, p_value in self.pack.items(): if p_name == "__opts__": continue mod_named_context = getattr(mod, p_name, None) if hasattr(mod_named_context, "default"): default = copy.deepcopy(mod_named_context.default) else: default = None named_context = loader_context.named_context(p_name, default) if mod_named_context is None: setattr(mod, p_name, named_context) elif named_context != mod_named_context: log.debug("Override %s: %s", p_name, mod) setattr(mod, p_name, named_context) else: setattr(mod, p_name, named_context) if self.pack_self is not None: mod_named_context = getattr(mod, self.pack_self, None) if hasattr(mod_named_context, "default"): default = copy.deepcopy(mod_named_context.default) else: default = None named_context = loader_context.named_context( self.pack_self, default) if mod_named_context is None: setattr(mod, self.pack_self, named_context) elif named_context != mod_named_context: log.debug("Override %s: %s", self.pack_self, mod) setattr(mod, self.pack_self, named_context) else: setattr(mod, self.pack_self, named_context) module_name = mod.__name__.rsplit(".", 1)[-1] # Call a module's initialization method if it exists module_init = getattr(mod, "__init__", None) if inspect.isfunction(module_init): try: self.run(module_init, self.opts) except TypeError as e: log.error(e) except Exception: # pylint: disable=broad-except err_string = "__init__ failed" log.debug( "Error loading %s.%s: %s", self.tag, module_name, err_string, exc_info=True, ) self.missing_modules[module_name] = err_string self.missing_modules[name] = err_string return False # if virtual modules are enabled, we need to look for the # __virtual__() function inside that module and run it. if self.virtual_enable: virtual_funcs_to_process = ["__virtual__"] + self.virtual_funcs for virtual_func in virtual_funcs_to_process: ( virtual_ret, module_name, virtual_err, virtual_aliases, ) = self._process_virtual(mod, module_name, virtual_func) if virtual_err is not None: log.trace("Error loading %s.%s: %s", self.tag, module_name, virtual_err) # if _process_virtual returned a non-True value then we are # supposed to not process this module if virtual_ret is not True and module_name not in self.missing_modules: # If a module has information about why it could not be loaded, record it self.missing_modules[module_name] = virtual_err self.missing_modules[name] = virtual_err return False else: virtual_aliases = () # If this is a proxy minion then MOST modules cannot work. Therefore, require that # any module that does work with salt-proxy-minion define __proxyenabled__ as a list # containing the names of the proxy types that the module supports. # # Render modules and state modules are OK though if "proxy" in self.opts: if self.tag in ["grains", "proxy"]: if not hasattr(mod, "__proxyenabled__") or ( self.opts["proxy"]["proxytype"] not in mod.__proxyenabled__ and "*" not in mod.__proxyenabled__): err_string = "not a proxy_minion enabled module" self.missing_modules[module_name] = err_string self.missing_modules[name] = err_string return False if getattr(mod, "__load__", False) is not False: log.info( "The functions from module '%s' are being loaded from the " "provided __load__ attribute", module_name, ) # If we had another module by the same virtual name, we should put any # new functions under the existing dictionary. mod_names = [module_name] + list(virtual_aliases) for attr in getattr(mod, "__load__", dir(mod)): if attr.startswith("_"): # private functions are skipped continue func = getattr(mod, attr) if not inspect.isfunction(func) and not isinstance( func, functools.partial): # Not a function!? Skip it!!! continue # Let's get the function name. # If the module has the __func_alias__ attribute, it must be a # dictionary mapping in the form of(key -> value): # <real-func-name> -> <desired-func-name> # # It default's of course to the found callable attribute name # if no alias is defined. funcname = getattr(mod, "__func_alias__", {}).get(attr, attr) for tgt_mod in mod_names: try: full_funcname = ".".join((tgt_mod, funcname)) except TypeError: full_funcname = "{}.{}".format(tgt_mod, funcname) # Save many references for lookups # Careful not to overwrite existing (higher priority) functions if full_funcname not in self._dict: self._dict[full_funcname] = func self._apply_outputter(func, mod) self.loaded_modules.add(tgt_mod) # enforce depends try: Depends.enforce_dependencies(self._dict, self.tag, name) except RuntimeError as exc: log.info( "Depends.enforce_dependencies() failed for the following reason: %s", exc, ) return True
def _load_module(self, name): mod = None fpath, suffix = self.file_mapping[name] self.loaded_files.add(name) try: sys.path.append(os.path.dirname(fpath)) if suffix == '.pyx': mod = self.pyximport.load_module(name, fpath, tempfile.gettempdir()) else: desc = self.suffix_map[suffix] # if it is a directory, we dont open a file if suffix == '': mod = imp.load_module( '{0}.{1}.{2}.{3}'.format( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name ), None, fpath, desc) # reload all submodules if necessary if not self.initial_load: self._reload_submodules(mod) else: with open(fpath, desc[1]) as fn_: mod = imp.load_module( '{0}.{1}.{2}.{3}'.format( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name ), fn_, fpath, desc) except IOError: raise except ImportError: log.debug( 'Failed to import {0} {1}:\n'.format( self.tag, name ), exc_info=True ) return mod except Exception: log.error( 'Failed to import {0} {1}, this is due most likely to a ' 'syntax error:\n'.format( self.tag, name ), exc_info=True ) return mod except SystemExit: log.error( 'Failed to import {0} {1} as the module called exit()\n'.format( self.tag, name ), exc_info=True ) return mod finally: sys.path.pop() if hasattr(mod, '__opts__'): mod.__opts__.update(self.opts) else: mod.__opts__ = self.opts mod.__grains__ = self._grains mod.__pillar__ = self._pillar # pack whatever other globals we were asked to for p_name, p_value in six.iteritems(self.pack): setattr(mod, p_name, p_value) # Call a module's initialization method if it exists module_init = getattr(mod, '__init__', None) if inspect.isfunction(module_init): try: module_init(self.opts) except TypeError: pass module_name = mod.__name__.rsplit('.', 1)[-1] # if virtual modules are enabled, we need to look for the # __virtual__() function inside that module and run it. if self.virtual_enable: (virtual_ret, module_name, virtual_err) = self.process_virtual( mod, module_name, ) if virtual_err is not None: log.debug('Error loading {0}.{1}: {2}'.format(self.tag, module_name, virtual_err, )) # if process_virtual returned a non-True value then we are # supposed to not process this module if virtual_ret is not True: # If a module has information about why it could not be loaded, record it self.missing_modules[module_name] = virtual_err self.missing_modules[name] = virtual_err return False # If this is a proxy minion then MOST modules cannot work. Therefore, require that # any module that does work with salt-proxy-minion define __proxyenabled__ as a list # containing the names of the proxy types that the module supports. # # Render modules and state modules are OK though if 'proxy' in self.opts: if self.tag not in ['render', 'states']: if not hasattr(mod, '__proxyenabled__') or \ (self.opts['proxy']['proxytype'] not in mod.__proxyenabled__ and '*' not in mod.__proxyenabled__): err_string = 'not a proxy_minion enabled module' self.missing_modules[module_name] = err_string self.missing_modules[name] = err_string return False if getattr(mod, '__load__', False) is not False: log.info( 'The functions from module {0!r} are being loaded from the ' 'provided __load__ attribute'.format( module_name ) ) mod_dict = salt.utils.odict.OrderedDict() for attr in getattr(mod, '__load__', dir(mod)): if attr.startswith('_'): # private functions are skipped continue func = getattr(mod, attr) if not inspect.isfunction(func): # Not a function!? Skip it!!! continue # Let's get the function name. # If the module has the __func_alias__ attribute, it must be a # dictionary mapping in the form of(key -> value): # <real-func-name> -> <desired-func-name> # # It default's of course to the found callable attribute name # if no alias is defined. funcname = getattr(mod, '__func_alias__', {}).get(attr, attr) # Save many references for lookups self._dict['{0}.{1}'.format(module_name, funcname)] = func setattr(mod_dict, funcname, func) mod_dict[funcname] = func self._apply_outputter(func, mod) # enforce depends try: Depends.enforce_dependencies(self._dict, self.tag) except RuntimeError as e: log.info('Depends.enforce_dependencies() failed ' 'for reasons: {0}'.format(e)) self.loaded_modules[module_name] = mod_dict return True