def target(cls, minion_instance, opts, data, connected): """ Handle targeting of the minion. Calling _thread_multi_return or _thread_return depending on a single or multiple commands. """ log.debug( "Deltaproxy minion_instance %s(ID: %s). Target: %s", minion_instance, minion_instance.opts["id"], opts["id"], ) if not hasattr(minion_instance, "serial"): minion_instance.serial = salt.payload.Serial(opts) if not hasattr(minion_instance, "proc_dir"): uid = salt.utils.user.get_uid(user=opts.get("user", None)) minion_instance.proc_dir = salt.minion.get_proc_dir(opts["cachedir"], uid=uid) with tornado.stack_context.StackContext(minion_instance.ctx): if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): ProxyMinion._thread_multi_return(minion_instance, opts, data) else: ProxyMinion._thread_return(minion_instance, opts, data)
def target(cls, minion_instance, opts, data, connected): if not minion_instance: minion_instance = cls(opts) minion_instance.connected = connected if not hasattr(minion_instance, 'functions'): # Need to load the modules so they get all the dunder variables functions, returners, function_errors, executors = ( minion_instance._load_modules(grains=opts['grains']) ) minion_instance.functions = functions minion_instance.returners = returners minion_instance.function_errors = function_errors minion_instance.executors = executors # Pull in the utils minion_instance.utils = salt.loader.utils(minion_instance.opts) # Then load the proxy module minion_instance.proxy = salt.loader.proxy(minion_instance.opts, utils=minion_instance.utils) # And re-load the modules so the __proxy__ variable gets injected functions, returners, function_errors, executors = ( minion_instance._load_modules(grains=opts['grains']) ) minion_instance.functions = functions minion_instance.returners = returners minion_instance.function_errors = function_errors minion_instance.executors = executors minion_instance.functions.pack['__proxy__'] = minion_instance.proxy minion_instance.proxy.pack['__salt__'] = minion_instance.functions minion_instance.proxy.pack['__ret__'] = minion_instance.returners minion_instance.proxy.pack['__pillar__'] = minion_instance.opts['pillar'] # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ minion_instance.utils = salt.loader.utils(minion_instance.opts, proxy=minion_instance.proxy) minion_instance.proxy.pack['__utils__'] = minion_instance.utils # Reload all modules so all dunder variables are injected minion_instance.proxy.reload_modules() fq_proxyname = opts['proxy']['proxytype'] minion_instance.module_executors = minion_instance.proxy.get('{0}.module_executors'.format(fq_proxyname), lambda: [])() proxy_init_fn = minion_instance.proxy[fq_proxyname + '.init'] proxy_init_fn(opts) if not hasattr(minion_instance, 'serial'): minion_instance.serial = salt.payload.Serial(opts) if not hasattr(minion_instance, 'proc_dir'): uid = salt.utils.user.get_uid(user=opts.get('user', None)) minion_instance.proc_dir = ( salt.minion.get_proc_dir(opts['cachedir'], uid=uid) ) with tornado.stack_context.StackContext(minion_instance.ctx): if isinstance(data['fun'], tuple) or isinstance(data['fun'], list): ProxyMinion._thread_multi_return(minion_instance, opts, data) else: ProxyMinion._thread_return(minion_instance, opts, data)
def target(cls, minion_instance, opts, data, connected): """ Handle targeting of the minion. Calling _thread_multi_return or _thread_return depending on a single or multiple commands. """ if not minion_instance: minion_instance = cls(opts) minion_instance.connected = connected if not hasattr(minion_instance, "functions"): # Need to load the modules so they get all the dunder variables ( functions, returners, function_errors, executors, ) = minion_instance._load_modules(grains=opts["grains"]) minion_instance.functions = functions minion_instance.returners = returners minion_instance.function_errors = function_errors minion_instance.executors = executors # Pull in the utils minion_instance.utils = salt.loader.utils(minion_instance.opts) # Then load the proxy module minion_instance.proxy = salt.loader.proxy( minion_instance.opts, utils=minion_instance.utils) # And re-load the modules so the __proxy__ variable gets injected ( functions, returners, function_errors, executors, ) = minion_instance._load_modules(grains=opts["grains"]) minion_instance.functions = functions minion_instance.returners = returners minion_instance.function_errors = function_errors minion_instance.executors = executors minion_instance.functions.pack["__proxy__"] = minion_instance.proxy minion_instance.proxy.pack["__salt__"] = minion_instance.functions minion_instance.proxy.pack["__ret__"] = minion_instance.returners minion_instance.proxy.pack["__pillar__"] = minion_instance.opts[ "pillar"] # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ minion_instance.utils = salt.loader.utils( minion_instance.opts, proxy=minion_instance.proxy) minion_instance.proxy.pack["__utils__"] = minion_instance.utils # Reload all modules so all dunder variables are injected minion_instance.proxy.reload_modules() fq_proxyname = opts["proxy"]["proxytype"] minion_instance.module_executors = minion_instance.proxy.get( "{}.module_executors".format(fq_proxyname), lambda: [])() proxy_init_fn = minion_instance.proxy[fq_proxyname + ".init"] proxy_init_fn(opts) if not hasattr(minion_instance, "proc_dir"): uid = salt.utils.user.get_uid(user=opts.get("user", None)) minion_instance.proc_dir = salt.minion.get_proc_dir( opts["cachedir"], uid=uid) with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx): if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): ProxyMinion._thread_multi_return(minion_instance, opts, data) else: ProxyMinion._thread_return(minion_instance, opts, data)
def post_master_init(self, master): """ Function to finish init after a deltaproxy proxy minion has finished connecting to a master. This is primarily loading modules, pillars, etc. (since they need to know which master they connected to) """ if self.connected: self.opts["pillar"] = yield salt.pillar.get_async_pillar( self.opts, self.opts["grains"], self.opts["id"], saltenv=self.opts["saltenv"], pillarenv=self.opts.get("pillarenv"), ).compile_pillar() # Ensure that the value of master is the one we passed in. # if pillar_opts is enabled then master could be overwritten # when compile_pillar is run. self.opts["master"] = master tag = "salt/deltaproxy/start" self._fire_master(tag=tag) if "proxy" not in self.opts["pillar"] and "proxy" not in self.opts: errmsg = ( "No proxy key found in pillar or opts for id {}. Check your pillar/opts " "configuration and contents. Salt-proxy aborted.".format( self.opts["id"])) log.error(errmsg) self._running = False raise SaltSystemExit(code=-1, msg=errmsg) if "proxy" not in self.opts: self.opts["proxy"] = self.opts["pillar"]["proxy"] self.opts = salt.utils.dictupdate.merge( self.opts, self.opts["pillar"], strategy=self.opts.get("proxy_merge_pillar_in_opts_strategy"), merge_lists=self.opts.get("proxy_deep_merge_pillar_in_opts", False), ) if self.opts.get("proxy_mines_pillar"): # Even when not required, some details such as mine configuration # should be merged anyway whenever possible. if "mine_interval" in self.opts["pillar"]: self.opts["mine_interval"] = self.opts["pillar"]["mine_interval"] if "mine_functions" in self.opts["pillar"]: general_proxy_mines = self.opts.get("mine_functions", []) specific_proxy_mines = self.opts["pillar"]["mine_functions"] try: self.opts[ "mine_functions"] = general_proxy_mines + specific_proxy_mines except TypeError as terr: log.error( "Unable to merge mine functions from the pillar in the opts, for proxy %s", self.opts["id"], ) fq_proxyname = self.opts["proxy"]["proxytype"] # Need to load the modules so they get all the dunder variables ( self.functions, self.returners, self.function_errors, self.executors, ) = self._load_modules() # we can then sync any proxymodules down from the master # we do a sync_all here in case proxy code was installed by # SPM or was manually placed in /srv/salt/_modules etc. self.functions["saltutil.sync_all"](saltenv=self.opts["saltenv"]) # Pull in the utils self.utils = salt.loader.utils(self.opts) # Then load the proxy module self.proxy = salt.loader.proxy(self.opts, utils=self.utils) # And re-load the modules so the __proxy__ variable gets injected ( self.functions, self.returners, self.function_errors, self.executors, ) = self._load_modules() self.functions.pack["__proxy__"] = self.proxy self.proxy.pack["__salt__"] = self.functions self.proxy.pack["__ret__"] = self.returners self.proxy.pack["__pillar__"] = self.opts["pillar"] # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ self.utils = salt.loader.utils(self.opts, proxy=self.proxy) self.proxy.pack["__utils__"] = self.utils # Reload all modules so all dunder variables are injected self.proxy.reload_modules() # Start engines here instead of in the Minion superclass __init__ # This is because we need to inject the __proxy__ variable but # it is not setup until now. self.io_loop.spawn_callback(salt.engines.start_engines, self.opts, self.process_manager, proxy=self.proxy) proxy_init_func_name = "{}.init".format(fq_proxyname) proxy_shutdown_func_name = "{}.shutdown".format(fq_proxyname) if (proxy_init_func_name not in self.proxy or proxy_shutdown_func_name not in self.proxy): errmsg = ( "Proxymodule {} is missing an init() or a shutdown() or both. " "Check your proxymodule. Salt-proxy aborted.".format(fq_proxyname) ) log.error(errmsg) self._running = False raise SaltSystemExit(code=-1, msg=errmsg) self.module_executors = self.proxy.get( "{}.module_executors".format(fq_proxyname), lambda: [])() proxy_init_fn = self.proxy[proxy_init_func_name] proxy_init_fn(self.opts) self.opts["grains"] = salt.loader.grains(self.opts, proxy=self.proxy) self.mod_opts = self._prep_mod_opts() self.matchers = salt.loader.matchers(self.opts) self.beacons = salt.beacons.Beacon(self.opts, self.functions) uid = salt.utils.user.get_uid(user=self.opts.get("user", None)) self.proc_dir = salt.minion.get_proc_dir(self.opts["cachedir"], uid=uid) if self.connected and self.opts["pillar"]: # The pillar has changed due to the connection to the master. # Reload the functions so that they can use the new pillar data. ( self.functions, self.returners, self.function_errors, self.executors, ) = self._load_modules() if hasattr(self, "schedule"): self.schedule.functions = self.functions self.schedule.returners = self.returners if not hasattr(self, "schedule"): self.schedule = salt.utils.schedule.Schedule( self.opts, self.functions, self.returners, cleanup=[salt.minion.master_event(type="alive")], proxy=self.proxy, _subprocess_list=self.subprocess_list, ) # add default scheduling jobs to the minions scheduler if self.opts["mine_enabled"] and "mine.update" in self.functions: self.schedule.add_job( { "__mine_interval": { "function": "mine.update", "minutes": self.opts["mine_interval"], "jid_include": True, "maxrunning": 2, "run_on_start": True, "return_job": self.opts.get("mine_return_job", False), } }, persist=True, ) log.info("Added mine.update to scheduler") else: self.schedule.delete_job("__mine_interval", persist=True) # add master_alive job if enabled if self.opts["transport"] != "tcp" and self.opts[ "master_alive_interval"] > 0: self.schedule.add_job( { salt.minion.master_event(type="alive", master=self.opts["master"]): { "function": "status.master", "seconds": self.opts["master_alive_interval"], "jid_include": True, "maxrunning": 1, "return_job": False, "kwargs": { "master": self.opts["master"], "connected": True }, } }, persist=True, ) if (self.opts["master_failback"] and "master_list" in self.opts and self.opts["master"] != self.opts["master_list"][0]): self.schedule.add_job( { salt.minion.master_event(type="failback"): { "function": "status.ping_master", "seconds": self.opts["master_failback_interval"], "jid_include": True, "maxrunning": 1, "return_job": False, "kwargs": { "master": self.opts["master_list"][0] }, } }, persist=True, ) else: self.schedule.delete_job(salt.minion.master_event(type="failback"), persist=True) else: self.schedule.delete_job( salt.minion.master_event(type="alive", master=self.opts["master"]), persist=True, ) self.schedule.delete_job(salt.minion.master_event(type="failback"), persist=True) # proxy keepalive proxy_alive_fn = fq_proxyname + ".alive" if (proxy_alive_fn in self.proxy and "status.proxy_reconnect" in self.functions and self.opts.get("proxy_keep_alive", True)): # if `proxy_keep_alive` is either not specified, either set to False does not retry reconnecting self.schedule.add_job( { "__proxy_keepalive": { "function": "status.proxy_reconnect", "minutes": self.opts.get("proxy_keep_alive_interval", 1), # by default, check once per minute "jid_include": True, "maxrunning": 1, "return_job": False, "kwargs": { "proxy_name": fq_proxyname }, } }, persist=True, ) self.schedule.enable_schedule() else: self.schedule.delete_job("__proxy_keepalive", persist=True) # Sync the grains here so the proxy can communicate them to the master self.functions["saltutil.sync_grains"](saltenv="base") self.grains_cache = self.opts["grains"] # Now setup the deltaproxies self.deltaproxy = {} self.deltaproxy_opts = {} self.deltaproxy_objs = {} self.proxy_grains = {} self.proxy_pillar = {} self.proxy_context = {} self.add_periodic_callback("cleanup", self.cleanup_subprocesses) for _id in self.opts["proxy"].get("ids", []): control_id = self.opts["id"] proxyopts = self.opts.copy() proxyopts["id"] = _id proxyopts = salt.config.proxy_config(self.opts["conf_file"], defaults=proxyopts, minion_id=_id) proxyopts["id"] = proxyopts["proxyid"] = _id proxyopts["subproxy"] = True self.proxy_context[_id] = {"proxy_id": _id} # We need grains first to be able to load pillar, which is where we keep the proxy # configurations self.proxy_grains[_id] = salt.loader.grains( proxyopts, proxy=self.proxy, context=self.proxy_context[_id]) self.proxy_pillar[_id] = yield salt.pillar.get_async_pillar( proxyopts, self.proxy_grains[_id], _id, saltenv=proxyopts["saltenv"], pillarenv=proxyopts.get("pillarenv"), ).compile_pillar() proxyopts["proxy"] = self.proxy_pillar[_id].get("proxy", {}) if not proxyopts["proxy"]: log.warning( "Pillar data for proxy minion %s could not be loaded, skipping.", _id) continue # Remove ids proxyopts["proxy"].pop("ids", None) proxyopts["pillar"] = self.proxy_pillar[_id] proxyopts["grains"] = self.proxy_grains[_id] proxyopts["hash_id"] = self.opts["id"] _proxy_minion = ProxyMinion(proxyopts) _proxy_minion.proc_dir = salt.minion.get_proc_dir( proxyopts["cachedir"], uid=uid) _proxy_minion.proxy = salt.loader.proxy( proxyopts, utils=self.utils, context=self.proxy_context[_id]) _proxy_minion.subprocess_list = self.subprocess_list # And load the modules ( _proxy_minion.functions, _proxy_minion.returners, _proxy_minion.function_errors, _proxy_minion.executors, ) = _proxy_minion._load_modules(opts=proxyopts, grains=proxyopts["grains"], context=self.proxy_context[_id]) # we can then sync any proxymodules down from the master # we do a sync_all here in case proxy code was installed by # SPM or was manually placed in /srv/salt/_modules etc. _proxy_minion.functions["saltutil.sync_all"]( saltenv=self.opts["saltenv"]) # And re-load the modules so the __proxy__ variable gets injected ( _proxy_minion.functions, _proxy_minion.returners, _proxy_minion.function_errors, _proxy_minion.executors, ) = _proxy_minion._load_modules(opts=proxyopts, grains=proxyopts["grains"], context=self.proxy_context[_id]) _proxy_minion.functions.pack["__proxy__"] = _proxy_minion.proxy _proxy_minion.proxy.pack["__salt__"] = _proxy_minion.functions _proxy_minion.proxy.pack["__ret__"] = _proxy_minion.returners _proxy_minion.proxy.pack["__pillar__"] = proxyopts["pillar"] _proxy_minion.proxy.pack["__grains__"] = proxyopts["grains"] # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ _proxy_minion.proxy.utils = salt.loader.utils( proxyopts, proxy=_proxy_minion.proxy, context=self.proxy_context[_id]) _proxy_minion.proxy.pack["__utils__"] = _proxy_minion.proxy.utils # Reload all modules so all dunder variables are injected _proxy_minion.proxy.reload_modules() _proxy_minion.connected = True _fq_proxyname = proxyopts["proxy"]["proxytype"] proxy_init_fn = _proxy_minion.proxy[_fq_proxyname + ".init"] try: proxy_init_fn(proxyopts) except Exception as exc: # pylint: disable=broad-except log.error( "An exception occured during the initialization of minion %s: %s", _id, exc, exc_info=True, ) continue # Reload the grains self.proxy_grains[_id] = salt.loader.grains( proxyopts, proxy=_proxy_minion.proxy, context=self.proxy_context[_id]) proxyopts["grains"] = self.proxy_grains[_id] if not hasattr(_proxy_minion, "schedule"): _proxy_minion.schedule = salt.utils.schedule.Schedule( proxyopts, _proxy_minion.functions, _proxy_minion.returners, cleanup=[salt.minion.master_event(type="alive")], proxy=_proxy_minion.proxy, new_instance=True, _subprocess_list=_proxy_minion.subprocess_list, ) self.deltaproxy_objs[_id] = _proxy_minion self.deltaproxy_opts[_id] = copy.deepcopy(proxyopts) # proxy keepalive _proxy_alive_fn = _fq_proxyname + ".alive" if (_proxy_alive_fn in _proxy_minion.proxy and "status.proxy_reconnect" in self.deltaproxy_objs[_id].functions and proxyopts.get("proxy_keep_alive", True)): # if `proxy_keep_alive` is either not specified, either set to False does not retry reconnecting _proxy_minion.schedule.add_job( { "__proxy_keepalive": { "function": "status.proxy_reconnect", "minutes": proxyopts.get("proxy_keep_alive_interval", 1), # by default, check once per minute "jid_include": True, "maxrunning": 1, "return_job": False, "kwargs": { "proxy_name": _fq_proxyname }, } }, persist=True, ) _proxy_minion.schedule.enable_schedule() else: _proxy_minion.schedule.delete_job("__proxy_keepalive", persist=True) self.ready = True