def start(self): log.debug("starting monitor with {} task{}".format(len(self.tasks), "" if len(self.tasks) == 1 else "s")) if self.tasks: for task in self.tasks: threading.Thread(target=task.run).start() else: log.error("no monitor tasks to run")
def ec2_tags(): boto_version = StrictVersion(boto.__version__) required_boto_version = StrictVersion('2.8.0') if boto_version < required_boto_version: log.error("Installed boto version %s < %s, can't find ec2_tags", boto_version, required_boto_version) return None if not _on_ec2(): log.info("Not an EC2 instance, skipping") return None instance_id, region = _get_instance_info() credentials = _get_credentials() # Connect to EC2 and parse the Roles tags for this instance try: conn = boto.ec2.connect_to_region( region, aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key'], ) except Exception, e: log.error("Could not get AWS connection: %s", e) return None
def daemonize(): ''' Daemonize a process ''' try: pid = os.fork() if pid > 0: # exit first parent sys.exit(0) except OSError as exc: log.error( 'fork #1 failed: {0} ({1})'.format(exc.errno, exc.strerror) ) sys.exit(1) # decouple from parent environment os.chdir('/') os.setsid() os.umask(18) # do second fork try: pid = os.fork() if pid > 0: sys.exit(0) except OSError as exc: log.error( 'fork #2 failed: {0} ({1})'.format( exc.errno, exc.strerror ) ) sys.exit(1)
def dns_check(addr, safe=False): ''' Return the ip resolved by dns, but do not exit on failure, only raise an exception. ''' try: socket.inet_aton(addr) except socket.error: # Not a valid ip adder, check DNS try: addr = socket.gethostbyname(addr) except socket.gaierror: err = ('This master address: \'{0}\' was previously resolvable but ' 'now fails to resolve! The previously resolved ip addr ' 'will continue to be used').format(addr) if safe: import salt.log if salt.log.is_console_configured(): # If logging is not configured it also means that either # the master or minion instance calling this hasn't even # started running log.error(err) raise SaltClientError else: err = err.format(addr) sys.stderr.write(err) sys.exit(42) return addr
def load_agents(config): ''' Load the agents specified in /etc/salt/alert from the salt.ext.alert.agents package. Each module must define a load_agents() function that accepts the parsed YAML configuration for the agents. ''' ignore_modules = ['alert.time', 'alert.subscriptions', 'alert.verbs'] agents = {} for key, value in config.iteritems(): if key.startswith('alert.') and key not in ignore_modules: modname = AGENTS_MODULE + '._' + key[6:] log.trace('load %s', modname) try: mod = __import__(modname, fromlist=[AGENTS_MODULE]) except ImportError, ex: log.trace('not an agent module: %s', modname, exc_info=ex) continue try: new_agents = mod.load_agents(value) except AttributeError, ex: log.error('not an agent module: %s', modname, exc_info=ex) continue common = set(agents.keys()) & set(new_agents.keys()) if len(common) != 0: raise ValueError( 'agent name(s) collide in config: {}'.format( ', '.join(["'{}'".format(x) for x in common]))) agents.update(new_agents) log.trace('loaded alert agent(s): %s', new_agents.keys())
def __init__(self, tgt='', expr_form='glob', env=None, use_cached_grains=True, use_cached_pillar=True, grains_fallback=True, pillar_fallback=True, opts=None): log.debug('New instance of {0} created.'.format( self.__class__.__name__)) if opts is None: log.error('{0}: Missing master opts init arg.'.format( self.__class__.__name__)) raise SaltException('{0}: Missing master opts init arg.'.format( self.__class__.__name__)) else: self.opts = opts self.tgt = tgt self.expr_form = expr_form self.env = env self.use_cached_grains = use_cached_grains self.use_cached_pillar = use_cached_pillar self.grains_fallback = grains_fallback self.pillar_fallback = pillar_fallback log.debug('Init settings: tgt: \"{0}\", expr_form: \"{1}\", env: \"{2}\", use_cached_grains: {3}, use_cached_pillar: {4}, grains_fallback: {5}, pillar_fallback: {6}'.format(tgt, expr_form, env, use_cached_grains, use_cached_pillar, grains_fallback, pillar_fallback))
def main(): """ Start computer service. .. code-block:: bash hs-computer --server|--node|--all """ salt.log.setup_console_logger() parser = option_parser() shell = Shell(parser) try: shell.exec_command() except Exception as err: log.error("shell command failed: {0!r}".format( err) ) except KeyboardInterrupt: raise SystemExit('\nExiting gracefully on Ctrl-c') finally: # TODO: need cleanup log.trace("TODO cleanup work") return
def ec2_tags(): log = logging.getLogger(__name__) # ************* REQUEST VALUES ************* instanceid = _get_instance_id() method = 'GET' service = 'ec2' region = _get_region() host = 'ec2.'+region+'.amazonaws.com' endpoint = 'https://ec2.'+region+'.amazonaws.com' params = [('Action','DescribeTags')] params.append( ('Filter.1.Name','resource-id') ) params.append( ('Filter.1.Value.1',instanceid) ) params.append( ('Version','2015-04-15') ) request_parameters = urllib.urlencode(params) creds = _get_role_credentials() access_key = creds['AccessKeyId'] secret_key = creds['SecretAccessKey'] token = creds['Token'] if access_key is None or secret_key is None or token is None: log.error('No role credentials found.') return None # Create a date for headers and the credential string t = datetime.datetime.utcnow() amzdate = t.strftime('%Y%m%dT%H%M%SZ') datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope # Calculate AWS Signature V4 canonical_uri = '/' canonical_querystring = request_parameters canonical_headers = 'host:' + host + '\n' + 'x-amz-date:' + amzdate + '\n' + 'x-amz-security-token:' + token + '\n' signed_headers = 'host;x-amz-date;x-amz-security-token' payload_hash = hashlib.sha256('').hexdigest() canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash algorithm = 'AWS4-HMAC-SHA256' credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request' string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request).hexdigest() signing_key = _getSignatureKey(secret_key, datestamp, region, service) signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest() authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature request_url = endpoint + '?' + canonical_querystring r = urllib2.Request(request_url) r.add_header('x-amz-date',amzdate) r.add_header('Authorization',authorization_header) r.add_header('x-amz-security-token',token) try: result = urllib2.urlopen(r) except Exception, e: log.error('Could not complete EC2 API request.') return None
def _match(target, pattern, regex_match=False): if regex_match: try: return re.match(pattern.lower(), str(target).lower()) except Exception: log.error('Invalid regex \'{0}\' in match'.format(pattern)) return False else: return fnmatch.fnmatch(str(target).lower(), pattern.lower())
def _thread_multi_return(class_, minion_instance, opts, data): ''' This method should be used as a threading target, start the actual minion side execution. ''' # this seems awkward at first, but it's a workaround for Windows # multiprocessing communication. if not minion_instance: minion_instance = class_(opts) ret = { 'return': {}, 'success': {}, } for ind in range(0, len(data['fun'])): for index in range(0, len(data['arg'][ind])): try: arg = eval(data['arg'][ind][index]) if isinstance(arg, bool): data['arg'][ind][index] = str(data['arg'][ind][index]) elif isinstance(arg, (dict, int, list, string_types)): data['arg'][ind][index] = arg else: data['arg'][ind][index] = str(data['arg'][ind][index]) except Exception: pass ret['success'][data['fun'][ind]] = False try: func = minion_instance.functions[data['fun'][ind]] args, kwargs = detect_kwargs(func, data['arg'][ind], data) ret['return'][data['fun'][ind]] = func(*args, **kwargs) ret['success'][data['fun'][ind]] = True except Exception as exc: trb = traceback.format_exc() log.warning( 'The minion function caused an exception: {0}'.format( exc ) ) ret['return'][data['fun'][ind]] = trb ret['jid'] = data['jid'] minion_instance._return_pub(ret) if data['ret']: for returner in set(data['ret'].split(',')): ret['id'] = opts['id'] try: minion_instance.returners['{0}.returner'.format( returner )](ret) except Exception as exc: log.error( 'The return failed for job {0} {1}'.format( data['jid'], exc ) )
def do_image(self): try: name = "{0}_image".format(self.CRUD_MAP[self.args.mode]) function = getattr(self.image, name) function() except Exception as err: log.error('call function{0} failed: {1!r}'.format( name, err) ) raise RunTimeFailture("call do_image failure")
def get_id(): ''' Guess the id of the minion. - If socket.getfqdn() returns us something other than localhost, use it - Check /etc/hosts for something that isn't localhost that maps to 127.* - Look for a routeable / public IP - A private IP is better than a loopback IP - localhost may be better than killing the minion ''' log.debug('Guessing ID. The id can be explicitly in set {0}' .format('/etc/salt/minion')) fqdn = socket.getfqdn() if 'localhost' != fqdn: log.info('Found minion id from getfqdn(): {0}'.format(fqdn)) return fqdn, False # Can /etc/hosts help us? try: # TODO Add Windows host file support with open('/etc/hosts') as f: line = f.readline() while line: names = line.split() ip = names.pop(0) if ip.startswith('127.'): for name in names: if name != 'localhost': log.info('Found minion id in hosts file: {0}' .format(name)) return name, False line = f.readline() except Exception: pass # What IP addresses do we have? ip_addresses = [salt.utils.socket_util.IPv4Address(a) for a in salt.utils.socket_util.ip4_addrs() if not a.startswith('127.')] for a in ip_addresses: if not a.is_private: log.info('Using public ip address for id: {0}'.format(a)) return str(a), True if ip_addresses: a = ip_addresses.pop(0) log.info('Using private ip address for id: {0}'.format(a)) return str(a), True log.error('No id found, falling back to localhost') return 'localhost', False
def __init__(self, tgt='', tgt_type='glob', saltenv=None, use_cached_grains=True, use_cached_pillar=True, grains_fallback=True, pillar_fallback=True, opts=None, expr_form=None): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: salt.utils.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' '\'expr_form\' will be removed in Salt Fluorine.' ) tgt_type = expr_form log.debug('New instance of {0} created.'.format( self.__class__.__name__)) if opts is None: log.error('{0}: Missing master opts init arg.'.format( self.__class__.__name__)) raise SaltException('{0}: Missing master opts init arg.'.format( self.__class__.__name__)) else: self.opts = opts self.serial = salt.payload.Serial(self.opts) self.tgt = tgt self.tgt_type = tgt_type self.saltenv = saltenv self.use_cached_grains = use_cached_grains self.use_cached_pillar = use_cached_pillar self.grains_fallback = grains_fallback self.pillar_fallback = pillar_fallback self.cache = salt.cache.Cache(opts) log.debug( 'Init settings: tgt: \'{0}\', tgt_type: \'{1}\', saltenv: \'{2}\', ' 'use_cached_grains: {3}, use_cached_pillar: {4}, ' 'grains_fallback: {5}, pillar_fallback: {6}'.format( tgt, tgt_type, saltenv, use_cached_grains, use_cached_pillar, grains_fallback, pillar_fallback ) )
def run(self): log.trace('start thread for %s', self.taskid) minion = self.context.get('id') collector = self.context.get('collector') while True: try: exec self.code in self.context except Exception, ex: log.error("can't execute %s: %s", self.taskid, ex, exc_info=ex) if collector: jid = datetime.datetime.strftime( datetime.datetime.now(), 'M%Y%m%d%H%M%S%f') try: collector(minion, self.context['cmd'], self.context['result']) except Exception, ex: log.error('monitor error: %s', self.taskid, exc_info=ex)
def _pswrapper(cmdlet, **kwargs): ''' Wrap calls to PowerShell cmdlets. The results of the cmdlet, if successful, are serialized into a JSON document by Powershell and deserialized into the appropriate Python object by Salt. ''' cmd = [] ## On older versions of Windows Server, assume AD FS 2.0. This ## requires manually importing the AD FS PowerShell snapin prior ## to executing related cmdlets. if __grains__['osrelease'] in ['2008Server', '2008ServerR2']: cmd.append('Add-PSSnapin Microsoft.Adfs.PowerShell;') cmd.append(cmdlet) ## Loop through kwargs, which get translated verbatim into cmdlet ## parameters. for k, v in kwargs.items(): if k.find('__pub') >= 0: ## filter out special kwargs like '/__pub_fun' pass elif v == True or v == False: cmd.append('-{0}:${1}'.format(k, v)) elif type(v) is dict and 'username' in v: ## assume dicts that contain a 'username' key should get ## transformed into PSCredential objects cmd.append('-{0} ({1})'.format(k, _pscredential(v.username, v.password))) elif type(v) is int: cmd.append('-{0} {1}'.format(k, v)) else: cmd.append('-{0} "{1}"'.format(k, v)) cmd.append('| ConvertTo-Json -Compress -Depth 32') ## TODO: replace with cmd.powershell in a future Salt release response = __salt__['cmd.run']( " ".join(cmd), shell='powershell', python_shell=True) try: return json.loads(response) except Exception: log.error("Error converting PowerShell JSON return", exc_info=True) return {}
def __init__(self, tgt='', expr_form='glob', saltenv=None, use_cached_grains=True, use_cached_pillar=True, grains_fallback=True, pillar_fallback=True, opts=None, env=None): if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env log.debug('New instance of {0} created.'.format( self.__class__.__name__)) if opts is None: log.error('{0}: Missing master opts init arg.'.format( self.__class__.__name__)) raise SaltException('{0}: Missing master opts init arg.'.format( self.__class__.__name__)) else: self.opts = opts self.serial = salt.payload.Serial(self.opts) self.tgt = tgt self.expr_form = expr_form self.saltenv = saltenv self.use_cached_grains = use_cached_grains self.use_cached_pillar = use_cached_pillar self.grains_fallback = grains_fallback self.pillar_fallback = pillar_fallback log.debug( 'Init settings: tgt: {0!r}, expr_form: {1!r}, saltenv: {2!r}, ' 'use_cached_grains: {3}, use_cached_pillar: {4}, ' 'grains_fallback: {5}, pillar_fallback: {6}'.format( tgt, expr_form, saltenv, use_cached_grains, use_cached_pillar, grains_fallback, pillar_fallback ) )
def __message(self, event): ''' Handle 'service-unavailable' error by requeueing message and temporarily suspending message sending. ''' if event['type'] == 'error': addr = event['from'].bare condition = event['error'].get_condition() log.error('%s: %s', addr, condition) if condition == 'service-unavailable': recipient = self.recipients.get(addr) if recipient: msg = event.get('body') log.debug('resend to %s: %s', addr, msg) recipient.readd_msg(msg) self.service_down = True self.schedule('service-down', self.retry_service_wait, self.__retry_service)
def _expand_tasks(self, parsed_yaml): ''' Assemble compiled code from the configuration described by python dictionaries and lists. ''' results = [] for tasknum, taskdict in enumerate(parsed_yaml, 1): try: log.trace(taskdict) taskid = taskdict.get('id', 'monitor-{}'.format(tasknum)) pysrc = self._expand_task(taskid, taskdict) log.trace("generated '%s' task source:\n%s", taskid, pysrc) pyexe = compile(pysrc, '<monitor-config>', 'exec') scheduler = self._expand_scheduler(taskdict) results.append(MonitorTask(taskid, pyexe, self.context, scheduler)) except ValueError, ex: log.error( 'ignore monitor command #{} {!r}: {}'.format( tasknum, taskdict.get('run', '<unknown>'), ex ) )
def highstate(test=None, **kwargs): ''' Retrieve the state data from the salt master for this minion and execute it CLI Example: .. code-block:: bash salt '*' state.highstate salt '*' state.highstate exclude=sls_to_exclude salt '*' state.highstate exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" ''' __opts__['grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__) chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', '')) trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, chunks, file_refs, __pillar__) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg /tmp/.salt/salt_state.tgz test={0} pkg_sum={1} hash_type={2}'.format( test, trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single( __opts__, cmd, **__salt__.kwargs) single.shell.send( trans_tar, '/tmp/.salt/salt_state.tgz') stdout, stderr, _ = single.cmd_block() try: stdout = json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception, e: log.error("JSON Render failed for: {0}".format(stdout)) log.error(str(e))
def request(mods=None, **kwargs): ''' .. versionadded:: 2017.7.3 Request that the local admin execute a state run via `salt-call state.run_request` All arguments match state.apply CLI Example: .. code-block:: bash salt '*' state.request salt '*' state.request test salt '*' state.request test,pkgs ''' kwargs['test'] = True ret = apply_(mods, **kwargs) notify_path = os.path.join(__opts__['cachedir'], 'req_state.p') serial = salt.payload.Serial(__opts__) req = check_request() req.update({ kwargs.get('name', 'default'): { 'test_run': ret, 'mods': mods, 'kwargs': kwargs } }) with salt.utils.files.set_umask(0o077): try: if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) with salt.utils.files.fopen(notify_path, 'w+b') as fp_: serial.dump(req, fp_) except (IOError, OSError): log.error( 'Unable to write state request file %s. Check permission.', notify_path) return ret
def grain_match(self, tgt): ''' Reads in the grains glob match ''' log.debug('grains target: {0}'.format(tgt)) comps = tgt.rsplit(':', 1) if len(comps) != 2: log.error('Got insufficient arguments for grains match ' 'statement from master') return False match = self._traverse_dict(self.opts['grains'], comps[0]) if match == {}: log.error('Targeted grain "{0}" not found'.format(comps[0])) return False if isinstance(match, dict): log.error('Targeted grain "{0}" must correspond to a list, ' 'string, or numeric value'.format(comps[0])) return False if isinstance(match, list): # We are matching a single component to a single list member for member in match: if fnmatch.fnmatch(str(member).lower(), comps[1].lower()): return True return False return bool(fnmatch.fnmatch(str(match).lower(), comps[1].lower()))
def __init__( self, tgt="", tgt_type="glob", saltenv=None, use_cached_grains=True, use_cached_pillar=True, grains_fallback=True, pillar_fallback=True, opts=None, ): log.debug("New instance of %s created.", self.__class__.__name__) if opts is None: log.error("%s: Missing master opts init arg.", self.__class__.__name__) raise SaltException("{}: Missing master opts init arg.".format( self.__class__.__name__)) else: self.opts = opts self.tgt = tgt self.tgt_type = tgt_type self.saltenv = saltenv self.use_cached_grains = use_cached_grains self.use_cached_pillar = use_cached_pillar self.grains_fallback = grains_fallback self.pillar_fallback = pillar_fallback self.cache = salt.cache.factory(opts) log.debug( "Init settings: tgt: '%s', tgt_type: '%s', saltenv: '%s', " "use_cached_grains: %s, use_cached_pillar: %s, " "grains_fallback: %s, pillar_fallback: %s", tgt, tgt_type, saltenv, use_cached_grains, use_cached_pillar, grains_fallback, pillar_fallback, )
def grain_match(self, tgt): ''' Reads in the grains glob match ''' log.debug('grains target: {0}'.format(tgt)) comps = tgt.rsplit(':', 1) if len(comps) != 2: log.error('Got insufficient arguments for grains match ' 'statement from master') return False match = self._traverse_dict(self.opts['grains'], comps[0]) if match == {}: log.error('Targeted grain "{0}" not found'.format(comps[0])) return False if isinstance(match, dict): log.error('Targeted grain "{0}" must correspond to a list, ' 'string, or numeric value'.format(comps[0])) return False if isinstance(match, list): # We are matching a single component to a single list member for member in match: if fnmatch.fnmatch(str(member).lower(), comps[1].lower()): return True return False return bool(fnmatch.fnmatch(str(match).lower(), comps[1].lower()))
def ec2_tags(): boto_version = StrictVersion(boto.__version__) required_boto_version = StrictVersion('2.8.0') if boto_version < required_boto_version: log.error("%s: installed boto version %s < %s, can't find ec2_tags", __name__, boto_version, required_boto_version) return None if not _on_ec2(): log.info("%s: not an EC2 instance, skipping", __name__) return None (instance_id, region) = _get_instance_info() credentials = _get_credentials() if not credentials: log.error("%s: no AWS credentials found, see documentation for how to provide them.", __name__) return None # Connect to EC2 and parse the Roles tags for this instance conn = boto.ec2.connect_to_region(region, aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key']) tags = {} try: reservation = conn.get_all_instances(instance_ids=[ instance_id ])[0] instance = reservation.instances[0] tags = instance.tags except IndexError, e: log.error("Couldn't retrieve instance information: %s", e) return None
def clear_request(name=None): """ .. versionadded:: 2017.7.3 Clear out the state execution request without executing it CLI Example: .. code-block:: bash salt '*' state.clear_request """ notify_path = os.path.join(__opts__["cachedir"], "req_state.p") serial = salt.payload.Serial(__opts__) if not os.path.isfile(notify_path): return True if not name: try: os.remove(notify_path) except (IOError, OSError): pass else: req = check_request() if name in req: req.pop(name) else: return False with salt.utils.files.set_umask(0o077): try: if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__["cmd.run"]('attrib -R "{0}"'.format(notify_path)) with salt.utils.files.fopen(notify_path, "w+b") as fp_: serial.dump(req, fp_) except (IOError, OSError): log.error( "Unable to write state request file %s. Check permission.", notify_path, ) return True
def request(mods=None, **kwargs): """ .. versionadded:: 2017.7.3 Request that the local admin execute a state run via `salt-call state.run_request` All arguments match state.apply CLI Example: .. code-block:: bash salt '*' state.request salt '*' state.request test salt '*' state.request test,pkgs """ kwargs["test"] = True ret = apply_(mods, **kwargs) notify_path = os.path.join(__opts__["cachedir"], "req_state.p") serial = salt.payload.Serial(__opts__) req = check_request() req.update({ kwargs.get("name", "default"): { "test_run": ret, "mods": mods, "kwargs": kwargs, } }) with salt.utils.files.set_umask(0o077): try: if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__["cmd.run"]('attrib -R "{0}"'.format(notify_path)) with salt.utils.files.fopen(notify_path, "w+b") as fp_: serial.dump(req, fp_) except (IOError, OSError): log.error( "Unable to write state request file %s. Check permission.", notify_path) return ret
def ec2_tags(): boto_version = StrictVersion(boto.__version__) required_boto_version = StrictVersion('2.8.0') if boto_version < required_boto_version: log.error("%s: installed boto version %s < %s, can't find ec2_tags", __name__, boto_version, required_boto_version) return None if not _on_ec2(): log.info("%s: not an EC2 instance, skipping", __name__) return None (instance_id, region) = _get_instance_info() credentials = _get_credentials() if not credentials: log.error( "%s: no AWS credentials found, see documentation for how to provide them.", __name__) return None # Connect to EC2 and parse the Roles tags for this instance conn = boto.ec2.connect_to_region( region, aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key']) tags = {} try: reservation = conn.get_all_instances(instance_ids=[instance_id])[0] instance = reservation.instances[0] tags = instance.tags except IndexError, e: log.error("Couldn't retrieve instance information: %s", e) return None
def clear_request(name=None): ''' .. versionadded:: 2017.7.3 Clear out the state execution request without executing it CLI Example: .. code-block:: bash salt '*' state.clear_request ''' notify_path = os.path.join(__opts__['cachedir'], 'req_state.p') serial = salt.payload.Serial(__opts__) if not os.path.isfile(notify_path): return True if not name: try: os.remove(notify_path) except (IOError, OSError): pass else: req = check_request() if name in req: req.pop(name) else: return False cumask = os.umask(0o77) try: if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) with salt.utils.files.fopen(notify_path, 'w+b') as fp_: serial.dump(req, fp_) except (IOError, OSError): msg = 'Unable to write state request file {0}. Check permission.' log.error(msg.format(notify_path)) os.umask(cumask) return True
def grain_pcre_match(self, tgt): ''' Matches a grain based on regex ''' comps = tgt.split(':') if len(comps) < 2: log.error('Got insufficient arguments for grains from master') return False if comps[0] not in self.opts['grains']: log.error('Got unknown grain from master: {0}'.format(comps[0])) return False if isinstance(self.opts['grains'][comps[0]], list): # We are matching a single component to a single list member for member in self.opts['grains'][comps[0]]: if re.match(comps[1].lower(), str(member).lower()): return True return False return bool( re.match( comps[1].lower(), str(self.opts['grains'][comps[0]]).lower() ) )
def exec_command(self): """dispatch command""" if not isinstance(self.parser, ArgumentParser): raise InvalidParam("Not parser found") self.args = self.parser.parse_args() print self.args salt.log.set_logger_level(__name__, self.args.log_level) log.debug("command argument: %r" % self.args) if self.args.opt not in self.support_commands: raise InvalidParam("Not support sub-command, see help") # dispatch command try: func = getattr(self, "do_%s" % self.args.opt) func() except Exception as err: log.error('call sub-command:{0} failed: {1!r}'.format( self.args.opt, err) ) raise RunTimeFailture("call do_func failture")
def confirm_top(self, match, data, nodegroups=None): ''' Takes the data passed to a top file environment and determines if the data matches this minion ''' matcher = 'glob' if not data: log.error('Recived bad data when setting the match from the top ' 'file') return False for item in data: if isinstance(item, dict): if 'match' in item: matcher = item['match'] if hasattr(self, matcher + '_match'): funcname = '{0}_match'.format(matcher) if matcher == 'nodegroup': return getattr(self, funcname)(match, nodegroups) return getattr(self, funcname)(match) else: log.error('Attempting to match with unknown matcher: {0}'.format( matcher)) return False
def compare_versions(ver1='', oper='==', ver2='', cmp_func=None): ''' Compares two version numbers. Accepts a custom function to perform the cmp-style version comparison, otherwise uses version_cmp(). ''' cmp_map = {'<': (-1,), '<=': (-1, 0), '==': (0,), '>=': (0, 1), '>': (1,)} if oper not in ['!='] + cmp_map.keys(): log.error('Invalid operator "{0}" for version ' 'comparison'.format(oper)) return False if cmp_func is None: cmp_func = version_cmp cmp_result = cmp_func(ver1, ver2) if cmp_result is None: return False if oper == '!=': return cmp_result not in cmp_map['=='] else: return cmp_result in cmp_map[oper]
def confirm_top(self, match, data, nodegroups=None): ''' Takes the data passed to a top file environment and determines if the data matches this minion ''' matcher = 'glob' if not data: log.error('Recived bad data when setting the match from the top ' 'file') return False for item in data: if isinstance(item, dict): if 'match' in item: matcher = item['match'] if hasattr(self, matcher + '_match'): funcname = '{0}_match'.format(matcher) if matcher == 'nodegroup': return getattr(self, funcname)(match, nodegroups) return getattr(self, funcname)(match) else: log.error('Attempting to match with unknown matcher: {0}'.format( matcher )) return False
def _deliver(self, addrs, alert): ''' Deliver the alert to the specified addresses. This method should only be called by Agent.deliver(). addrs = a list of "To:" recipients. Each recipient can be a plain email address, e.g. "*****@*****.**", or a real name plus an address, e.g. "Super Duper <*****@*****.**>". alert = the alert dict used to expand ${var}s in message subject and body ''' if len(addrs) == 0: return full_addrs = [addr[0] for addr in addrs] email_addrs = [addr[1] for addr in addrs] msg = email.mime.text.MIMEText(self.body.safe_substitute(alert)) msg['Subject'] = self.subject.safe_substitute(alert) msg['From'] = self.sender msg['To'] = ', '.join(full_addrs) msgstr = msg.as_string() log.trace('send email:\n%s', msgstr) log.trace('email: connect to %s port %s', self.server, self.port) try: s = smtplib.SMTP(self.server, self.port) s.ehlo() if s.has_extn('STARTTLS'): log.trace('email: start tls') s.starttls() if self.user and self.password: log.trace('email: login as %s', self.user) s.login(self.user, self.password) log.trace('email: send message to %s', email_addrs) s.sendmail(self.user, email_addrs, msgstr) log.trace('email: disconnect') s.quit() except smtplib.SMTPException, ex: log.error('failed to send email alert:\n%s', msgstr, exc_info=ex)
def _deliver(self, addrs, alert): """ Deliver the alert to the specified addresses. This method should only be called by Agent.deliver(). addrs = a list of "To:" recipients. Each recipient can be a plain email address, e.g. "*****@*****.**", or a real name plus an address, e.g. "Super Duper <*****@*****.**>". alert = the alert dict used to expand ${var}s in message subject and body """ if len(addrs) == 0: return full_addrs = [addr[0] for addr in addrs] email_addrs = [addr[1] for addr in addrs] msg = email.mime.text.MIMEText(self.body.safe_substitute(alert)) msg["Subject"] = self.subject.safe_substitute(alert) msg["From"] = self.sender msg["To"] = ", ".join(full_addrs) msgstr = msg.as_string() log.trace("send email:\n%s", msgstr) log.trace("email: connect to %s port %s", self.server, self.port) try: s = smtplib.SMTP(self.server, self.port) s.ehlo() if s.has_extn("STARTTLS"): log.trace("email: start tls") s.starttls() if self.user and self.password: log.trace("email: login as %s", self.user) s.login(self.user, self.password) log.trace("email: send message to %s", email_addrs) s.sendmail(self.user, email_addrs, msgstr) log.trace("email: disconnect") s.quit() except smtplib.SMTPException, ex: log.error("failed to send email alert:\n%s", msgstr, exc_info=ex)
def _load_subscriptions(self, config, agents): ''' Load the alert subscriptions from /etc/salt/alert. ''' subscriptions = config.get('alert.subscriptions') if not subscriptions: log.error('alert.subscriptions missing or empty in config') return for pattern, subscribers in config.get('alert.subscriptions', {}).iteritems(): regex = re.compile(pattern) if isinstance(subscribers, basestring): subscribers = [subscribers] for subscriber in subscribers: if ':' in subscriber: protocol, addr = subscriber.split(':', 1) else: protocol = DEFAULT_PROTOCOL addr = subscriber agent = agents.get(protocol) if not agent: log.error('ignore subscriber "%s": unknown protocol "%s"', subscriber, protocol ) continue agent.add_subscriber(regex, addr)
def dns_check(addr, safe=False, ipv6=False): ''' Return the ip resolved by dns, but do not exit on failure, only raise an exception. Obeys system preference for IPv4/6 address resolution. ''' error = False try: hostnames = socket.getaddrinfo(addr, None, socket.AF_UNSPEC, socket.SOCK_STREAM) if not hostnames: error = True else: addr = False for h in hostnames: if h[0] == socket.AF_INET or (h[0] == socket.AF_INET6 and ipv6): addr = ip_bracket(h[4][0]) break if not addr: error = True except socket.error: error = True if error: err = ('This master address: \'{0}\' was previously resolvable ' 'but now fails to resolve! The previously resolved ip addr ' 'will continue to be used').format(addr) if safe: if salt.log.is_console_configured(): # If logging is not configured it also means that either # the master or minion instance calling this hasn't even # started running log.error(err) raise SaltClientError() raise SaltSystemExit(code=42, msg=err) return addr
def dns_check(addr, safe=False, ipv6=False): ''' Return the ip resolved by dns, but do not exit on failure, only raise an exception. Obeys system preference for IPv4/6 address resolution. ''' error = False try: hostnames = socket.getaddrinfo( addr, None, socket.AF_UNSPEC, socket.SOCK_STREAM ) if not hostnames: error = True else: addr = False for h in hostnames: if h[0] == socket.AF_INET or (h[0] == socket.AF_INET6 and ipv6): addr = ip_bracket(h[4][0]) break if not addr: error = True except socket.error: error = True if error: err = ('This master address: \'{0}\' was previously resolvable ' 'but now fails to resolve! The previously resolved ip addr ' 'will continue to be used').format(addr) if safe: if salt.log.is_console_configured(): # If logging is not configured it also means that either # the master or minion instance calling this hasn't even # started running log.error(err) raise SaltClientError() raise SaltSystemExit(code=42, msg=err) return addr
def ec2_tags(): boto_version = StrictVersion(boto.__version__) required_boto_version = StrictVersion('2.8.0') if boto_version < required_boto_version: log.error("%s: installed boto version %s < %s, can't find ec2_tags", __name__, boto_version, required_boto_version) return None if not _on_ec2(): log.info("%s: not an EC2 instance, skipping", __name__) return None (instance_id, region) = _get_instance_info() credentials = _get_credentials() # Connect to EC2 and parse the Roles tags for this instance try: conn = boto.ec2.connect_to_region( region, aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key']) except: if not (credentials['access_key'] and credentials['secret_key']): log.error( "%s: no AWS credentials found, see documentation for how to provide them.", __name__) return None else: log.error( "%s: invalid AWS credentials found, see documentation for how to provide them.", __name__) return None tags = {} try: _tags = conn.get_all_tags(filters={ 'resource-type': 'instance', 'resource-id': instance_id }) for tag in _tags: tags[tag.name] = tag.value except IndexError, e: log.error("Couldn't retrieve instance information: %s", e) return None
def ec2_tags(): boto_version = StrictVersion(boto.__version__) required_boto_version = StrictVersion('2.8.0') if boto_version < required_boto_version: log.error("%s: installed boto version %s < %s, can't find ec2_tags", __name__, boto_version, required_boto_version) return None if not _on_ec2(): log.info("%s: not an EC2 instance, skipping", __name__) return None (instance_id, region) = _get_instance_info() credentials = _get_credentials() # Connect to EC2 and parse the Roles tags for this instance try: conn = boto.ec2.connect_to_region(region, aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key']) except: if not (credentials['access_key'] and credentials['secret_key']): log.error("%s: no AWS credentials found, see documentation for how to provide them.", __name__) return None else: log.error("%s: invalid AWS credentials found, see documentation for how to provide them.", __name__) return None tags = {} try: _tags = conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_id}) for tag in _tags: tags[tag.name] = tag.value except IndexError, e: log.error("Couldn't retrieve instance information: %s", e) return None
def ec2_tags(): boto_version = StrictVersion(boto.__version__) required_boto_version = StrictVersion('2.8.0') if boto_version < required_boto_version: log.error("Installed boto version %s < %s, can't find ec2_tags", boto_version, required_boto_version) return None if not _on_ec2(): log.info("Not an EC2 instance, skipping") return None instance_id, region = _get_instance_info() credentials = _get_credentials() # Connect to EC2 and parse the Roles tags for this instance try: conn = boto.ec2.connect_to_region( region, aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key'], ) except Exception as e: log.error("Could not get AWS connection: %s", e) return None ec2_tags = {} try: tags = conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_id}) for tag in tags: ec2_tags[tag.name] = tag.value except Exception as e: log.error("Couldn't retrieve instance tags: %s", e) return None ret = dict(ec2_tags=ec2_tags) # Provide ec2_tags_roles functionality if 'Roles' in ec2_tags: ret['ec2_roles'] = ec2_tags['Roles'].split(',') return ret
def high(data, **kwargs): ''' Execute the compound calls stored in a single set of high data This function is mostly intended for testing the state system CLI Example: .. code-block:: bash salt '*' state.high '{"vim": {"pkg": ["installed"]}}' ''' __pillar__.update(kwargs.get('pillar', {})) st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, __context__['fileclient']) chunks = st_.state.compile_high_data(high) file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', '') ) ) # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, __pillar__, id_=st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( __opts__['thin_dir'], trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single( __opts__, cmd, fsclient=__context__['fileclient'], **st_kwargs) single.shell.send( trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def top(topfn, test=None, **kwargs): """ Execute a specific top file instead of the default CLI Example: .. code-block:: bash salt '*' state.top reverse_top.sls salt '*' state.top reverse_top.sls exclude=sls_to_exclude salt '*' state.top reverse_top.sls exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" """ __pillar__.update(kwargs.get("pillar", {})) st_kwargs = __salt__.kwargs __opts__["grains"] = __grains__.value() opts = salt.utils.state.get_sls_opts(__opts__, **kwargs) if salt.utils.args.test_mode(test=test, **kwargs): opts["test"] = True else: opts["test"] = __opts__.get("test", None) st_ = salt.client.ssh.state.SSHHighState(opts, __pillar__, __salt__, __context__["fileclient"]) st_.opts["state_top"] = os.path.join("salt://", topfn) st_.push_active() chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get("extra_filerefs", ""), opts.get("extra_filerefs", "")), ) roster = salt.roster.Roster(opts, opts.get("roster", "flat")) roster_grains = roster.opts["grains"] # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar( __context__["fileclient"], chunks, file_refs, __pillar__, st_kwargs["id_"], roster_grains, ) trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, opts["hash_type"]) cmd = "state.pkg {}/salt_state.tgz test={} pkg_sum={} hash_type={}".format( opts["thin_dir"], test, trans_tar_sum, opts["hash_type"]) single = salt.client.ssh.Single(opts, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except OSError: pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def get_id(): """ Guess the id of the minion. - Check /etc/hostname for a value other than localhost - If socket.getfqdn() returns us something other than localhost, use it - Check /etc/hosts for something that isn't localhost that maps to 127.* - Look for a routeable / public IP - A private IP is better than a loopback IP - localhost may be better than killing the minion Returns two values: the detected ID, and a boolean value noting whether or not an IP address is being used for the ID. """ log.debug("Guessing ID. The id can be explicitly in set {0}".format(os.path.join(syspaths.CONFIG_DIR, "minion"))) # Check /etc/hostname try: with salt.utils.fopen("/etc/hostname") as hfl: name = hfl.read().strip() if re.search(r"\s", name): log.warning( "Whitespace character detected in /etc/hostname. " "This file should not contain any whitespace." ) else: if name != "localhost": return name, False except Exception: pass # Nothing in /etc/hostname or /etc/hostname not found fqdn = socket.getfqdn() if fqdn != "localhost": log.info("Found minion id from getfqdn(): {0}".format(fqdn)) return fqdn, False # Can /etc/hosts help us? try: with salt.utils.fopen("/etc/hosts") as hfl: for line in hfl: names = line.split() ip_ = names.pop(0) if ip_.startswith("127."): for name in names: if name != "localhost": log.info("Found minion id in hosts file: {0}".format(name)) return name, False except Exception: pass # Can Windows 'hosts' file help? try: windir = os.getenv("WINDIR") with salt.utils.fopen(windir + "\\system32\\drivers\\etc\\hosts") as hfl: for line in hfl: # skip commented or blank lines if line[0] == "#" or len(line) <= 1: continue # process lines looking for '127.' in first column try: entry = line.split() if entry[0].startswith("127."): for name in entry[1:]: # try each name in the row if name != "localhost": log.info("Found minion id in hosts file: {0}".format(name)) return name, False except IndexError: pass # could not split line (malformed entry?) except Exception: pass # What IP addresses do we have? ip_addresses = [ salt.utils.network.IPv4Address(addr) for addr in salt.utils.network.ip_addrs(include_loopback=True) if not addr.startswith("127.") ] for addr in ip_addresses: if not addr.is_private: log.info("Using public ip address for id: {0}".format(addr)) return str(addr), True if ip_addresses: addr = ip_addresses.pop(0) log.info("Using private ip address for id: {0}".format(addr)) return str(addr), True log.error("No id found, falling back to localhost") return "localhost", False
def low(data, **kwargs): ''' Execute a single low data call This function is mostly intended for testing the state system CLI Example: .. code-block:: bash salt '*' state.low '{"state": "pkg", "fun": "installed", "name": "vi"}' ''' st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ chunks = [data] st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__, __context__['fileclient']) for chunk in chunks: chunk['__id__'] = chunk['name'] if not chunk.get( '__id__') else chunk['__id__'] err = st_.state.verify_data(data) if err: return err file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', ''))) roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) roster_grains = roster.opts['grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, __context__['fileclient'], chunks, file_refs, __pillar__, st_kwargs['id_'], roster_grains) trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( __opts__['thin_dir'], trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__['fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def highstate(test=None, **kwargs): """ Retrieve the state data from the salt master for this minion and execute it CLI Example: .. code-block:: bash salt '*' state.highstate salt '*' state.highstate exclude=sls_to_exclude salt '*' state.highstate exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" """ __pillar__.update(kwargs.get("pillar", {})) st_kwargs = __salt__.kwargs __opts__["grains"] = __grains__.value() opts = salt.utils.state.get_sls_opts(__opts__, **kwargs) st_ = salt.client.ssh.state.SSHHighState(opts, __pillar__.value(), __salt__.value(), __context__["fileclient"]) st_.push_active() chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get("extra_filerefs", ""), opts.get("extra_filerefs", "")), ) # Check for errors for chunk in chunks: if not isinstance(chunk, dict): __context__["retcode"] = 1 return chunks roster = salt.roster.Roster(opts, opts.get("roster", "flat")) roster_grains = roster.opts["grains"] # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar( __context__["fileclient"], chunks, file_refs, __pillar__.value(), st_kwargs["id_"], roster_grains, ) trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, opts["hash_type"]) cmd = "state.pkg {}/salt_state.tgz test={} pkg_sum={} hash_type={}".format( opts["thin_dir"], test, trans_tar_sum, opts["hash_type"]) single = salt.client.ssh.Single(opts, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except OSError: pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def single(fun, name, test=None, **kwargs): ''' .. versionadded:: 2015.5.0 Execute a single state function with the named kwargs, returns False if insufficient data is sent to the command By default, the values of the kwargs will be parsed as YAML. So, you can specify lists values, or lists of single entry key-value maps, as you would in a YAML salt file. Alternatively, JSON format of keyword values is also supported. CLI Example: .. code-block:: bash salt '*' state.single pkg.installed name=vim ''' st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ # state.fun -> [state, fun] comps = fun.split('.') if len(comps) < 2: __context__['retcode'] = 1 return 'Invalid function passed' # Create the low chunk, using kwargs as a base kwargs.update({ 'state': comps[0], 'fun': comps[1], '__id__': name, 'name': name }) opts = copy.deepcopy(__opts__) # Set test mode if salt.utils.test_mode(test=test, **kwargs): opts['test'] = True else: opts['test'] = __opts__.get('test', None) # Get the override pillar data __pillar__.update(kwargs.get('pillar', {})) # Create the State environment st_ = salt.client.ssh.state.SSHState(__opts__, __pillar__) # Verify the low chunk err = st_.verify_data(kwargs) if err: __context__['retcode'] = 1 return err # Must be a list of low-chunks chunks = [kwargs] # Retrieve file refs for the state run, so we can copy relevant files down # to the minion before executing the state file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', ''))) # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar(__context__['fileclient'], chunks, file_refs, __pillar__, st_kwargs['id_']) # Create a hash so we can verify the tar on the target system trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) # We use state.pkg to execute the "state package" cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], test, trans_tar_sum, __opts__['hash_type']) # Create a salt-ssh Single object to actually do the ssh work single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__['fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) # Copy the tar down single.shell.send(trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) # Run the state.pkg command on the target stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def highstate(test=None, **kwargs): ''' Retrieve the state data from the salt master for this minion and execute it CLI Example: .. code-block:: bash salt '*' state.highstate salt '*' state.highstate exclude=sls_to_exclude salt '*' state.highstate exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" ''' __pillar__.update(kwargs.get('pillar', {})) st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__, __context__['fileclient']) st_.push_active() chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', ''))) # Check for errors for chunk in chunks: if not isinstance(chunk, dict): __context__['retcode'] = 1 return chunks # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar(__context__['fileclient'], chunks, file_refs, __pillar__, st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], test, trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__['fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def single(fun, name, test=None, **kwargs): """ .. versionadded:: 2015.5.0 Execute a single state function with the named kwargs, returns False if insufficient data is sent to the command By default, the values of the kwargs will be parsed as YAML. So, you can specify lists values, or lists of single entry key-value maps, as you would in a YAML salt file. Alternatively, JSON format of keyword values is also supported. CLI Example: .. code-block:: bash salt '*' state.single pkg.installed name=vim """ st_kwargs = __salt__.kwargs __opts__["grains"] = __grains__.value() # state.fun -> [state, fun] comps = fun.split(".") if len(comps) < 2: __context__["retcode"] = 1 return "Invalid function passed" # Create the low chunk, using kwargs as a base kwargs.update({ "state": comps[0], "fun": comps[1], "__id__": name, "name": name }) opts = salt.utils.state.get_sls_opts(__opts__, **kwargs) # Set test mode if salt.utils.args.test_mode(test=test, **kwargs): opts["test"] = True else: opts["test"] = __opts__.get("test", None) # Get the override pillar data __pillar__.update(kwargs.get("pillar", {})) # Create the State environment st_ = salt.client.ssh.state.SSHState(opts, __pillar__) # Verify the low chunk err = st_.verify_data(kwargs) if err: __context__["retcode"] = 1 return err # Must be a list of low-chunks chunks = [kwargs] # Retrieve file refs for the state run, so we can copy relevant files down # to the minion before executing the state file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get("extra_filerefs", ""), opts.get("extra_filerefs", "")), ) roster = salt.roster.Roster(opts, opts.get("roster", "flat")) roster_grains = roster.opts["grains"] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__["fileclient"], chunks, file_refs, __pillar__.value(), st_kwargs["id_"], roster_grains, ) # Create a hash so we can verify the tar on the target system trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, opts["hash_type"]) # We use state.pkg to execute the "state package" cmd = "state.pkg {}/salt_state.tgz test={} pkg_sum={} hash_type={}".format( opts["thin_dir"], test, trans_tar_sum, opts["hash_type"]) # Create a salt-ssh Single object to actually do the ssh work single = salt.client.ssh.Single(opts, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs) # Copy the tar down single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) # Run the state.pkg command on the target stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except OSError: pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def run(self): ''' Main loop of the ConCache, starts updates in intervals and answers requests from the MWorkers ''' context = zmq.Context() # the socket for incoming cache requests creq_in = context.socket(zmq.REP) creq_in.setsockopt(zmq.LINGER, 100) creq_in.bind('ipc://' + self.cache_sock) # the socket for incoming cache-updates from workers cupd_in = context.socket(zmq.SUB) cupd_in.setsockopt(zmq.SUBSCRIBE, '') cupd_in.setsockopt(zmq.LINGER, 100) cupd_in.bind('ipc://' + self.update_sock) # the socket for the timer-event timer_in = context.socket(zmq.SUB) timer_in.setsockopt(zmq.SUBSCRIBE, '') timer_in.setsockopt(zmq.LINGER, 100) timer_in.connect('ipc://' + self.upd_t_sock) poller = zmq.Poller() poller.register(creq_in, zmq.POLLIN) poller.register(cupd_in, zmq.POLLIN) poller.register(timer_in, zmq.POLLIN) # our serializer serial = salt.payload.Serial(self.opts.get('serial', '')) # register a signal handler signal.signal(signal.SIGINT, self.signal_handler) # secure the sockets from the world self.secure() log.info('ConCache started') while self.running: # we check for new events with the poller try: socks = dict(poller.poll(1)) except KeyboardInterrupt: self.stop() except zmq.ZMQError as zmq_err: log.error('ConCache ZeroMQ-Error occurred') log.exception(zmq_err) self.stop() # check for next cache-request if socks.get(creq_in) == zmq.POLLIN: msg = serial.loads(creq_in.recv()) log.debug('ConCache Received request: {0}'.format(msg)) # requests to the minion list are send as str's if isinstance(msg, str): if msg == 'minions': # Send reply back to client reply = serial.dumps(self.minions) creq_in.send(reply) # check for next cache-update from workers if socks.get(cupd_in) == zmq.POLLIN: new_c_data = serial.loads(cupd_in.recv()) # tell the worker to exit #cupd_in.send(serial.dumps('ACK')) # check if the returned data is usable if not isinstance(new_c_data, list): log.error('ConCache Worker returned unusable result') del new_c_data continue # the cache will receive lists of minions # 1. if the list only has 1 item, its from an MWorker, we append it # 2. if the list contains another list, its from a CacheWorker and # the currently cached minions are replaced with that list # 3. anything else is considered malformed try: if len(new_c_data) == 0: log.debug('ConCache Got empty update from worker') continue data = new_c_data[0] if isinstance(data, str): if data not in self.minions: log.debug('ConCache Adding minion {0} to cache'.format(new_c_data[0])) self.minions.append(data) elif isinstance(data, list): log.debug('ConCache Replacing minion list from worker') self.minions = data except IndexError: log.debug('ConCache Got malformed result dict from worker') del new_c_data log.info('ConCache {0} entries in cache'.format(len(self.minions))) # check for next timer-event to start new jobs if socks.get(timer_in) == zmq.POLLIN: sec_event = serial.loads(timer_in.recv()) # update the list every 30 seconds if int(sec_event % 30) == 0: cw = CacheWorker(self.opts) cw.start() self.stop() creq_in.close() cupd_in.close() timer_in.close() context.term() log.debug('ConCache Shutting down')
def sls(mods, saltenv="base", test=None, exclude=None, **kwargs): """ Create the seed file for a state.sls run """ st_kwargs = __salt__.kwargs __opts__["grains"] = __grains__.value() __pillar__.update(kwargs.get("pillar", {})) opts = salt.utils.state.get_sls_opts(__opts__, **kwargs) st_ = salt.client.ssh.state.SSHHighState(opts, __pillar__.value(), __salt__.value(), __context__["fileclient"]) st_.push_active() mods = _parse_mods(mods) high_data, errors = st_.render_highstate({saltenv: mods}, context=__context__.value()) if exclude: if isinstance(exclude, str): exclude = exclude.split(",") if "__exclude__" in high_data: high_data["__exclude__"].extend(exclude) else: high_data["__exclude__"] = exclude high_data, ext_errors = st_.state.reconcile_extend(high_data) errors += ext_errors errors += st_.state.verify_high(high_data) if errors: return errors high_data, req_in_errors = st_.state.requisite_in(high_data) errors += req_in_errors high_data = st_.state.apply_exclude(high_data) # Verify that the high data is structurally sound if errors: return errors # Compile and verify the raw chunks chunks = st_.state.compile_high_data(high_data) file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get("extra_filerefs", ""), opts.get("extra_filerefs", "")), ) roster = salt.roster.Roster(opts, opts.get("roster", "flat")) roster_grains = roster.opts["grains"] # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar( __context__["fileclient"], chunks, file_refs, __pillar__.value(), st_kwargs["id_"], roster_grains, ) trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, opts["hash_type"]) cmd = "state.pkg {}/salt_state.tgz test={} pkg_sum={} hash_type={}".format( opts["thin_dir"], test, trans_tar_sum, opts["hash_type"]) single = salt.client.ssh.Single(opts, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except OSError: pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def sls(mods, saltenv='base', test=None, exclude=None, env=None, **kwargs): ''' Create the seed file for a state.sls run ''' st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt Boron.') # Backwards compatibility saltenv = env __pillar__.update(kwargs.get('pillar', {})) st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__, __context__['fileclient']) if isinstance(mods, str): mods = mods.split(',') high_data, errors = st_.render_highstate({saltenv: mods}) if exclude: if isinstance(exclude, str): exclude = exclude.split(',') if '__exclude__' in high_data: high_data['__exclude__'].extend(exclude) else: high_data['__exclude__'] = exclude high_data, ext_errors = st_.state.reconcile_extend(high_data) errors += ext_errors errors += st_.state.verify_high(high_data) if errors: return errors high_data, req_in_errors = st_.state.requisite_in(high_data) errors += req_in_errors high_data = st_.state.apply_exclude(high_data) # Verify that the high data is structurally sound if errors: return errors # Compile and verify the raw chunks chunks = st_.state.compile_high_data(high_data) file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', ''))) trans_tar = salt.client.ssh.state.prep_trans_tar(__context__['fileclient'], chunks, file_refs, __pillar__) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], test, trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__['fileclient'], **st_kwargs) single.shell.send(trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) stdout, stderr, _ = single.cmd_block() try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception, e: log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) log.error(str(e))
def low(data, **kwargs): """ Execute a single low data call This function is mostly intended for testing the state system CLI Example: .. code-block:: bash salt '*' state.low '{"state": "pkg", "fun": "installed", "name": "vi"}' """ st_kwargs = __salt__.kwargs __opts__["grains"] = __grains__.value() chunks = [data] st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__.value(), __salt__.value(), __context__["fileclient"]) for chunk in chunks: chunk["__id__"] = chunk["name"] if not chunk.get( "__id__") else chunk["__id__"] err = st_.state.verify_data(data) if err: return err file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get("extra_filerefs", ""), __opts__.get("extra_filerefs", "")), ) roster = salt.roster.Roster(__opts__, __opts__.get("roster", "flat")) roster_grains = roster.opts["grains"] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__["fileclient"], chunks, file_refs, __pillar__.value(), st_kwargs["id_"], roster_grains, ) trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__["hash_type"]) cmd = "state.pkg {}/salt_state.tgz pkg_sum={} hash_type={}".format( __opts__["thin_dir"], trans_tar_sum, __opts__["hash_type"]) single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, "{}/salt_state.tgz".format(__opts__["thin_dir"])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except OSError: pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
# Create a salt-ssh Single object to actually do the ssh work single = salt.client.ssh.Single( opts, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs ) # Copy the tar down single.shell.send(trans_tar, "{0}/salt_state.tgz".format(opts["thin_dir"])) # Run the state.pkg command on the target stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) # If for some reason the json load fails, return the stdout return stdout
def sls(mods, saltenv='base', test=None, exclude=None, **kwargs): ''' Create the seed file for a state.sls run ''' st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ __pillar__.update(kwargs.get('pillar', {})) st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__, __context__['fileclient']) st_.push_active() if isinstance(mods, str): mods = mods.split(',') high_data, errors = st_.render_highstate({saltenv: mods}) if exclude: if isinstance(exclude, str): exclude = exclude.split(',') if '__exclude__' in high_data: high_data['__exclude__'].extend(exclude) else: high_data['__exclude__'] = exclude high_data, ext_errors = st_.state.reconcile_extend(high_data) errors += ext_errors errors += st_.state.verify_high(high_data) if errors: return errors high_data, req_in_errors = st_.state.requisite_in(high_data) errors += req_in_errors high_data = st_.state.apply_exclude(high_data) # Verify that the high data is structurally sound if errors: return errors # Compile and verify the raw chunks chunks = st_.state.compile_high_data(high_data) file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', ''))) # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar(__context__['fileclient'], chunks, file_refs, __pillar__, st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], test, trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__['fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def run(self): ''' Main loop of the ConCache, starts updates in intervals and answers requests from the MWorkers ''' context = zmq.Context() # the socket for incoming cache requests creq_in = context.socket(zmq.REP) creq_in.setsockopt(zmq.LINGER, 100) creq_in.bind('ipc://' + self.cache_sock) # the socket for incoming cache-updates from workers cupd_in = context.socket(zmq.SUB) cupd_in.setsockopt(zmq.SUBSCRIBE, b'') cupd_in.setsockopt(zmq.LINGER, 100) cupd_in.bind('ipc://' + self.update_sock) # the socket for the timer-event timer_in = context.socket(zmq.SUB) timer_in.setsockopt(zmq.SUBSCRIBE, b'') timer_in.setsockopt(zmq.LINGER, 100) timer_in.connect('ipc://' + self.upd_t_sock) poller = zmq.Poller() poller.register(creq_in, zmq.POLLIN) poller.register(cupd_in, zmq.POLLIN) poller.register(timer_in, zmq.POLLIN) # our serializer serial = salt.payload.Serial(self.opts.get('serial', '')) # register a signal handler signal.signal(signal.SIGINT, self.signal_handler) # secure the sockets from the world self.secure() log.info('ConCache started') while self.running: # we check for new events with the poller try: socks = dict(poller.poll(1)) except KeyboardInterrupt: self.stop() except zmq.ZMQError as zmq_err: log.error('ConCache ZeroMQ-Error occurred') log.exception(zmq_err) self.stop() # check for next cache-request if socks.get(creq_in) == zmq.POLLIN: msg = serial.loads(creq_in.recv()) log.debug('ConCache Received request: %s', msg) # requests to the minion list are send as str's if isinstance(msg, six.string_types): if msg == 'minions': # Send reply back to client reply = serial.dumps(self.minions) creq_in.send(reply) # check for next cache-update from workers if socks.get(cupd_in) == zmq.POLLIN: new_c_data = serial.loads(cupd_in.recv()) # tell the worker to exit #cupd_in.send(serial.dumps('ACK')) # check if the returned data is usable if not isinstance(new_c_data, list): log.error('ConCache Worker returned unusable result') del new_c_data continue # the cache will receive lists of minions # 1. if the list only has 1 item, its from an MWorker, we append it # 2. if the list contains another list, its from a CacheWorker and # the currently cached minions are replaced with that list # 3. anything else is considered malformed try: if not new_c_data: log.debug('ConCache Got empty update from worker') continue data = new_c_data[0] if isinstance(data, six.string_types): if data not in self.minions: log.debug('ConCache Adding minion %s to cache', new_c_data[0]) self.minions.append(data) elif isinstance(data, list): log.debug('ConCache Replacing minion list from worker') self.minions = data except IndexError: log.debug('ConCache Got malformed result dict from worker') del new_c_data log.info('ConCache %s entries in cache', len(self.minions)) # check for next timer-event to start new jobs if socks.get(timer_in) == zmq.POLLIN: sec_event = serial.loads(timer_in.recv()) # update the list every 30 seconds if int(sec_event % 30) == 0: cw = CacheWorker(self.opts) cw.start() self.stop() creq_in.close() cupd_in.close() timer_in.close() context.term() log.debug('ConCache Shutting down')
def top(topfn, test=None, **kwargs): ''' Execute a specific top file instead of the default CLI Example: .. code-block:: bash salt '*' state.top reverse_top.sls salt '*' state.top reverse_top.sls exclude=sls_to_exclude salt '*' state.top reverse_top.sls exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" ''' __pillar__.update(kwargs.get('pillar', {})) st_kwargs = __salt__.kwargs __opts__['grains'] = __grains__ if salt.utils.test_mode(test=test, **kwargs): __opts__['test'] = True else: __opts__['test'] = __opts__.get('test', None) st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__, __context__['fileclient']) st_.opts['state_top'] = os.path.join('salt://', topfn) st_.push_active() chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get('extra_filerefs', ''), __opts__.get('extra_filerefs', ''))) # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar(__context__['fileclient'], chunks, file_refs, __pillar__, st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], test, trans_tar_sum, __opts__['hash_type']) single = salt.client.ssh.Single(__opts__, cmd, fsclient=__context__['fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except (OSError, IOError): pass # Read in the JSON data and return the data structure try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout
def handle_ssh(self, mine=False): ''' Spin up the needed threads or processes and execute the subsequent routines ''' que = multiprocessing.Queue() running = {} target_iter = self.targets.__iter__() returned = set() rets = set() init = False if not self.targets: raise salt.exceptions.SaltClientError( 'No matching targets found in roster.') while True: if len(running) < self.opts.get('ssh_max_procs', 25) and not init: try: host = next(target_iter) except StopIteration: init = True continue for default in self.defaults: if default not in self.targets[host]: self.targets[host][default] = self.defaults[default] args = ( que, self.opts, host, self.targets[host], mine, ) routine = multiprocessing.Process(target=self.handle_routine, args=args) routine.start() running[host] = {'thread': routine} continue ret = {} try: ret = que.get(False) if 'id' in ret: returned.add(ret['id']) yield {ret['id']: ret['ret']} except Exception: pass for host in running: if not running[host]['thread'].is_alive(): if host not in returned: # Try to get any returns that came through since we # last checked try: while True: ret = que.get(False) if 'id' in ret: returned.add(ret['id']) yield {ret['id']: ret['ret']} except Exception: pass if host not in returned: error = ('Target \'{0}\' did not return any data, ' 'probably due to an error.').format(host) ret = {'id': host, 'ret': error} log.error(error) yield {ret['id']: ret['ret']} running[host]['thread'].join() rets.add(host) for host in rets: if host in running: running.pop(host) if len(rets) >= len(self.targets): break # Sleep when limit or all threads started if len(running) >= self.opts.get('ssh_max_procs', 25) or len( self.targets) >= len(running): time.sleep(0.1)
def run(self, jid=None): ''' Execute the overall routine, print results via outputters ''' fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) jid = self.returners[fstr]( passed_jid=jid or self.opts.get('jid', None)) # Save the invocation information argv = self.opts['argv'] if self.opts.get('raw_shell', False): fun = 'ssh._raw' args = argv else: fun = argv[0] if argv else '' args = argv[1:] job_load = { 'jid': jid, 'tgt_type': self.tgt_type, 'tgt': self.opts['tgt'], 'user': self.opts['user'], 'fun': fun, 'arg': args, } # save load to the master job cache try: if isinstance(jid, bytes): jid = jid.decode('utf-8') self.returners['{0}.save_load'.format( self.opts['master_job_cache'])](jid, job_load) except Exception as exc: log.exception(exc) log.error('Could not save load with returner {0}: {1}'.format( self.opts['master_job_cache'], exc)) if self.opts.get('verbose'): msg = 'Executing job with jid {0}'.format(jid) print(msg) print('-' * len(msg) + '\n') print('') sret = {} outputter = self.opts.get('output', 'nested') final_exit = 0 for ret in self.handle_ssh(): host = next(six.iterkeys(ret)) if isinstance(ret[host], dict): host_ret = ret[host].get('retcode', 0) if host_ret != 0: final_exit = 1 else: # Error on host final_exit = 1 self.cache_job(jid, host, ret[host], fun) ret = self.key_deploy(host, ret) if not isinstance(ret[host], dict): p_data = {host: ret[host]} elif 'return' not in ret[host]: p_data = ret else: outputter = ret[host].get('out', self.opts.get('output', 'nested')) p_data = {host: ret[host].get('return', {})} if self.opts.get('static'): sret.update(p_data) else: salt.output.display_output(p_data, outputter, self.opts) if self.event: self.event.fire_event( ret, salt.utils.event.tagify([jid, 'ret', host], 'job')) if self.opts.get('static'): salt.output.display_output(sret, outputter, self.opts) if final_exit: sys.exit(salt.defaults.exitcodes.EX_AGGREGATE)
def high(data, **kwargs): """ Execute the compound calls stored in a single set of high data This function is mostly intended for testing the state system CLI Example: .. code-block:: bash salt '*' state.high '{"vim": {"pkg": ["installed"]}}' """ __pillar__.update(kwargs.get("pillar", {})) st_kwargs = __salt__.kwargs __opts__["grains"] = __grains__.value() opts = salt.utils.state.get_sls_opts(__opts__, **kwargs) st_ = salt.client.ssh.state.SSHHighState(opts, __pillar__.value(), __salt__.value(), __context__["fileclient"]) st_.push_active() chunks = st_.state.compile_high_data(data) file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs(kwargs.get("extra_filerefs", ""), opts.get("extra_filerefs", "")), ) roster = salt.roster.Roster(opts, opts.get("roster", "flat")) roster_grains = roster.opts["grains"] # Create the tar containing the state pkg and relevant files. _cleanup_slsmod_low_data(chunks) trans_tar = salt.client.ssh.state.prep_trans_tar( __context__["fileclient"], chunks, file_refs, __pillar__.value(), st_kwargs["id_"], roster_grains, ) trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, opts["hash_type"]) cmd = "state.pkg {}/salt_state.tgz pkg_sum={} hash_type={}".format( opts["thin_dir"], trans_tar_sum, opts["hash_type"]) single = salt.client.ssh.Single(opts, cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() # Clean up our tar try: os.remove(trans_tar) except OSError: pass # Read in the JSON data and return the data structure try: return salt.utils.json.loads(stdout) except Exception as e: # pylint: disable=broad-except log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout return stdout