def test_compile_template_preserves_windows_newlines(self): """ Test to ensure that a file with Windows newlines, when rendered by a template renderer, does not eat the CR character. """ def _get_rend(renderer, value): """ We need a new MagicMock each time since we're dealing with StringIO objects which are read like files. """ return {renderer: MagicMock(return_value=StringIO(value))} input_data_windows = "foo\r\nbar\r\nbaz\r\n" input_data_non_windows = input_data_windows.replace("\r\n", "\n") renderer = "test" blacklist = whitelist = [] ret = template.compile_template( ":string:", _get_rend(renderer, input_data_non_windows), renderer, blacklist, whitelist, input_data=input_data_windows, ).read() # Even though the mocked renderer returned a string without the windows # newlines, the compiled template should still have them. self.assertEqual(ret, input_data_windows) # Now test that we aren't adding them in unnecessarily. ret = template.compile_template( ":string:", _get_rend(renderer, input_data_non_windows), renderer, blacklist, whitelist, input_data=input_data_non_windows, ).read() self.assertEqual(ret, input_data_non_windows) # Finally, ensure that we're not unnecessarily replacing the \n with # \r\n in the event that the renderer returned a string with the # windows newlines intact. ret = template.compile_template( ":string:", _get_rend(renderer, input_data_windows), renderer, blacklist, whitelist, input_data=input_data_windows, ).read() self.assertEqual(ret, input_data_windows)
def query(key, value=None, service=None, profile=None): # pylint: disable=W0613 """ Get a value from the REST interface """ comps = key.split("?") key = comps[0] key_vars = {} for pair in comps[1].split("&"): pair_key, pair_val = pair.split("=") key_vars[pair_key] = pair_val renderer = __opts__.get("renderer", "jinja|yaml") rend = salt.loader.render(__opts__, {}) blacklist = __opts__.get("renderer_blacklist") whitelist = __opts__.get("renderer_whitelist") url = compile_template(":string:", rend, renderer, blacklist, whitelist, input_data=profile[key]["url"], **key_vars) extras = {} for item in profile[key]: if item not in ("backend", "url"): extras[item] = profile[key][item] result = http.query(url, decode=True, **extras) return result["dict"]
def test_compile_template_bad_type(self): ''' Test to ensure that unsupported types cannot be passed to the template compiler ''' ret = template.compile_template(['1', '2', '3'], None, None, None, None) self.assertDictEqual(ret, {})
def test_compile_template_bad_type(self): """ Test to ensure that unsupported types cannot be passed to the template compiler """ ret = template.compile_template(["1", "2", "3"], None, None, None, None) self.assertDictEqual(ret, {})
def query(key, value=None, service=None, profile=None): # pylint: disable=W0613 ''' Get a value from the REST interface ''' comps = key.split('?') key = comps[0] key_vars = {} for pair in comps[1].split('&'): pair_key, pair_val = pair.split('=') key_vars[pair_key] = pair_val renderer = __opts__.get('renderer', 'yaml_jinja') rend = salt.loader.render(__opts__, {}) blacklist = __opts__.get('renderer_blacklist') whitelist = __opts__.get('renderer_whitelist') url = compile_template( ':string:', rend, renderer, blacklist, whitelist, input_data=profile[key]['url'], **key_vars ) result = http.query( url, decode=True, **key_vars ) return result['dict']
def query(key, value=None, service=None, profile=None): # pylint: disable=W0613 ''' Get a value from the REST interface ''' comps = key.split('?') key = comps[0] key_vars = {} for pair in comps[1].split('&'): pair_key, pair_val = pair.split('=') key_vars[pair_key] = pair_val renderer = __opts__.get('renderer', 'yaml_jinja') rend = salt.loader.render(__opts__, {}) blacklist = __opts__.get('renderer_blacklist') whitelist = __opts__.get('renderer_whitelist') url = compile_template(':string:', rend, renderer, blacklist, whitelist, input_data=profile[key]['url'], **key_vars) result = http.query(url, decode=True, **key_vars) return result['dict']
def render_pstate(self, sls, env, mods): """ Collect a single pillar sls file and render it """ err = "" errors = [] fn_ = self.client.get_state(sls, env) if not fn_: errors.append( ("Specified SLS {0} in environment {1} is not" " available on the salt master").format(sls, env) ) state = None try: state = compile_template(fn_, self.rend, self.opts["renderer"], env, sls) except Exception as exc: errors.append(("Rendering SLS {0} failed, render error:\n{1}".format(sls, exc))) mods.add(sls) nstate = None if state: if not isinstance(state, dict): errors.append(("SLS {0} does not render to a dictionary".format(sls))) else: if "include" in state: if not isinstance(state["include"], list): err = "Include Declaration in SLS {0} is not formed " "as a list".format(sls) errors.append(err) else: for sub_sls in state.pop("include"): if sub_sls not in mods: nstate, mods, err = self.render_pstate(sub_sls, env, mods) if nstate: state.update(nstate) if err: errors += err return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) state = None try: state = compile_template(fn_, self.rend, self.opts['renderer'], saltenv, sls, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc) log.critical(msg) errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS {0!r} does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS {0!r} is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = sub_sls.iteritems().next() defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults) if nstate: if key: state[key] = nstate else: state.update(nstate) if err: errors += err return state, mods, errors
def render(self, template, opts=None, filename=None): if opts: self.config.update(opts) if not filename: filename = ".".join([str(uuid.uuid4()), "sls"]) full_path = self.write_template_file(filename, template) state = salt.state.State(self.config) return compile_template(full_path, state.rend, state.opts["renderer"])
def render(self, template, opts=None, filename=None): if opts: self.config.update(opts) if not filename: filename = ".".join([str(uuid.uuid4()), "sls"]) full_path = self.write_template_file(filename, template) state = salt.state.State(self.config) return compile_template(full_path, state.rend, state.opts['renderer'])
def targets(tgt, tgt_type='glob', **kwargs): ''' Return the targets from the flat yaml file, checks opts for location but defaults to /etc/salt/roster ''' template = get_roster_file(__opts__) rend = salt.loader.render(__opts__, {}) raw = compile_template(template, rend, __opts__['renderer'], **kwargs) rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4') return rmatcher.targets()
def _process_stack_cfg(cfg, stack, minion_id, pillar): basedir = os.path.dirname(cfg) items = compile_template( cfg, salt.loader.render(__opts__, __salt__), __opts__['renderer'], __opts__['renderer_blacklist'], __opts__['renderer_whitelist'], __opts__['pillarenv'], minion_id=minion_id, stack=stack, ) try: _ = iter(items['stack']) except: return stack for item in items['stack']: if not item.strip(): continue # silently ignore whitespace or empty lines paths = glob(os.path.join(basedir, item)) if not paths: log.debug( 'Ignoring pillar stack template "{0}": can\'t find from ' 'root dir "{1}"'.format(item, basedir) ) continue for sls in sorted(paths): log.debug('Compiling SLS: "{0}"'.format(sls)) obj = compile_template( sls, salt.loader.render(__opts__, __salt__), __opts__['renderer'], __opts__['renderer_blacklist'], __opts__['renderer_whitelist'], __opts__['pillarenv'], minion_id=minion_id, stack=stack, ) if obj: stack = _merge_dict(stack, obj) return stack
def _render(template, render, renderer, template_dict, opts): ''' Render a template ''' if render: if template_dict is None: template_dict = {} if not renderer: renderer = opts.get('renderer', 'yaml_jinja') rend = salt.loader.render(opts, {}) return compile_template(template, rend, renderer, **template_dict) with salt.utils.fopen(template, 'r') as fh_: return fh_.read()
def render_pstate(self, sls, env, mods): ''' Collect a single pillar sls file and render it ''' err = '' errors = [] fn_ = self.client.get_state(sls, env).get('dest', False) if not fn_: msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, env) log.error(msg) errors.append(msg) state = None try: state = compile_template( fn_, self.rend, self.opts['renderer'], env, sls) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc ) log.error(msg) errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS {0!r} does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS {0!r} is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, env, mods ) if nstate: state.update(nstate) if err: errors += err return state, mods, errors
def targets(tgt, tgt_type='glob', **kwargs): ''' Return the targets from the flat yaml file, checks opts for location but defaults to /etc/salt/roster ''' if os.path.isfile(__opts__['conf_file']) or not os.path.exists(__opts__['conf_file']): template = os.path.join( os.path.dirname(__opts__['conf_file']), 'roster') else: template = os.path.join(__opts__['conf_file'], 'roster') rend = salt.loader.render(__opts__, {}) raw = compile_template(template, rend, __opts__['renderer'], **kwargs) rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4') return rmatcher.targets()
def targets(tgt, tgt_type='glob', **kwargs): ''' Return the targets from the flat yaml file, checks opts for location but defaults to /etc/salt/roster ''' template = get_roster_file(__opts__) rend = salt.loader.render(__opts__, {}) raw = compile_template(template, rend, __opts__['renderer'], __opts__['renderer_blacklist'], __opts__['renderer_whitelist'], **kwargs) conditioned_raw = {} for minion in raw: conditioned_raw[str(minion)] = salt.config.apply_sdb(raw[minion]) rmatcher = RosterMatcher(conditioned_raw, tgt, tgt_type, 'ipv4') return rmatcher.targets()
def targets(tgt, tgt_type='glob', **kwargs): ''' Return the targets from the flat yaml file, checks opts for location but defaults to /etc/salt/roster ''' if __opts__.get('roster_file'): template = __opts__.get('roster_file') elif os.path.isfile(__opts__['conf_file']) or not os.path.exists(__opts__['conf_file']): template = os.path.join( os.path.dirname(__opts__['conf_file']), 'roster') else: template = os.path.join(__opts__['conf_file'], 'roster') rend = salt.loader.render(__opts__, {}) raw = compile_template(template, rend, __opts__['renderer'], **kwargs) rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4') return rmatcher.targets()
def targets(tgt, tgt_type='glob', **kwargs): ''' Return the targets from the flat yaml file, checks opts for location but defaults to /etc/salt/roster ''' template = get_roster_file(__opts__) rend = salt.loader.render(__opts__, {}) raw = compile_template(template, rend, __opts__['renderer'], __opts__['renderer_blacklist'], __opts__['renderer_whitelist'], **kwargs) conditioned_raw = {} for minion in raw: conditioned_raw[str(minion)] = raw[minion] rmatcher = RosterMatcher(conditioned_raw, tgt, tgt_type, 'ipv4') return rmatcher.targets()
def _render(self, data, formula_def): ''' Render a [pre|post]_local_state or [pre|post]_tgt_state script ''' # FORMULA can contain a renderer option renderer = formula_def.get('renderer', self.opts.get('renderer', 'jinja|yaml')) rend = salt.loader.render(self.opts, {}) blacklist = self.opts.get('renderer_blacklist') whitelist = self.opts.get('renderer_whitelist') template_vars = formula_def.copy() template_vars['opts'] = self.opts.copy() return compile_template(':string:', rend, renderer, blacklist, whitelist, input_data=data, **template_vars)
def _render(self, data, formula_def): """ Render a [pre|post]_local_state or [pre|post]_tgt_state script """ # FORMULA can contain a renderer option renderer = formula_def.get("renderer", self.opts.get("renderer", "jinja|yaml")) rend = salt.loader.render(self.opts, {}) blacklist = self.opts.get("renderer_blacklist") whitelist = self.opts.get("renderer_whitelist") template_vars = formula_def.copy() template_vars["opts"] = self.opts.copy() return compile_template(":string:", rend, renderer, blacklist, whitelist, input_data=data, **template_vars)
def _render(template, render, renderer, template_dict, opts): ''' Render a template ''' if render: if template_dict is None: template_dict = {} if not renderer: renderer = opts.get('renderer', 'yaml_jinja') rend = salt.loader.render(opts, {}) blacklist = opts.get('renderer_blacklist') whitelist = opts.get('renderer_whitelist') ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) ret = ret.read() if str(ret).startswith('#!') and not str(ret).startswith('#!/'): ret = str(ret).split('\n', 1)[1] return ret with salt.utils.files.fopen(template, 'r') as fh_: return fh_.read()
def render_pstate(self, sls, env, mods): ''' Collect a single pillar sls file and render it ''' err = '' errors = [] fn_ = self.client.get_state(sls, env) if not fn_: errors.append(('Specified SLS {0} in environment {1} is not' ' available on the salt master').format(sls, env)) state = None try: state = compile_template( fn_, self.rend, self.opts['renderer'], env, sls) except Exception as exc: errors.append(('Rendering SLS {0} failed, render error:\n{1}' .format(sls, exc))) mods.add(sls) nstate = None if state: if not isinstance(state, dict): errors.append(('SLS {0} does not render to a dictionary' .format(sls))) else: if 'include' in state: if not isinstance(state['include'], list): err = ('Include Declaration in SLS {0} is not formed ' 'as a list'.format(sls)) errors.append(err) else: for sub_sls in state.pop('include'): if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, env, mods ) if nstate: state.update(nstate) if err: errors += err return state, mods, errors
def targets(tgt, tgt_type="glob", **kwargs): """ Return the targets from the flat yaml file, checks opts for location but defaults to /etc/salt/roster """ template = get_roster_file(__opts__) rend = salt.loader.render(__opts__, {}) raw = compile_template(template, rend, __opts__["renderer"], __opts__["renderer_blacklist"], __opts__["renderer_whitelist"], mask_value="passw*", **kwargs) conditioned_raw = {} for minion in raw: conditioned_raw[str(minion)] = salt.config.apply_sdb(raw[minion]) return __utils__["roster_matcher.targets"](conditioned_raw, tgt, tgt_type, "ipv4")
def _render(template, render, renderer, template_dict, opts): """ Render a template """ if render: if template_dict is None: template_dict = {} if not renderer: renderer = opts.get("renderer", "jinja|yaml") rend = salt.loader.render(opts, {}) blacklist = opts.get("renderer_blacklist") whitelist = opts.get("renderer_whitelist") ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) if salt.utils.stringio.is_readable(ret): ret = ret.read() if str(ret).startswith("#!") and not str(ret).startswith("#!/"): ret = str(ret).split("\n", 1)[1] return ret with salt.utils.files.fopen(template, "r") as fh_: return fh_.read()
def targets(tgt, tgt_type='glob', **kwargs): ''' Return the targets from the sls file, checks opts for location but defaults to /etc/salt/roster ''' template = get_roster_file(__opts__) rend = salt.loader.render(__opts__, {}) kwargs['__salt__'] = __runner__ pool = compile_template(template, rend, __opts__['renderer'], __opts__['renderer_blacklist'], __opts__['renderer_whitelist'], mask_value='passw*', **kwargs) pool = {host: {'minion_opts': conf} for host, conf in pool.items()} pool = salt_sproxy._roster.load_cache(pool, __runner__, __opts__, tgt, tgt_type=tgt_type) engine = salt_sproxy._roster.TGT_FUN[tgt_type] return engine(pool, tgt, opts=__opts__)
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug('Skipping ignored and missing SLS {0!r} in' ' environment {1!r}'.format(sls, saltenv)) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: log.debug('Specified SLS {0!r} in environment {1!r} is not' ' found, which might be due to environment {1!r}' ' not being present in "pillar_roots" yet!' .format(sls, saltenv)) # return state, mods, errors return None, mods, errors state = None try: state = compile_template( fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc ) log.critical(msg) if self.opts.get('pillar_safe_render_error', True): errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls)) else: errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS {0!r} does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS {0!r} is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults ) if nstate: if key: nstate = { key: nstate } state = merge( state, nstate, self.merge_strategy, self.opts.get('renderer', 'yaml')) if err: errors += err return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug('Skipping ignored and missing SLS {0!r} in' ' environment {1!r}'.format(sls, saltenv)) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: log.debug('Specified SLS {0!r} in environment {1!r} is not' ' found, which might be due to environment {1!r}' ' not being present in "pillar_roots" yet!'.format( sls, saltenv)) # return state, mods, errors return None, mods, errors state = None try: state = compile_template(fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc) log.critical(msg) if self.opts.get('pillar_safe_render_error', True): errors.append( 'Rendering SLS \'{0}\' failed. Please see master log for details.' .format(sls)) else: errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS {0!r} does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS {0!r} is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults) if nstate: if key: nstate = {key: nstate} state = merge( state, nstate, self.merge_strategy, self.opts.get('renderer', 'yaml')) if err: errors += err return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug( 'Skipping ignored and missing SLS \'%s\' in ' 'environment \'%s\'', sls, saltenv) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' 'found. '.format(sls, saltenv)) if self.opts.get('__git_pillar', False) is True: msg += ( 'This is likely caused by a git_pillar top file ' 'containing an environment other than the one for the ' 'branch in which it resides. Each git_pillar ' 'branch/tag must have its own top file.') else: msg += ( 'This could be because SLS \'{0}\' is in an ' 'environment other than \'{1}\', but \'{1}\' is ' 'included in that environment\'s Pillar top file. It ' 'could also be due to environment \'{1}\' not being ' 'defined in \'pillar_roots\'.'.format(sls, saltenv)) log.debug(msg) # return state, mods, errors return None, mods, errors state = None try: state = compile_template(fn_, self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format( sls, exc) log.critical(msg, exc_info=True) if self.opts.get('pillar_safe_render_error', True): errors.append( 'Rendering SLS \'{0}\' failed. Please see master log for ' 'details.'.format(sls)) else: errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS \'{0}\' is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: # render included state(s) include_states = [] matched_pstates = [] for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None try: matched_pstates += fnmatch.filter( self.avail[saltenv], sub_sls) except KeyError: errors.extend([ 'No matching pillar environment for environment ' '\'{0}\' found'.format(saltenv) ]) for sub_sls in matched_pstates: if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults) if nstate: if key: # If key is x:y, convert it to {x: {y: nstate}} for key_fragment in reversed( key.split(":")): nstate = {key_fragment: nstate} if not self.opts.get( 'pillar_includes_override_sls', False): include_states.append(nstate) else: state = merge( state, nstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get( 'pillar_merge_lists', False)) if err: errors += err if not self.opts.get('pillar_includes_override_sls', False): # merge included state(s) with the current state # merged last to ensure that its values are # authoritative. include_states.append(state) state = None for s in include_states: if state is None: state = s else: state = merge( state, s, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) return state, mods, errors
def get_tops(self): ''' Gather the top files ''' tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: saltenvs = set() if self.opts['pillarenv']: # If the specified pillarenv is not present in the available # pillar environments, do not cache the pillar top file. if self.opts['pillarenv'] not in self.opts['pillar_roots']: log.debug( 'pillarenv \'%s\' not found in the configured pillar ' 'environments (%s)', self.opts['pillarenv'], ', '.join(self.opts['pillar_roots'])) else: saltenvs.add(self.opts['pillarenv']) else: saltenvs = self._get_envs() if self.opts.get('pillar_source_merging_strategy', None) == "none": saltenvs &= set([self.saltenv or 'base']) for saltenv in saltenvs: top = self.client.cache_file(self.opts['state_top'], saltenv) if top: tops[saltenv].append( compile_template( top, self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv=saltenv, _pillar_rend=True, )) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}'. format(exc))) log.exception('Pillar rendering failed for minion %s', self.minion_id) # Search initial top files for includes for saltenv, ctops in six.iteritems(tops): for ctop in ctops: if 'include' not in ctop: continue for sls in ctop['include']: include[saltenv].append(sls) ctop.pop('include') # Go through the includes and pull out the extra tops and add them while include: pops = [] for saltenv, states in six.iteritems(include): pops.append(saltenv) if not states: continue for sls in states: if sls in done[saltenv]: continue try: tops[saltenv].append( compile_template( self.client.get_state(sls, saltenv).get( 'dest', False), self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv=saltenv, _pillar_rend=True, )) except Exception as exc: errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: include.pop(saltenv) return tops, errors
def get_tops(self): ''' Gather the top files ''' tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: if self.opts['environment']: tops[self.opts['environment']] = [ compile_template( self.client.cache_file( self.opts['state_top'], self.opts['environment'] ), self.rend, self.opts['renderer'], self.opts['environment'] ) ] else: for env in self._get_envs(): tops[env].append( compile_template( self.client.cache_file( self.opts['state_top'], env ), self.rend, self.opts['renderer'], env=env ) ) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) # Search initial top files for includes for env, ctops in tops.items(): for ctop in ctops: if not 'include' in ctop: continue for sls in ctop['include']: include[env].append(sls) ctop.pop('include') # Go through the includes and pull out the extra tops and add them while include: pops = [] for env, states in include.items(): pops.append(env) if not states: continue for sls in states: if sls in done[env]: continue try: tops[env].append( compile_template( self.client.get_state( sls, env ), self.rend, self.opts['renderer'], env=env ) ) except Exception as exc: errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) done[env].append(sls) for env in pops: if env in include: include.pop(env) return tops, errors
def returner(ret): ''' Send an email with the data ''' _options = _get_options(ret) from_addr = _options.get('from') to_addrs = _options.get('to') host = _options.get('host') port = _options.get('port') user = _options.get('username') passwd = _options.get('password') subject = _options.get('subject') gpgowner = _options.get('gpgowner') fields = _options.get('fields').split(',') if 'fields' in _options else [] smtp_tls = _options.get('tls') renderer = _options.get('renderer', __opts__.get('renderer', 'yaml_jinja')) rend = salt.loader.render(__opts__, {}) if not port: port = 25 log.debug('SMTP port has been set to {0}'.format(port)) for field in fields: if field in ret: subject += ' {0}'.format(ret[field]) subject = compile_template(':string:', rend, renderer, input_data=subject, **ret) log.debug("smtp_return: Subject is '{0}'".format(subject)) template = _options.get('template') if template: content = compile_template(template, rend, renderer, **ret) else: template = ('id: {{id}}\r\n' 'function: {{fun}}\r\n' 'function args: {{fun_args}}\r\n' 'jid: {{jid}}\r\n' 'return: {{return}}\r\n') content = compile_template(':string:', rend, renderer, input_data=template, **ret) if HAS_GNUPG and gpgowner: gpg = gnupg.GPG(gnupghome=os.path.expanduser('~{0}/.gnupg'.format(gpgowner)), options=['--trust-model always']) encrypted_data = gpg.encrypt(content, to_addrs) if encrypted_data.ok: log.debug('smtp_return: Encryption successful') content = str(encrypted_data) else: log.error('smtp_return: Encryption failed, only an error message will be sent') content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format( encrypted_data.status, encrypted_data.stderr) message = ('From: {0}\r\n' 'To: {1}\r\n' 'Date: {2}\r\n' 'Subject: {3}\r\n' '\r\n' '{4}').format(from_addr, to_addrs, formatdate(localtime=True), subject, content) log.debug('smtp_return: Connecting to the server...') server = smtplib.SMTP(host, int(port)) server.set_debuglevel = 'debug' if smtp_tls is True: server.starttls() log.debug('smtp_return: TLS enabled') if user and passwd: server.login(user, passwd) log.debug('smtp_return: Authenticated') server.sendmail(from_addr, to_addrs, message) log.debug('smtp_return: Message sent.') server.quit()
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: log.debug('Specified SLS {0!r} in environment {1!r} is not' ' found, which might be due to environment {1!r}' ' not being present in "pillar_roots" yet!' .format(sls, saltenv)) # return state, mods, errors return None, mods, errors state = None try: state = compile_template( fn_, self.rend, self.opts['renderer'], saltenv, sls, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc ) log.critical(msg) errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS {0!r} does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS {0!r} is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = sub_sls.iteritems().next() defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults ) if nstate: if key: state[key] = nstate else: state.update(nstate) if err: errors += err return state, mods, errors
def get_tops(self): ''' Gather the top files ''' tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: if self.opts['environment']: tops[self.opts['environment']] = [ compile_template( self.client.cache_file(self.opts['state_top'], self.opts['environment']), self.rend, self.opts['renderer'], self.opts['environment']) ] else: for saltenv in self._get_envs(): tops[saltenv].append( compile_template(self.client.cache_file( self.opts['state_top'], saltenv), self.rend, self.opts['renderer'], saltenv=saltenv)) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}'. format(exc))) # Search initial top files for includes for saltenv, ctops in tops.items(): for ctop in ctops: if 'include' not in ctop: continue for sls in ctop['include']: include[saltenv].append(sls) ctop.pop('include') # Go through the includes and pull out the extra tops and add them while include: pops = [] for saltenv, states in include.items(): pops.append(saltenv) if not states: continue for sls in states: if sls in done[saltenv]: continue try: tops[saltenv].append( compile_template(self.client.get_state( sls, saltenv).get('dest', False), self.rend, self.opts['renderer'], saltenv=saltenv)) except Exception as exc: errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: include.pop(saltenv) return tops, errors
def get_tops(self): ''' Gather the top files ''' tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: if self.opts['pillarenv']: tops[self.opts['pillarenv']] = [ compile_template( self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']), self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], self.opts['pillarenv'], _pillar_rend=True, ) ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": if self.saltenv and saltenv != self.saltenv: continue if not self.saltenv and not saltenv == 'base': continue top = self.client.cache_file(self.opts['state_top'], saltenv) if top: tops[saltenv].append( compile_template( top, self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv=saltenv, _pillar_rend=True, )) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}'. format(exc))) log.error('Pillar rendering failed for minion {0}: '.format( self.minion_id), exc_info=True) # Search initial top files for includes for saltenv, ctops in six.iteritems(tops): for ctop in ctops: if 'include' not in ctop: continue for sls in ctop['include']: include[saltenv].append(sls) ctop.pop('include') # Go through the includes and pull out the extra tops and add them while include: pops = [] for saltenv, states in six.iteritems(include): pops.append(saltenv) if not states: continue for sls in states: if sls in done[saltenv]: continue try: tops[saltenv].append( compile_template( self.client.get_state(sls, saltenv).get( 'dest', False), self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv=saltenv, _pillar_rend=True, )) except Exception as exc: errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: include.pop(saltenv) return tops, errors
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug('Skipping ignored and missing SLS \'{0}\' in' ' environment \'{1}\''.format(sls, saltenv)) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' 'found. '.format(sls, saltenv)) if self.opts.get('__git_pillar', False) is True: msg += ( 'This is likely caused by a git_pillar top file ' 'containing an environment other than the one for the ' 'branch in which it resides. Each git_pillar ' 'branch/tag must have its own top file.' ) else: msg += ( 'This could be because SLS \'{0}\' is in an ' 'environment other than \'{1}\', but \'{1}\' is ' 'included in that environment\'s Pillar top file. It ' 'could also be due to environment \'{1}\' not being ' 'defined in \'pillar_roots\'.'.format(sls, saltenv) ) log.debug(msg) # return state, mods, errors return None, mods, errors state = None try: state = compile_template(fn_, self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format( sls, exc ) log.critical(msg) if self.opts.get('pillar_safe_render_error', True): errors.append( 'Rendering SLS \'{0}\' failed. Please see master log for ' 'details.'.format(sls) ) else: errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS \'{0}\' is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults ) if nstate: if key: nstate = { key: nstate } state = merge( state, nstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) if err: errors += err return state, mods, errors
def get_tops(self): ''' Gather the top files ''' tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: if self.opts['pillarenv']: tops[self.opts['pillarenv']] = [ compile_template( self.client.cache_file( self.opts['state_top'], self.opts['pillarenv'] ), self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], self.opts['pillarenv'], _pillar_rend=True, ) ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": if self.saltenv and saltenv != self.saltenv: continue if not self.saltenv and not saltenv == 'base': continue top = self.client.cache_file( self.opts['state_top'], saltenv ) if top: tops[saltenv].append( compile_template( top, self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv=saltenv, _pillar_rend=True, ) ) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) log.error('Pillar rendering failed for minion {0}: '.format(self.minion_id), exc_info=True) # Search initial top files for includes for saltenv, ctops in six.iteritems(tops): for ctop in ctops: if 'include' not in ctop: continue for sls in ctop['include']: include[saltenv].append(sls) ctop.pop('include') # Go through the includes and pull out the extra tops and add them while include: pops = [] for saltenv, states in six.iteritems(include): pops.append(saltenv) if not states: continue for sls in states: if sls in done[saltenv]: continue try: tops[saltenv].append( compile_template( self.client.get_state( sls, saltenv ).get('dest', False), self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], saltenv=saltenv, _pillar_rend=True, ) ) except Exception as exc: errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: include.pop(saltenv) return tops, errors
def render_pstate(self, sls, saltenv, mods, defaults=None): """ Collect a single pillar sls file and render it """ if defaults is None: defaults = {} err = "" errors = [] fn_ = self.client.get_state(sls, saltenv).get("dest", False) if not fn_: if self.opts["pillar_roots"].get(saltenv): msg = ("Specified SLS {0!r} in environment {1!r} is not" " available on the salt master").format( sls, saltenv ) log.error(msg) errors.append(msg) else: log.debug( "Specified SLS {0!r} in environment {1!r} is not" " found, which might be due to environment {1!r}" ' not being present in "pillar_roots" yet!'.format(sls, saltenv) ) # return state, mods, errors return None, mods, errors state = None try: state = compile_template(fn_, self.rend, self.opts["renderer"], saltenv, sls, **defaults) except Exception as exc: msg = "Rendering SLS {0!r} failed, render error:\n{1}".format(sls, exc) log.critical(msg) errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = "SLS {0!r} does not render to a dictionary".format(sls) log.error(msg) errors.append(msg) else: if "include" in state: if not isinstance(state["include"], list): msg = "Include Declaration in SLS {0!r} is not " "formed as a list".format(sls) log.error(msg) errors.append(msg) else: for sub_sls in state.pop("include"): if isinstance(sub_sls, dict): sub_sls, v = sub_sls.iteritems().next() defaults = v.get("defaults", {}) key = v.get("key", None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate(sub_sls, saltenv, mods, defaults) if nstate: if key: state[key] = nstate else: state.update(nstate) if err: errors += err return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' if defaults is None: defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug('Skipping ignored and missing SLS {0!r} in' ' environment {1!r}'.format(sls, saltenv)) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' 'found. '.format(sls, saltenv)) if self.opts.get('__git_pillar', False) is True: msg += ( 'This is likely caused by a git_pillar top file ' 'containing an environment other than the one for the ' 'branch in which it resides. Each git_pillar ' 'branch/tag must have its own top file.') else: msg += ( 'This could be because SLS \'{0}\' is in an ' 'environment other than \'{1}\', but \'{1}\' is ' 'included in that environment\'s Pillar top file. It ' 'could also be due to environment \'{1}\' not being ' 'defined in \'pillar_roots\'.'.format(sls, saltenv)) log.debug(msg) # return state, mods, errors return None, mods, errors state = None try: state = compile_template(fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc) log.critical(msg) if self.opts.get('pillar_safe_render_error', True): errors.append( 'Rendering SLS \'{0}\' failed. Please see master log for details.' .format(sls)) else: errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): msg = 'SLS {0!r} does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): msg = ('Include Declaration in SLS {0!r} is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) defaults = v.get('defaults', {}) key = v.get('key', None) else: key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, saltenv, mods, defaults) if nstate: if key: nstate = {key: nstate} state = merge( state, nstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) if err: errors += err return state, mods, errors
def returner(ret): ''' Send an email with the data ''' _options = _get_options(ret) from_addr = _options.get('from') to_addrs = _options.get('to').split(',') host = _options.get('host') port = _options.get('port') user = _options.get('username') passwd = _options.get('password') subject = _options.get('subject') or 'Email from Salt' gpgowner = _options.get('gpgowner') fields = _options.get('fields').split(',') if 'fields' in _options else [] smtp_tls = _options.get('tls') renderer = _options.get('renderer') or 'jinja' rend = salt.loader.render(__opts__, {}) blacklist = __opts__.get('renderer_blacklist') whitelist = __opts__.get('renderer_whitelist') if not port: port = 25 log.debug('SMTP port has been set to {0}'.format(port)) for field in fields: if field in ret: subject += ' {0}'.format(ret[field]) subject = compile_template(':string:', rend, renderer, blacklist, whitelist, input_data=subject, **ret) if isinstance(subject, six.moves.StringIO): subject = subject.read() log.debug("smtp_return: Subject is '{0}'".format(subject)) template = _options.get('template') if template: content = compile_template(template, rend, renderer, blacklist, whitelist, **ret) else: template = ('id: {{id}}\r\n' 'function: {{fun}}\r\n' 'function args: {{fun_args}}\r\n' 'jid: {{jid}}\r\n' 'return: {{return}}\r\n') content = compile_template(':string:', rend, renderer, blacklist, whitelist, input_data=template, **ret) if gpgowner: if HAS_GNUPG: gpg = gnupg.GPG(gnupghome=os.path.expanduser( '~{0}/.gnupg'.format(gpgowner)), options=['--trust-model always']) encrypted_data = gpg.encrypt(content, to_addrs) if encrypted_data.ok: log.debug('smtp_return: Encryption successful') content = str(encrypted_data) else: log.error( 'smtp_return: Encryption failed, only an error message will be sent' ) content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format( encrypted_data.status, encrypted_data.stderr) else: log.error( "gnupg python module is required in order to user gpgowner in smtp returner ; ignoring gpgowner configuration for now" ) if isinstance(content, six.moves.StringIO): content = content.read() message = ('From: {0}\r\n' 'To: {1}\r\n' 'Date: {2}\r\n' 'Subject: {3}\r\n' '\r\n' '{4}').format(from_addr, ', '.join(to_addrs), formatdate(localtime=True), subject, content) log.debug('smtp_return: Connecting to the server...') server = smtplib.SMTP(host, int(port)) if smtp_tls is True: server.starttls() log.debug('smtp_return: TLS enabled') if user and passwd: server.login(user, passwd) log.debug('smtp_return: Authenticated') # enable logging SMTP session after the login credentials were passed server.set_debuglevel(1) server.sendmail(from_addr, to_addrs, message) log.debug('smtp_return: Message sent.') server.quit()
def get_tops(self): """ Gather the top files """ tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: saltenvs = set() if self.opts["pillarenv"]: # If the specified pillarenv is not present in the available # pillar environments, do not cache the pillar top file. if self.opts["pillarenv"] not in self.opts["pillar_roots"]: log.debug( "pillarenv '%s' not found in the configured pillar " "environments (%s)", self.opts["pillarenv"], ", ".join(self.opts["pillar_roots"]), ) else: saltenvs.add(self.opts["pillarenv"]) else: saltenvs = self._get_envs() if self.opts.get("pillar_source_merging_strategy", None) == "none": saltenvs &= {self.saltenv or "base"} for saltenv in saltenvs: top = self.client.cache_file(self.opts["state_top"], saltenv) if top: tops[saltenv].append( compile_template( top, self.rend, self.opts["renderer"], self.opts["renderer_blacklist"], self.opts["renderer_whitelist"], saltenv=saltenv, _pillar_rend=True, )) except Exception as exc: # pylint: disable=broad-except errors.append( "Rendering Primary Top file failed, render error:\n{}".format( exc)) log.exception("Pillar rendering failed for minion %s", self.minion_id) # Search initial top files for includes for saltenv, ctops in tops.items(): for ctop in ctops: if "include" not in ctop: continue for sls in ctop["include"]: include[saltenv].append(sls) ctop.pop("include") # Go through the includes and pull out the extra tops and add them while include: pops = [] for saltenv, states in include.items(): pops.append(saltenv) if not states: continue for sls in states: if sls in done[saltenv]: continue try: tops[saltenv].append( compile_template( self.client.get_state(sls, saltenv).get( "dest", False), self.rend, self.opts["renderer"], self.opts["renderer_blacklist"], self.opts["renderer_whitelist"], saltenv=saltenv, _pillar_rend=True, )) except Exception as exc: # pylint: disable=broad-except errors.append( "Rendering Top file {} failed, render error:\n{}". format(sls, exc)) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: include.pop(saltenv) return tops, errors
def returner(ret): ''' Send an email with the data ''' _options = _get_options(ret) from_addr = _options.get('from') to_addrs = _options.get('to') host = _options.get('host') port = _options.get('port') user = _options.get('username') passwd = _options.get('password') subject = _options.get('subject') gpgowner = _options.get('gpgowner') fields = _options.get('fields').split(',') if 'fields' in _options else [] smtp_tls = _options.get('tls') renderer = _options.get('renderer', __opts__.get('renderer', 'yaml_jinja')) rend = salt.loader.render(__opts__, {}) if not port: port = 25 log.debug('SMTP port has been set to {0}'.format(port)) for field in fields: if field in ret: subject += ' {0}'.format(ret[field]) subject = compile_template(':string:', rend, renderer, input_data=subject, **ret) log.debug("smtp_return: Subject is '{0}'".format(subject)) template = _options.get('template') if template: content = compile_template(template, rend, renderer, **ret) else: template = ('id: {{id}}\r\n' 'function: {{fun}}\r\n' 'function args: {{fun_args}}\r\n' 'jid: {{jid}}\r\n' 'return: {{return}}\r\n') content = compile_template(':string:', rend, renderer, input_data=template, **ret) if HAS_GNUPG and gpgowner: gpg = gnupg.GPG(gnupghome=os.path.expanduser( '~{0}/.gnupg'.format(gpgowner)), options=['--trust-model always']) encrypted_data = gpg.encrypt(content, to_addrs) if encrypted_data.ok: log.debug('smtp_return: Encryption successful') content = str(encrypted_data) else: log.error( 'smtp_return: Encryption failed, only an error message will be sent' ) content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format( encrypted_data.status, encrypted_data.stderr) message = ('From: {0}\r\n' 'To: {1}\r\n' 'Date: {2}\r\n' 'Subject: {3}\r\n' '\r\n' '{4}').format(from_addr, to_addrs, formatdate(localtime=True), subject, content) log.debug('smtp_return: Connecting to the server...') server = smtplib.SMTP(host, int(port)) server.set_debuglevel = 'debug' if smtp_tls is True: server.starttls() log.debug('smtp_return: TLS enabled') if user and passwd: server.login(user, passwd) log.debug('smtp_return: Authenticated') server.sendmail(from_addr, to_addrs, message) log.debug('smtp_return: Message sent.') server.quit()
def returner(ret): """ Send an email with the data """ _options = _get_options(ret) from_addr = _options.get("from") to_addrs = _options.get("to").split(",") host = _options.get("host") port = _options.get("port") user = _options.get("username") passwd = _options.get("password") subject = _options.get("subject") or "Email from Salt" gpgowner = _options.get("gpgowner") fields = _options.get("fields").split(",") if "fields" in _options else [] smtp_tls = _options.get("tls") renderer = _options.get("renderer") or "jinja" rend = salt.loader.render(__opts__, {}) blacklist = __opts__.get("renderer_blacklist") whitelist = __opts__.get("renderer_whitelist") if not port: port = 25 log.debug("SMTP port has been set to %s", port) for field in fields: if field in ret: subject += " {0}".format(ret[field]) subject = compile_template( ":string:", rend, renderer, blacklist, whitelist, input_data=subject, **ret ) if isinstance(subject, six.moves.StringIO): subject = subject.read() log.debug("smtp_return: Subject is '%s'", subject) template = _options.get("template") if template: content = compile_template( template, rend, renderer, blacklist, whitelist, **ret ) else: template = ( "id: {{id}}\r\n" "function: {{fun}}\r\n" "function args: {{fun_args}}\r\n" "jid: {{jid}}\r\n" "return: {{return}}\r\n" ) content = compile_template( ":string:", rend, renderer, blacklist, whitelist, input_data=template, **ret ) if gpgowner: if HAS_GNUPG: gpg = gnupg.GPG( gnupghome=os.path.expanduser("~{0}/.gnupg".format(gpgowner)), options=["--trust-model always"], ) encrypted_data = gpg.encrypt(content, to_addrs) if encrypted_data.ok: log.debug("smtp_return: Encryption successful") content = six.text_type(encrypted_data) else: log.error( "smtp_return: Encryption failed, only an error message will be sent" ) content = "Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}".format( encrypted_data.status, encrypted_data.stderr ) else: log.error( "gnupg python module is required in order to user gpgowner in smtp returner ; ignoring gpgowner configuration for now" ) if isinstance(content, six.moves.StringIO): content = content.read() message = ( "From: {0}\r\n" "To: {1}\r\n" "Date: {2}\r\n" "Subject: {3}\r\n" "\r\n" "{4}" ).format( from_addr, ", ".join(to_addrs), formatdate(localtime=True), subject, content ) log.debug("smtp_return: Connecting to the server...") server = smtplib.SMTP(host, int(port)) if smtp_tls is True: server.starttls() log.debug("smtp_return: TLS enabled") if user and passwd: server.login(user, passwd) log.debug("smtp_return: Authenticated") # enable logging SMTP session after the login credentials were passed server.set_debuglevel(1) server.sendmail(from_addr, to_addrs, message) log.debug("smtp_return: Message sent.") server.quit()
def render_pstate(self, sls, saltenv, mods, defaults=None): """ Collect a single pillar sls file and render it """ if defaults is None: defaults = {} err = "" errors = [] state_data = self.client.get_state(sls, saltenv) fn_ = state_data.get("dest", False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug( "Skipping ignored and missing SLS '%s' in environment '%s'", sls, saltenv, ) return None, mods, errors elif self.opts["pillar_roots"].get(saltenv): msg = ("Specified SLS '{}' in environment '{}' is not" " available on the salt master".format(sls, saltenv)) log.error(msg) errors.append(msg) else: msg = "Specified SLS '{}' in environment '{}' was not found. ".format( sls, saltenv) if self.opts.get("__git_pillar", False) is True: msg += ( "This is likely caused by a git_pillar top file " "containing an environment other than the one for the " "branch in which it resides. Each git_pillar " "branch/tag must have its own top file.") else: msg += ( "This could be because SLS '{0}' is in an " "environment other than '{1}', but '{1}' is " "included in that environment's Pillar top file. It " "could also be due to environment '{1}' not being " "defined in 'pillar_roots'.".format(sls, saltenv)) log.debug(msg) # return state, mods, errors return None, mods, errors state = None try: state = compile_template(fn_, self.rend, self.opts["renderer"], self.opts["renderer_blacklist"], self.opts["renderer_whitelist"], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: # pylint: disable=broad-except msg = "Rendering SLS '{}' failed, render error:\n{}".format( sls, exc) log.critical(msg, exc_info=True) if self.opts.get("pillar_safe_render_error", True): errors.append( "Rendering SLS '{}' failed. Please see master log for " "details.".format(sls)) else: errors.append(msg) mods[sls] = state nstate = None if state: if not isinstance(state, dict): msg = "SLS '{}' does not render to a dictionary".format(sls) log.error(msg) errors.append(msg) else: if "include" in state: if not isinstance(state["include"], list): msg = ("Include Declaration in SLS '{}' is not " "formed as a list".format(sls)) log.error(msg) errors.append(msg) else: # render included state(s) include_states = [] for sub_sls in state.pop("include"): if isinstance(sub_sls, dict): sub_sls, v = next(iter(sub_sls.items())) defaults = v.get("defaults", {}) key = v.get("key", None) else: key = None try: matched_pstates = fnmatch.filter( self.avail[saltenv], sub_sls.lstrip(".").replace("/", "."), ) if sub_sls.startswith("."): if state_data.get( "source", "").endswith("/init.sls"): include_parts = sls.split(".") else: include_parts = sls.split(".")[:-1] sub_sls = ".".join(include_parts + [sub_sls[1:]]) matches = fnmatch.filter( self.avail[saltenv], sub_sls, ) matched_pstates.extend(matches) except KeyError: errors.extend([ "No matching pillar environment for environment" " '{}' found".format(saltenv) ]) matched_pstates = [sub_sls] # If matched_pstates is empty, set to sub_sls if len(matched_pstates) < 1: matched_pstates = [sub_sls] for m_sub_sls in matched_pstates: if m_sub_sls not in mods: nstate, mods, err = self.render_pstate( m_sub_sls, saltenv, mods, defaults) else: nstate = mods[m_sub_sls] if nstate: if key: # If key is x:y, convert it to {x: {y: nstate}} for key_fragment in reversed( key.split(":")): nstate = {key_fragment: nstate} if not self.opts.get( "pillar_includes_override_sls", False): include_states.append(nstate) else: state = merge( state, nstate, self.merge_strategy, self.opts.get("renderer", "yaml"), self.opts.get( "pillar_merge_lists", False), ) if err: errors += err if not self.opts.get("pillar_includes_override_sls", False): # merge included state(s) with the current state # merged last to ensure that its values are # authoritative. include_states.append(state) state = None for s in include_states: if state is None: state = s else: state = merge( state, s, self.merge_strategy, self.opts.get("renderer", "yaml"), self.opts.get("pillar_merge_lists", False), ) return state, mods, errors
def test_compile_template_bad_type(self): ''' Test to ensure that unsupported types cannot be passed to the template compiler ''' ret = template.compile_template(['1', '2', '3'], None, None) self.assertDictEqual(ret, {})
def get_tops(self): """ Gather the top files """ tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) errors = [] # Gather initial top files try: if self.opts["environment"]: tops[self.opts["environment"]] = [ compile_template( self.client.cache_file(self.opts["state_top"], self.opts["environment"]), self.rend, self.opts["renderer"], self.opts["environment"], ) ] else: for saltenv in self._get_envs(): tops[saltenv].append( compile_template( self.client.cache_file(self.opts["state_top"], saltenv), self.rend, self.opts["renderer"], saltenv=saltenv, ) ) except Exception as exc: errors.append(("Rendering Primary Top file failed, render error:\n{0}".format(exc))) # Search initial top files for includes for saltenv, ctops in tops.items(): for ctop in ctops: if "include" not in ctop: continue for sls in ctop["include"]: include[saltenv].append(sls) ctop.pop("include") # Go through the includes and pull out the extra tops and add them while include: pops = [] for saltenv, states in include.items(): pops.append(saltenv) if not states: continue for sls in states: if sls in done[saltenv]: continue try: tops[saltenv].append( compile_template( self.client.get_state(sls, saltenv).get("dest", False), self.rend, self.opts["renderer"], saltenv=saltenv, ) ) except Exception as exc: errors.append(("Rendering Top file {0} failed, render error" ":\n{1}").format(sls, exc)) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: include.pop(saltenv) return tops, errors