def test_compile_template_preserves_windows_newlines(self):
"""
Test to ensure that a file with Windows newlines, when rendered by a
template renderer, does not eat the CR character.
"""
def _get_rend(renderer, value):
"""
We need a new MagicMock each time since we're dealing with StringIO
objects which are read like files.
"""
return {renderer: MagicMock(return_value=StringIO(value))}
input_data_windows = "foo\r\nbar\r\nbaz\r\n"
input_data_non_windows = input_data_windows.replace("\r\n", "\n")
renderer = "test"
blacklist = whitelist = []
ret = template.compile_template(
":string:",
_get_rend(renderer, input_data_non_windows),
renderer,
blacklist,
whitelist,
input_data=input_data_windows,
).read()
# Even though the mocked renderer returned a string without the windows
# newlines, the compiled template should still have them.
self.assertEqual(ret, input_data_windows)
# Now test that we aren't adding them in unnecessarily.
ret = template.compile_template(
":string:",
_get_rend(renderer, input_data_non_windows),
renderer,
blacklist,
whitelist,
input_data=input_data_non_windows,
).read()
self.assertEqual(ret, input_data_non_windows)
# Finally, ensure that we're not unnecessarily replacing the \n with
# \r\n in the event that the renderer returned a string with the
# windows newlines intact.
ret = template.compile_template(
":string:",
_get_rend(renderer, input_data_windows),
renderer,
blacklist,
whitelist,
input_data=input_data_windows,
).read()
self.assertEqual(ret, input_data_windows)
def query(key, value=None, service=None, profile=None): # pylint: disable=W0613
"""
Get a value from the REST interface
"""
comps = key.split("?")
key = comps[0]
key_vars = {}
for pair in comps[1].split("&"):
pair_key, pair_val = pair.split("=")
key_vars[pair_key] = pair_val
renderer = __opts__.get("renderer", "jinja|yaml")
rend = salt.loader.render(__opts__, {})
blacklist = __opts__.get("renderer_blacklist")
whitelist = __opts__.get("renderer_whitelist")
url = compile_template(":string:",
rend,
renderer,
blacklist,
whitelist,
input_data=profile[key]["url"],
**key_vars)
extras = {}
for item in profile[key]:
if item not in ("backend", "url"):
extras[item] = profile[key][item]
result = http.query(url, decode=True, **extras)
return result["dict"]
def test_compile_template_bad_type(self):
'''
Test to ensure that unsupported types cannot be passed to the template compiler
'''
ret = template.compile_template(['1', '2', '3'], None, None, None,
None)
self.assertDictEqual(ret, {})
def test_compile_template_bad_type(self):
"""
Test to ensure that unsupported types cannot be passed to the template compiler
"""
ret = template.compile_template(["1", "2", "3"], None, None, None,
None)
self.assertDictEqual(ret, {})
def query(key, value=None, service=None, profile=None): # pylint: disable=W0613
'''
Get a value from the REST interface
'''
comps = key.split('?')
key = comps[0]
key_vars = {}
for pair in comps[1].split('&'):
pair_key, pair_val = pair.split('=')
key_vars[pair_key] = pair_val
renderer = __opts__.get('renderer', 'yaml_jinja')
rend = salt.loader.render(__opts__, {})
blacklist = __opts__.get('renderer_blacklist')
whitelist = __opts__.get('renderer_whitelist')
url = compile_template(
':string:',
rend,
renderer,
blacklist,
whitelist,
input_data=profile[key]['url'],
**key_vars
)
result = http.query(
url,
decode=True,
**key_vars
)
return result['dict']
def query(key, value=None, service=None, profile=None): # pylint: disable=W0613
'''
Get a value from the REST interface
'''
comps = key.split('?')
key = comps[0]
key_vars = {}
for pair in comps[1].split('&'):
pair_key, pair_val = pair.split('=')
key_vars[pair_key] = pair_val
renderer = __opts__.get('renderer', 'yaml_jinja')
rend = salt.loader.render(__opts__, {})
blacklist = __opts__.get('renderer_blacklist')
whitelist = __opts__.get('renderer_whitelist')
url = compile_template(':string:',
rend,
renderer,
blacklist,
whitelist,
input_data=profile[key]['url'],
**key_vars)
result = http.query(url, decode=True, **key_vars)
return result['dict']
def render_pstate(self, sls, env, mods):
"""
Collect a single pillar sls file and render it
"""
err = ""
errors = []
fn_ = self.client.get_state(sls, env)
if not fn_:
errors.append(
("Specified SLS {0} in environment {1} is not" " available on the salt master").format(sls, env)
)
state = None
try:
state = compile_template(fn_, self.rend, self.opts["renderer"], env, sls)
except Exception as exc:
errors.append(("Rendering SLS {0} failed, render error:\n{1}".format(sls, exc)))
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
errors.append(("SLS {0} does not render to a dictionary".format(sls)))
else:
if "include" in state:
if not isinstance(state["include"], list):
err = "Include Declaration in SLS {0} is not formed " "as a list".format(sls)
errors.append(err)
else:
for sub_sls in state.pop("include"):
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(sub_sls, env, mods)
if nstate:
state.update(nstate)
if err:
errors += err
return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
state = None
try:
state = compile_template(fn_, self.rend, self.opts['renderer'],
saltenv, sls, **defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg)
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = sub_sls.iteritems().next()
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
state[key] = nstate
else:
state.update(nstate)
if err:
errors += err
return state, mods, errors
def render(self, template, opts=None, filename=None):
if opts:
self.config.update(opts)
if not filename:
filename = ".".join([str(uuid.uuid4()), "sls"])
full_path = self.write_template_file(filename, template)
state = salt.state.State(self.config)
return compile_template(full_path, state.rend, state.opts["renderer"])
def render(self, template, opts=None, filename=None):
if opts:
self.config.update(opts)
if not filename:
filename = ".".join([str(uuid.uuid4()), "sls"])
full_path = self.write_template_file(filename, template)
state = salt.state.State(self.config)
return compile_template(full_path, state.rend, state.opts['renderer'])
def targets(tgt, tgt_type='glob', **kwargs):
'''
Return the targets from the flat yaml file, checks opts for location but
defaults to /etc/salt/roster
'''
template = get_roster_file(__opts__)
rend = salt.loader.render(__opts__, {})
raw = compile_template(template, rend, __opts__['renderer'], **kwargs)
rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4')
return rmatcher.targets()
def _process_stack_cfg(cfg, stack, minion_id, pillar):
basedir = os.path.dirname(cfg)
items = compile_template(
cfg,
salt.loader.render(__opts__, __salt__),
__opts__['renderer'],
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'],
__opts__['pillarenv'],
minion_id=minion_id,
stack=stack,
)
try:
_ = iter(items['stack'])
except:
return stack
for item in items['stack']:
if not item.strip():
continue # silently ignore whitespace or empty lines
paths = glob(os.path.join(basedir, item))
if not paths:
log.debug(
'Ignoring pillar stack template "{0}": can\'t find from '
'root dir "{1}"'.format(item, basedir)
)
continue
for sls in sorted(paths):
log.debug('Compiling SLS: "{0}"'.format(sls))
obj = compile_template(
sls,
salt.loader.render(__opts__, __salt__),
__opts__['renderer'],
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'],
__opts__['pillarenv'],
minion_id=minion_id,
stack=stack,
)
if obj:
stack = _merge_dict(stack, obj)
return stack
def _render(template, render, renderer, template_dict, opts):
'''
Render a template
'''
if render:
if template_dict is None:
template_dict = {}
if not renderer:
renderer = opts.get('renderer', 'yaml_jinja')
rend = salt.loader.render(opts, {})
return compile_template(template, rend, renderer, **template_dict)
with salt.utils.fopen(template, 'r') as fh_:
return fh_.read()
def render_pstate(self, sls, env, mods):
'''
Collect a single pillar sls file and render it
'''
err = ''
errors = []
fn_ = self.client.get_state(sls, env).get('dest', False)
if not fn_:
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, env)
log.error(msg)
errors.append(msg)
state = None
try:
state = compile_template(
fn_, self.rend, self.opts['renderer'], env, sls)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc
)
log.error(msg)
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
env,
mods
)
if nstate:
state.update(nstate)
if err:
errors += err
return state, mods, errors
def targets(tgt, tgt_type='glob', **kwargs):
'''
Return the targets from the flat yaml file, checks opts for location but
defaults to /etc/salt/roster
'''
if os.path.isfile(__opts__['conf_file']) or not os.path.exists(__opts__['conf_file']):
template = os.path.join(
os.path.dirname(__opts__['conf_file']),
'roster')
else:
template = os.path.join(__opts__['conf_file'], 'roster')
rend = salt.loader.render(__opts__, {})
raw = compile_template(template, rend, __opts__['renderer'], **kwargs)
rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4')
return rmatcher.targets()
def targets(tgt, tgt_type='glob', **kwargs):
'''
Return the targets from the flat yaml file, checks opts for location but
defaults to /etc/salt/roster
'''
template = get_roster_file(__opts__)
rend = salt.loader.render(__opts__, {})
raw = compile_template(template, rend, __opts__['renderer'],
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'], **kwargs)
conditioned_raw = {}
for minion in raw:
conditioned_raw[str(minion)] = salt.config.apply_sdb(raw[minion])
rmatcher = RosterMatcher(conditioned_raw, tgt, tgt_type, 'ipv4')
return rmatcher.targets()
def targets(tgt, tgt_type='glob', **kwargs):
'''
Return the targets from the flat yaml file, checks opts for location but
defaults to /etc/salt/roster
'''
if __opts__.get('roster_file'):
template = __opts__.get('roster_file')
elif os.path.isfile(__opts__['conf_file']) or not os.path.exists(__opts__['conf_file']):
template = os.path.join(
os.path.dirname(__opts__['conf_file']),
'roster')
else:
template = os.path.join(__opts__['conf_file'], 'roster')
rend = salt.loader.render(__opts__, {})
raw = compile_template(template, rend, __opts__['renderer'], **kwargs)
rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4')
return rmatcher.targets()
def targets(tgt, tgt_type='glob', **kwargs):
'''
Return the targets from the flat yaml file, checks opts for location but
defaults to /etc/salt/roster
'''
template = get_roster_file(__opts__)
rend = salt.loader.render(__opts__, {})
raw = compile_template(template,
rend,
__opts__['renderer'],
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'],
**kwargs)
conditioned_raw = {}
for minion in raw:
conditioned_raw[str(minion)] = raw[minion]
rmatcher = RosterMatcher(conditioned_raw, tgt, tgt_type, 'ipv4')
return rmatcher.targets()
def _render(self, data, formula_def):
'''
Render a [pre|post]_local_state or [pre|post]_tgt_state script
'''
# FORMULA can contain a renderer option
renderer = formula_def.get('renderer',
self.opts.get('renderer', 'jinja|yaml'))
rend = salt.loader.render(self.opts, {})
blacklist = self.opts.get('renderer_blacklist')
whitelist = self.opts.get('renderer_whitelist')
template_vars = formula_def.copy()
template_vars['opts'] = self.opts.copy()
return compile_template(':string:',
rend,
renderer,
blacklist,
whitelist,
input_data=data,
**template_vars)
def _render(self, data, formula_def):
"""
Render a [pre|post]_local_state or [pre|post]_tgt_state script
"""
# FORMULA can contain a renderer option
renderer = formula_def.get("renderer",
self.opts.get("renderer", "jinja|yaml"))
rend = salt.loader.render(self.opts, {})
blacklist = self.opts.get("renderer_blacklist")
whitelist = self.opts.get("renderer_whitelist")
template_vars = formula_def.copy()
template_vars["opts"] = self.opts.copy()
return compile_template(":string:",
rend,
renderer,
blacklist,
whitelist,
input_data=data,
**template_vars)
def _render(template, render, renderer, template_dict, opts):
'''
Render a template
'''
if render:
if template_dict is None:
template_dict = {}
if not renderer:
renderer = opts.get('renderer', 'yaml_jinja')
rend = salt.loader.render(opts, {})
blacklist = opts.get('renderer_blacklist')
whitelist = opts.get('renderer_whitelist')
ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict)
ret = ret.read()
if str(ret).startswith('#!') and not str(ret).startswith('#!/'):
ret = str(ret).split('\n', 1)[1]
return ret
with salt.utils.files.fopen(template, 'r') as fh_:
return fh_.read()
def render_pstate(self, sls, env, mods):
'''
Collect a single pillar sls file and render it
'''
err = ''
errors = []
fn_ = self.client.get_state(sls, env)
if not fn_:
errors.append(('Specified SLS {0} in environment {1} is not'
' available on the salt master').format(sls, env))
state = None
try:
state = compile_template(
fn_, self.rend, self.opts['renderer'], env, sls)
except Exception as exc:
errors.append(('Rendering SLS {0} failed, render error:\n{1}'
.format(sls, exc)))
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
errors.append(('SLS {0} does not render to a dictionary'
.format(sls)))
else:
if 'include' in state:
if not isinstance(state['include'], list):
err = ('Include Declaration in SLS {0} is not formed '
'as a list'.format(sls))
errors.append(err)
else:
for sub_sls in state.pop('include'):
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
env,
mods
)
if nstate:
state.update(nstate)
if err:
errors += err
return state, mods, errors
def targets(tgt, tgt_type="glob", **kwargs):
"""
Return the targets from the flat yaml file, checks opts for location but
defaults to /etc/salt/roster
"""
template = get_roster_file(__opts__)
rend = salt.loader.render(__opts__, {})
raw = compile_template(template,
rend,
__opts__["renderer"],
__opts__["renderer_blacklist"],
__opts__["renderer_whitelist"],
mask_value="passw*",
**kwargs)
conditioned_raw = {}
for minion in raw:
conditioned_raw[str(minion)] = salt.config.apply_sdb(raw[minion])
return __utils__["roster_matcher.targets"](conditioned_raw, tgt, tgt_type,
"ipv4")
def _render(template, render, renderer, template_dict, opts):
"""
Render a template
"""
if render:
if template_dict is None:
template_dict = {}
if not renderer:
renderer = opts.get("renderer", "jinja|yaml")
rend = salt.loader.render(opts, {})
blacklist = opts.get("renderer_blacklist")
whitelist = opts.get("renderer_whitelist")
ret = compile_template(template, rend, renderer, blacklist, whitelist,
**template_dict)
if salt.utils.stringio.is_readable(ret):
ret = ret.read()
if str(ret).startswith("#!") and not str(ret).startswith("#!/"):
ret = str(ret).split("\n", 1)[1]
return ret
with salt.utils.files.fopen(template, "r") as fh_:
return fh_.read()
def targets(tgt, tgt_type='glob', **kwargs):
'''
Return the targets from the sls file, checks opts for location but
defaults to /etc/salt/roster
'''
template = get_roster_file(__opts__)
rend = salt.loader.render(__opts__, {})
kwargs['__salt__'] = __runner__
pool = compile_template(template,
rend,
__opts__['renderer'],
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'],
mask_value='passw*',
**kwargs)
pool = {host: {'minion_opts': conf} for host, conf in pool.items()}
pool = salt_sproxy._roster.load_cache(pool,
__runner__,
__opts__,
tgt,
tgt_type=tgt_type)
engine = salt_sproxy._roster.TGT_FUN[tgt_type]
return engine(pool, tgt, opts=__opts__)
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS {0!r} in'
' environment {1!r}'.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
log.debug('Specified SLS {0!r} in environment {1!r} is not'
' found, which might be due to environment {1!r}'
' not being present in "pillar_roots" yet!'
.format(sls, saltenv))
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(
fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc
)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
saltenv,
mods,
defaults
)
if nstate:
if key:
nstate = {
key: nstate
}
state = merge(
state,
nstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
if err:
errors += err
return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS {0!r} in'
' environment {1!r}'.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
log.debug('Specified SLS {0!r} in environment {1!r} is not'
' found, which might be due to environment {1!r}'
' not being present in "pillar_roots" yet!'.format(
sls, saltenv))
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for details.'
.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
nstate = {key: nstate}
state = merge(
state, nstate, self.merge_strategy,
self.opts.get('renderer', 'yaml'))
if err:
errors += err
return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug(
'Skipping ignored and missing SLS \'%s\' in '
'environment \'%s\'', sls, saltenv)
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
'found. '.format(sls, saltenv))
if self.opts.get('__git_pillar', False) is True:
msg += (
'This is likely caused by a git_pillar top file '
'containing an environment other than the one for the '
'branch in which it resides. Each git_pillar '
'branch/tag must have its own top file.')
else:
msg += (
'This could be because SLS \'{0}\' is in an '
'environment other than \'{1}\', but \'{1}\' is '
'included in that environment\'s Pillar top file. It '
'could also be due to environment \'{1}\' not being '
'defined in \'pillar_roots\'.'.format(sls, saltenv))
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg, exc_info=True)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for '
'details.'.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS \'{0}\' is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
# render included state(s)
include_states = []
matched_pstates = []
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
try:
matched_pstates += fnmatch.filter(
self.avail[saltenv], sub_sls)
except KeyError:
errors.extend([
'No matching pillar environment for environment '
'\'{0}\' found'.format(saltenv)
])
for sub_sls in matched_pstates:
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
# If key is x:y, convert it to {x: {y: nstate}}
for key_fragment in reversed(
key.split(":")):
nstate = {key_fragment: nstate}
if not self.opts.get(
'pillar_includes_override_sls',
False):
include_states.append(nstate)
else:
state = merge(
state, nstate, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get(
'pillar_merge_lists', False))
if err:
errors += err
if not self.opts.get('pillar_includes_override_sls',
False):
# merge included state(s) with the current state
# merged last to ensure that its values are
# authoritative.
include_states.append(state)
state = None
for s in include_states:
if state is None:
state = s
else:
state = merge(
state, s, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists',
False))
return state, mods, errors
def get_tops(self):
'''
Gather the top files
'''
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
saltenvs = set()
if self.opts['pillarenv']:
# If the specified pillarenv is not present in the available
# pillar environments, do not cache the pillar top file.
if self.opts['pillarenv'] not in self.opts['pillar_roots']:
log.debug(
'pillarenv \'%s\' not found in the configured pillar '
'environments (%s)', self.opts['pillarenv'],
', '.join(self.opts['pillar_roots']))
else:
saltenvs.add(self.opts['pillarenv'])
else:
saltenvs = self._get_envs()
if self.opts.get('pillar_source_merging_strategy',
None) == "none":
saltenvs &= set([self.saltenv or 'base'])
for saltenv in saltenvs:
top = self.client.cache_file(self.opts['state_top'], saltenv)
if top:
tops[saltenv].append(
compile_template(
top,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv=saltenv,
_pillar_rend=True,
))
except Exception as exc:
errors.append(
('Rendering Primary Top file failed, render error:\n{0}'.
format(exc)))
log.exception('Pillar rendering failed for minion %s',
self.minion_id)
# Search initial top files for includes
for saltenv, ctops in six.iteritems(tops):
for ctop in ctops:
if 'include' not in ctop:
continue
for sls in ctop['include']:
include[saltenv].append(sls)
ctop.pop('include')
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in six.iteritems(include):
pops.append(saltenv)
if not states:
continue
for sls in states:
if sls in done[saltenv]:
continue
try:
tops[saltenv].append(
compile_template(
self.client.get_state(sls, saltenv).get(
'dest', False),
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv=saltenv,
_pillar_rend=True,
))
except Exception as exc:
errors.append(
('Rendering Top file {0} failed, render error'
':\n{1}').format(sls, exc))
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops, errors
def get_tops(self):
'''
Gather the top files
'''
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
if self.opts['environment']:
tops[self.opts['environment']] = [
compile_template(
self.client.cache_file(
self.opts['state_top'],
self.opts['environment']
),
self.rend,
self.opts['renderer'],
self.opts['environment']
)
]
else:
for env in self._get_envs():
tops[env].append(
compile_template(
self.client.cache_file(
self.opts['state_top'],
env
),
self.rend,
self.opts['renderer'],
env=env
)
)
except Exception as exc:
errors.append(
('Rendering Primary Top file failed, render error:\n{0}'
.format(exc)))
# Search initial top files for includes
for env, ctops in tops.items():
for ctop in ctops:
if not 'include' in ctop:
continue
for sls in ctop['include']:
include[env].append(sls)
ctop.pop('include')
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for env, states in include.items():
pops.append(env)
if not states:
continue
for sls in states:
if sls in done[env]:
continue
try:
tops[env].append(
compile_template(
self.client.get_state(
sls,
env
),
self.rend,
self.opts['renderer'],
env=env
)
)
except Exception as exc:
errors.append(
('Rendering Top file {0} failed, render error'
':\n{1}').format(sls, exc))
done[env].append(sls)
for env in pops:
if env in include:
include.pop(env)
return tops, errors
def returner(ret):
'''
Send an email with the data
'''
_options = _get_options(ret)
from_addr = _options.get('from')
to_addrs = _options.get('to')
host = _options.get('host')
port = _options.get('port')
user = _options.get('username')
passwd = _options.get('password')
subject = _options.get('subject')
gpgowner = _options.get('gpgowner')
fields = _options.get('fields').split(',') if 'fields' in _options else []
smtp_tls = _options.get('tls')
renderer = _options.get('renderer', __opts__.get('renderer', 'yaml_jinja'))
rend = salt.loader.render(__opts__, {})
if not port:
port = 25
log.debug('SMTP port has been set to {0}'.format(port))
for field in fields:
if field in ret:
subject += ' {0}'.format(ret[field])
subject = compile_template(':string:', rend, renderer, input_data=subject, **ret)
log.debug("smtp_return: Subject is '{0}'".format(subject))
template = _options.get('template')
if template:
content = compile_template(template, rend, renderer, **ret)
else:
template = ('id: {{id}}\r\n'
'function: {{fun}}\r\n'
'function args: {{fun_args}}\r\n'
'jid: {{jid}}\r\n'
'return: {{return}}\r\n')
content = compile_template(':string:', rend, renderer, input_data=template, **ret)
if HAS_GNUPG and gpgowner:
gpg = gnupg.GPG(gnupghome=os.path.expanduser('~{0}/.gnupg'.format(gpgowner)),
options=['--trust-model always'])
encrypted_data = gpg.encrypt(content, to_addrs)
if encrypted_data.ok:
log.debug('smtp_return: Encryption successful')
content = str(encrypted_data)
else:
log.error('smtp_return: Encryption failed, only an error message will be sent')
content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format(
encrypted_data.status, encrypted_data.stderr)
message = ('From: {0}\r\n'
'To: {1}\r\n'
'Date: {2}\r\n'
'Subject: {3}\r\n'
'\r\n'
'{4}').format(from_addr,
to_addrs,
formatdate(localtime=True),
subject,
content)
log.debug('smtp_return: Connecting to the server...')
server = smtplib.SMTP(host, int(port))
server.set_debuglevel = 'debug'
if smtp_tls is True:
server.starttls()
log.debug('smtp_return: TLS enabled')
if user and passwd:
server.login(user, passwd)
log.debug('smtp_return: Authenticated')
server.sendmail(from_addr, to_addrs, message)
log.debug('smtp_return: Message sent.')
server.quit()
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
log.debug('Specified SLS {0!r} in environment {1!r} is not'
' found, which might be due to environment {1!r}'
' not being present in "pillar_roots" yet!'
.format(sls, saltenv))
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(
fn_, self.rend, self.opts['renderer'], saltenv, sls, **defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc
)
log.critical(msg)
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = sub_sls.iteritems().next()
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
saltenv,
mods,
defaults
)
if nstate:
if key:
state[key] = nstate
else:
state.update(nstate)
if err:
errors += err
return state, mods, errors
def get_tops(self):
'''
Gather the top files
'''
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
if self.opts['environment']:
tops[self.opts['environment']] = [
compile_template(
self.client.cache_file(self.opts['state_top'],
self.opts['environment']),
self.rend, self.opts['renderer'],
self.opts['environment'])
]
else:
for saltenv in self._get_envs():
tops[saltenv].append(
compile_template(self.client.cache_file(
self.opts['state_top'], saltenv),
self.rend,
self.opts['renderer'],
saltenv=saltenv))
except Exception as exc:
errors.append(
('Rendering Primary Top file failed, render error:\n{0}'.
format(exc)))
# Search initial top files for includes
for saltenv, ctops in tops.items():
for ctop in ctops:
if 'include' not in ctop:
continue
for sls in ctop['include']:
include[saltenv].append(sls)
ctop.pop('include')
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in include.items():
pops.append(saltenv)
if not states:
continue
for sls in states:
if sls in done[saltenv]:
continue
try:
tops[saltenv].append(
compile_template(self.client.get_state(
sls, saltenv).get('dest', False),
self.rend,
self.opts['renderer'],
saltenv=saltenv))
except Exception as exc:
errors.append(
('Rendering Top file {0} failed, render error'
':\n{1}').format(sls, exc))
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops, errors
def get_tops(self):
'''
Gather the top files
'''
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
if self.opts['pillarenv']:
tops[self.opts['pillarenv']] = [
compile_template(
self.client.cache_file(self.opts['state_top'],
self.opts['pillarenv']),
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
self.opts['pillarenv'],
_pillar_rend=True,
)
]
else:
for saltenv in self._get_envs():
if self.opts.get('pillar_source_merging_strategy',
None) == "none":
if self.saltenv and saltenv != self.saltenv:
continue
if not self.saltenv and not saltenv == 'base':
continue
top = self.client.cache_file(self.opts['state_top'],
saltenv)
if top:
tops[saltenv].append(
compile_template(
top,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv=saltenv,
_pillar_rend=True,
))
except Exception as exc:
errors.append(
('Rendering Primary Top file failed, render error:\n{0}'.
format(exc)))
log.error('Pillar rendering failed for minion {0}: '.format(
self.minion_id),
exc_info=True)
# Search initial top files for includes
for saltenv, ctops in six.iteritems(tops):
for ctop in ctops:
if 'include' not in ctop:
continue
for sls in ctop['include']:
include[saltenv].append(sls)
ctop.pop('include')
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in six.iteritems(include):
pops.append(saltenv)
if not states:
continue
for sls in states:
if sls in done[saltenv]:
continue
try:
tops[saltenv].append(
compile_template(
self.client.get_state(sls, saltenv).get(
'dest', False),
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv=saltenv,
_pillar_rend=True,
))
except Exception as exc:
errors.append(
('Rendering Top file {0} failed, render error'
':\n{1}').format(sls, exc))
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS \'{0}\' in'
' environment \'{1}\''.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
'found. '.format(sls, saltenv))
if self.opts.get('__git_pillar', False) is True:
msg += (
'This is likely caused by a git_pillar top file '
'containing an environment other than the one for the '
'branch in which it resides. Each git_pillar '
'branch/tag must have its own top file.'
)
else:
msg += (
'This could be because SLS \'{0}\' is in an '
'environment other than \'{1}\', but \'{1}\' is '
'included in that environment\'s Pillar top file. It '
'could also be due to environment \'{1}\' not being '
'defined in \'pillar_roots\'.'.format(sls, saltenv)
)
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format(
sls, exc
)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for '
'details.'.format(sls)
)
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS \'{0}\' is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
saltenv,
mods,
defaults
)
if nstate:
if key:
nstate = {
key: nstate
}
state = merge(
state,
nstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
if err:
errors += err
return state, mods, errors
def get_tops(self):
'''
Gather the top files
'''
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
if self.opts['pillarenv']:
tops[self.opts['pillarenv']] = [
compile_template(
self.client.cache_file(
self.opts['state_top'],
self.opts['pillarenv']
),
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
self.opts['pillarenv'],
_pillar_rend=True,
)
]
else:
for saltenv in self._get_envs():
if self.opts.get('pillar_source_merging_strategy', None) == "none":
if self.saltenv and saltenv != self.saltenv:
continue
if not self.saltenv and not saltenv == 'base':
continue
top = self.client.cache_file(
self.opts['state_top'],
saltenv
)
if top:
tops[saltenv].append(
compile_template(
top,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv=saltenv,
_pillar_rend=True,
)
)
except Exception as exc:
errors.append(
('Rendering Primary Top file failed, render error:\n{0}'
.format(exc)))
log.error('Pillar rendering failed for minion {0}: '.format(self.minion_id),
exc_info=True)
# Search initial top files for includes
for saltenv, ctops in six.iteritems(tops):
for ctop in ctops:
if 'include' not in ctop:
continue
for sls in ctop['include']:
include[saltenv].append(sls)
ctop.pop('include')
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in six.iteritems(include):
pops.append(saltenv)
if not states:
continue
for sls in states:
if sls in done[saltenv]:
continue
try:
tops[saltenv].append(
compile_template(
self.client.get_state(
sls,
saltenv
).get('dest', False),
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv=saltenv,
_pillar_rend=True,
)
)
except Exception as exc:
errors.append(
('Rendering Top file {0} failed, render error'
':\n{1}').format(sls, exc))
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
"""
Collect a single pillar sls file and render it
"""
if defaults is None:
defaults = {}
err = ""
errors = []
fn_ = self.client.get_state(sls, saltenv).get("dest", False)
if not fn_:
if self.opts["pillar_roots"].get(saltenv):
msg = ("Specified SLS {0!r} in environment {1!r} is not" " available on the salt master").format(
sls, saltenv
)
log.error(msg)
errors.append(msg)
else:
log.debug(
"Specified SLS {0!r} in environment {1!r} is not"
" found, which might be due to environment {1!r}"
' not being present in "pillar_roots" yet!'.format(sls, saltenv)
)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_, self.rend, self.opts["renderer"], saltenv, sls, **defaults)
except Exception as exc:
msg = "Rendering SLS {0!r} failed, render error:\n{1}".format(sls, exc)
log.critical(msg)
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = "SLS {0!r} does not render to a dictionary".format(sls)
log.error(msg)
errors.append(msg)
else:
if "include" in state:
if not isinstance(state["include"], list):
msg = "Include Declaration in SLS {0!r} is not " "formed as a list".format(sls)
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop("include"):
if isinstance(sub_sls, dict):
sub_sls, v = sub_sls.iteritems().next()
defaults = v.get("defaults", {})
key = v.get("key", None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(sub_sls, saltenv, mods, defaults)
if nstate:
if key:
state[key] = nstate
else:
state.update(nstate)
if err:
errors += err
return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS {0!r} in'
' environment {1!r}'.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
'found. '.format(sls, saltenv))
if self.opts.get('__git_pillar', False) is True:
msg += (
'This is likely caused by a git_pillar top file '
'containing an environment other than the one for the '
'branch in which it resides. Each git_pillar '
'branch/tag must have its own top file.')
else:
msg += (
'This could be because SLS \'{0}\' is in an '
'environment other than \'{1}\', but \'{1}\' is '
'included in that environment\'s Pillar top file. It '
'could also be due to environment \'{1}\' not being '
'defined in \'pillar_roots\'.'.format(sls, saltenv))
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for details.'
.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
nstate = {key: nstate}
state = merge(
state, nstate, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists',
False))
if err:
errors += err
return state, mods, errors
def returner(ret):
'''
Send an email with the data
'''
_options = _get_options(ret)
from_addr = _options.get('from')
to_addrs = _options.get('to').split(',')
host = _options.get('host')
port = _options.get('port')
user = _options.get('username')
passwd = _options.get('password')
subject = _options.get('subject') or 'Email from Salt'
gpgowner = _options.get('gpgowner')
fields = _options.get('fields').split(',') if 'fields' in _options else []
smtp_tls = _options.get('tls')
renderer = _options.get('renderer') or 'jinja'
rend = salt.loader.render(__opts__, {})
blacklist = __opts__.get('renderer_blacklist')
whitelist = __opts__.get('renderer_whitelist')
if not port:
port = 25
log.debug('SMTP port has been set to {0}'.format(port))
for field in fields:
if field in ret:
subject += ' {0}'.format(ret[field])
subject = compile_template(':string:',
rend,
renderer,
blacklist,
whitelist,
input_data=subject,
**ret)
if isinstance(subject, six.moves.StringIO):
subject = subject.read()
log.debug("smtp_return: Subject is '{0}'".format(subject))
template = _options.get('template')
if template:
content = compile_template(template, rend, renderer, blacklist,
whitelist, **ret)
else:
template = ('id: {{id}}\r\n'
'function: {{fun}}\r\n'
'function args: {{fun_args}}\r\n'
'jid: {{jid}}\r\n'
'return: {{return}}\r\n')
content = compile_template(':string:',
rend,
renderer,
blacklist,
whitelist,
input_data=template,
**ret)
if gpgowner:
if HAS_GNUPG:
gpg = gnupg.GPG(gnupghome=os.path.expanduser(
'~{0}/.gnupg'.format(gpgowner)),
options=['--trust-model always'])
encrypted_data = gpg.encrypt(content, to_addrs)
if encrypted_data.ok:
log.debug('smtp_return: Encryption successful')
content = str(encrypted_data)
else:
log.error(
'smtp_return: Encryption failed, only an error message will be sent'
)
content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format(
encrypted_data.status, encrypted_data.stderr)
else:
log.error(
"gnupg python module is required in order to user gpgowner in smtp returner ; ignoring gpgowner configuration for now"
)
if isinstance(content, six.moves.StringIO):
content = content.read()
message = ('From: {0}\r\n'
'To: {1}\r\n'
'Date: {2}\r\n'
'Subject: {3}\r\n'
'\r\n'
'{4}').format(from_addr, ', '.join(to_addrs),
formatdate(localtime=True), subject, content)
log.debug('smtp_return: Connecting to the server...')
server = smtplib.SMTP(host, int(port))
if smtp_tls is True:
server.starttls()
log.debug('smtp_return: TLS enabled')
if user and passwd:
server.login(user, passwd)
log.debug('smtp_return: Authenticated')
# enable logging SMTP session after the login credentials were passed
server.set_debuglevel(1)
server.sendmail(from_addr, to_addrs, message)
log.debug('smtp_return: Message sent.')
server.quit()
def get_tops(self):
"""
Gather the top files
"""
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
saltenvs = set()
if self.opts["pillarenv"]:
# If the specified pillarenv is not present in the available
# pillar environments, do not cache the pillar top file.
if self.opts["pillarenv"] not in self.opts["pillar_roots"]:
log.debug(
"pillarenv '%s' not found in the configured pillar "
"environments (%s)",
self.opts["pillarenv"],
", ".join(self.opts["pillar_roots"]),
)
else:
saltenvs.add(self.opts["pillarenv"])
else:
saltenvs = self._get_envs()
if self.opts.get("pillar_source_merging_strategy",
None) == "none":
saltenvs &= {self.saltenv or "base"}
for saltenv in saltenvs:
top = self.client.cache_file(self.opts["state_top"], saltenv)
if top:
tops[saltenv].append(
compile_template(
top,
self.rend,
self.opts["renderer"],
self.opts["renderer_blacklist"],
self.opts["renderer_whitelist"],
saltenv=saltenv,
_pillar_rend=True,
))
except Exception as exc: # pylint: disable=broad-except
errors.append(
"Rendering Primary Top file failed, render error:\n{}".format(
exc))
log.exception("Pillar rendering failed for minion %s",
self.minion_id)
# Search initial top files for includes
for saltenv, ctops in tops.items():
for ctop in ctops:
if "include" not in ctop:
continue
for sls in ctop["include"]:
include[saltenv].append(sls)
ctop.pop("include")
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in include.items():
pops.append(saltenv)
if not states:
continue
for sls in states:
if sls in done[saltenv]:
continue
try:
tops[saltenv].append(
compile_template(
self.client.get_state(sls, saltenv).get(
"dest", False),
self.rend,
self.opts["renderer"],
self.opts["renderer_blacklist"],
self.opts["renderer_whitelist"],
saltenv=saltenv,
_pillar_rend=True,
))
except Exception as exc: # pylint: disable=broad-except
errors.append(
"Rendering Top file {} failed, render error:\n{}".
format(sls, exc))
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops, errors
def returner(ret):
'''
Send an email with the data
'''
_options = _get_options(ret)
from_addr = _options.get('from')
to_addrs = _options.get('to')
host = _options.get('host')
port = _options.get('port')
user = _options.get('username')
passwd = _options.get('password')
subject = _options.get('subject')
gpgowner = _options.get('gpgowner')
fields = _options.get('fields').split(',') if 'fields' in _options else []
smtp_tls = _options.get('tls')
renderer = _options.get('renderer', __opts__.get('renderer', 'yaml_jinja'))
rend = salt.loader.render(__opts__, {})
if not port:
port = 25
log.debug('SMTP port has been set to {0}'.format(port))
for field in fields:
if field in ret:
subject += ' {0}'.format(ret[field])
subject = compile_template(':string:',
rend,
renderer,
input_data=subject,
**ret)
log.debug("smtp_return: Subject is '{0}'".format(subject))
template = _options.get('template')
if template:
content = compile_template(template, rend, renderer, **ret)
else:
template = ('id: {{id}}\r\n'
'function: {{fun}}\r\n'
'function args: {{fun_args}}\r\n'
'jid: {{jid}}\r\n'
'return: {{return}}\r\n')
content = compile_template(':string:',
rend,
renderer,
input_data=template,
**ret)
if HAS_GNUPG and gpgowner:
gpg = gnupg.GPG(gnupghome=os.path.expanduser(
'~{0}/.gnupg'.format(gpgowner)),
options=['--trust-model always'])
encrypted_data = gpg.encrypt(content, to_addrs)
if encrypted_data.ok:
log.debug('smtp_return: Encryption successful')
content = str(encrypted_data)
else:
log.error(
'smtp_return: Encryption failed, only an error message will be sent'
)
content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format(
encrypted_data.status, encrypted_data.stderr)
message = ('From: {0}\r\n'
'To: {1}\r\n'
'Date: {2}\r\n'
'Subject: {3}\r\n'
'\r\n'
'{4}').format(from_addr, to_addrs, formatdate(localtime=True),
subject, content)
log.debug('smtp_return: Connecting to the server...')
server = smtplib.SMTP(host, int(port))
server.set_debuglevel = 'debug'
if smtp_tls is True:
server.starttls()
log.debug('smtp_return: TLS enabled')
if user and passwd:
server.login(user, passwd)
log.debug('smtp_return: Authenticated')
server.sendmail(from_addr, to_addrs, message)
log.debug('smtp_return: Message sent.')
server.quit()
def returner(ret):
"""
Send an email with the data
"""
_options = _get_options(ret)
from_addr = _options.get("from")
to_addrs = _options.get("to").split(",")
host = _options.get("host")
port = _options.get("port")
user = _options.get("username")
passwd = _options.get("password")
subject = _options.get("subject") or "Email from Salt"
gpgowner = _options.get("gpgowner")
fields = _options.get("fields").split(",") if "fields" in _options else []
smtp_tls = _options.get("tls")
renderer = _options.get("renderer") or "jinja"
rend = salt.loader.render(__opts__, {})
blacklist = __opts__.get("renderer_blacklist")
whitelist = __opts__.get("renderer_whitelist")
if not port:
port = 25
log.debug("SMTP port has been set to %s", port)
for field in fields:
if field in ret:
subject += " {0}".format(ret[field])
subject = compile_template(
":string:", rend, renderer, blacklist, whitelist, input_data=subject, **ret
)
if isinstance(subject, six.moves.StringIO):
subject = subject.read()
log.debug("smtp_return: Subject is '%s'", subject)
template = _options.get("template")
if template:
content = compile_template(
template, rend, renderer, blacklist, whitelist, **ret
)
else:
template = (
"id: {{id}}\r\n"
"function: {{fun}}\r\n"
"function args: {{fun_args}}\r\n"
"jid: {{jid}}\r\n"
"return: {{return}}\r\n"
)
content = compile_template(
":string:", rend, renderer, blacklist, whitelist, input_data=template, **ret
)
if gpgowner:
if HAS_GNUPG:
gpg = gnupg.GPG(
gnupghome=os.path.expanduser("~{0}/.gnupg".format(gpgowner)),
options=["--trust-model always"],
)
encrypted_data = gpg.encrypt(content, to_addrs)
if encrypted_data.ok:
log.debug("smtp_return: Encryption successful")
content = six.text_type(encrypted_data)
else:
log.error(
"smtp_return: Encryption failed, only an error message will be sent"
)
content = "Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}".format(
encrypted_data.status, encrypted_data.stderr
)
else:
log.error(
"gnupg python module is required in order to user gpgowner in smtp returner ; ignoring gpgowner configuration for now"
)
if isinstance(content, six.moves.StringIO):
content = content.read()
message = (
"From: {0}\r\n" "To: {1}\r\n" "Date: {2}\r\n" "Subject: {3}\r\n" "\r\n" "{4}"
).format(
from_addr, ", ".join(to_addrs), formatdate(localtime=True), subject, content
)
log.debug("smtp_return: Connecting to the server...")
server = smtplib.SMTP(host, int(port))
if smtp_tls is True:
server.starttls()
log.debug("smtp_return: TLS enabled")
if user and passwd:
server.login(user, passwd)
log.debug("smtp_return: Authenticated")
# enable logging SMTP session after the login credentials were passed
server.set_debuglevel(1)
server.sendmail(from_addr, to_addrs, message)
log.debug("smtp_return: Message sent.")
server.quit()
def render_pstate(self, sls, saltenv, mods, defaults=None):
"""
Collect a single pillar sls file and render it
"""
if defaults is None:
defaults = {}
err = ""
errors = []
state_data = self.client.get_state(sls, saltenv)
fn_ = state_data.get("dest", False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug(
"Skipping ignored and missing SLS '%s' in environment '%s'",
sls,
saltenv,
)
return None, mods, errors
elif self.opts["pillar_roots"].get(saltenv):
msg = ("Specified SLS '{}' in environment '{}' is not"
" available on the salt master".format(sls, saltenv))
log.error(msg)
errors.append(msg)
else:
msg = "Specified SLS '{}' in environment '{}' was not found. ".format(
sls, saltenv)
if self.opts.get("__git_pillar", False) is True:
msg += (
"This is likely caused by a git_pillar top file "
"containing an environment other than the one for the "
"branch in which it resides. Each git_pillar "
"branch/tag must have its own top file.")
else:
msg += (
"This could be because SLS '{0}' is in an "
"environment other than '{1}', but '{1}' is "
"included in that environment's Pillar top file. It "
"could also be due to environment '{1}' not being "
"defined in 'pillar_roots'.".format(sls, saltenv))
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts["renderer"],
self.opts["renderer_blacklist"],
self.opts["renderer_whitelist"],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc: # pylint: disable=broad-except
msg = "Rendering SLS '{}' failed, render error:\n{}".format(
sls, exc)
log.critical(msg, exc_info=True)
if self.opts.get("pillar_safe_render_error", True):
errors.append(
"Rendering SLS '{}' failed. Please see master log for "
"details.".format(sls))
else:
errors.append(msg)
mods[sls] = state
nstate = None
if state:
if not isinstance(state, dict):
msg = "SLS '{}' does not render to a dictionary".format(sls)
log.error(msg)
errors.append(msg)
else:
if "include" in state:
if not isinstance(state["include"], list):
msg = ("Include Declaration in SLS '{}' is not "
"formed as a list".format(sls))
log.error(msg)
errors.append(msg)
else:
# render included state(s)
include_states = []
for sub_sls in state.pop("include"):
if isinstance(sub_sls, dict):
sub_sls, v = next(iter(sub_sls.items()))
defaults = v.get("defaults", {})
key = v.get("key", None)
else:
key = None
try:
matched_pstates = fnmatch.filter(
self.avail[saltenv],
sub_sls.lstrip(".").replace("/", "."),
)
if sub_sls.startswith("."):
if state_data.get(
"source",
"").endswith("/init.sls"):
include_parts = sls.split(".")
else:
include_parts = sls.split(".")[:-1]
sub_sls = ".".join(include_parts +
[sub_sls[1:]])
matches = fnmatch.filter(
self.avail[saltenv],
sub_sls,
)
matched_pstates.extend(matches)
except KeyError:
errors.extend([
"No matching pillar environment for environment"
" '{}' found".format(saltenv)
])
matched_pstates = [sub_sls]
# If matched_pstates is empty, set to sub_sls
if len(matched_pstates) < 1:
matched_pstates = [sub_sls]
for m_sub_sls in matched_pstates:
if m_sub_sls not in mods:
nstate, mods, err = self.render_pstate(
m_sub_sls, saltenv, mods, defaults)
else:
nstate = mods[m_sub_sls]
if nstate:
if key:
# If key is x:y, convert it to {x: {y: nstate}}
for key_fragment in reversed(
key.split(":")):
nstate = {key_fragment: nstate}
if not self.opts.get(
"pillar_includes_override_sls",
False):
include_states.append(nstate)
else:
state = merge(
state,
nstate,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get(
"pillar_merge_lists", False),
)
if err:
errors += err
if not self.opts.get("pillar_includes_override_sls",
False):
# merge included state(s) with the current state
# merged last to ensure that its values are
# authoritative.
include_states.append(state)
state = None
for s in include_states:
if state is None:
state = s
else:
state = merge(
state,
s,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get("pillar_merge_lists",
False),
)
return state, mods, errors
def test_compile_template_bad_type(self):
'''
Test to ensure that unsupported types cannot be passed to the template compiler
'''
ret = template.compile_template(['1', '2', '3'], None, None)
self.assertDictEqual(ret, {})
def get_tops(self):
"""
Gather the top files
"""
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
errors = []
# Gather initial top files
try:
if self.opts["environment"]:
tops[self.opts["environment"]] = [
compile_template(
self.client.cache_file(self.opts["state_top"], self.opts["environment"]),
self.rend,
self.opts["renderer"],
self.opts["environment"],
)
]
else:
for saltenv in self._get_envs():
tops[saltenv].append(
compile_template(
self.client.cache_file(self.opts["state_top"], saltenv),
self.rend,
self.opts["renderer"],
saltenv=saltenv,
)
)
except Exception as exc:
errors.append(("Rendering Primary Top file failed, render error:\n{0}".format(exc)))
# Search initial top files for includes
for saltenv, ctops in tops.items():
for ctop in ctops:
if "include" not in ctop:
continue
for sls in ctop["include"]:
include[saltenv].append(sls)
ctop.pop("include")
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in include.items():
pops.append(saltenv)
if not states:
continue
for sls in states:
if sls in done[saltenv]:
continue
try:
tops[saltenv].append(
compile_template(
self.client.get_state(sls, saltenv).get("dest", False),
self.rend,
self.opts["renderer"],
saltenv=saltenv,
)
)
except Exception as exc:
errors.append(("Rendering Top file {0} failed, render error" ":\n{1}").format(sls, exc))
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops, errors
