def ext_pillar(self, pillar, pillar_dirs, errors=None):
'''
Render the external pillar data
'''
if errors is None:
errors = []
if 'ext_pillar' not in self.opts:
return pillar, errors
if not isinstance(self.opts['ext_pillar'], list):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return pillar, errors
ext = None
# Bring in CLI pillar data
if self.pillar_override and isinstance(self.pillar_override, dict):
pillar = merge(pillar,
self.pillar_override,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
for run in self.opts['ext_pillar']:
if not isinstance(run, dict):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return {}, errors
if next(six.iterkeys(run)) in self.opts.get('exclude_ext_pillar', []):
continue
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
log.critical(
'Specified ext_pillar interface {0} is '
'unavailable'.format(key)
)
continue
try:
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except Exception as exc:
errors.append('Failed to load ext_pillar {0}: {1}'.format(
key, exc))
if ext:
pillar = merge(
pillar,
ext,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
ext = None
return pillar, errors
def compile_pillar(self, ext=True, pillar_dirs=None):
'''
Render the pillar data and return
'''
top, top_errors = self.get_top()
if ext:
if self.opts.get('ext_pillar_first', False):
self.opts['pillar'] = self.ext_pillar({}, pillar_dirs)
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
pillar = merge(pillar,
self.opts['pillar'],
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
pillar = self.ext_pillar(pillar, pillar_dirs)
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
errors.extend(top_errors)
if self.opts.get('pillar_opts', True):
mopts = dict(self.opts)
if 'grains' in mopts:
mopts.pop('grains')
# Restore the actual file_roots path. Issue 5449
mopts['file_roots'] = self.actual_file_roots
mopts['saltversion'] = __version__
pillar['master'] = mopts
if errors:
for error in errors:
log.critical('Pillar render error: {0}'.format(error))
pillar['_errors'] = errors
return pillar
def render_pillar(self, matches):
'''
Extract the sls pillar files from the matches and render them into the
pillar
'''
pillar = copy.copy(self.pillar_override)
errors = []
for saltenv, pstates in matches.items():
mods = set()
for sls in pstates:
pstate, mods, err = self.render_pstate(sls, saltenv, mods)
if err:
errors += err
if pstate is not None:
if not isinstance(pstate, dict):
log.error(
'The rendered pillar sls file, {0!r} state did '
'not return the expected data format. This is '
'a sign of a malformed pillar sls file. Returned '
'errors: {1}'.format(
sls,
', '.join(['{0!r}'.format(e) for e in errors])
)
)
continue
pillar = merge(
pillar,
pstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
return pillar, errors
def compile_pillar(self, ext=True):
'''
Render the pillar data and return
'''
top, top_errors = self.get_top()
if ext:
if self.opts.get('ext_pillar_first', False):
self.opts['pillar'], errors = self.ext_pillar(
self.pillar_override)
self.rend = salt.loader.render(self.opts, self.functions)
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches, errors=errors)
pillar = merge(self.opts['pillar'], pillar,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
pillar, errors = self.ext_pillar(pillar, errors=errors)
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
errors.extend(top_errors)
if self.opts.get('pillar_opts', False):
mopts = dict(self.opts)
if 'grains' in mopts:
mopts.pop('grains')
# Restore the actual file_roots path. Issue 5449
mopts['file_roots'] = self.actual_file_roots
mopts['saltversion'] = __version__
pillar['master'] = mopts
if errors:
for error in errors:
log.critical('Pillar render error: %s', error)
pillar['_errors'] = errors
if self.pillar_override:
pillar = merge(pillar, self.pillar_override, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
decrypt_errors = self.decrypt_pillar(pillar)
if decrypt_errors:
pillar.setdefault('_errors', []).extend(decrypt_errors)
return pillar
def ext_pillar(self, pillar, pillar_dirs):
'''
Render the external pillar data
'''
if 'ext_pillar' not in self.opts:
return pillar
if not isinstance(self.opts['ext_pillar'], list):
log.critical('The "ext_pillar" option is malformed')
return pillar
ext = None
# Bring in CLI pillar data
pillar.update(self.pillar_override)
for run in self.opts['ext_pillar']:
if run in self.opts.get('exclude_ext_pillar', []):
continue
if not isinstance(run, dict):
log.critical('The "ext_pillar" option is malformed')
return {}
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
err = ('Specified ext_pillar interface {0} is '
'unavailable').format(key)
log.critical(err)
continue
try:
try:
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except TypeError as exc:
if str(exc).startswith('ext_pillar() takes exactly '):
log.warning('Deprecation warning: ext_pillar "{0}"'
' needs to accept minion_id as first'
' argument'.format(key))
else:
raise
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except Exception as exc:
log.exception(
'Failed to load ext_pillar {0}: {1}'.format(
key,
exc
)
)
if ext:
pillar = merge(
pillar,
ext,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', 'False'))
ext = None
return pillar
def __merge(self, *items):
'''
Merge 'items' according to Pillar's merge strategy and other options.
'''
return six.moves.reduce(
lambda a, b: merge(a, b, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False)),
items)
def init(opts):
log.debug('heroku proxy init() called...')
log.debug('Validating heroku proxy input')
proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {}))
log.trace('proxy_conf = %s', proxy_conf)
DETAILS['proxytype'] = proxy_conf['proxytype']
DETAILS['initialized'] = True
return True
def ext_pillar(self, pillar, pillar_dirs, errors=None):
'''
Render the external pillar data
'''
if errors is None:
errors = []
if 'ext_pillar' not in self.opts:
return pillar, errors
if not isinstance(self.opts['ext_pillar'], list):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return pillar, errors
ext = None
# Bring in CLI pillar data
if self.pillar_override and isinstance(self.pillar_override, dict):
pillar = merge(pillar, self.pillar_override, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
for run in self.opts['ext_pillar']:
if not isinstance(run, dict):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return {}, errors
if next(six.iterkeys(run)) in self.opts.get(
'exclude_ext_pillar', []):
continue
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
log.critical('Specified ext_pillar interface {0} is '
'unavailable'.format(key))
continue
try:
ext = self._external_pillar_data(pillar, val, pillar_dirs,
key)
except Exception as exc:
errors.append('Failed to load ext_pillar {0}: {1}'.format(
key, exc))
if ext:
pillar = merge(pillar, ext, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
ext = None
return pillar, errors
def init(opts):
'''
This function gets called when the proxy starts up. For
login the protocol and port are cached.
'''
log.debug('Initting esxvm proxy module in process '
'{}'.format(os.getpid()))
log.debug('Validating esxvm proxy input')
proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {}))
log.trace('proxy_conf = {0}'.format(proxy_conf))
# TODO json schema validation
# Save mandatory fields in cache
for key in ('vcenter', 'datacenter', 'mechanism'):
DETAILS[key] = proxy_conf[key]
# Additional validation
if DETAILS['mechanism'] == 'userpass':
if 'username' not in proxy_conf:
raise excs.InvalidProxyInputError(
'Mechanism is set to \'userpass\' , but no '
'\'username\' key found in pillar for this proxy.')
if 'passwords' not in proxy_conf:
raise excs.InvalidProxyInputError(
'Mechanism is set to \'userpass\' , but no '
'\'passwords\' key found in pillar for this proxy.')
for key in ('username', 'passwords'):
DETAILS[key] = proxy_conf[key]
else:
if 'domain' not in proxy_conf:
raise excs.InvalidProxyInputError(
'Mechanism is set to \'sspi\' , but no '
'\'domain\' key found in pillar for this proxy.')
if 'principal' not in proxy_conf:
raise excs.InvalidProxyInputError(
'Mechanism is set to \'sspi\' , but no '
'\'principal\' key found in pillar for this proxy.')
for key in ('domain', 'principal'):
DETAILS[key] = proxy_conf[key]
# Save optional
DETAILS['protocol'] = proxy_conf.get('protocol')
DETAILS['port'] = proxy_conf.get('port')
# Test connection
if DETAILS['mechanism'] == 'userpass':
# Get the correct login details
log.debug('Retrieving credentials and testing vCenter connection for '
'mehchanism \'userpass\'')
try:
username, password = find_credentials()
DETAILS['password'] = password
except excs.SaltSystemExit as err:
log.critical('Error: {0}'.format(err))
return False
return True
def compile_pillar(self, ext=True, pillar_dirs=None):
'''
Render the pillar data and return
'''
top, top_errors = self.get_top()
if ext:
if self.opts.get('ext_pillar_first', False):
self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs)
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches, errors=errors)
pillar = merge(self.opts['pillar'], pillar,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
pillar, errors = self.ext_pillar(pillar,
pillar_dirs,
errors=errors)
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)
errors.extend(top_errors)
if self.opts.get('pillar_opts', False):
mopts = dict(self.opts)
if 'grains' in mopts:
mopts.pop('grains')
# Restore the actual file_roots path. Issue 5449
mopts['file_roots'] = self.actual_file_roots
mopts['saltversion'] = __version__
pillar['master'] = mopts
if errors:
for error in errors:
log.critical('Pillar render error: {0}'.format(error))
pillar['_errors'] = errors
if self.pillar_override and isinstance(self.pillar_override, dict):
pillar = merge(pillar, self.pillar_override, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
return pillar
def init(opts):
"""
This function gets called when the proxy starts up. For
login the protocol and port are cached.
"""
log.debug("Initting esxvm proxy module in process %s", os.getpid())
log.debug("Validating esxvm proxy input")
proxy_conf = merge(opts.get("proxy", {}), __pillar__.get("proxy", {}))
log.trace("proxy_conf = %s", proxy_conf)
# TODO json schema validation
# Save mandatory fields in cache
for key in ("vcenter", "datacenter", "mechanism"):
DETAILS[key] = proxy_conf[key]
# Additional validation
if DETAILS["mechanism"] == "userpass":
if "username" not in proxy_conf:
raise excs.InvalidProxyInputError(
"Mechanism is set to 'userpass' , but no "
"'username' key found in pillar for this proxy.")
if "passwords" not in proxy_conf:
raise excs.InvalidProxyInputError(
"Mechanism is set to 'userpass' , but no "
"'passwords' key found in pillar for this proxy.")
for key in ("username", "passwords"):
DETAILS[key] = proxy_conf[key]
else:
if "domain" not in proxy_conf:
raise excs.InvalidProxyInputError(
"Mechanism is set to 'sspi' , but no "
"'domain' key found in pillar for this proxy.")
if "principal" not in proxy_conf:
raise excs.InvalidProxyInputError(
"Mechanism is set to 'sspi' , but no "
"'principal' key found in pillar for this proxy.")
for key in ("domain", "principal"):
DETAILS[key] = proxy_conf[key]
# Save optional
DETAILS["protocol"] = proxy_conf.get("protocol")
DETAILS["port"] = proxy_conf.get("port")
# Test connection
if DETAILS["mechanism"] == "userpass":
# Get the correct login details
log.debug("Retrieving credentials and testing vCenter connection for "
"mehchanism 'userpass'")
try:
username, password = find_credentials()
DETAILS["password"] = password
except excs.SaltSystemExit as err:
log.critical("Error: %s", err)
return False
return True
def ext_pillar(self, pillar, pillar_dirs):
'''
Render the external pillar data
'''
if 'ext_pillar' not in self.opts:
return pillar
if not isinstance(self.opts['ext_pillar'], list):
log.critical('The "ext_pillar" option is malformed')
return pillar
ext = None
# Bring in CLI pillar data
pillar.update(self.pillar_override)
for run in self.opts['ext_pillar']:
if not isinstance(run, dict):
log.critical('The "ext_pillar" option is malformed')
return {}
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
err = ('Specified ext_pillar interface {0} is '
'unavailable').format(key)
log.critical(err)
continue
try:
try:
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except TypeError as exc:
if str(exc).startswith('ext_pillar() takes exactly '):
log.warning('Deprecation warning: ext_pillar "{0}"'
' needs to accept minion_id as first'
' argument'.format(key))
else:
raise
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except Exception as exc:
log.exception(
'Failed to load ext_pillar {0}: {1}'.format(
key,
exc
)
)
if ext:
pillar = merge(
pillar,
ext,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
ext = None
return pillar
def render_pillar(self, matches, errors=None):
"""
Extract the sls pillar files from the matches and render them into the
pillar
"""
pillar = copy.copy(self.pillar_override)
if errors is None:
errors = []
for saltenv, pstates in matches.items():
pstatefiles = []
mods = {}
for sls_match in pstates:
matched_pstates = []
try:
matched_pstates = fnmatch.filter(self.avail[saltenv], sls_match)
except KeyError:
errors.extend(
[
"No matching pillar environment for environment "
"'{}' found".format(saltenv)
]
)
if matched_pstates:
pstatefiles.extend(matched_pstates)
else:
pstatefiles.append(sls_match)
for sls in pstatefiles:
pstate, mods, err = self.render_pstate(sls, saltenv, mods)
if err:
errors += err
if pstate is not None:
if not isinstance(pstate, dict):
log.error(
"The rendered pillar sls file, '%s' state did "
"not return the expected data format. This is "
"a sign of a malformed pillar sls file. Returned "
"errors: %s",
sls,
", ".join(["'{}'".format(e) for e in errors]),
)
continue
pillar = merge(
pillar,
pstate,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get("pillar_merge_lists", False),
)
return pillar, errors
def render_pillar(self, matches, errors=None):
'''
Extract the sls pillar files from the matches and render them into the
pillar
'''
pillar = copy.copy(self.pillar_override)
if errors is None:
errors = []
for saltenv, pstates in six.iteritems(matches):
pstatefiles = []
mods = set()
for sls_match in pstates:
matched_pstates = []
try:
matched_pstates = fnmatch.filter(self.avail[saltenv], sls_match)
except KeyError:
errors.extend(
['No matching pillar environment for environment '
'\'{0}\' found'.format(saltenv)]
)
if matched_pstates:
pstatefiles.extend(matched_pstates)
else:
pstatefiles.append(sls_match)
for sls in pstatefiles:
pstate, mods, err = self.render_pstate(sls, saltenv, mods)
if err:
errors += err
if pstate is not None:
if not isinstance(pstate, dict):
log.error(
'The rendered pillar sls file, \'{0}\' state did '
'not return the expected data format. This is '
'a sign of a malformed pillar sls file. Returned '
'errors: {1}'.format(
sls,
', '.join(
['\'{0}\''.format(e) for e in errors]
)
)
)
continue
pillar = merge(
pillar,
pstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
return pillar, errors
def compile_pillar(self, *args,
**kwargs): # Will likely just be pillar_dirs
'''
Compile pillar and set it to the cache, if not found.
:param args:
:param kwargs:
:return:
'''
log.debug(
'Scanning pillar cache for information about minion %s and pillarenv %s',
self.minion_id, self.pillarenv)
log.debug('Scanning cache for minion %s: %s', self.minion_id,
self.cache[self.minion_id] or '*empty*')
# Check the cache!
if self.minion_id in self.cache: # Keyed by minion_id
# TODO Compare grains, etc?
if self.pillarenv in self.cache[self.minion_id]:
# We have a cache hit! Send it back.
log.debug('Pillar cache hit for minion %s and pillarenv %s',
self.minion_id, self.pillarenv)
pillar_data = self.cache[self.minion_id][self.pillarenv]
else:
# We found the minion but not the env. Store it.
pillar_data = self.fetch_pillar()
self.cache[self.minion_id][self.pillarenv] = pillar_data
self.cache.store()
log.debug('Pillar cache miss for pillarenv %s for minion %s',
self.pillarenv, self.minion_id)
else:
# We haven't seen this minion yet in the cache. Store it.
pillar_data = self.fetch_pillar()
self.cache[self.minion_id] = {self.pillarenv: pillar_data}
log.debug('Pillar cache has been added for minion %s',
self.minion_id)
log.debug('Current pillar cache: %s', self.cache[self.minion_id])
# we dont want the pillar_override baked into the cached fetch_pillar from above
if self.pillar_override:
pillar_data = merge(
pillar_data, self.pillar_override,
self.opts.get('pillar_source_merging_strategy', 'smart'),
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
pillar_data.update(self.pillar_override)
return pillar_data
def ext_pillar(self, pillar, pillar_dirs):
'''
Render the external pillar data
'''
if 'ext_pillar' not in self.opts:
return pillar
if not isinstance(self.opts['ext_pillar'], list):
log.critical('The "ext_pillar" option is malformed')
return pillar
ext = None
# Bring in CLI pillar data
pillar.update(self.pillar_override)
for run in self.opts['ext_pillar']:
if run in self.opts.get('exclude_ext_pillar', []):
continue
if not isinstance(run, dict):
log.critical('The "ext_pillar" option is malformed')
return {}
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
log.critical(
'Specified ext_pillar interface {0} is '
'unavailable'.format(key)
)
continue
try:
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except Exception as exc: # pylint: disable=broad-except
log.exception(
'Failed to load ext_pillar {0}: {1}'.format(
key,
exc
)
)
if ext:
pillar = merge(
pillar,
ext,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', 'False'))
ext = None
return pillar
def render_pillar(self, matches, errors=None):
'''
Extract the sls pillar files from the matches and render them into the
pillar
'''
pillar = copy.copy(self.pillar_override)
if errors is None:
errors = []
for saltenv, pstates in six.iteritems(matches):
mods = set()
for sls in pstates:
pstate, mods, err = self.render_pstate(sls, saltenv, mods)
if err:
errors += err
if pstate is not None:
if not isinstance(pstate, dict):
log.error(
'The rendered pillar sls file, \'{0}\' state did '
'not return the expected data format. This is '
'a sign of a malformed pillar sls file. Returned '
'errors: {1}'.format(
sls,
', '.join(
['\'{0}\''.format(e) for e in errors]
)
)
)
continue
pillar = merge(
pillar,
pstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
return pillar, errors
def init(opts):
'''
This function gets called when the proxy starts up. For
login
the protocol and port are cached.
'''
log.debug('Initting esxcluster proxy module in process %s', os.getpid())
log.debug('Validating esxcluster proxy input')
schema = EsxclusterProxySchema.serialize()
log.trace('schema = %s', schema)
proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {}))
log.trace('proxy_conf = %s', proxy_conf)
try:
jsonschema.validate(proxy_conf, schema)
except jsonschema.exceptions.ValidationError as exc:
raise salt.exceptions.InvalidConfigError(exc)
# Save mandatory fields in cache
for key in ('vcenter', 'datacenter', 'cluster', 'mechanism'):
DETAILS[key] = proxy_conf[key]
# Additional validation
if DETAILS['mechanism'] == 'userpass':
if 'username' not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
'Mechanism is set to \'userpass\', but no '
'\'username\' key found in proxy config.')
if 'passwords' not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
'Mechanism is set to \'userpass\', but no '
'\'passwords\' key found in proxy config.')
for key in ('username', 'passwords'):
DETAILS[key] = proxy_conf[key]
else:
if 'domain' not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
'Mechanism is set to \'sspi\', but no '
'\'domain\' key found in proxy config.')
if 'principal' not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
'Mechanism is set to \'sspi\', but no '
'\'principal\' key found in proxy config.')
for key in ('domain', 'principal'):
DETAILS[key] = proxy_conf[key]
# Save optional
DETAILS['protocol'] = proxy_conf.get('protocol')
DETAILS['port'] = proxy_conf.get('port')
# Test connection
if DETAILS['mechanism'] == 'userpass':
# Get the correct login details
log.debug('Retrieving credentials and testing vCenter connection for '
'mehchanism \'userpass\'')
try:
username, password = find_credentials()
DETAILS['password'] = password
except salt.exceptions.SaltSystemExit as err:
log.critical('Error: %s', err)
return False
return True
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS {0!r} in'
' environment {1!r}'.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
log.debug('Specified SLS {0!r} in environment {1!r} is not'
' found, which might be due to environment {1!r}'
' not being present in "pillar_roots" yet!'.format(
sls, saltenv))
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for details.'
.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
nstate = {key: nstate}
state = merge(
state, nstate, self.merge_strategy,
self.opts.get('renderer', 'yaml'))
if err:
errors += err
return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug(
'Skipping ignored and missing SLS \'%s\' in '
'environment \'%s\'', sls, saltenv)
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
'found. '.format(sls, saltenv))
if self.opts.get('__git_pillar', False) is True:
msg += (
'This is likely caused by a git_pillar top file '
'containing an environment other than the one for the '
'branch in which it resides. Each git_pillar '
'branch/tag must have its own top file.')
else:
msg += (
'This could be because SLS \'{0}\' is in an '
'environment other than \'{1}\', but \'{1}\' is '
'included in that environment\'s Pillar top file. It '
'could also be due to environment \'{1}\' not being '
'defined in \'pillar_roots\'.'.format(sls, saltenv))
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg, exc_info=True)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for '
'details.'.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS \'{0}\' is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
# render included state(s)
include_states = []
matched_pstates = []
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
try:
matched_pstates += fnmatch.filter(
self.avail[saltenv], sub_sls)
except KeyError:
errors.extend([
'No matching pillar environment for environment '
'\'{0}\' found'.format(saltenv)
])
for sub_sls in matched_pstates:
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
# If key is x:y, convert it to {x: {y: nstate}}
for key_fragment in reversed(
key.split(":")):
nstate = {key_fragment: nstate}
if not self.opts.get(
'pillar_includes_override_sls',
False):
include_states.append(nstate)
else:
state = merge(
state, nstate, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get(
'pillar_merge_lists', False))
if err:
errors += err
if not self.opts.get('pillar_includes_override_sls',
False):
# merge included state(s) with the current state
# merged last to ensure that its values are
# authoritative.
include_states.append(state)
state = None
for s in include_states:
if state is None:
state = s
else:
state = merge(
state, s, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists',
False))
return state, mods, errors
def ext_pillar(self, pillar, errors=None):
'''
Render the external pillar data
'''
if errors is None:
errors = []
try:
# Make sure that on-demand git_pillar is fetched before we try to
# compile the pillar data. git_pillar will fetch a remote when
# the git ext_pillar() func is run, but only for masterless.
if self.ext and 'git' in self.ext \
and self.opts.get('__role') != 'minion':
# Avoid circular import
import salt.utils.gitfs
import salt.pillar.git_pillar
git_pillar = salt.utils.gitfs.GitPillar(
self.opts,
self.ext['git'],
per_remote_overrides=salt.pillar.git_pillar.
PER_REMOTE_OVERRIDES,
per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY,
global_only=salt.pillar.git_pillar.GLOBAL_ONLY)
git_pillar.fetch_remotes()
except TypeError:
# Handle malformed ext_pillar
pass
if 'ext_pillar' not in self.opts:
return pillar, errors
if not isinstance(self.opts['ext_pillar'], list):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return pillar, errors
ext = None
# Bring in CLI pillar data
if self.pillar_override:
pillar = merge(pillar, self.pillar_override, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
for run in self.opts['ext_pillar']:
if not isinstance(run, dict):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return {}, errors
if next(six.iterkeys(run)) in self.opts.get(
'exclude_ext_pillar', []):
continue
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
log.critical(
'Specified ext_pillar interface %s is unavailable',
key)
continue
try:
ext = self._external_pillar_data(pillar, val, key)
except Exception as exc:
errors.append('Failed to load ext_pillar {0}: {1}'.format(
key,
exc.__str__(),
))
log.error(
'Exception caught loading ext_pillar \'%s\':\n%s', key,
''.join(traceback.format_tb(sys.exc_info()[2])))
if ext:
pillar = merge(pillar, ext, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
ext = None
return pillar, errors
def ext_pillar(self, pillar, pillar_dirs, errors=None):
'''
Render the external pillar data
'''
if errors is None:
errors = []
try:
# Make sure that on-demand git_pillar is fetched before we try to
# compile the pillar data. git_pillar will fetch a remote when
# the git ext_pillar() func is run, but only for masterless.
if self.ext and 'git' in self.ext \
and self.opts.get('__role') != 'minion':
# Avoid circular import
import salt.utils.gitfs
from salt.pillar.git_pillar import PER_REMOTE_OVERRIDES
git_pillar = salt.utils.gitfs.GitPillar(self.opts)
git_pillar.init_remotes(self.ext['git'], PER_REMOTE_OVERRIDES)
git_pillar.fetch_remotes()
except TypeError:
# Handle malformed ext_pillar
pass
if 'ext_pillar' not in self.opts:
return pillar, errors
if not isinstance(self.opts['ext_pillar'], list):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return pillar, errors
ext = None
# Bring in CLI pillar data
if self.pillar_override and isinstance(self.pillar_override, dict):
pillar = merge(pillar,
self.pillar_override,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
for run in self.opts['ext_pillar']:
if not isinstance(run, dict):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return {}, errors
if next(six.iterkeys(run)) in self.opts.get('exclude_ext_pillar', []):
continue
for key, val in six.iteritems(run):
if key not in self.ext_pillars:
log.critical(
'Specified ext_pillar interface {0} is '
'unavailable'.format(key)
)
continue
try:
ext = self._external_pillar_data(pillar,
val,
pillar_dirs,
key)
except Exception as exc:
errors.append('Failed to load ext_pillar {0}: {1}'.format(
key, exc))
if ext:
pillar = merge(
pillar,
ext,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
ext = None
return pillar, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS \'{0}\' in'
' environment \'{1}\''.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
'found. '.format(sls, saltenv))
if self.opts.get('__git_pillar', False) is True:
msg += (
'This is likely caused by a git_pillar top file '
'containing an environment other than the one for the '
'branch in which it resides. Each git_pillar '
'branch/tag must have its own top file.'
)
else:
msg += (
'This could be because SLS \'{0}\' is in an '
'environment other than \'{1}\', but \'{1}\' is '
'included in that environment\'s Pillar top file. It '
'could also be due to environment \'{1}\' not being '
'defined in \'pillar_roots\'.'.format(sls, saltenv)
)
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
self.opts['renderer_blacklist'],
self.opts['renderer_whitelist'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format(
sls, exc
)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for '
'details.'.format(sls)
)
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS \'{0}\' is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
saltenv,
mods,
defaults
)
if nstate:
if key:
nstate = {
key: nstate
}
state = merge(
state,
nstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
if err:
errors += err
return state, mods, errors
def init(opts):
'''
This function gets called when the proxy starts up. For
ESXi devices, the host, login credentials, and, if configured,
the protocol and port are cached.
'''
log.debug('Initting esxi proxy module in process \'{}\''
''.format(os.getpid()))
log.debug('Validating esxi proxy input')
schema = EsxiProxySchema.serialize()
log.trace('esxi_proxy_schema = {}'.format(schema))
proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {}))
log.trace('proxy_conf = {0}'.format(proxy_conf))
try:
jsonschema.validate(proxy_conf, schema)
except jsonschema.exceptions.ValidationError as exc:
raise InvalidConfigError(exc)
DETAILS['proxytype'] = proxy_conf['proxytype']
if ('host' not in proxy_conf) and ('vcenter' not in proxy_conf):
log.critical('Neither \'host\' nor \'vcenter\' keys found in pillar '
'for this proxy.')
return False
if 'host' in proxy_conf:
# We have started the proxy by connecting directly to the host
if 'username' not in proxy_conf:
log.critical('No \'username\' key found in pillar for this proxy.')
return False
if 'passwords' not in proxy_conf:
log.critical(
'No \'passwords\' key found in pillar for this proxy.')
return False
host = proxy_conf['host']
# Get the correct login details
try:
username, password = find_credentials(host)
except SaltSystemExit as err:
log.critical('Error: {0}'.format(err))
return False
# Set configuration details
DETAILS['host'] = host
DETAILS['username'] = username
DETAILS['password'] = password
DETAILS['protocol'] = proxy_conf.get('protocol')
DETAILS['port'] = proxy_conf.get('port')
return True
if 'vcenter' in proxy_conf:
vcenter = proxy_conf['vcenter']
if not proxy_conf.get('esxi_host'):
log.critical(
'No \'esxi_host\' key found in pillar for this proxy.')
DETAILS['esxi_host'] = proxy_conf['esxi_host']
# We have started the proxy by connecting via the vCenter
if 'mechanism' not in proxy_conf:
log.critical(
'No \'mechanism\' key found in pillar for this proxy.')
return False
mechanism = proxy_conf['mechanism']
# Save mandatory fields in cache
for key in ('vcenter', 'mechanism'):
DETAILS[key] = proxy_conf[key]
if mechanism == 'userpass':
if 'username' not in proxy_conf:
log.critical('No \'username\' key found in pillar for this '
'proxy.')
return False
if 'passwords' not in proxy_conf and \
len(proxy_conf['passwords']) > 0:
log.critical('Mechanism is set to \'userpass\' , but no '
'\'passwords\' key found in pillar for this '
'proxy.')
return False
for key in ('username', 'passwords'):
DETAILS[key] = proxy_conf[key]
elif mechanism == 'sspi':
if 'domain' not in proxy_conf:
log.critical('Mechanism is set to \'sspi\' , but no '
'\'domain\' key found in pillar for this proxy.')
return False
if 'principal' not in proxy_conf:
log.critical('Mechanism is set to \'sspi\' , but no '
'\'principal\' key found in pillar for this '
'proxy.')
return False
for key in ('domain', 'principal'):
DETAILS[key] = proxy_conf[key]
if mechanism == 'userpass':
# Get the correct login details
log.debug('Retrieving credentials and testing vCenter connection'
' for mehchanism \'userpass\'')
try:
username, password = find_credentials(DETAILS['vcenter'])
DETAILS['password'] = password
except SaltSystemExit as err:
log.critical('Error: {0}'.format(err))
return False
# Save optional
DETAILS['protocol'] = proxy_conf.get('protocol', 'https')
DETAILS['port'] = proxy_conf.get('port', '443')
DETAILS['credstore'] = proxy_conf.get('credstore')
def init(opts):
"""
This function gets called when the proxy starts up. For
ESXi devices, the host, login credentials, and, if configured,
the protocol and port are cached.
"""
log.debug("Initting esxi proxy module in process %s", os.getpid())
log.debug("Validating esxi proxy input")
schema = EsxiProxySchema.serialize()
log.trace("esxi_proxy_schema = %s", schema)
proxy_conf = merge(opts.get("proxy", {}), __pillar__.get("proxy", {}))
log.trace("proxy_conf = %s", proxy_conf)
try:
jsonschema.validate(proxy_conf, schema)
except jsonschema.exceptions.ValidationError as exc:
raise InvalidConfigError(exc)
DETAILS["proxytype"] = proxy_conf["proxytype"]
if ("host" not in proxy_conf) and ("vcenter" not in proxy_conf):
log.critical("Neither 'host' nor 'vcenter' keys found in pillar "
"for this proxy.")
return False
if "host" in proxy_conf:
# We have started the proxy by connecting directly to the host
if "username" not in proxy_conf:
log.critical("No 'username' key found in pillar for this proxy.")
return False
if "passwords" not in proxy_conf:
log.critical("No 'passwords' key found in pillar for this proxy.")
return False
host = proxy_conf["host"]
# Get the correct login details
try:
username, password = find_credentials(host)
except SaltSystemExit as err:
log.critical("Error: %s", err)
return False
# Set configuration details
DETAILS["host"] = host
DETAILS["username"] = username
DETAILS["password"] = password
DETAILS["protocol"] = proxy_conf.get("protocol")
DETAILS["port"] = proxy_conf.get("port")
return True
if "vcenter" in proxy_conf:
vcenter = proxy_conf["vcenter"]
if not proxy_conf.get("esxi_host"):
log.critical("No 'esxi_host' key found in pillar for this proxy.")
DETAILS["esxi_host"] = proxy_conf["esxi_host"]
# We have started the proxy by connecting via the vCenter
if "mechanism" not in proxy_conf:
log.critical("No 'mechanism' key found in pillar for this proxy.")
return False
mechanism = proxy_conf["mechanism"]
# Save mandatory fields in cache
for key in ("vcenter", "mechanism"):
DETAILS[key] = proxy_conf[key]
if mechanism == "userpass":
if "username" not in proxy_conf:
log.critical("No 'username' key found in pillar for this "
"proxy.")
return False
if "passwords" not in proxy_conf and len(
proxy_conf["passwords"]) > 0:
log.critical("Mechanism is set to 'userpass' , but no "
"'passwords' key found in pillar for this "
"proxy.")
return False
for key in ("username", "passwords"):
DETAILS[key] = proxy_conf[key]
elif mechanism == "sspi":
if "domain" not in proxy_conf:
log.critical("Mechanism is set to 'sspi' , but no "
"'domain' key found in pillar for this proxy.")
return False
if "principal" not in proxy_conf:
log.critical("Mechanism is set to 'sspi' , but no "
"'principal' key found in pillar for this "
"proxy.")
return False
for key in ("domain", "principal"):
DETAILS[key] = proxy_conf[key]
if mechanism == "userpass":
# Get the correct login details
log.debug("Retrieving credentials and testing vCenter connection"
" for mehchanism 'userpass'")
try:
username, password = find_credentials(DETAILS["vcenter"])
DETAILS["password"] = password
except SaltSystemExit as err:
log.critical("Error: %s", err)
return False
# Save optional
DETAILS["protocol"] = proxy_conf.get("protocol", "https")
DETAILS["port"] = proxy_conf.get("port", "443")
DETAILS["credstore"] = proxy_conf.get("credstore")
def init(opts):
"""
This function gets called when the proxy starts up.
For login the protocol and port are cached.
"""
log.info("Initting vcenter proxy module in process %s", os.getpid())
log.trace("VCenter Proxy Validating vcenter proxy input")
schema = VCenterProxySchema.serialize()
log.trace("schema = %s", schema)
proxy_conf = merge(opts.get("proxy", {}), __pillar__.get("proxy", {}))
log.trace("proxy_conf = %s", proxy_conf)
try:
jsonschema.validate(proxy_conf, schema)
except jsonschema.exceptions.ValidationError as exc:
raise salt.exceptions.InvalidConfigError(exc)
# Save mandatory fields in cache
for key in ("vcenter", "mechanism"):
DETAILS[key] = proxy_conf[key]
# Additional validation
if DETAILS["mechanism"] == "userpass":
if "username" not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
"Mechanism is set to 'userpass' , but no "
"'username' key found in proxy config")
if "passwords" not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
"Mechanism is set to 'userpass' , but no "
"'passwords' key found in proxy config")
for key in ("username", "passwords"):
DETAILS[key] = proxy_conf[key]
else:
if "domain" not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
"Mechanism is set to 'sspi' , but no "
"'domain' key found in proxy config")
if "principal" not in proxy_conf:
raise salt.exceptions.InvalidConfigError(
"Mechanism is set to 'sspi' , but no "
"'principal' key found in proxy config")
for key in ("domain", "principal"):
DETAILS[key] = proxy_conf[key]
# Save optional
DETAILS["protocol"] = proxy_conf.get("protocol")
DETAILS["port"] = proxy_conf.get("port")
# Test connection
if DETAILS["mechanism"] == "userpass":
# Get the correct login details
log.info("Retrieving credentials and testing vCenter connection for "
"mehchanism 'userpass'")
try:
username, password = find_credentials()
except salt.exceptions.SaltSystemExit as err:
log.critical("Error: %s", err)
return False
else:
DETAILS["password"] = password
return True
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug('Skipping ignored and missing SLS {0!r} in'
' environment {1!r}'.format(sls, saltenv))
return None, mods, errors
elif self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
'found. '.format(sls, saltenv))
if self.opts.get('__git_pillar', False) is True:
msg += (
'This is likely caused by a git_pillar top file '
'containing an environment other than the one for the '
'branch in which it resides. Each git_pillar '
'branch/tag must have its own top file.')
else:
msg += (
'This could be because SLS \'{0}\' is in an '
'environment other than \'{1}\', but \'{1}\' is '
'included in that environment\'s Pillar top file. It '
'could also be due to environment \'{1}\' not being '
'defined in \'pillar_roots\'.'.format(sls, saltenv))
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts['renderer'],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append(
'Rendering SLS \'{0}\' failed. Please see master log for details.'
.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(six.iteritems(sub_sls))
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls, saltenv, mods, defaults)
if nstate:
if key:
nstate = {key: nstate}
state = merge(
state, nstate, self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists',
False))
if err:
errors += err
return state, mods, errors
def ext_pillar(self, pillar, errors=None):
"""
Render the external pillar data
"""
if errors is None:
errors = []
try:
# Make sure that on-demand git_pillar is fetched before we try to
# compile the pillar data. git_pillar will fetch a remote when
# the git ext_pillar() func is run, but only for masterless.
if self.ext and "git" in self.ext and self.opts.get(
"__role") != "minion":
# Avoid circular import
import salt.utils.gitfs
import salt.pillar.git_pillar
git_pillar = salt.utils.gitfs.GitPillar(
self.opts,
self.ext["git"],
per_remote_overrides=salt.pillar.git_pillar.
PER_REMOTE_OVERRIDES,
per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY,
global_only=salt.pillar.git_pillar.GLOBAL_ONLY,
)
git_pillar.fetch_remotes()
except TypeError:
# Handle malformed ext_pillar
pass
if "ext_pillar" not in self.opts:
return pillar, errors
if not isinstance(self.opts["ext_pillar"], list):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return pillar, errors
ext = None
# Bring in CLI pillar data
if self.pillar_override:
pillar = merge(
pillar,
self.pillar_override,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get("pillar_merge_lists", False),
)
for run in self.opts["ext_pillar"]:
if not isinstance(run, dict):
errors.append('The "ext_pillar" option is malformed')
log.critical(errors[-1])
return {}, errors
if next(iter(run.keys())) in self.opts.get("exclude_ext_pillar",
[]):
continue
for key, val in run.items():
if key not in self.ext_pillars:
log.critical(
"Specified ext_pillar interface %s is unavailable",
key)
continue
try:
ext = self._external_pillar_data(pillar, val, key)
except Exception as exc: # pylint: disable=broad-except
errors.append("Failed to load ext_pillar {}: {}".format(
key,
exc.__str__(),
))
log.error(
"Exception caught loading ext_pillar '%s':\n%s",
key,
"".join(traceback.format_tb(sys.exc_info()[2])),
)
if ext:
pillar = merge(
pillar,
ext,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get("pillar_merge_lists", False),
)
ext = None
return pillar, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
"""
Collect a single pillar sls file and render it
"""
if defaults is None:
defaults = {}
err = ""
errors = []
state_data = self.client.get_state(sls, saltenv)
fn_ = state_data.get("dest", False)
if not fn_:
if sls in self.ignored_pillars.get(saltenv, []):
log.debug(
"Skipping ignored and missing SLS '%s' in environment '%s'",
sls,
saltenv,
)
return None, mods, errors
elif self.opts["pillar_roots"].get(saltenv):
msg = ("Specified SLS '{}' in environment '{}' is not"
" available on the salt master".format(sls, saltenv))
log.error(msg)
errors.append(msg)
else:
msg = "Specified SLS '{}' in environment '{}' was not found. ".format(
sls, saltenv)
if self.opts.get("__git_pillar", False) is True:
msg += (
"This is likely caused by a git_pillar top file "
"containing an environment other than the one for the "
"branch in which it resides. Each git_pillar "
"branch/tag must have its own top file.")
else:
msg += (
"This could be because SLS '{0}' is in an "
"environment other than '{1}', but '{1}' is "
"included in that environment's Pillar top file. It "
"could also be due to environment '{1}' not being "
"defined in 'pillar_roots'.".format(sls, saltenv))
log.debug(msg)
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(fn_,
self.rend,
self.opts["renderer"],
self.opts["renderer_blacklist"],
self.opts["renderer_whitelist"],
saltenv,
sls,
_pillar_rend=True,
**defaults)
except Exception as exc: # pylint: disable=broad-except
msg = "Rendering SLS '{}' failed, render error:\n{}".format(
sls, exc)
log.critical(msg, exc_info=True)
if self.opts.get("pillar_safe_render_error", True):
errors.append(
"Rendering SLS '{}' failed. Please see master log for "
"details.".format(sls))
else:
errors.append(msg)
mods[sls] = state
nstate = None
if state:
if not isinstance(state, dict):
msg = "SLS '{}' does not render to a dictionary".format(sls)
log.error(msg)
errors.append(msg)
else:
if "include" in state:
if not isinstance(state["include"], list):
msg = ("Include Declaration in SLS '{}' is not "
"formed as a list".format(sls))
log.error(msg)
errors.append(msg)
else:
# render included state(s)
include_states = []
for sub_sls in state.pop("include"):
if isinstance(sub_sls, dict):
sub_sls, v = next(iter(sub_sls.items()))
defaults = v.get("defaults", {})
key = v.get("key", None)
else:
key = None
try:
matched_pstates = fnmatch.filter(
self.avail[saltenv],
sub_sls.lstrip(".").replace("/", "."),
)
if sub_sls.startswith("."):
if state_data.get(
"source",
"").endswith("/init.sls"):
include_parts = sls.split(".")
else:
include_parts = sls.split(".")[:-1]
sub_sls = ".".join(include_parts +
[sub_sls[1:]])
matches = fnmatch.filter(
self.avail[saltenv],
sub_sls,
)
matched_pstates.extend(matches)
except KeyError:
errors.extend([
"No matching pillar environment for environment"
" '{}' found".format(saltenv)
])
matched_pstates = [sub_sls]
# If matched_pstates is empty, set to sub_sls
if len(matched_pstates) < 1:
matched_pstates = [sub_sls]
for m_sub_sls in matched_pstates:
if m_sub_sls not in mods:
nstate, mods, err = self.render_pstate(
m_sub_sls, saltenv, mods, defaults)
else:
nstate = mods[m_sub_sls]
if nstate:
if key:
# If key is x:y, convert it to {x: {y: nstate}}
for key_fragment in reversed(
key.split(":")):
nstate = {key_fragment: nstate}
if not self.opts.get(
"pillar_includes_override_sls",
False):
include_states.append(nstate)
else:
state = merge(
state,
nstate,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get(
"pillar_merge_lists", False),
)
if err:
errors += err
if not self.opts.get("pillar_includes_override_sls",
False):
# merge included state(s) with the current state
# merged last to ensure that its values are
# authoritative.
include_states.append(state)
state = None
for s in include_states:
if state is None:
state = s
else:
state = merge(
state,
s,
self.merge_strategy,
self.opts.get("renderer", "yaml"),
self.opts.get("pillar_merge_lists",
False),
)
return state, mods, errors
def render_pstate(self, sls, saltenv, mods, defaults=None):
'''
Collect a single pillar sls file and render it
'''
if defaults is None:
defaults = {}
err = ''
errors = []
fn_ = self.client.get_state(sls, saltenv).get('dest', False)
if not fn_:
if self.opts['pillar_roots'].get(saltenv):
msg = ('Specified SLS {0!r} in environment {1!r} is not'
' available on the salt master').format(sls, saltenv)
log.error(msg)
errors.append(msg)
else:
log.debug('Specified SLS {0!r} in environment {1!r} is not'
' found, which might be due to environment {1!r}'
' not being present in "pillar_roots" yet!'
.format(sls, saltenv))
# return state, mods, errors
return None, mods, errors
state = None
try:
state = compile_template(
fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults)
except Exception as exc:
msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format(
sls, exc
)
log.critical(msg)
if self.opts.get('pillar_safe_render_error', True):
errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls))
else:
errors.append(msg)
mods.add(sls)
nstate = None
if state:
if not isinstance(state, dict):
msg = 'SLS {0!r} does not render to a dictionary'.format(sls)
log.error(msg)
errors.append(msg)
else:
if 'include' in state:
if not isinstance(state['include'], list):
msg = ('Include Declaration in SLS {0!r} is not '
'formed as a list'.format(sls))
log.error(msg)
errors.append(msg)
else:
for sub_sls in state.pop('include'):
if isinstance(sub_sls, dict):
sub_sls, v = next(sub_sls.iteritems())
defaults = v.get('defaults', {})
key = v.get('key', None)
else:
key = None
if sub_sls not in mods:
nstate, mods, err = self.render_pstate(
sub_sls,
saltenv,
mods,
defaults
)
if nstate:
if key:
nstate = {
key: nstate
}
state = merge(
state,
nstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
if err:
errors += err
return state, mods, errors