def get_tops(self, tops=None, env_order=None, state_top_saltenv=None):
'''
A test helper to emulate salt.state.HighState.get_tops() but just to
construct an appropriate data structure for top files from multiple
environments
'''
if tops is None:
tops = self.tops
if state_top_saltenv:
append_order = [state_top_saltenv]
elif env_order:
append_order = env_order
else:
append_order = self.env_order
ret = DefaultOrderedDict(list)
for env in append_order:
item = tops[env]
if env_order:
for remove in [
x for x in self.env_order if x not in env_order
]:
# Remove this env from the tops from the tops since this
# env is not part of env_order.
item.pop(remove)
ret[env].append(tops[env])
return ret
def tops(self, saltenv=None):
'''
Return a list of tops files in the specified environment or a dictionary
of all results if saltenv is None.
:param saltenv:
'''
# Only return SALTENV if provided, otherwise ALL
try:
if saltenv:
return {saltenv: self._tops[saltenv]}
return self._tops
except AttributeError:
pass
tops = DefaultOrderedDict(list)
toplist = self.files(saltenv=saltenv)
for info in toplist:
toppath = self.toppath(info, verify=False)
tops[self.get(info, 'saltenv')].append(toppath)
if saltenv:
return {saltenv: tops[saltenv]}
return tops
def disable(self, paths=None, saltenv='base'):
'''
:param paths:
:param saltenv:
'''
results = DefaultOrderedDict(list)
toppaths, unseen = self.prepare_paths(paths)
if toppaths:
tops = self.enabled(paths=paths, saltenv=saltenv, view='raw')
for topinfo in tops:
if os.path.exists(topinfo.abspath):
os.remove(topinfo.abspath)
results['disabled'].append(topinfo.toppath)
if toppaths:
tops = self.disabled(paths=toppaths, saltenv=saltenv, view='raw')
for topinfo in tops:
results['unchanged'].append(topinfo.toppath)
toppaths.remove(topinfo.toppath)
if unseen:
for path in unseen:
results['error'].append(path)
return results
def _get_tops(self):
'''
A test helper to emulate HighState.get_tops() but just to construct
an appropriate data structure for top files from multiple environments
'''
tops = DefaultOrderedDict(list)
tops['a'].append(self.env1)
tops['b'].append(self.env2)
tops['c'].append(self.env3)
return tops
def test_basic_merge(self):
'''
This is the default approach for Salt. Merge the top files with the
earlier appends taking precendence. Since Salt does the appends
lexecographically, this is effectively a test against the default
lexecographical behaviour.
'''
merged_tops = self.highstate.merge_tops(self._get_tops())
expected_merge = DefaultOrderedDict(OrderedDict)
expected_merge['base']['*'] = ['e1_c', 'e1_b', 'e1_a']
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_same_state_top_saltenv_baz(self):
'''
Test the 'state_top_saltenv' parameter to load states exclusively from
the 'baz' saltenv, with the 'same' merging strategy. This should
result in an empty dictionary since this environment has not top file.
'''
tops = self.get_tops(state_top_saltenv='baz')
merged_tops = self.highstate(
top_file_merging_strategy='same').merge_tops(tops)
expected_merge = DefaultOrderedDict(OrderedDict)
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_merge(self):
'''
Test the default merge strategy for top files, in an instance where the
base top file contains sections for all envs and the other envs' top
files are therefore ignored.
'''
merged_tops = self.highstate().merge_tops(self.get_tops())
expected_merge = DefaultOrderedDict(OrderedDict)
for env in self.env_order:
expected_merge[env]['*'] = ['base_{0}'.format(env)]
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_default_limited_base(self):
'''
Test the default merge strategy for top files when
'''
tops = self.get_tops(tops=self.tops_limited_base)
merged_tops = self.highstate().merge_tops(tops)
# No baz in the expected results because baz has no top file
expected_merge = DefaultOrderedDict(OrderedDict)
for env in self.env_order[:-1]:
expected_merge[env]['*'] = ['_'.join((env, env))]
self.assertEqual(merged_tops, expected_merge)
def __init__(self, opts, pillar=False, *varargs, **kwargs): # pylint: disable=W0613
self.opts = opts
self.pillar = pillar
if pillar and not self.is_pillar():
opts = dict(opts)
opts['file_roots'] = opts['pillar_roots']
self.opts = opts
self.client = salt.fileclient.get_file_client(self.opts,
self.is_pillar())
self._states = DefaultOrderedDict(list)
self._saltenvs = self.client.envs()
def test_merge_tops_merge_state_top_saltenv_foo(self):
'''
Test the 'state_top_saltenv' parameter to load states exclusively from
the 'foo' saltenv, with the default merging strategy. This should
result in just the 'foo' environment's states from the 'foo' top file
being in the merged result.
'''
env = 'foo'
tops = self.get_tops(state_top_saltenv=env)
merged_tops = self.highstate().merge_tops(tops)
expected_merge = DefaultOrderedDict(OrderedDict)
expected_merge[env]['*'] = ['_'.join((env, env))]
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_same_limited_base_without_default_top(self):
'''
Test to see if the top file that corresponds to the requested env is
the one that is used by the state system. default_top will not be set
(falling back to 'base'), and since we are using a limited base top
file, the 'baz' environment should not appear in the merged tops.
'''
tops = self.get_tops(tops=self.tops_limited_base)
merged_tops = \
self.highstate(top_file_merging_strategy='same').merge_tops(tops)
expected_merge = DefaultOrderedDict(OrderedDict)
for env in self.env_order[:-1]:
expected_merge[env]['*'] = ['_'.join((env, env))]
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_merge_state_top_saltenv_base(self):
'''
Test the 'state_top_saltenv' parameter to load states exclusively from
the 'base' saltenv, with the default merging strategy. This should
result in all states from the 'base' top file being in the merged
result.
'''
env = 'base'
tops = self.get_tops(state_top_saltenv=env)
merged_tops = self.highstate().merge_tops(tops)
expected_merge = DefaultOrderedDict(OrderedDict)
for env2 in self.env_order:
expected_merge[env2]['*'] = ['_'.join((env, env2))]
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_merge_all_with_env_order(self):
'''
Test an altered env_order with the 'merge_all' strategy.
'''
env_order = ['bar', 'foo', 'base']
tops = self.get_tops(env_order=env_order)
merged_tops = self.highstate(top_file_merging_strategy='merge_all',
env_order=env_order).merge_tops(tops)
expected_merge = DefaultOrderedDict(OrderedDict)
for env in [x for x in self.env_order if x in env_order]:
states = []
for top_env in env_order:
states.extend(tops[top_env][0][env]['*'])
expected_merge[env]['*'] = states
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_merge_all(self):
'''
Test the merge_all strategy
'''
tops = self.get_tops()
merged_tops = self.highstate(
top_file_merging_strategy='merge_all').merge_tops(tops)
expected_merge = DefaultOrderedDict(OrderedDict)
for env in self.env_order:
states = []
for top_env in self.env_order:
if top_env in tops[top_env][0]:
states.extend(tops[top_env][0][env]['*'])
expected_merge[env]['*'] = states
self.assertEqual(merged_tops, expected_merge)
def test_merge_tops_same_without_default_top(self):
'''
Test to see if the top file that corresponds to the requested env is
the one that is used by the state system. default_top will not be set
(falling back to 'base'), so the 'baz' environment should pull its
states from the 'base' top file.
'''
merged_tops = self.highstate(
top_file_merging_strategy='same').merge_tops(self.get_tops())
expected_merge = DefaultOrderedDict(OrderedDict)
for env in self.env_order[:-1]:
expected_merge[env]['*'] = ['_'.join((env, env))]
# The 'baz' env should be using the foo top file because baz lacks a
# top file, and default_top == 'base'
expected_merge['baz']['*'] = ['base_baz']
self.assertEqual(merged_tops, expected_merge)
def merge_tops(tops):
'''
Cleanly merge the top files
Top structure
OrderedDict - str(saltenv)
OrderedDict - str(target)
list [(str state...}]
list [(OrderedDict matches), (str state..)]
:param tops:
'''
top = DefaultOrderedDict(OrderedDict)
# List of complied tops
for _top in tops:
# Compiled tops of one tops file
for ctops in six.itervalues(_top):
# Targets in a list
for ctop in ctops:
for saltenv, targets in six.iteritems(ctop):
if saltenv == 'include':
continue
try:
for tgt in targets:
if tgt not in top[saltenv]:
top[saltenv][tgt] = ctop[saltenv][tgt]
continue
matches = []
states = set()
for comp in top[saltenv][tgt] + ctop[saltenv][tgt]:
if isinstance(
comp, dict
) and comp not in matches:
matches.append(comp)
if isinstance(comp, six.string_types):
states.add(comp)
top[saltenv][tgt] = matches
top[saltenv][tgt].extend(list(states))
except TypeError:
raise SaltRenderError(
'Unable to render top file. No targets found.'
)
return top
def test_merge_tops_same_with_default_top(self):
'''
Test to see if the top file that corresponds to the requested env is
the one that is used by the state system. Also test the 'default_top'
option for env 'baz', which has no top file and should pull its states
from the 'foo' top file.
'''
merged_tops = self.highstate(top_file_merging_strategy='same',
default_top='foo').merge_tops(
self.get_tops())
expected_merge = DefaultOrderedDict(OrderedDict)
for env in self.env_order[:-1]:
expected_merge[env]['*'] = ['_'.join((env, env))]
# The 'baz' env should be using the foo top file because baz lacks a
# top file, and default_top has been seet to 'foo'
expected_merge['baz']['*'] = ['foo_baz']
self.assertEqual(merged_tops, expected_merge)
def render_top(opts, toputils): # pylint: disable=W0621
'''
Gather the top files
:param toputils:
:param opts:
'''
tops = DefaultOrderedDict(list)
include = DefaultOrderedDict(list)
done = DefaultOrderedDict(list)
environment = get_environment(opts)
# Gather initial top files
if opts['top_file_merging_strategy'] == 'same' and not opts[environment]:
if not opts['default_top']:
raise SaltRenderError(
'Top file merge strategy set to same, but no default_top '
'configuration option was set'
)
opts[environment] = opts['default_top']
if opts.get(environment, None):
salt_data = render(
opts['state_top'],
opts=opts,
saltenv=opts[environment]
)
if salt_data:
tops[opts[environment]] = salt_data
elif opts['top_file_merging_strategy'] == 'merge':
if opts.get('state_top_saltenv', False):
saltenv = opts['state_top_saltenv']
salt_data = render(opts['state_top'], opts=opts, saltenv=saltenv)
if salt_data:
tops[saltenv].append(salt_data)
else:
log.debug('No contents loaded for env: {0}'.format(saltenv))
else:
for saltenv in get_envs(opts):
salt_data = render(
opts['state_top'],
opts=opts,
saltenv=saltenv
)
if salt_data:
tops[saltenv].append(salt_data)
else:
log.debug(
'No contents loaded for env: {0}'.format(
saltenv
)
)
# Search initial top files for includes
for saltenv, ctops in six.iteritems(tops):
for ctop in ctops:
if 'include' not in ctop:
continue
for sls in ctop['include']:
include[saltenv].append(sls)
ctop.pop('include')
# Go through the includes and pull out the extra tops and add them
while include:
pops = []
for saltenv, states in six.iteritems(include):
pops.append(saltenv)
if not states:
continue
for sls_match in states:
states = toputils.states(saltenv)
for sls in fnmatch.filter(states[saltenv], sls_match):
if sls in done[saltenv]:
continue
salt_data = render(sls, opts=opts, saltenv=saltenv)
if salt_data:
tops[saltenv].append(salt_data)
else:
log.debug(
'No contents loaded for include {0} env: {1}'
.format(sls, saltenv)
)
done[saltenv].append(sls)
for saltenv in pops:
if saltenv in include:
include.pop(saltenv)
return tops
