def merge_tops(self, tops):
'''
Cleanly merge the top files
'''
top = collections.defaultdict(dict)
orders = collections.defaultdict(dict)
for ctops in tops.values():
for ctop in ctops:
for saltenv, targets in ctop.items():
if saltenv == 'include':
continue
for tgt in targets:
matches = []
states = OrderedDict()
orders[saltenv][tgt] = 0
for comp in ctop[saltenv][tgt]:
if isinstance(comp, dict):
if 'match' in comp:
matches.append(comp)
if 'order' in comp:
order = comp['order']
if not isinstance(order, int):
try:
order = int(order)
except ValueError:
order = 0
orders[saltenv][tgt] = order
if isinstance(comp, string_types):
states[comp] = True
top[saltenv][tgt] = matches
top[saltenv][tgt].extend(list(states.keys()))
return self.sort_top_targets(top, orders)
def _prompt_choice(var_name, options):
'''
Prompt the user to choose between a list of options, index each one by adding an enumerator
based on https://github.com/audreyr/cookiecutter/blob/master/cookiecutter/prompt.py#L51
:param var_name: The question to ask the user
:type var_name: ``str``
:param options: A list of options
:type options: ``list`` of ``tupple``
:rtype: ``tuple``
:returns: The selected user
'''
choice_map = OrderedDict(
(u'{0}'.format(i), value) for i, value in enumerate(options, 1) if value[0] != 'test'
)
choices = choice_map.keys()
default = u'1'
choice_lines = [u'{0} - {1} - {2}'.format(c[0], c[1][0], c[1][1]) for c in choice_map.items()]
prompt = u'\n'.join((
u'Select {0}:'.format(var_name),
u'\n'.join(choice_lines),
u'Choose from {0}'.format(u', '.join(choices))
))
user_choice = click.prompt(
prompt, type=click.Choice(choices), default=default
)
return choice_map[user_choice]
def list_vms(search=None, verbose=False):
'''
List all vms
search : string
filter vms, see the execution module
verbose : boolean
print additional information about the vm
CLI Example:
.. code-block:: bash
salt-run vmadm.list
salt-run vmadm.list search='type=KVM'
salt-run vmadm.list verbose=True
'''
ret = OrderedDict() if verbose else []
client = salt.client.get_local_client(__opts__['conf_file'])
try:
vmadm_args = {}
vmadm_args['order'] = 'uuid,alias,hostname,state,type,cpu_cap,vcpus,ram'
if search:
vmadm_args['search'] = search
for cn in client.cmd_iter('G@virtual:physical and G@os:smartos',
'vmadm.list', kwarg=vmadm_args,
tgt_type='compound'):
if not cn:
continue
node = next(cn.iterkeys())
if not isinstance(cn[node], dict) or \
'ret' not in cn[node] or \
not isinstance(cn[node]['ret'], dict):
continue
for vm in cn[node]['ret'].keys():
vmcfg = cn[node]['ret'][vm]
if verbose:
ret[vm] = OrderedDict()
ret[vm]['hostname'] = vmcfg['hostname']
ret[vm]['alias'] = vmcfg['alias']
ret[vm]['computenode'] = node
ret[vm]['state'] = vmcfg['state']
ret[vm]['resources'] = OrderedDict()
ret[vm]['resources']['memory'] = vmcfg['ram']
if vmcfg['type'] == 'KVM':
ret[vm]['resources']['cpu'] = "{0:.2f}".format(int(vmcfg['vcpus']))
else:
if vmcfg['cpu_cap'] != '':
ret[vm]['resources']['cpu'] = "{0:.2f}".format(int(vmcfg['cpu_cap'])/100)
else:
ret.append(vm)
except SaltClientError as client_error:
return "{0}".format(client_error)
if not verbose:
ret = sorted(ret)
return ret
def list_vms(search=None, sort=None, order='uuid,type,ram,state,alias', keyed=True):
'''
Return a list of VMs
search : string
vmadm filter property
sort : string
vmadm sort (-s) property
order : string
vmadm order (-o) property -- Default: uuid,type,ram,state,alias
keyed : boolean
specified if the output should be an array (False) or dict (True)
For a dict the key is the first item from the order parameter.
Note: If key is not unique last vm wins.
CLI Example:
.. code-block:: bash
salt '*' vmadm.list
salt '*' vmadm.list order=alias,ram,cpu_cap sort=-ram,-cpu_cap
salt '*' vmadm.list search='type=KVM'
'''
ret = {}
vmadm = _check_vmadm()
# vmadm list [-p] [-H] [-o field,...] [-s field,...] [field=value ...]
cmd = '{vmadm} list -p -H {order} {sort} {search}'.format(
vmadm=vmadm,
order='-o {0}'.format(order) if order else '',
sort='-s {0}'.format(sort) if sort else '',
search=search if search else ''
)
res = __salt__['cmd.run_all'](cmd)
retcode = res['retcode']
result = OrderedDict() if keyed else []
if retcode != 0:
ret['Error'] = res['stderr'] if 'stderr' in res else _exit_status(retcode)
return ret
fields = order.split(',')
for vm in res['stdout'].splitlines():
vm_data = OrderedDict()
vm = vm.split(':')
if keyed:
for field in fields:
if fields.index(field) == 0:
continue
vm_data[field.strip()] = vm[fields.index(field)].strip()
result[vm[0]] = vm_data
else:
if len(vm) > 1:
for field in fields:
vm_data[field.strip()] = vm[fields.index(field)].strip()
else:
vm_data = vm[0]
result.append(vm_data)
return result
def get_vms_per_type(target):
'''Return all vms indexed by virt_type for a special target'''
all_targets = OrderedDict()
for virt_type in VIRT_TYPES:
per_type = all_targets.setdefault(virt_type, set())
all_infos = get_targets_and_vms_for_virt_type(virt_type)
for vmname in all_infos.get(target, []):
per_type.add(vmname)
for i in [a for a in all_targets]:
all_targets[i] = [a for a in all_targets[i]]
return all_targets
def gen_ini(self):
yield '\n[{0}]\n'.format(self.name)
sections_dict = OrderedDict()
for name, value in self.iteritems():
if com_regx.match(name):
yield '{0}\n'.format(value)
elif isinstance(value, _Section):
sections_dict.update({name: value})
else:
yield '{0} {1} {2}\n'.format(name, self.sep, value)
for name, value in sections_dict.iteritems():
for line in value.gen_ini():
yield line
def test_compare_sls_vs_yaml_with_jinja(self):
tpl = '{{ data }}'
env = jinja2.Environment()
src = '{foo: 1, bar: 2, baz: {qux: true}}'
sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src))
yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))
sls_data = yamlex.deserialize(sls_src)
yml_data = yaml.deserialize(yml_src)
# ensure that sls & yaml have the same base
assert isinstance(sls_data, dict)
assert isinstance(yml_data, dict)
# The below has been commented out because something the loader test
# is modifying the yaml renderer to render things to unicode. Without
# running the loader test, the below passes. Even reloading the module
# from disk does not reset its internal state (per the Python docs).
##
#assert sls_data == yml_data
# ensure that sls is ordered, while yaml not
assert isinstance(sls_data, OrderedDict)
assert not isinstance(yml_data, OrderedDict)
# prove that yaml does not handle well with OrderedDict
# while sls is jinja friendly.
obj = OrderedDict([
('foo', 1),
('bar', 2),
('baz', {'qux': True})
])
sls_obj = yamlex.deserialize(yamlex.serialize(obj))
try:
yml_obj = yaml.deserialize(yaml.serialize(obj))
except SerializationError:
# BLAAM! yaml was unable to serialize OrderedDict,
# but it's not the purpose of the current test.
yml_obj = obj.copy()
sls_src = env.from_string(tpl).render(data=sls_obj)
yml_src = env.from_string(tpl).render(data=yml_obj)
final_obj = yaml.deserialize(sls_src)
assert obj == final_obj
# BLAAM! yml_src is not valid !
final_obj = OrderedDict(yaml.deserialize(yml_src))
assert obj != final_obj
def gen_ini(self):
yield "{0}[{1}]{0}".format(os.linesep, self.name)
sections_dict = OrderedDict()
for name, value in six.iteritems(self):
if com_regx.match(name):
yield "{0}{1}".format(value, os.linesep)
elif isinstance(value, _Section):
sections_dict.update({name: value})
else:
yield "{0}{1}{2}{3}".format(
name, (" {0} ".format(self.sep) if self.sep != " " else self.sep), value, os.linesep
)
for name, value in six.iteritems(sections_dict):
for line in value.gen_ini():
yield line
def get_vms():
'''Return all vms indexed by targets'''
data = OrderedDict()
for virt_type in VIRT_TYPES:
all_infos = get_targets_and_vms_for_virt_type(virt_type)
for t in all_infos:
target = data.setdefault(t, {})
vms = {}
vts = set()
for vmname in all_infos[t]:
vms.setdefault(vmname, virt_type)
vts.add(virt_type)
target['virt_types'] = [a for a in vts]
target['vms'] = vms
return data
def test_compare_sls_vs_yaml_with_jinja(self):
tpl = '{{ data }}'
env = jinja2.Environment()
src = '{foo: 1, bar: 2, baz: {qux: true}}'
sls_src = env.from_string(tpl).render(data=sls.deserialize(src))
yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))
sls_data = sls.deserialize(sls_src)
yml_data = yaml.deserialize(yml_src)
# ensure that sls & yaml have the same base
assert isinstance(sls_data, dict)
assert isinstance(yml_data, dict)
assert sls_data == yml_data
# ensure that sls is ordered, while yaml not
assert isinstance(sls_data, OrderedDict)
assert not isinstance(yml_data, OrderedDict)
# prove that yaml does not handle well with OrderedDict
# while sls is jinja friendly.
obj = OrderedDict([
('foo', 1),
('bar', 2),
('baz', {'qux': True})
])
sls_obj = sls.deserialize(sls.serialize(obj))
try:
yml_obj = yaml.deserialize(yaml.serialize(obj))
except SerializationError:
# BLAAM! yaml was unable to serialize OrderedDict,
# but it's not the purpose of the current test.
yml_obj = obj.copy()
sls_src = env.from_string(tpl).render(data=sls_obj)
yml_src = env.from_string(tpl).render(data=yml_obj)
final_obj = yaml.deserialize(sls_src)
assert obj == final_obj
# BLAAM! yml_src is not valid !
final_obj = yaml.deserialize(yml_src)
assert obj != final_obj
def element(self, root=None, abspath=None, **kwargs):
'''
kwargs contain extra information for custom methods.
This method must return a valid empty object if no vars are passed
to allow introspection to create patterns.
:param root:
:param abspath:
'''
if root is None and abspath is None:
root = os.path.abspath('.')
abspath = os.path.abspath('.')
relpath = os.path.relpath(abspath, root)
try:
element = self._elements.get(relpath, OrderedDict())
except AttributeError:
element = OrderedDict()
if not element:
for field in PATHINFO_FIELDS:
element.setdefault(field, '')
element['saltenv'] = kwargs.get('saltenv', 'base')
element['relpath'] = relpath
element['abspath'] = abspath
element['is_pillar'] = kwargs.get('is_pillar', False)
cachedir = kwargs.get('cachedir', '')
if cachedir and os.path.commonprefix([abspath, cachedir]) == cachedir:
element['cache_root'] = root
else:
element['file_root'] = root
element_hook = kwargs.get('_element_hook', None)
if element_hook:
element = element_hook(self, element, **kwargs)
return element
def merge_tops(self, tops):
'''
Cleanly merge the top files
'''
top = collections.defaultdict(OrderedDict)
orders = collections.defaultdict(OrderedDict)
for ctops in six.itervalues(tops):
for ctop in ctops:
for saltenv, targets in six.iteritems(ctop):
if saltenv == 'include':
continue
for tgt in targets:
matches = []
states = OrderedDict()
orders[saltenv][tgt] = 0
ignore_missing = False
for comp in ctop[saltenv][tgt]:
if isinstance(comp, dict):
if 'match' in comp:
matches.append(comp)
if 'order' in comp:
order = comp['order']
if not isinstance(order, int):
try:
order = int(order)
except ValueError:
order = 0
orders[saltenv][tgt] = order
if comp.get('ignore_missing', False):
ignore_missing = True
if isinstance(comp, six.string_types):
states[comp] = True
if ignore_missing:
if saltenv not in self.ignored_pillars:
self.ignored_pillars[saltenv] = []
self.ignored_pillars[saltenv].extend(states.keys())
top[saltenv][tgt] = matches
top[saltenv][tgt].extend(states)
return self.sort_top_targets(top, orders)
def _uncomment_if_commented(self, opt_key):
# should be called only if opt_key is not already present
# will uncomment the key if commented and create a place holder
# for the key where the correct value can be update later
# used to preserve the ordering of comments and commented options
# and to make sure options without sectons go above any section
options_backup = OrderedDict()
comment_index = None
for key, value in self.iteritems():
if comment_index is not None:
options_backup.update({key: value})
continue
if '#comment' not in key:
continue
opt_match = opt_regx.match(value.lstrip('#'))
if opt_match and opt_match.group(2) == opt_key:
comment_index = key
for key in options_backup:
self.pop(key)
self.pop(comment_index, None)
super(_Section, self).update({opt_key: None})
for key, value in options_backup.iteritems():
super(_Section, self).update({key: value})
def compile(labels, **patterns): # pylint: disable=W0622
'''
Compile patterns.
Args:
labels:
'''
pattern = patterns.pop('_pattern', None)
if pattern:
return pattern
regex = patterns.pop('_regex', False)
escape = patterns.pop('_escape', [])
if not patterns or not labels:
return None
for pattern in list(patterns.keys()):
if pattern not in labels:
patterns.pop(pattern)
default_pattern = get_default_pattern(regex)
escape = escape if escape else []
_escape_text = functools.partial(escape_text, regex=regex)
# Set default values and join patterns for each field
pattern = OrderedDict.fromkeys(labels, None)
for label in labels:
if label in patterns and patterns[label]:
field = patterns[label]
if isinstance(field, re._pattern_type): # pylint: disable=W0212
field = [field.pattern]
if isinstance(field, six.string_types):
field = [field]
if label in escape or not regex:
field = [_escape_text(text) for text in field]
else:
field = default_pattern
pattern[label] = r'(?:{0})'.format(r'|'.join(field))
try:
return re.compile(
r'\n'.join(six.itervalues(pattern)), re.MULTILINE | re.DOTALL
)
except NameError:
raise
def __init__(self, match_each=True, **patterns):
'''
match_each:
If True, each file path is matched which prevents uses less memory
but sacrifices performance a little bit. If False, the complete
list is matched after all the file information has been added to
pathinfo.
patterns:
Contains the patterns to match.
Example:
{ 'saltenv': 'base', 'relpath': ['*.sls'] }
'''
super(PathInfoDict, self).__init__(
fields=PATHINFO_FIELDS,
match_each=match_each,
**patterns
)
self._elements = OrderedDict()
class StateRegistry(object):
"""
The StateRegistry holds all of the states that have been created.
"""
def __init__(self):
self.empty()
def empty(self):
self.states = OrderedDict()
self.requisites = []
def salt_data(self):
states = OrderedDict([
(id_, state())
for id_, state in self.states.iteritems()
])
self.empty()
return states
def add(self, id_, state):
if id_ in self.states:
raise DuplicateState("A state with id '%s' already exists" % id_)
# if we have requisites in our stack then add them to the state
if len(self.requisites) > 0:
for req in self.requisites:
if req.requisite not in state.kwargs:
state.kwargs[req.requisite] = []
state.kwargs[req.requisite].append(req())
self.states[id_] = state
def push_requisite(self, requisite):
self.requisites.append(requisite)
def pop_requisite(self):
del self.requisites[-1]
def iostat(zpool=None, sample_time=0):
'''
.. versionchanged:: 2016.3.0
Display I/O statistics for the given pools
zpool : string
optional name of storage pool
sample_time : int
seconds to capture data before output
CLI Example:
.. code-block:: bash
salt '*' zpool.iostat myzpool
'''
ret = OrderedDict()
# get zpool list data
zpool_cmd = _check_zpool()
cmd = '{zpool_cmd} iostat -v{zpool}{sample_time}'.format(
zpool_cmd=zpool_cmd,
zpool=' {0}'.format(zpool) if zpool else '',
sample_time=' {0} 2'.format(sample_time) if sample_time else ''
)
res = __salt__['cmd.run_all'](cmd, python_shell=False)
if res['retcode'] != 0:
ret['error'] = res['stderr'] if 'stderr' in res else res['stdout']
return ret
# note: hardcoded header fields, the double header is hard to parse
# capacity operations bandwidth
#pool alloc free read write read write
header = [
'pool',
'capacity-alloc',
'capacity-free',
'operations-read',
'operations-write',
'bandwith-read',
'bandwith-write'
]
root_vdev = None
vdev = None
dev = None
config_data = None
current_pool = None
for line in res['stdout'].splitlines():
if line.strip() == '':
continue
# ignore header
if line.startswith('pool') and line.endswith('write'):
continue
if line.endswith('bandwidth'):
continue
if line.startswith('-') and line.endswith('-'):
if config_data:
ret[current_pool] = config_data
config_data = OrderedDict()
current_pool = None
else:
if not isinstance(config_data, salt.utils.odict.OrderedDict):
continue
stat_data = OrderedDict()
stats = [x for x in line.strip().split(' ') if x not in ['']]
for prop in header:
if header.index(prop) < len(stats):
if prop == 'pool':
if not current_pool:
current_pool = stats[header.index(prop)]
continue
if stats[header.index(prop)] == '-':
continue
stat_data[prop] = stats[header.index(prop)]
dev = line.strip().split()[0]
if line[0:4] != ' ':
if line[0:2] == ' ':
vdev = line.strip().split()[0]
dev = None
else:
root_vdev = line.strip().split()[0]
vdev = None
dev = None
if root_vdev:
if not config_data.get(root_vdev):
config_data[root_vdev] = {}
if len(stat_data) > 0:
config_data[root_vdev] = stat_data
if vdev:
if vdev not in config_data[root_vdev]:
config_data[root_vdev][vdev] = {}
if len(stat_data) > 0:
config_data[root_vdev][vdev] = stat_data
if dev and dev not in config_data[root_vdev][vdev]:
config_data[root_vdev][vdev][dev] = {}
if len(stat_data) > 0:
config_data[root_vdev][vdev][dev] = stat_data
return ret
def handle_func(self, func, data):
'''
Execute this method in a multiprocess or thread
'''
if salt.utils.is_windows():
# Since function references can't be pickled and pickling
# is required when spawning new processes on Windows, regenerate
# the functions and returners.
self.functions = salt.loader.minion_mods(self.opts)
self.returners = salt.loader.returners(self.opts, self.functions)
ret = {'id': self.opts.get('id', 'master'),
'fun': func,
'schedule': data['name'],
'jid': salt.utils.jid.gen_jid()}
if 'metadata' in data:
if isinstance(data['metadata'], dict):
ret['metadata'] = data['metadata']
ret['metadata']['_TOS'] = self.time_offset
ret['metadata']['_TS'] = time.ctime()
ret['metadata']['_TT'] = time.strftime('%Y %B %d %a %H %m', time.gmtime())
else:
log.warning('schedule: The metadata parameter must be '
'specified as a dictionary. Ignoring.')
salt.utils.appendproctitle(ret['jid'])
proc_fn = os.path.join(
salt.minion.get_proc_dir(self.opts['cachedir']),
ret['jid']
)
# Check to see if there are other jobs with this
# signature running. If there are more than maxrunning
# jobs present then don't start another.
# If jid_include is False for this job we can ignore all this
# NOTE--jid_include defaults to True, thus if it is missing from the data
# dict we treat it like it was there and is True
if 'jid_include' not in data or data['jid_include']:
jobcount = 0
for basefilename in os.listdir(salt.minion.get_proc_dir(self.opts['cachedir'])):
fn_ = os.path.join(salt.minion.get_proc_dir(self.opts['cachedir']), basefilename)
if not os.path.exists(fn_):
log.debug('schedule.handle_func: {0} was processed '
'in another thread, skipping.'.format(
basefilename))
continue
with salt.utils.fopen(fn_, 'rb') as fp_:
job = salt.payload.Serial(self.opts).load(fp_)
if job:
if 'schedule' in job:
log.debug('schedule.handle_func: Checking job against '
'fun {0}: {1}'.format(ret['fun'], job))
if ret['schedule'] == job['schedule'] and os_is_running(job['pid']):
jobcount += 1
log.debug(
'schedule.handle_func: Incrementing jobcount, now '
'{0}, maxrunning is {1}'.format(
jobcount, data['maxrunning']))
if jobcount >= data['maxrunning']:
log.debug(
'schedule.handle_func: The scheduled job {0} '
'was not started, {1} already running'.format(
ret['schedule'], data['maxrunning']))
return False
else:
try:
log.info('Invalid job file found. Removing.')
os.remove(fn_)
except OSError:
log.info('Unable to remove file: {0}.'.format(fn_))
salt.utils.daemonize_if(self.opts)
ret['pid'] = os.getpid()
if 'jid_include' not in data or data['jid_include']:
log.debug('schedule.handle_func: adding this job to the jobcache '
'with data {0}'.format(ret))
# write this to /var/cache/salt/minion/proc
with salt.utils.fopen(proc_fn, 'w+b') as fp_:
fp_.write(salt.payload.Serial(self.opts).dumps(ret))
args = tuple()
if 'args' in data:
args = data['args']
kwargs = {}
if 'kwargs' in data:
kwargs = data['kwargs']
# if the func support **kwargs, lets pack in the pub data we have
# TODO: pack the *same* pub data as a minion?
argspec = salt.utils.args.get_function_argspec(self.functions[func])
if argspec.keywords:
# this function accepts **kwargs, pack in the publish data
for key, val in six.iteritems(ret):
kwargs['__pub_{0}'.format(key)] = val
try:
ret['return'] = self.functions[func](*args, **kwargs)
data_returner = data.get('returner', None)
if data_returner or self.schedule_returner:
if 'returner_config' in data:
ret['ret_config'] = data['returner_config']
rets = []
for returner in [data_returner, self.schedule_returner]:
if isinstance(returner, str):
rets.append(returner)
elif isinstance(returner, list):
rets.extend(returner)
# simple de-duplication with order retained
for returner in OrderedDict.fromkeys(rets):
ret_str = '{0}.returner'.format(returner)
if ret_str in self.returners:
ret['success'] = True
self.returners[ret_str](ret)
else:
log.info(
'Job {0} using invalid returner: {1}. Ignoring.'.format(
func, returner
)
)
if 'return_job' in data and not data['return_job']:
pass
else:
# Send back to master so the job is included in the job list
mret = ret.copy()
mret['jid'] = 'req'
channel = salt.transport.Channel.factory(self.opts, usage='salt_schedule')
load = {'cmd': '_return', 'id': self.opts['id']}
for key, value in six.iteritems(mret):
load[key] = value
channel.send(load)
except Exception:
log.exception("Unhandled exception running {0}".format(ret['fun']))
# Although catch-all exception handlers are bad, the exception here
# is to let the exception bubble up to the top of the thread context,
# where the thread will die silently, which is worse.
finally:
try:
log.debug('schedule.handle_func: Removing {0}'.format(proc_fn))
os.unlink(proc_fn)
except OSError as exc:
if exc.errno == errno.EEXIST or exc.errno == errno.ENOENT:
# EEXIST and ENOENT are OK because the file is gone and that's what
# we wanted
pass
else:
log.error("Failed to delete '{0}': {1}".format(proc_fn, exc.errno))
# Otherwise, failing to delete this file is not something
# we can cleanly handle.
raise
def format_resolve(value,
original_dict=None,
global_tries=50,
this_call=0, topdb=False):
"""Resolve a dict of formatted strings, mappings & list to a valued dict
Please also read the associated test::
{"a": ["{b}", "{c}", "{e}"],
"b": 1,
"c": "{d}",
"d": "{b}",
"e": "{d}",
}
====>
{"a": ["1", "1", "{e}"],
"b": 1,
"c": "{d}",
"d": "{b}",
"e": "{d}",
}
"""
if not original_dict:
original_dict = OrderedDict()
if this_call == 0 and not original_dict and isinstance(value, dict):
original_dict = value
left = False
cycle = True
if isinstance(value, dict):
new = OrderedDict()
for key, val in value.items():
val = format_resolve(val, original_dict, this_call=this_call + 1, topdb=topdb)
new[key] = val
elif isinstance(value, (list, tuple)):
new = type(value)()
for v in value:
val = format_resolve(v, original_dict, this_call=this_call + 1, topdb=topdb)
new = new + type(value)([val])
elif isinstance(value, basestring):
new = value
if '/downloads' in new:
topdb= True
# do not directly call format to handle keyerror in original mapping
# where we may have yet keyerrors
if isinstance(original_dict, dict):
for k in original_dict:
reprk = k
if not isinstance(reprk, basestring):
reprk = '{0}'.format(k)
subst = '{' + reprk + '}'
subst_val = original_dict[k]
if subst in new:
if isinstance(subst_val, (list, dict)):
inner_new = format_resolve(
subst_val, original_dict, this_call=this_call + 1, topdb=topdb)
# composed, we take the repr
if new != subst:
new = new.replace(subst, str(inner_new))
# no composed value, take the original list
else:
new = inner_new
else:
if new != subst_val:
new = new.replace(subst,
str(subst_val))
if ('{' in new) and ('}' in new):
i = 0
while True:
try:
this_call += 1
if this_call > 1000:
raise _CycleError('cycle')
new_val = format_resolve(
new, original_dict, this_call=this_call + 1, topdb=topdb)
new_braces = new.count('{'), new.count('}')
newval_braces = new_val.count('{'), new_val.count('}')
if new_braces == newval_braces:
break
else:
new = new_val
except _CycleError:
cycle = True
break
if ('{' in new) and ('}' in new):
left = True
else:
new = value
if left:
if this_call == 0:
for i in global_tries:
new_val = format_resolve(
new, original_dict, this_call=this_call + 1, topdb=topdb)
if (new == new_val) or cycle:
break
else:
new = new_val
else:
while not cycle:
new_val = format_resolve(
new, original_dict, this_call=this_call + 1, topdb=topdb)
if (new == new_val) or (cycle):
break
else:
new = new_val
return new
def register_dns_masters(only_domains=None, only_providers=None):
'''
Use registrar apis to switch the nameservers to the ones
we manage on salt
only_domains
list of domains to act on, if empty all managed domains
will be checked
only_providers
limit action to one or more providers (gandi, ovh)
CLI Examples::
salt-call mc_dns.register_dns_masters only_providers=ovh
salt-call mc_dns.register_dns_masters only_providers=gandi
salt-call mc_dns.register_dns_masters foo.net
'''
register_checks()
_s = __salt__
if isinstance(only_providers, six.string_types):
only_providers = only_providers.split(',')
if isinstance(only_domains, six.string_types):
only_domains = only_domains.split(',')
if not only_domains:
only_domains = []
managed_domains = _s['mc_pillar.query']('managed_dns_zones')[:]
if not only_domains:
only_domains = managed_domains
errors = [a for a in only_domains if a not in managed_domains]
if errors:
raise ValueError('{0} are not managed'.format(errors))
dnss = {}
for i in _s['mc_pillar.get_nss']()['slaves']:
dnss[i] = ips_for(i, fail_over=True)
# for each name server, registrer its glue record
# in the domain it belongs
glues = {}
crets = OrderedDict()
crets['set_ns'] = OrderedDict()
for dns in dnss:
domain = '.'.join(dns.split('.')[-2:])
glues.setdefault(domain, {})
glues[domain][dns] = dnss[dns]
try:
skipped = _s['mc_pillar.query']('skipped_updated_domains')
except Exception:
skipped = []
for domain, dnss in six.iteritems(glues):
provider = domain_registrar(domain)
fun = 'mc_dns.{0}_glues'.format(provider)
if fun not in _s:
log.error('Registrar {0} switcher doest exist'
' for {1}'.format(fun, domain))
continue
if not _s['mc_provider.get_{0}_opt'.format(provider)]('activated',
domain=domain):
log.error('{0} not configured'.format(provider))
continue
crets['glues'] = _s[fun](domain, dnss)
for domain in only_domains:
if domain in skipped:
log.error('{0} SKIPPED'.format(domain))
continue
provider = domain_registrar(domain)
fun = 'mc_dns.{0}_set_nss'.format(provider)
if not _s['mc_provider.get_{0}_opt'.format(provider)]('activated',
domain=domain):
log.error('{0} not configured'.format(provider))
continue
if fun not in _s:
log.error('Registrar {0} switcher doest exist'
' for {1}'.format(fun, domain))
continue
if only_providers:
if provider not in only_providers:
continue
crets['set_ns'][domain] = _s[fun](domain, dnss)
return crets
def gen_functions(self, pack=None, virtual_enable=True, whitelist=None,
provider_overrides=False):
'''
Return a dict of functions found in the defined module_dirs
'''
funcs = OrderedDict()
self.load_modules()
for mod in self.modules:
# If this is a proxy minion then MOST modules cannot work. Therefore, require that
# any module that does work with salt-proxy-minion define __proxyenabled__ as a list
# containing the names of the proxy types that the module supports.
if not hasattr(mod, 'render') and 'proxy' in self.opts:
if not hasattr(mod, '__proxyenabled__'):
# This is a proxy minion but this module doesn't support proxy
# minions at all
continue
if not (self.opts['proxy']['proxytype'] in mod.__proxyenabled__ or
'*' in mod.__proxyenabled__):
# This is a proxy minion, this module supports proxy
# minions, but not this particular minion
log.debug(mod)
continue
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
if not isinstance(chunk, dict):
continue
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
# Trim the full pathname to just the module
# this will be the short name that other salt modules and state
# will refer to it as.
module_name = mod.__name__.rsplit('.', 1)[-1]
if virtual_enable:
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
(virtual_ret, virtual_name) = self.process_virtual(mod,
module_name)
# if process_virtual returned a non-True value then we are
# supposed to not process this module
if virtual_ret is not True:
continue
# update our module name to reflect the virtual name
module_name = virtual_name
if whitelist:
# If a whitelist is defined then only load the module if it is
# in the whitelist
if module_name not in whitelist:
continue
# load the functions from the module and update our dict
funcs.update(self.load_functions(mod, module_name))
# Handle provider overrides
if provider_overrides and self.opts.get('providers', False):
if isinstance(self.opts['providers'], dict):
for mod, provider in self.opts['providers'].items():
newfuncs = raw_mod(self.opts, provider, funcs)
if newfuncs:
for newfunc in newfuncs:
f_key = '{0}{1}'.format(
mod, newfunc[newfunc.rindex('.'):]
)
funcs[f_key] = newfuncs[newfunc]
# now that all the functions have been collected, iterate back over
# the available modules and inject the special __salt__ namespace that
# contains these functions.
for mod in self.modules:
if not hasattr(mod, '__salt__') or (
not in_pack(pack, '__salt__') and
(not str(mod.__name__).startswith('salt.loaded.int.grain') and
not str(mod.__name__).startswith('salt.loaded.ext.grain'))
):
mod.__salt__ = funcs
elif not in_pack(pack, '__salt__') and \
(str(mod.__name__).startswith('salt.loaded.int.grain') or
str(mod.__name__).startswith('salt.loaded.ext.grain')):
mod.__salt__.update(funcs)
return funcs
def hold_present(name, snapshot, recursive=False):
'''
ensure hold is present on the system
name : string
name of holdt
snapshot : string
name of snapshot
recursive : boolean
recursively add hold with the given tag on the snapshots of all descendent file systems.
'''
ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
## log configuration
log.debug('zfs.hold_present::%s::config::snapshot = %s', name, snapshot)
log.debug('zfs.hold_present::%s::config::recursive = %s', name, recursive)
## check we have a snapshot/tag name
if not __utils__['zfs.is_snapshot'](snapshot):
ret['result'] = False
ret['comment'] = 'invalid snapshot name: {0}'.format(snapshot)
return ret
if __utils__['zfs.is_snapshot'](name) or \
__utils__['zfs.is_bookmark'](name) or \
name == 'error':
ret['result'] = False
ret['comment'] = 'invalid tag name: {0}'.format(name)
return ret
## place hold if required
holds = __salt__['zfs.holds'](snapshot)
if name in holds:
## NOTE: hold with name already exists for snapshot
ret['comment'] = 'hold {0} is present for {1}'.format(
name,
snapshot,
)
else:
## NOTE: no hold found with name for snapshot
if not __opts__['test']:
mod_res = __salt__['zfs.hold'](name, snapshot, **{
'recursive': recursive
})
else:
mod_res = OrderedDict([('held', True)])
ret['result'] = mod_res['held']
if ret['result']:
ret['changes'] = OrderedDict([
(snapshot, OrderedDict([
(name, 'held'),
])),
])
ret['comment'] = 'hold {0} added to {1}'.format(
name,
snapshot,
)
else:
ret['comment'] = 'failed to add hold {0} to {1}'.format(
name,
snapshot,
)
if 'error' in mod_res:
ret['comment'] = mod_res['error']
return ret
def test_include(self):
ret = self.render(include_template)
self.assertEqual(ret, OrderedDict([("include", ["http"])]))
def test_check_state_result(self):
self.assertFalse(utils.check_state_result(None),
"Failed to handle None as an invalid data type.")
self.assertFalse(utils.check_state_result([]),
"Failed to handle an invalid data type.")
self.assertFalse(utils.check_state_result({}),
"Failed to handle an empty dictionary.")
self.assertFalse(utils.check_state_result({'host1': []}),
"Failed to handle an invalid host data structure.")
test_valid_state = {
'host1': {
'test_state': {
'result': 'We have liftoff!'
}
}
}
self.assertTrue(utils.check_state_result(test_valid_state))
test_valid_false_states = {
'test1':
OrderedDict([
('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': False
}),
])),
]),
'test2':
OrderedDict([
('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])),
('host2',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': False
}),
])),
]),
'test3': ['a'],
'test4':
OrderedDict([('asup',
OrderedDict([('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])),
('host2',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': False
}),
]))]))]),
'test5':
OrderedDict([('asup',
OrderedDict([('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])), ('host2', [])]))])
}
for test, data in six.iteritems(test_valid_false_states):
self.assertFalse(utils.check_state_result(data),
msg='{0} failed'.format(test))
test_valid_true_states = {
'test1':
OrderedDict([
('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])),
]),
'test3':
OrderedDict([
('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])),
('host2',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])),
]),
'test4':
OrderedDict([('asup',
OrderedDict([('host1',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
])),
('host2',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': True
}),
]))]))]),
'test2':
OrderedDict([('host1',
OrderedDict([
('test_state0', {
'result': None
}),
('test_state', {
'result': True
}),
])),
('host2',
OrderedDict([
('test_state0', {
'result': True
}),
('test_state', {
'result': 'abc'
}),
]))])
}
for test, data in six.iteritems(test_valid_true_states):
self.assertTrue(utils.check_state_result(data),
msg='{0} failed'.format(test))
test_valid_false_state = {'host1': {'test_state': {'result': False}}}
self.assertFalse(utils.check_state_result(test_valid_false_state))
def process_results(self, rows):
'''
This function takes a list of database results and iterates over,
merging them in to a dict form.
'''
listify = OrderedDict()
listify_dicts = OrderedDict()
for ret in rows:
# crd is the Current Return Data level, to make this non-recursive.
crd = self.focus
# Walk and create dicts above the final layer
for i in range(0, self.depth-1):
# At the end we'll use listify to find values to make a list of
if i+1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[i] not in listify[id(crd)]:
listify[id(crd)].append(ret[i])
if ret[i] not in crd:
# Key missing
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# Check type of collision
ty = type(crd[ret[i]])
if ty is list:
# Already made list
temp = {}
crd[ret[i]].append(temp)
crd = temp
elif ty is not dict:
# Not a list, not a dict
if self.as_list:
# Make list
temp = {}
crd[ret[i]] = [crd[ret[i]], temp]
crd = temp
else:
# Overwrite
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# dict, descend.
crd = crd[ret[i]]
# If this test is true, the penultimate field is the key
if self.depth == self.num_fields - 1:
nk = self.num_fields-2 # Aka, self.depth-1
# Should we and will we have a list at the end?
if ((self.as_list and (ret[nk] in crd)) or
(nk+1 in self.with_lists)):
if ret[nk] in crd:
if not isinstance(crd[ret[nk]], list):
crd[ret[nk]] = [crd[ret[nk]]]
# if it's already a list, do nothing
else:
crd[ret[nk]] = []
crd[ret[nk]].append(ret[self.num_fields-1])
else:
if not self.ignore_null or ret[self.num_fields-1]:
crd[ret[nk]] = ret[self.num_fields-1]
else:
# Otherwise, the field name is the key but we have a spare.
# The spare results because of {c: d} vs {c: {"d": d, "e": e }}
# So, make that last dict
if ret[self.depth-1] not in crd:
crd[ret[self.depth-1]] = {}
# This bit doesn't escape listify
if self.depth in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[self.depth-1] not in listify[id(crd)]:
listify[id(crd)].append(ret[self.depth-1])
crd = crd[ret[self.depth-1]]
# Now for the remaining keys, we put them in to the dict
for i in range(self.depth, self.num_fields):
nk = self.field_names[i]
# Listify
if i+1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if nk not in listify[id(crd)]:
listify[id(crd)].append(nk)
# Collision detection
if self.as_list and (nk in crd):
# Same as before...
if isinstance(crd[nk], list):
crd[nk].append(ret[i])
else:
crd[nk] = [crd[nk], ret[i]]
else:
if not self.ignore_null or ret[i]:
crd[nk] = ret[i]
# Get key list and work backwards. This is inner-out processing
ks = list(listify_dicts.keys())
ks.reverse()
for i in ks:
d = listify_dicts[i]
for k in listify[i]:
if isinstance(d[k], dict):
d[k] = list(d[k].values())
elif isinstance(d[k], list):
d[k] = [d[k]]
def _repr(self, context=None):
return OrderedDict(m._repr(context) for m in self)
def pytest_sessionstart(session):
session.stats_processes = OrderedDict((
#('Log Server', test_daemon.log_server),
(' Test Suite Run', psutil.Process(os.getpid())), ))
def diff(name_a, name_b=None, **kwargs):
'''
Display the difference between a snapshot of a given filesystem and
another snapshot of that filesystem from a later time or the current
contents of the filesystem.
name_a : string
name of snapshot
name_b : string
(optional) name of snapshot or filesystem
show_changetime : boolean
display the path's inode change time as the first column of output. (default = True)
show_indication : boolean
display an indication of the type of file. (default = True)
parsable : boolean
if true we don't parse the timestamp to a more readable date (default = True)
.. versionadded:: 2016.3.0
CLI Example:
.. code-block:: bash
salt '*' zfs.diff myzpool/mydataset@yesterday myzpool/mydataset
'''
## Configure command
# NOTE: initialize the defaults
flags = ['-H']
target = []
# NOTE: set extra config from kwargs
if kwargs.get('show_changetime', True):
flags.append('-t')
if kwargs.get('show_indication', True):
flags.append('-F')
# NOTE: update target
target.append(name_a)
if name_b:
target.append(name_b)
## Diff filesystem/snapshot
res = __salt__['cmd.run_all'](
__utils__['zfs.zfs_command'](
command='diff',
flags=flags,
target=target,
),
python_shell=False,
)
if res['retcode'] != 0:
return __utils__['zfs.parse_command_result'](res)
else:
if not kwargs.get('parsable', True) and kwargs.get(
'show_changetime', True):
ret = OrderedDict()
for entry in res['stdout'].splitlines():
entry = entry.split()
entry_timestamp = __utils__['dateutils.strftime'](
entry[0], '%Y-%m-%d.%H:%M:%S.%f')
entry_data = "\t\t".join(entry[1:])
ret[entry_timestamp] = entry_data
else:
ret = res['stdout'].splitlines()
return ret
def list_(name=None, **kwargs):
'''
Return a list of all datasets or a specified dataset on the system and the
values of their used, available, referenced, and mountpoint properties.
name : string
name of dataset, volume, or snapshot
recursive : boolean
recursively list children
depth : int
limit recursion to depth
properties : string
comma-separated list of properties to list, the name property will always be added
type : string
comma-separated list of types to display, where type is one of
filesystem, snapshot, volume, bookmark, or all.
sort : string
property to sort on (default = name)
order : string [ascending|descending]
sort order (default = ascending)
parsable : boolean
display numbers in parsable (exact) values
.. versionadded:: 2018.3.0
.. versionadded:: 2015.5.0
CLI Example:
.. code-block:: bash
salt '*' zfs.list
salt '*' zfs.list myzpool/mydataset [recursive=True|False]
salt '*' zfs.list myzpool/mydataset properties="sharenfs,mountpoint"
'''
ret = OrderedDict()
## update properties
# NOTE: properties should be a list
properties = kwargs.get('properties', 'used,avail,refer,mountpoint')
if not isinstance(properties, list):
properties = properties.split(',')
# NOTE: name should be first property
# we loop here because there 'name' can be in the list
# multiple times.
while 'name' in properties:
properties.remove('name')
properties.insert(0, 'name')
## Configure command
# NOTE: initialize the defaults
flags = ['-H', '-p']
opts = {}
# NOTE: set extra config from kwargs
if kwargs.get('recursive', False):
flags.append('-r')
if kwargs.get('recursive', False) and kwargs.get('depth', False):
opts['-d'] = kwargs.get('depth')
if kwargs.get('type', False):
opts['-t'] = kwargs.get('type')
kwargs_sort = kwargs.get('sort', False)
if kwargs_sort and kwargs_sort in properties:
if kwargs.get('order', 'ascending').startswith('a'):
opts['-s'] = kwargs_sort
else:
opts['-S'] = kwargs_sort
if isinstance(properties, list):
# NOTE: There can be only one -o and it takes a comma-seperated list
opts['-o'] = ','.join(properties)
else:
opts['-o'] = properties
## parse zfs list
res = __salt__['cmd.run_all'](
__utils__['zfs.zfs_command'](
command='list',
flags=flags,
opts=opts,
target=name,
),
python_shell=False,
)
if res['retcode'] == 0:
for ds in res['stdout'].splitlines():
if kwargs.get('parsable', True):
ds_data = __utils__['zfs.from_auto_dict'](OrderedDict(
list(zip(properties, ds.split("\t")))), )
else:
ds_data = __utils__['zfs.to_auto_dict'](
OrderedDict(list(zip(properties, ds.split("\t")))),
convert_to_human=True,
)
ret[ds_data['name']] = ds_data
del ret[ds_data['name']]['name']
else:
return __utils__['zfs.parse_command_result'](res)
return ret
def get(*dataset, **kwargs):
'''
Displays properties for the given datasets.
*dataset : string
name of snapshot(s), filesystem(s), or volume(s)
properties : string
comma-separated list of properties to list, defaults to all
recursive : boolean
recursively list children
depth : int
recursively list children to depth
fields : string
comma-separated list of fields to include, the name and property field will always be added
type : string
comma-separated list of types to display, where type is one of
filesystem, snapshot, volume, bookmark, or all.
source : string
comma-separated list of sources to display. Must be one of the following:
local, default, inherited, temporary, and none. The default value is all sources.
parsable : boolean
display numbers in parsable (exact) values (default = True)
.. versionadded:: 2018.3.0
.. note::
If no datasets are specified, then the command displays properties
for all datasets on the system.
.. versionadded:: 2016.3.0
CLI Example:
.. code-block:: bash
salt '*' zfs.get
salt '*' zfs.get myzpool/mydataset [recursive=True|False]
salt '*' zfs.get myzpool/mydataset properties="sharenfs,mountpoint" [recursive=True|False]
salt '*' zfs.get myzpool/mydataset myzpool/myotherdataset properties=available fields=value depth=1
'''
## Configure command
# NOTE: initialize the defaults
flags = ['-H', '-p']
opts = {}
# NOTE: set extra config from kwargs
if kwargs.get('depth', False):
opts['-d'] = kwargs.get('depth')
elif kwargs.get('recursive', False):
flags.append('-r')
fields = kwargs.get('fields', 'value,source').split(',')
if 'name' in fields: # ensure name is first
fields.remove('name')
if 'property' in fields: # ensure property is second
fields.remove('property')
fields.insert(0, 'name')
fields.insert(1, 'property')
opts['-o'] = ",".join(fields)
if kwargs.get('type', False):
opts['-t'] = kwargs.get('type')
if kwargs.get('source', False):
opts['-s'] = kwargs.get('source')
# NOTE: set property_name
property_name = kwargs.get('properties', 'all')
## Get properties
res = __salt__['cmd.run_all'](
__utils__['zfs.zfs_command'](
command='get',
flags=flags,
opts=opts,
property_name=property_name,
target=list(dataset),
),
python_shell=False,
)
ret = __utils__['zfs.parse_command_result'](res)
if res['retcode'] == 0:
for ds in res['stdout'].splitlines():
ds_data = OrderedDict(list(zip(fields, ds.split("\t"))))
if 'value' in ds_data:
if kwargs.get('parsable', True):
ds_data['value'] = __utils__['zfs.from_auto'](
ds_data['property'],
ds_data['value'],
)
else:
ds_data['value'] = __utils__['zfs.to_auto'](
ds_data['property'],
ds_data['value'],
convert_to_human=True,
)
if ds_data['name'] not in ret:
ret[ds_data['name']] = OrderedDict()
ret[ds_data['name']][ds_data['property']] = ds_data
del ds_data['name']
del ds_data['property']
return ret
def serialize(cls, id_=None):
# The order matters
serialized = OrderedDict()
if id_ is not None:
# This is meant as a configuration section, sub json schema
serialized['id'] = '{0}/{1}.json#'.format(BASE_SCHEMA_URL, id_)
else:
# Main configuration block, json schema
serialized['$schema'] = 'http://json-schema.org/draft-04/schema#'
if cls.title is not None:
serialized['title'] = cls.title
if cls.description is not None:
if cls.description == cls.__doc__:
serialized['description'] = textwrap.dedent(cls.description).strip()
else:
serialized['description'] = cls.description
required = []
ordering = []
serialized['type'] = 'object'
properties = OrderedDict()
cls.after_items_update = []
for name in cls._order:
skip_order = False
if name in cls._sections:
section = cls._sections[name]
serialized_section = section.serialize(None if section.__flatten__ is True else name)
if section.__flatten__ is True:
# Flatten the configuration section into the parent
# configuration
properties.update(serialized_section['properties'])
if 'x-ordering' in serialized_section:
ordering.extend(serialized_section['x-ordering'])
if 'required' in serialized_section:
required.extend(serialized_section['required'])
if hasattr(section, 'after_items_update'):
cls.after_items_update.extend(section.after_items_update)
skip_order = True
else:
# Store it as a configuration section
properties[name] = serialized_section
if name in cls._items:
config = cls._items[name]
# Handle the configuration items defined in the class instance
if config.__flatten__ is True:
serialized_config = config.serialize()
cls.after_items_update.append(serialized_config)
skip_order = True
else:
properties[name] = config.serialize()
if config.required:
# If it's a required item, add it to the required list
required.append(name)
if skip_order is False:
# Store the order of the item
if name not in ordering:
ordering.append(name)
if properties:
serialized['properties'] = properties
# Update the serialized object with any items to include after properties
if cls.after_items_update:
after_items_update = {}
for entry in cls.after_items_update:
name, data = next(six.iteritems(entry))
if name in after_items_update:
after_items_update[name].extend(data)
else:
after_items_update[name] = data
serialized.update(after_items_update)
if required:
# Only include required if not empty
serialized['required'] = required
if ordering:
# Only include ordering if not empty
serialized['x-ordering'] = ordering
serialized['additionalProperties'] = cls.__allow_additional_items__
return serialized
def list_all(
prefix=None,
app=None,
owner=None,
description_contains=None,
name_not_contains=None,
profile="splunk",
):
"""
Get all splunk search details. Produces results that can be used to create
an sls file.
if app or owner are specified, results will be limited to matching saved
searches.
if description_contains is specified, results will be limited to those
where "description_contains in description" is true if name_not_contains is
specified, results will be limited to those where "name_not_contains not in
name" is true.
If prefix parameter is given, alarm names in the output will be prepended
with the prefix; alarms that have the prefix will be skipped. This can be
used to convert existing alarms to be managed by salt, as follows:
CLI example:
1. Make a "backup" of all existing searches
$ salt-call splunk_search.list_all --out=txt | sed "s/local: //" > legacy_searches.sls
2. Get all searches with new prefixed names
$ salt-call splunk_search.list_all "prefix=**MANAGED BY SALT** " --out=txt | sed "s/local: //" > managed_searches.sls
3. Insert the managed searches into splunk
$ salt-call state.sls managed_searches.sls
4. Manually verify that the new searches look right
5. Delete the original searches
$ sed s/present/absent/ legacy_searches.sls > remove_legacy_searches.sls
$ salt-call state.sls remove_legacy_searches.sls
6. Get all searches again, verify no changes
$ salt-call splunk_search.list_all --out=txt | sed "s/local: //" > final_searches.sls
$ diff final_searches.sls managed_searches.sls
"""
client = _get_splunk(profile)
# splunklib doesn't provide the default settings for saved searches.
# so, in order to get the defaults, we create a search with no
# configuration, get that search, and then delete it. We use its contents
# as the default settings
name = "splunk_search.list_all get defaults"
try:
client.saved_searches.delete(name)
except Exception: # pylint: disable=broad-except
pass
search = client.saved_searches.create(name, search="nothing")
defaults = dict(search.content)
client.saved_searches.delete(name)
# stuff that splunk returns but that you should not attempt to set.
# cf http://dev.splunk.com/view/python-sdk/SP-CAAAEK2
readonly_keys = (
"triggered_alert_count",
"action.email",
"action.populate_lookup",
"action.rss",
"action.script",
"action.summary_index",
"qualifiedSearch",
"next_scheduled_time",
)
results = OrderedDict()
# sort the splunk searches by name, so we get consistent output
searches = sorted([(s.name, s) for s in client.saved_searches])
for name, search in searches:
if app and search.access.app != app:
continue
if owner and search.access.owner != owner:
continue
if name_not_contains and name_not_contains in name:
continue
if prefix:
if name.startswith(prefix):
continue
name = prefix + name
# put name in the OrderedDict first
d = [{"name": name}]
# add the rest of the splunk settings, ignoring any defaults
description = ""
for (k, v) in sorted(search.content.items()):
if k in readonly_keys:
continue
if k.startswith("display."):
continue
if not v:
continue
if k in defaults and defaults[k] == v:
continue
d.append({k: v})
if k == "description":
description = v
if description_contains and description_contains not in description:
continue
results["manage splunk search " + name] = {"splunk_search.present": d}
return salt.utils.yaml.safe_dump(results,
default_flow_style=False,
width=120)
def _dataset_present(dataset_type,
name,
volume_size=None,
sparse=False,
create_parent=False,
properties=None,
cloned_from=None):
'''
internal handler for filesystem_present/volume_present
dataset_type : string
volume or filesystem
name : string
name of volume
volume_size : string
size of volume
sparse : boolean
create sparse volume
create_parent : boolean
creates all the non-existing parent datasets.
any property specified on the command line using the -o option is ignored.
cloned_from : string
name of snapshot to clone
properties : dict
additional zfs properties (-o)
.. note::
``cloned_from`` is only use if the volume does not exist yet,
when ``cloned_from`` is set after the volume exists it will be ignored.
.. note::
Properties do not get cloned, if you specify the properties in the state file
they will be applied on a subsequent run.
``volume_size`` is considered a property, so the volume's size will be
corrected when the properties get updated if it differs from the
original volume.
The sparse parameter is ignored when using ``cloned_from``.
'''
ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
## fallback dataset_type to filesystem if out of range
if dataset_type not in ['filesystem', 'volume']:
dataset_type = 'filesystem'
## ensure properties are zfs values
if volume_size:
volume_size = __utils__['zfs.from_size'](volume_size)
if properties:
properties = __utils__['zfs.from_auto_dict'](properties)
elif properties is None:
properties = {}
## log configuration
log.debug('zfs.%s_present::%s::config::volume_size = %s', dataset_type,
name, volume_size)
log.debug('zfs.%s_present::%s::config::sparse = %s', dataset_type, name,
sparse)
log.debug('zfs.%s_present::%s::config::create_parent = %s', dataset_type,
name, create_parent)
log.debug('zfs.%s_present::%s::config::cloned_from = %s', dataset_type,
name, cloned_from)
log.debug('zfs.%s_present::%s::config::properties = %s', dataset_type,
name, properties)
## check we have valid filesystem name/volume name/clone snapshot
if not __utils__['zfs.is_dataset'](name):
ret['result'] = False
ret['comment'] = 'invalid dataset name: {0}'.format(name)
return ret
if cloned_from and not __utils__['zfs.is_snapshot'](cloned_from):
ret['result'] = False
ret['comment'] = '{0} is not a snapshot'.format(cloned_from)
return ret
## ensure dataset is in correct state
## NOTE: update the dataset
if __salt__['zfs.exists'](name, **{'type': dataset_type}):
## NOTE: fetch current volume properties
properties_current = __salt__['zfs.get'](
name,
type=dataset_type,
fields='value',
depth=0,
parsable=True,
).get(name, OrderedDict())
## NOTE: add volsize to properties
if volume_size:
properties['volsize'] = volume_size
## NOTE: build list of properties to update
properties_update = []
for prop in properties:
## NOTE: skip unexisting properties
if prop not in properties_current:
log.warning(
'zfs.%s_present::%s::update - unknown property: %s',
dataset_type, name, prop)
continue
## NOTE: compare current and wanted value
if properties_current[prop]['value'] != properties[prop]:
properties_update.append(prop)
## NOTE: update pool properties
for prop in properties_update:
if not __opts__['test']:
mod_res = __salt__['zfs.set'](name, **{prop: properties[prop]})
else:
mod_res = OrderedDict([('set', True)])
if mod_res['set']:
if name not in ret['changes']:
ret['changes'][name] = {}
ret['changes'][name][prop] = properties[prop]
else:
ret['result'] = False
if ret['comment'] == '':
ret['comment'] = 'The following properties were not updated:'
ret['comment'] = '{0} {1}'.format(ret['comment'], prop)
## NOTE: update comment
if ret['result'] and name in ret['changes']:
ret['comment'] = '{0} {1} was updated'.format(dataset_type, name)
elif ret['result']:
ret['comment'] = '{0} {1} is uptodate'.format(dataset_type, name)
else:
ret['comment'] = '{0} {1} failed to be updated'.format(
dataset_type, name)
## NOTE: create or clone the dataset
else:
mod_res_action = 'cloned' if cloned_from else 'created'
if __opts__['test']:
## NOTE: pretend to create/clone
mod_res = OrderedDict([
(mod_res_action, True),
])
elif cloned_from:
## NOTE: add volsize to properties
if volume_size:
properties['volsize'] = volume_size
## NOTE: clone the dataset
mod_res = __salt__['zfs.clone'](cloned_from, name, **{
'create_parent': create_parent,
'properties': properties,
})
else:
## NOTE: create the dataset
mod_res = __salt__['zfs.create'](name, **{
'create_parent': create_parent,
'properties': properties,
'volume_size': volume_size,
'sparse': sparse,
})
ret['result'] = mod_res[mod_res_action]
if ret['result']:
ret['changes'][name] = mod_res_action
if properties:
ret['changes'][name] = properties
ret['comment'] = '{0} {1} was {2}'.format(
dataset_type,
name,
mod_res_action,
)
else:
ret['comment'] = 'failed to {0} {1} {2}'.format(
mod_res_action[:-1],
dataset_type,
name,
)
if 'error' in mod_res:
ret['comment'] = mod_res['error']
return ret
def test_present_create_passthrough_fail(self):
"""
Test zpool present with non existing pool (without a layout)
"""
ret = {
"name":
"myzpool",
"result":
False,
"comment":
"\n".join([
"invalid vdev specification",
"use 'force=True' to override the following errors:",
"/data/salt/vdisk0 is part of exported pool 'zsalt'",
"/data/salt/vdisk1 is part of exported pool 'zsalt'",
]),
"changes": {},
}
config = {
"force": False,
"import": False,
}
layout = [
OrderedDict([("mirror", ["disk0", "disk1"])]),
OrderedDict([("mirror", ["disk2", "disk3"])]),
]
properties = {
"autoexpand": True,
}
filesystem_properties = {
"quota": "5G",
}
mock_exists = MagicMock(return_value=False)
mock_create = MagicMock(return_value=OrderedDict([
("created", False),
(
"error",
"\n".join([
"invalid vdev specification",
"use 'force=True' to override the following errors:",
"/data/salt/vdisk0 is part of exported pool 'zsalt'",
"/data/salt/vdisk1 is part of exported pool 'zsalt'",
]),
),
]))
with patch.dict(zpool.__salt__,
{"zpool.exists": mock_exists}), patch.dict(
zpool.__salt__,
{"zpool.create": mock_create}), patch.dict(
zpool.__utils__, self.utils_patch):
self.assertEqual(
zpool.present(
"myzpool",
config=config,
layout=layout,
properties=properties,
filesystem_properties=filesystem_properties,
),
ret,
)
"mode": "0755"
},
{
"require": [{
"file": "/usr/local/bin"
}]
},
{
"source": "salt://debian/files/pydmesg.py"
},
{
"user": "******"
},
]
}
pydmesg_salt_expected = OrderedDict([("/usr/local/bin/pydmesg",
pydmesg_expected)])
pydmesg_kwargs = dict(user="******",
group="root",
mode="0755",
source="salt://debian/files/pydmesg.py")
basic_template = """#!pyobjects
File.directory('/tmp', mode='1777', owner='root', group='root')
"""
invalid_template = """#!pyobjects
File.fail('/tmp')
"""
include_template = """#!pyobjects
include('http')
def managed(name, entries, connect_spec=None):
'''Ensure the existance (or not) of LDAP entries and their attributes
Example:
.. code-block:: yaml
ldapi:///:
ldap.managed:
- connect_spec:
bind:
method: sasl
- entries:
# make sure the entry doesn't exist
- cn=foo,ou=users,dc=example,dc=com:
- delete_others: True
# make sure the entry exists with only the specified
# attribute values
- cn=admin,dc=example,dc=com:
- delete_others: True
- replace:
cn:
- admin
description:
- LDAP administrator
objectClass:
- simpleSecurityObject
- organizationalRole
userPassword:
- {{pillar.ldap_admin_password}}
# make sure the entry exists, its olcRootDN attribute
# has only the specified value, the olcRootDN attribute
# doesn't exist, and all other attributes are ignored
- 'olcDatabase={1}hdb,cn=config':
- replace:
olcRootDN:
- cn=admin,dc=example,dc=com
# the admin entry has its own password attribute
olcRootPW: []
# note the use of 'default'. also note how you don't
# have to use list syntax if there is only one attribute
# value
- cn=foo,ou=users,dc=example,dc=com:
- delete_others: True
- default:
userPassword: changeme
shadowLastChange: 0
# keep sshPublicKey if present, but don't create
# the attribute if it is missing
sshPublicKey: []
- replace:
cn: foo
uid: foo
uidNumber: 1000
gidNumber: 1000
gecos: Foo Bar
givenName: Foo
sn: Bar
homeDirectory: /home/foo
loginShell: /bin/bash
objectClass:
- inetOrgPerson
- posixAccount
- top
- ldapPublicKey
- shadowAccount
:param name:
The URL of the LDAP server. This is ignored if
``connect_spec`` is either a connection object or a dict with
a ``'url'`` entry.
:param entries:
A description of the desired state of zero or more LDAP
entries.
``entries`` is an iterable of dicts. Each of these dict's
keys are the distinguished names (DNs) of LDAP entries to
manage. Each of these dicts is processed in order. A later
dict can reference an LDAP entry that was already mentioned in
an earlier dict, which makes it possible for later dicts to
enhance or alter the desired state of an LDAP entry.
The DNs are mapped to a description of the LDAP entry's
desired state. These LDAP entry descriptions are themselves
iterables of dicts. Each dict in the iterable is processed in
order. They contain directives controlling the entry's state.
The key names the directive type and the value is state
information for the directive. The specific structure of the
state information depends on the directive type.
The structure of ``entries`` looks like this::
[{dn1: [{directive1: directive1_state,
directive2: directive2_state},
{directive3: directive3_state}],
dn2: [{directive4: directive4_state,
directive5: directive5_state}]},
{dn3: [{directive6: directive6_state}]}]
These are the directives:
* ``'delete_others'``
Boolean indicating whether to delete attributes not
mentioned in this dict or any of the other directive
dicts for this DN. Defaults to ``False``.
If you don't want to delete an attribute if present, but
you also don't want to add it if it is missing or modify
it if it is present, you can use either the ``'default'``
directive or the ``'add'`` directive with an empty value
list.
* ``'default'``
A dict mapping an attribute name to an iterable of default
values for that attribute. If the attribute already
exists, it is left alone. If not, it is created using the
given list of values.
An empty value list is useful when you don't want to
create an attribute if it is missing but you do want to
preserve it if the ``'delete_others'`` key is ``True``.
* ``'add'``
Attribute values to add to the entry. This is a dict
mapping an attribute name to an iterable of values to add.
An empty value list is useful when you don't want to
create an attribute if it is missing but you do want to
preserve it if the ``'delete_others'`` key is ``True``.
* ``'delete'``
Attribute values to remove from the entry. This is a dict
mapping an attribute name to an iterable of values to
delete from the attribute. If the iterable is empty, all
of the attribute's values are deleted.
* ``'replace'``
Attributes to replace. This is a dict mapping an
attribute name to an iterable of values. Any existing
values for the attribute are deleted, then the given
values are added. The iterable may be empty.
In the above directives, the iterables of attribute values may
instead be ``None``, in which case an empty list is used, or a
scalar such as a string or number, in which case a new list
containing the scalar is used.
Note that if all attribute values are removed from an entry,
the entire entry is deleted.
:param connect_spec:
See the description of the ``connect_spec`` parameter of the
:py:func:`ldap3.connect <salt.modules.ldap3.connect>` function
in the :py:mod:`ldap3 <salt.modules.ldap3>` execution module.
If this is a dict and the ``'url'`` entry is not specified,
the ``'url'`` entry is set to the value of the ``name``
parameter.
:returns:
A dict with the following keys:
* ``'name'``
This is the same object passed to the ``name`` parameter.
* ``'changes'``
This is a dict describing the changes made (or, in test
mode, the changes that would have been attempted). If no
changes were made (or no changes would have been
attempted), then this dict is empty. Only successful
changes are included.
Each key is a DN of an entry that was changed (or would
have been changed). Entries that were not changed (or
would not have been changed) are not included. The value
is a dict with two keys:
* ``'old'``
The state of the entry before modification. If the
entry did not previously exist, this key maps to
``None``. Otherwise, the value is a dict mapping each
of the old entry's attributes to a list of its values
before any modifications were made. Unchanged
attributes are excluded from this dict.
* ``'new'``
The state of the entry after modification. If the
entry was deleted, this key maps to ``None``.
Otherwise, the value is a dict mapping each of the
entry's attributes to a list of its values after the
modifications were made. Unchanged attributes are
excluded from this dict.
Example ``'changes'`` dict where a new entry was created
with a single attribute containing two values::
{'dn1': {'old': None,
'new': {'attr1': ['val1', 'val2']}}}
Example ``'changes'`` dict where a new attribute was added
to an existing entry::
{'dn1': {'old': {},
'new': {'attr2': ['val3']}}}
* ``'result'``
One of the following values:
* ``True`` if no changes were necessary or if all changes
were applied successfully.
* ``False`` if at least one change was unable to be applied.
* ``None`` if changes would be applied but it is in test
mode.
'''
if connect_spec is None:
connect_spec = {}
try:
connect_spec.setdefault('url', name)
except AttributeError:
# already a connection object
pass
connect = __salt__['ldap3.connect']
# hack to get at the ldap3 module to access the ldap3.LDAPError
# exception class. https://github.com/saltstack/salt/issues/27578
ldap3 = inspect.getmodule(connect)
with connect(connect_spec) as l:
old, new = _process_entries(l, entries)
# collect all of the affected entries (only the key is
# important in this dict; would have used an OrderedSet if
# there was one)
dn_set = OrderedDict()
dn_set.update(old)
dn_set.update(new)
# do some cleanup
dn_to_delete = set()
for dn in dn_set:
o = old.get(dn, {})
n = new.get(dn, {})
for x in o, n:
to_delete = set()
for attr, vals in six.iteritems(x):
if not len(vals):
# clean out empty attribute lists
to_delete.add(attr)
for attr in to_delete:
del x[attr]
if o == n:
# clean out unchanged entries
dn_to_delete.add(dn)
for dn in dn_to_delete:
for x in old, new:
x.pop(dn, None)
del dn_set[dn]
ret = {
'name': name,
'changes': {},
'result': None,
'comment': '',
}
if old == new:
ret['comment'] = 'LDAP entries already set'
ret['result'] = True
return ret
if __opts__['test']:
ret['comment'] = 'Would change LDAP entries'
changed_old = old
changed_new = new
success_dn_set = dn_set
else:
# execute the changes
changed_old = OrderedDict()
changed_new = OrderedDict()
# assume success; these will be changed on error
ret['result'] = True
ret['comment'] = 'Successfully updated LDAP entries'
errs = []
success_dn_set = OrderedDict()
for dn in dn_set:
o = old.get(dn, {})
n = new.get(dn, {})
try:
# perform the operation
if len(o):
if len(n):
op = 'modify'
assert o != n
__salt__['ldap3.change'](l, dn, o, n)
else:
op = 'delete'
__salt__['ldap3.delete'](l, dn)
else:
op = 'add'
assert len(n)
__salt__['ldap3.add'](l, dn, n)
# update these after the op in case an exception
# is raised
changed_old[dn] = o
changed_new[dn] = n
success_dn_set[dn] = True
except ldap3.LDAPError:
log.exception('failed to %s entry %s', op, dn)
errs.append((op, dn))
continue
if len(errs):
ret['result'] = False
ret['comment'] = 'failed to ' \
+ ', '.join((op + ' entry ' + dn
for op, dn in errs))
# set ret['changes']. filter out any unchanged attributes, and
# convert the value sets to lists before returning them to the
# user (sorted for easier comparisons)
for dn in success_dn_set:
o = changed_old.get(dn, {})
n = changed_new.get(dn, {})
changes = {}
ret['changes'][dn] = changes
for x, xn in ((o, 'old'), (n, 'new')):
if not len(x):
changes[xn] = None
continue
changes[xn] = dict(((attr, sorted(vals))
for attr, vals in six.iteritems(x)
if o.get(attr, ()) != n.get(attr, ())))
return ret
def ovh_set_nss(domain, dnss, dnssec=False):
register_checks()
log.info('{0}: Setting up DNS'.format(domain))
if not dnss:
raise ValueError('no dns for {0}'.format(domain))
_s = __salt__
crets = OrderedDict()
client = __salt__['mc_provider.ovh_client'](domain=domain)
dnssec_ep = '/domain/zone/{0}/dnssec'.format(domain)
dnssec_status = client.get(dnssec_ep)['status'] != 'disabled'
if dnssec:
if not dnssec_status:
client.post(dnssec_ep)
log.info('{1}: activated dnssec')
else:
if dnssec_status:
client.delete(dnssec_ep)
log.info('{1}: deactivated dnssec')
hosted_status = client.get('/domain/{0}'.format(domain))
if hosted_status['nameServerType'] == 'hosted':
client.put('/domain/{0}'.format(domain), nameServerType='external')
current_nss = {}
todelete = set()
skip = []
crets['skipped'] = OrderedDict()
for nsid in client.get('/domain/{0}/nameServer'.format(domain)):
ns = client.get('/domain/{0}/nameServer/{1}'.format(domain, nsid))
host = ns['host']
dns = current_nss.setdefault(host, {})
cns = current_nss[host] = __salt__['mc_utils.dictupdate'](dns, ns)
if host not in dnss:
todelete.add(nsid)
if host in dnss and cns['ip']:
if cns['ip'] == dnss[host][0]:
crets['skipped'][host] = cns
skip.append(host)
else:
todelete.add(nsid)
def remove_remaining(remaining, log=False):
can_delete = True
for ns in [a for a in todelete]:
if not can_delete:
continue
ret = client.get('/domain/{0}/nameServer/{1}'.format(domain, ns))
if ret['toDelete']:
continue
try:
client.delete('/domain/{0}/nameServer/{1}'.format(domain, ns))
log.info('{1}: deleted ns: {0}'.format(ns, domain))
todelete.remove(ns)
except (Exception, ) as exc:
if log:
print(exc)
log.error(traceback.format_exc())
can_delete = False
continue
return todelete
todelete = remove_remaining(todelete)
for ns, data in six.iteritems(dnss):
if ns in skip:
continue
nameservers = [{'host': ns, 'ip': a} for a in data]
crets['ns_{0}'.format(ns)] = ret = client.post(
'/domain/{0}/nameServer'.format(domain), nameServer=nameservers)
if ret['status'] != 'todo':
log.error("{0} unexpected result".format(ns))
log.error(pprint.pformat(ret))
else:
log.info('{1}: Created ns: {0}'.format(ns, domain))
# try to delete remaining deleted servers
todelete = remove_remaining(todelete)
todelete = remove_remaining(todelete, log=True)
if todelete:
log.error('{0}: {1} were not deleted'.format(ns, todelete))
return crets
def _absent(name,
dataset_type,
force=False,
recursive=False,
recursive_all=False):
'''
internal shared function for *_absent
name : string
name of dataset
dataset_type : string [filesystem, volume, snapshot, or bookmark]
type of dataset to remove
force : boolean
try harder to destroy the dataset
recursive : boolean
also destroy all the child datasets
recursive_all : boolean
recursively destroy all dependents, including cloned file systems
outside the target hierarchy. (-R)
'''
ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
## log configuration
dataset_type = dataset_type.lower()
log.debug('zfs.%s_absent::%s::config::force = %s', dataset_type, name,
force)
log.debug('zfs.%s_absent::%s::config::recursive = %s', dataset_type, name,
recursive)
## destroy dataset if needed
if __salt__['zfs.exists'](name, **{'type': dataset_type}):
## NOTE: dataset found with the name and dataset_type
if not __opts__['test']:
mod_res = __salt__['zfs.destroy'](name, **{
'force': force,
'recursive': recursive,
'recursive_all': recursive_all
})
else:
mod_res = OrderedDict([('destroyed', True)])
ret['result'] = mod_res['destroyed']
if ret['result']:
ret['changes'][name] = 'destroyed'
ret['comment'] = '{0} {1} was destroyed'.format(
dataset_type,
name,
)
else:
ret['comment'] = 'failed to destroy {0} {1}'.format(
dataset_type,
name,
)
if 'error' in mod_res:
ret['comment'] = mod_res['error']
else:
## NOTE: no dataset found with name of the dataset_type
ret['comment'] = '{0} {1} is absent'.format(dataset_type, name)
return ret
def gandi_glues(domain, dnss):
register_checks()
_s = __salt__
api, apikey = _s['mc_provider.gandi_client'](domain=domain)
current_glues = api.domain.host.list(apikey, domain)
ret = OrderedDict()
ret['add'] = OrderedDict()
ret['update'] = OrderedDict()
ret['delete'] = OrderedDict()
ret['ns'] = OrderedDict()
toadd = {}
toupdate = {}
todelete = {}
for a in current_glues:
if a['name'] not in dnss:
ips = todelete.setdefault(a['name'], [])
for ip in a['ips']:
if ip not in ips:
ips.append(ip)
for a in current_glues:
if a['name'] in dnss:
aips = dnss[a['name']][:]
aips.sort()
bips = a['ips'][:]
bips.sort()
if aips != bips:
toupdate[a['name']] = aips
for dns in dnss:
if dns not in [a['name'] for a in current_glues]:
toadd[dns] = dnss[dns][:]
error = False
for ns, ips in six.iteritems(toadd):
try:
ret['add'][ns] = api.domain.host.create(apikey, ns, ips)
assert ret['add'][ns]['errortype'] is None
except KeyboardInterrupt:
raise
except Exception:
print(traceback.format_exc())
error = True
if not error:
for ns, ips in six.iteritems(toupdate):
try:
ret['update'][ns] = api.domain.host.update(apikey, ns, ips)
assert ret['update'][ns]['errortype'] is None
except KeyboardInterrupt:
raise
except Exception:
print(traceback.format_exc())
error = True
if not error:
try:
ret['ns'] = api.domain.nameservers.set(apikey, domain,
[a for a in dnss])
assert ret['ns']['errortype'] is None
except KeyboardInterrupt:
raise
except Exception:
print(traceback.format_exc())
error = True
if not error:
for ns, ips in six.iteritems(todelete):
try:
ret['delete'][ns] = api.domain.host.delete(apikey, ns)
assert ret['delete'][ns]['errortype'] is None
except KeyboardInterrupt:
raise
except Exception:
print(traceback.format_exc())
error = True
ret['error'] = error
return ret
class PathInfoDict(fileinfo.FileInfo):
def __init__(self, match_each=True, **patterns):
'''
match_each:
If True, each file path is matched which prevents uses less memory
but sacrifices performance a little bit. If False, the complete
list is matched after all the file information has been added to
pathinfo.
patterns:
Contains the patterns to match.
Example:
{ 'saltenv': 'base', 'relpath': ['*.sls'] }
'''
super(PathInfoDict, self).__init__(
fields=PATHINFO_FIELDS,
match_each=match_each,
**patterns
)
self._elements = OrderedDict()
@property
def as_sequence(self):
if self.pattern and not self.match_each:
return list(
matcher.ifilter(
self._elements.values(),
_pattern=self.pattern
)
)
return self._elements.values()
def element(self, root=None, abspath=None, **kwargs):
'''
kwargs contain extra information for custom methods.
This method must return a valid empty object if no vars are passed
to allow introspection to create patterns.
:param root:
:param abspath:
'''
if root is None and abspath is None:
root = os.path.abspath('.')
abspath = os.path.abspath('.')
relpath = os.path.relpath(abspath, root)
try:
element = self._elements.get(relpath, OrderedDict())
except AttributeError:
element = OrderedDict()
if not element:
for field in PATHINFO_FIELDS:
element.setdefault(field, '')
element['saltenv'] = kwargs.get('saltenv', 'base')
element['relpath'] = relpath
element['abspath'] = abspath
element['is_pillar'] = kwargs.get('is_pillar', False)
cachedir = kwargs.get('cachedir', '')
if cachedir and os.path.commonprefix([abspath, cachedir]) == cachedir:
element['cache_root'] = root
else:
element['file_root'] = root
element_hook = kwargs.get('_element_hook', None)
if element_hook:
element = element_hook(self, element, **kwargs)
return element
def add_element(self, element, **kwargs):
add_hook = kwargs.get('_add_hook', None)
if add_hook:
element = add_hook(self, element, **kwargs)
if element['relpath'] not in self._elements:
self._elements[element['relpath']] = element
def filelist(self, roots, **kwargs):
'''
:param roots:
file_roots, pillar_roots, cache_roots, etc to walk.
kwargs:
Contains any extra variables to pass to element.
'''
for env, destdirs in six.iteritems(roots):
kwargs['saltenv'] = env
super(PathInfoDict, self).filelist(destdirs, **kwargs)
return self.as_sequence
def test_present_update_nochange_success(self):
"""
Test zpool present with non existing pool
"""
ret = {
"name": "myzpool",
"result": True,
"comment": "no update needed",
"changes": {},
}
config = {
"import": False,
}
layout = [
OrderedDict([("mirror", ["disk0", "disk1"])]),
OrderedDict([("mirror", ["disk2", "disk3"])]),
]
properties = {
"autoexpand": True,
}
mock_exists = MagicMock(return_value=True)
mock_get = MagicMock(return_value=OrderedDict([
("comment", "salt managed pool"),
("freeing", 0),
("listsnapshots", False),
("leaked", 0),
("feature@obsolete_counts", "enabled"),
("feature@sha512", "enabled"),
("delegation", True),
("dedupditto", "0"),
("dedupratio", "1.00x"),
("autoexpand", True),
("feature@bookmarks", "enabled"),
("allocated", 115712),
("guid", 1591906802560842214),
("feature@large_blocks", "enabled"),
("size", 2113929216),
("feature@enabled_txg", "active"),
("feature@hole_birth", "active"),
("capacity", 0),
("feature@multi_vdev_crash_dump", "enabled"),
("feature@extensible_dataset", "enabled"),
("cachefile", "-"),
("bootfs", "-"),
("autoreplace", True),
("readonly", False),
("version", "-"),
("health", "ONLINE"),
("expandsize", "-"),
("feature@embedded_data", "active"),
("feature@lz4_compress", "active"),
("feature@async_destroy", "enabled"),
("feature@skein", "enabled"),
("feature@empty_bpobj", "enabled"),
("feature@spacemap_histogram", "active"),
("bootsize", "-"),
("free", 2113813504),
("feature@device_removal", "enabled"),
("failmode", "wait"),
("feature@filesystem_limits", "enabled"),
("feature@edonr", "enabled"),
("altroot", "-"),
("fragmentation", "0%"),
]))
with patch.dict(zpool.__salt__,
{"zpool.exists": mock_exists}), patch.dict(
zpool.__salt__,
{"zpool.get": mock_get}), patch.dict(
zpool.__utils__, self.utils_patch):
self.assertEqual(
zpool.present(
"myzpool",
config=config,
layout=layout,
properties=properties,
),
ret,
)
# Run state with test=true
ret = {
"name": "myzpool",
"result": True,
"comment": "storage pool myzpool is uptodate",
"changes": {},
}
with patch.dict(zpool.__salt__,
{"zpool.exists": mock_exists}), patch.dict(
zpool.__salt__,
{"zpool.get": mock_get}), patch.dict(
zpool.__utils__, self.utils_patch), patch.dict(
zpool.__opts__, {"test": True}):
self.assertEqual(
zpool.present(
"myzpool",
config=config,
layout=layout,
properties=properties,
),
ret,
)
def scheduled_snapshot(name, prefix, recursive=True, schedule=None):
'''
maintain a set of snapshots based on a schedule
name : string
name of filesystem or volume
prefix : string
prefix for the snapshots
e.g. 'test' will result in snapshots being named 'test-yyyymmdd_hhmm'
recursive : boolean
create snapshots for all children also
schedule : dict
dict holding the schedule, the following keys are available (minute, hour,
day, month, and year) by default all are set to 0 the value indicated the
number of snapshots of that type to keep around.
.. warning::
snapshots will only be created and pruned every time the state runs.
a schedule must be setup to automatically run the state. this means that if
you run the state daily the hourly snapshot will only be made once per day!
.. versionchanged:: 2018.3.0
switched to localtime from gmtime so times now take into account timezones.
'''
ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
## initialize defaults
schedule_holds = ['minute', 'hour', 'day', 'month', 'year']
snapshots = OrderedDict([
('_create', OrderedDict()),
('_prunable', []),
('_schedule', OrderedDict()),
])
## strict configuration validation
## NOTE: we need a valid dataset
if not __utils__['zfs.is_dataset'](name):
ret['result'] = False
ret['comment'] = 'invalid dataset name: {0}'.format(name)
if not __salt__['zfs.exists'](name, **{'type': 'filesystem,volume'}):
ret['comment'] = 'dataset {0} does not exist'.format(name)
ret['result'] = False
## NOTE: prefix must be 4 or longer
if not prefix or len(prefix) < 4:
ret['comment'] = 'prefix ({0}) must be at least 4 long'.format(prefix)
ret['result'] = False
## NOTE: validate schedule
total_count = 0
for hold in schedule_holds:
snapshots[hold] = []
if hold not in schedule:
snapshots['_schedule'][hold] = 0
elif isinstance(schedule[hold], int):
snapshots['_schedule'][hold] = schedule[hold]
else:
ret['result'] = False
ret['comment'] = 'schedule value for {0} is not an integer'.format(
hold, )
break
total_count += snapshots['_schedule'][hold]
if ret['result'] and total_count == 0:
ret['result'] = False
ret['comment'] = 'schedule is not valid, you need to keep atleast 1 snapshot'
## NOTE: return if configuration is not valid
if not ret['result']:
return ret
## retrieve existing snapshots
snapshots = _schedule_snapshot_retrieve(name, prefix, snapshots)
## prepare snapshot
snapshots = _schedule_snapshot_prepare(name, prefix, snapshots)
## log configuration
log.debug('zfs.scheduled_snapshot::%s::config::recursive = %s', name,
recursive)
log.debug('zfs.scheduled_snapshot::%s::config::prefix = %s', name, prefix)
log.debug('zfs.scheduled_snapshot::%s::snapshots = %s', name, snapshots)
## create snapshot(s)
for snapshot_name, snapshot_holds in snapshots['_create'].items():
## NOTE: skip if new snapshot has no holds
if not snapshot_holds:
continue
## NOTE: create snapshot
if not __opts__['test']:
mod_res = __salt__['zfs.snapshot'](snapshot_name, **{
'recursive': recursive
})
else:
mod_res = OrderedDict([('snapshotted', True)])
if not mod_res['snapshotted']:
ret['result'] = False
ret['comment'] = 'error creating snapshot ({0})'.format(
snapshot_name)
else:
## NOTE: create holds (if we have a snapshot)
for hold in snapshot_holds:
if not __opts__['test']:
mod_res = __salt__['zfs.hold'](hold, snapshot_name, **{
'recursive': recursive
})
else:
mod_res = OrderedDict([('held', True)])
if not mod_res['held']:
ret['result'] = False
ret['comment'] = "error adding hold ({0}) to snapshot ({1})".format(
hold,
snapshot_name,
)
break
snapshots[hold].append(snapshot_name)
if ret['result']:
ret['comment'] = 'scheduled snapshots updated'
if 'created' not in ret['changes']:
ret['changes']['created'] = []
ret['changes']['created'].append(snapshot_name)
## prune hold(s)
for hold, hold_count in snapshots['_schedule'].items():
while ret['result'] and len(snapshots[hold]) > hold_count:
## NOTE: pop oldest snapshot
snapshot_name = snapshots[hold].pop(0)
## NOTE: release hold for snapshot
if not __opts__['test']:
mod_res = __salt__['zfs.release'](hold, snapshot_name, **{
'recursive': recursive
})
else:
mod_res = OrderedDict([('released', True)])
if not mod_res['released']:
ret['result'] = False
ret['comment'] = "error adding hold ({0}) to snapshot ({1})".format(
hold,
snapshot_name,
)
## NOTE: mark as prunable
if not __salt__['zfs.holds'](snapshot_name):
snapshots['_prunable'].append(snapshot_name)
## prune snapshot(s)
for snapshot_name in snapshots['_prunable']:
## NOTE: destroy snapshot
if not __opts__['test']:
mod_res = __salt__['zfs.destroy'](snapshot_name, **{
'recursive': recursive
})
else:
mod_res = OrderedDict([('destroyed', True)])
if not mod_res['destroyed']:
ret['result'] = False
ret['comment'] = "error prunding snapshot ({1})".format(
snapshot_name, )
break
if ret['result'] and snapshots['_prunable']:
ret['comment'] = 'scheduled snapshots updated'
ret['changes']['pruned'] = snapshots['_prunable']
if ret['result'] and not ret['changes']:
ret['comment'] = 'scheduled snapshots are up to date'
return ret
def _process_entries(l, entries):
'''Helper for managed() to process entries and return before/after views
Collect the current database state and update it according to the
data in :py:func:`managed`'s ``entries`` parameter. Return the
current database state and what it will look like after
modification.
:param l:
the LDAP connection object
:param entries:
the same object passed to the ``entries`` parameter of
:py:func:`manage`
:return:
an ``(old, new)`` tuple that describes the current state of
the entries and what they will look like after modification.
Each item in the tuple is an OrderedDict that maps an entry DN
to another dict that maps an attribute name to a set of its
values (it's a set because according to the LDAP spec,
attribute value ordering is unspecified and there can't be
duplicates). The structure looks like this:
{dn1: {attr1: set([val1])},
dn2: {attr1: set([val2]), attr2: set([val3, val4])}}
All of an entry's attributes and values will be included, even
if they will not be modified. If an entry mentioned in the
entries variable doesn't yet exist in the database, the DN in
``old`` will be mapped to an empty dict. If an entry in the
database will be deleted, the DN in ``new`` will be mapped to
an empty dict. All value sets are non-empty: An attribute
that will be added to an entry is not included in ``old``, and
an attribute that will be deleted frm an entry is not included
in ``new``.
These are OrderedDicts to ensure that the user-supplied
entries are processed in the user-specified order (in case
there are dependencies, such as ACL rules specified in an
early entry that make it possible to modify a later entry).
'''
old = OrderedDict()
new = OrderedDict()
for entries_dict in entries:
for dn, directives_seq in six.iteritems(entries_dict):
# get the old entry's state. first check to see if we've
# previously processed the entry.
olde = new.get(dn, None)
if olde is None:
# next check the database
results = __salt__['ldap3.search'](l, dn, 'base')
if len(results) == 1:
attrs = results[dn]
olde = dict(((attr, set(attrs[attr]))
for attr in attrs
if len(attrs[attr])))
else:
# nothing, so it must be a brand new entry
assert len(results) == 0
olde = {}
old[dn] = olde
# copy the old entry to create the new (don't do a simple
# assignment or else modifications to newe will affect
# olde)
newe = copy.deepcopy(olde)
new[dn] = newe
# process the directives
entry_status = {
'delete_others': False,
'mentioned_attributes': set(),
}
for directives in directives_seq:
_update_entry(newe, entry_status, directives)
if entry_status['delete_others']:
to_delete = set()
for attr in newe:
if attr not in entry_status['mentioned_attributes']:
to_delete.add(attr)
for attr in to_delete:
del newe[attr]
return old, new
def _process_entries(l, entries):
'''Helper for managed() to process entries and return before/after views
Collect the current database state and update it according to the
data in :py:func:`managed`'s ``entries`` parameter. Return the
current database state and what it will look like after
modification.
:param l:
the LDAP connection object
:param entries:
the same object passed to the ``entries`` parameter of
:py:func:`manage`
:return:
an ``(old, new)`` tuple that describes the current state of
the entries and what they will look like after modification.
Each item in the tuple is an OrderedDict that maps an entry DN
to another dict that maps an attribute name to a set of its
values (it's a set because according to the LDAP spec,
attribute value ordering is unspecified and there can't be
duplicates). The structure looks like this:
{dn1: {attr1: set([val1])},
dn2: {attr1: set([val2]), attr2: set([val3, val4])}}
All of an entry's attributes and values will be included, even
if they will not be modified. If an entry mentioned in the
entries variable doesn't yet exist in the database, the DN in
``old`` will be mapped to an empty dict. If an entry in the
database will be deleted, the DN in ``new`` will be mapped to
an empty dict. All value sets are non-empty: An attribute
that will be added to an entry is not included in ``old``, and
an attribute that will be deleted frm an entry is not included
in ``new``.
These are OrderedDicts to ensure that the user-supplied
entries are processed in the user-specified order (in case
there are dependencies, such as ACL rules specified in an
early entry that make it possible to modify a later entry).
'''
old = OrderedDict()
new = OrderedDict()
for entries_dict in entries:
for dn, directives_seq in six.iteritems(entries_dict):
# get the old entry's state. first check to see if we've
# previously processed the entry.
olde = new.get(dn, None)
if olde is None:
# next check the database
results = __salt__['ldap3.search'](l, dn, 'base')
if len(results) == 1:
attrs = results[dn]
olde = dict(((attr, OrderedSet(attrs[attr]))
for attr in attrs if len(attrs[attr])))
else:
# nothing, so it must be a brand new entry
assert not results
olde = {}
old[dn] = olde
# copy the old entry to create the new (don't do a simple
# assignment or else modifications to newe will affect
# olde)
newe = copy.deepcopy(olde)
new[dn] = newe
# process the directives
entry_status = {
'delete_others': False,
'mentioned_attributes': set(),
}
for directives in directives_seq:
_update_entry(newe, entry_status, directives)
if entry_status['delete_others']:
to_delete = set()
for attr in newe:
if attr not in entry_status['mentioned_attributes']:
to_delete.add(attr)
for attr in to_delete:
del newe[attr]
return old, new
# -*- coding: utf-8 -*-
'''
Application Kinds of Salt apps.
These are used to indicate what kind of Application is using RAET
'''
from __future__ import absolute_import
from collections import namedtuple
from salt.utils.odict import OrderedDict
# Python equivalent of an enum
APPL_KINDS = OrderedDict([('master', 0),
('minion', 1),
('syndic', 2),
('caller', 3)])
APPL_KIND_NAMES = OrderedDict((v, k) for k, v in list(APPL_KINDS.items())) # inverse map
ApplKind = namedtuple('ApplKind', list(APPL_KINDS.keys()))
applKinds = ApplKind(**APPL_KINDS)
def get_local_registry(name,
cached=True,
cachetime=60,
registry_format='yaml'):
'''Get local registry
Masteralt & Salt share the local registries
unless for the main ones:
- controllers
- services
- nodetypes
- localsettings
- cloud
For backward compatibility, we take care to load and merge
shared registries in mastersalt & salt prefix if any is found.
'''
not_shared = ['controllers', 'services', 'nodetypes',
'localsettings', 'cloud']
mastersalt_registryf = '{0}/makina-states/{1}.{2}'.format(
'/etc/mastersalt', name, registry_format)
salt_registryf = '{0}/makina-states/{1}.{2}'.format(
'/etc/salt', name, registry_format)
shared_registryf = os.path.join(
'/etc/makina-states/{0}.{1}'.format(name, registry_format))
registry = OrderedDict()
# cache local registries one minute
pkey = '{0}____'.format(name)
key = '{0}{1}'.format(pkey, time.time() // cachetime)
if name not in not_shared:
to_load = [mastersalt_registryf,
salt_registryf,
shared_registryf]
else:
to_load = [
'{0}/makina-states/{1}.{2}'.format(
__opts__['config_dir'], name, registry_format)
]
if (key not in _LOCAL_REG_CACHE) or (not cached):
invalidate_cached_registry(name)
for registryf in to_load:
dregistry = os.path.dirname(registryf)
if not os.path.exists(dregistry):
os.makedirs(dregistry)
if os.path.exists(registryf):
_LOCAL_REG_CACHE[key] = registry = __salt__[
'mc_utils.dictupdate'](
registry,
__salt__[
'mc_macros.{0}_load_local_registry'.format(
registry_format)](name, registryf))
# unprefix local simple registries
loc_k = DEFAULT_LOCAL_REG_NAME.format(name)
for k in [t for t in registry if t.startswith(loc_k)]:
spl = loc_k + '.'
nk = spl.join(k.split(spl)[1:])
registry[nk] = registry[k]
registry.pop(k)
elif cached:
registry = _LOCAL_REG_CACHE[key]
return registry
def empty(self):
self.states = OrderedDict()
self.requisites = []
def managed(name, entries, connect_spec=None):
'''Ensure the existence (or not) of LDAP entries and their attributes
Example:
.. code-block:: yaml
ldapi:///:
ldap.managed:
- connect_spec:
bind:
method: sasl
- entries:
# make sure the entry doesn't exist
- cn=foo,ou=users,dc=example,dc=com:
- delete_others: True
# make sure the entry exists with only the specified
# attribute values
- cn=admin,dc=example,dc=com:
- delete_others: True
- replace:
cn:
- admin
description:
- LDAP administrator
objectClass:
- simpleSecurityObject
- organizationalRole
userPassword:
- {{pillar.ldap_admin_password}}
# make sure the entry exists, its olcRootDN attribute
# has only the specified value, the olcRootDN attribute
# doesn't exist, and all other attributes are ignored
- 'olcDatabase={1}hdb,cn=config':
- replace:
olcRootDN:
- cn=admin,dc=example,dc=com
# the admin entry has its own password attribute
olcRootPW: []
# note the use of 'default'. also note how you don't
# have to use list syntax if there is only one attribute
# value
- cn=foo,ou=users,dc=example,dc=com:
- delete_others: True
- default:
userPassword: changeme
shadowLastChange: 0
# keep sshPublicKey if present, but don't create
# the attribute if it is missing
sshPublicKey: []
- replace:
cn: foo
uid: foo
uidNumber: 1000
gidNumber: 1000
gecos: Foo Bar
givenName: Foo
sn: Bar
homeDirectory: /home/foo
loginShell: /bin/bash
objectClass:
- inetOrgPerson
- posixAccount
- top
- ldapPublicKey
- shadowAccount
:param name:
The URL of the LDAP server. This is ignored if
``connect_spec`` is either a connection object or a dict with
a ``'url'`` entry.
:param entries:
A description of the desired state of zero or more LDAP
entries.
``entries`` is an iterable of dicts. Each of these dict's
keys are the distinguished names (DNs) of LDAP entries to
manage. Each of these dicts is processed in order. A later
dict can reference an LDAP entry that was already mentioned in
an earlier dict, which makes it possible for later dicts to
enhance or alter the desired state of an LDAP entry.
The DNs are mapped to a description of the LDAP entry's
desired state. These LDAP entry descriptions are themselves
iterables of dicts. Each dict in the iterable is processed in
order. They contain directives controlling the entry's state.
The key names the directive type and the value is state
information for the directive. The specific structure of the
state information depends on the directive type.
The structure of ``entries`` looks like this::
[{dn1: [{directive1: directive1_state,
directive2: directive2_state},
{directive3: directive3_state}],
dn2: [{directive4: directive4_state,
directive5: directive5_state}]},
{dn3: [{directive6: directive6_state}]}]
These are the directives:
* ``'delete_others'``
Boolean indicating whether to delete attributes not
mentioned in this dict or any of the other directive
dicts for this DN. Defaults to ``False``.
If you don't want to delete an attribute if present, but
you also don't want to add it if it is missing or modify
it if it is present, you can use either the ``'default'``
directive or the ``'add'`` directive with an empty value
list.
* ``'default'``
A dict mapping an attribute name to an iterable of default
values for that attribute. If the attribute already
exists, it is left alone. If not, it is created using the
given list of values.
An empty value list is useful when you don't want to
create an attribute if it is missing but you do want to
preserve it if the ``'delete_others'`` key is ``True``.
* ``'add'``
Attribute values to add to the entry. This is a dict
mapping an attribute name to an iterable of values to add.
An empty value list is useful when you don't want to
create an attribute if it is missing but you do want to
preserve it if the ``'delete_others'`` key is ``True``.
* ``'delete'``
Attribute values to remove from the entry. This is a dict
mapping an attribute name to an iterable of values to
delete from the attribute. If the iterable is empty, all
of the attribute's values are deleted.
* ``'replace'``
Attributes to replace. This is a dict mapping an
attribute name to an iterable of values. Any existing
values for the attribute are deleted, then the given
values are added. The iterable may be empty.
In the above directives, the iterables of attribute values may
instead be ``None``, in which case an empty list is used, or a
scalar such as a string or number, in which case a new list
containing the scalar is used.
Note that if all attribute values are removed from an entry,
the entire entry is deleted.
:param connect_spec:
See the description of the ``connect_spec`` parameter of the
:py:func:`ldap3.connect <salt.modules.ldap3.connect>` function
in the :py:mod:`ldap3 <salt.modules.ldap3>` execution module.
If this is a dict and the ``'url'`` entry is not specified,
the ``'url'`` entry is set to the value of the ``name``
parameter.
:returns:
A dict with the following keys:
* ``'name'``
This is the same object passed to the ``name`` parameter.
* ``'changes'``
This is a dict describing the changes made (or, in test
mode, the changes that would have been attempted). If no
changes were made (or no changes would have been
attempted), then this dict is empty. Only successful
changes are included.
Each key is a DN of an entry that was changed (or would
have been changed). Entries that were not changed (or
would not have been changed) are not included. The value
is a dict with two keys:
* ``'old'``
The state of the entry before modification. If the
entry did not previously exist, this key maps to
``None``. Otherwise, the value is a dict mapping each
of the old entry's attributes to a list of its values
before any modifications were made. Unchanged
attributes are excluded from this dict.
* ``'new'``
The state of the entry after modification. If the
entry was deleted, this key maps to ``None``.
Otherwise, the value is a dict mapping each of the
entry's attributes to a list of its values after the
modifications were made. Unchanged attributes are
excluded from this dict.
Example ``'changes'`` dict where a new entry was created
with a single attribute containing two values::
{'dn1': {'old': None,
'new': {'attr1': ['val1', 'val2']}}}
Example ``'changes'`` dict where a new attribute was added
to an existing entry::
{'dn1': {'old': {},
'new': {'attr2': ['val3']}}}
* ``'result'``
One of the following values:
* ``True`` if no changes were necessary or if all changes
were applied successfully.
* ``False`` if at least one change was unable to be applied.
* ``None`` if changes would be applied but it is in test
mode.
'''
if connect_spec is None:
connect_spec = {}
try:
connect_spec.setdefault('url', name)
except AttributeError:
# already a connection object
pass
connect = __salt__['ldap3.connect']
# hack to get at the ldap3 module to access the ldap3.LDAPError
# exception class. https://github.com/saltstack/salt/issues/27578
ldap3 = inspect.getmodule(connect)
with connect(connect_spec) as l:
old, new = _process_entries(l, entries)
# collect all of the affected entries (only the key is
# important in this dict; would have used an OrderedSet if
# there was one)
dn_set = OrderedDict()
dn_set.update(old)
dn_set.update(new)
# do some cleanup
dn_to_delete = set()
for dn in dn_set:
o = old.get(dn, {})
n = new.get(dn, {})
for x in o, n:
to_delete = set()
for attr, vals in six.iteritems(x):
if not vals:
# clean out empty attribute lists
to_delete.add(attr)
for attr in to_delete:
del x[attr]
if o == n:
# clean out unchanged entries
dn_to_delete.add(dn)
for dn in dn_to_delete:
for x in old, new:
x.pop(dn, None)
del dn_set[dn]
ret = {
'name': name,
'changes': {},
'result': None,
'comment': '',
}
if old == new:
ret['comment'] = 'LDAP entries already set'
ret['result'] = True
return ret
if __opts__['test']:
ret['comment'] = 'Would change LDAP entries'
changed_old = old
changed_new = new
success_dn_set = dn_set
else:
# execute the changes
changed_old = OrderedDict()
changed_new = OrderedDict()
# assume success; these will be changed on error
ret['result'] = True
ret['comment'] = 'Successfully updated LDAP entries'
errs = []
success_dn_set = OrderedDict()
for dn in dn_set:
o = old.get(dn, {})
n = new.get(dn, {})
try:
# perform the operation
if o:
if n:
op = 'modify'
assert o != n
__salt__['ldap3.change'](l, dn, o, n)
else:
op = 'delete'
__salt__['ldap3.delete'](l, dn)
else:
op = 'add'
assert n
__salt__['ldap3.add'](l, dn, n)
# update these after the op in case an exception
# is raised
changed_old[dn] = o
changed_new[dn] = n
success_dn_set[dn] = True
except ldap3.LDAPError as err:
log.exception('failed to %s entry %s (%s)', op, dn, err)
errs.append((op, dn, err))
continue
if errs:
ret['result'] = False
ret['comment'] = 'failed to ' \
+ ', '.join((op + ' entry ' + dn + '(' + six.text_type(err) + ')'
for op, dn, err in errs))
# set ret['changes']. filter out any unchanged attributes, and
# convert the value sets to lists before returning them to the
# user (sorted for easier comparisons)
for dn in success_dn_set:
o = changed_old.get(dn, {})
n = changed_new.get(dn, {})
changes = {}
ret['changes'][dn] = changes
for x, xn in ((o, 'old'), (n, 'new')):
if not x:
changes[xn] = None
continue
changes[xn] = dict(((attr, sorted(vals))
for attr, vals in six.iteritems(x)
if o.get(attr, ()) != n.get(attr, ())))
return ret
def handle_func(self, func, data):
'''
Execute this method in a multiprocess or thread
'''
if salt.utils.is_windows():
self.functions = salt.loader.minion_mods(self.opts)
self.returners = salt.loader.returners(self.opts, self.functions)
ret = {'id': self.opts.get('id', 'master'),
'fun': func,
'schedule': data['name'],
'jid': '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())}
if 'metadata' in data:
if isinstance(data['metadata'], dict):
ret['metadata'] = data['metadata']
ret['metadata']['_TOS'] = self.time_offset
ret['metadata']['_TS'] = time.ctime()
ret['metadata']['_TT'] = time.strftime('%Y %B %d %a %H %m', time.gmtime())
else:
log.warning('schedule: The metadata parameter must be '
'specified as a dictionary. Ignoring.')
salt.utils.appendproctitle(ret['jid'])
proc_fn = os.path.join(
salt.minion.get_proc_dir(self.opts['cachedir']),
ret['jid']
)
# Check to see if there are other jobs with this
# signature running. If there are more than maxrunning
# jobs present then don't start another.
# If jid_include is False for this job we can ignore all this
# NOTE--jid_include defaults to True, thus if it is missing from the data
# dict we treat it like it was there and is True
if 'jid_include' not in data or data['jid_include']:
jobcount = 0
for basefilename in os.listdir(salt.minion.get_proc_dir(self.opts['cachedir'])):
fn_ = os.path.join(salt.minion.get_proc_dir(self.opts['cachedir']), basefilename)
if not os.path.exists(fn_):
log.debug('schedule.handle_func: {0} was processed '
'in another thread, skipping.'.format(
basefilename))
continue
with salt.utils.fopen(fn_, 'rb') as fp_:
job = salt.payload.Serial(self.opts).load(fp_)
if job:
if 'schedule' in job:
log.debug('schedule.handle_func: Checking job against '
'fun {0}: {1}'.format(ret['fun'], job))
if ret['schedule'] == job['schedule'] and os_is_running(job['pid']):
jobcount += 1
log.debug(
'schedule.handle_func: Incrementing jobcount, now '
'{0}, maxrunning is {1}'.format(
jobcount, data['maxrunning']))
if jobcount >= data['maxrunning']:
log.debug(
'schedule.handle_func: The scheduled job {0} '
'was not started, {1} already running'.format(
ret['schedule'], data['maxrunning']))
return False
else:
try:
log.info('Invalid job file found. Removing.')
os.remove(fn_)
except OSError:
log.info('Unable to remove file: {0}.'.format(fn_))
salt.utils.daemonize_if(self.opts)
ret['pid'] = os.getpid()
if 'jid_include' not in data or data['jid_include']:
log.debug('schedule.handle_func: adding this job to the jobcache '
'with data {0}'.format(ret))
# write this to /var/cache/salt/minion/proc
with salt.utils.fopen(proc_fn, 'wb+') as fp_:
fp_.write(salt.payload.Serial(self.opts).dumps(ret))
args = tuple()
if 'args' in data:
args = data['args']
kwargs = {}
if 'kwargs' in data:
kwargs = data['kwargs']
# if the func support **kwargs, lets pack in the pub data we have
# TODO: pack the *same* pub data as a minion?
argspec = salt.utils.args.get_function_argspec(self.functions[func])
if argspec.keywords:
# this function accepts **kwargs, pack in the publish data
for key, val in ret.iteritems():
kwargs['__pub_{0}'.format(key)] = val
try:
ret['return'] = self.functions[func](*args, **kwargs)
data_returner = data.get('returner', None)
if data_returner or self.schedule_returner:
if 'returner_config' in data:
ret['ret_config'] = data['returner_config']
rets = []
for returner in [data_returner, self.schedule_returner]:
if isinstance(returner, str):
rets.append(returner)
elif isinstance(returner, list):
rets.extend(returner)
# simple de-duplication with order retained
for returner in OrderedDict.fromkeys(rets):
ret_str = '{0}.returner'.format(returner)
if ret_str in self.returners:
ret['success'] = True
self.returners[ret_str](ret)
else:
log.info(
'Job {0} using invalid returner: {1}. Ignoring.'.format(
func, returner
)
)
if 'return_job' in data and not data['return_job']:
pass
else:
# Send back to master so the job is included in the job list
mret = ret.copy()
mret['jid'] = 'req'
channel = salt.transport.Channel.factory(self.opts, usage='salt_schedule')
load = {'cmd': '_return', 'id': self.opts['id']}
for key, value in mret.items():
load[key] = value
channel.send(load)
except Exception:
log.exception("Unhandled exception running {0}".format(ret['fun']))
# Although catch-all exception handlers are bad, the exception here
# is to let the exception bubble up to the top of the thread context,
# where the thread will die silently, which is worse.
finally:
try:
os.unlink(proc_fn)
except OSError as exc:
if exc.errno == errno.EEXIST or exc.errno == errno.ENOENT:
# EEXIST and ENOENT are OK because the file is gone and that's what
# we wanted
pass
else:
log.error("Failed to delete '{0}': {1}".format(proc_fn, exc.errno))
# Otherwise, failing to delete this file is not something
# we can cleanly handle.
raise
def get(zpool, prop=None, show_source=False, parsable=True):
"""
.. versionadded:: 2016.3.0
Retrieves the given list of properties
zpool : string
Name of storage pool
prop : string
Optional name of property to retrieve
show_source : boolean
Show source of property
parsable : boolean
Display numbers in parsable (exact) values
.. versionadded:: 2018.3.0
CLI Example:
.. code-block:: bash
salt '*' zpool.get myzpool
"""
ret = OrderedDict()
value_properties = ["name", "property", "value", "source"]
## collect get output
res = __salt__["cmd.run_all"](
__utils__["zfs.zpool_command"](
command="get",
flags=["-H"],
property_name=prop if prop else "all",
target=zpool,
),
python_shell=False,
)
if res["retcode"] != 0:
return __utils__["zfs.parse_command_result"](res)
# NOTE: command output for reference
# ========================================================================
# ...
# data mountpoint /data local
# data compression off default
# ...
# =========================================================================
# parse get output
for line in res["stdout"].splitlines():
# NOTE: transform data into dict
prop_data = OrderedDict(
list(
zip(
value_properties,
[x for x in line.strip().split("\t") if x not in [""]],
)))
# NOTE: older zfs does not have -o, fall back to manually stipping the name field
del prop_data["name"]
# NOTE: normalize values
if parsable:
# NOTE: raw numbers and pythonic types
prop_data["value"] = __utils__["zfs.from_auto"](
prop_data["property"], prop_data["value"])
else:
# NOTE: human readable zfs types
prop_data["value"] = __utils__["zfs.to_auto"](
prop_data["property"], prop_data["value"])
# NOTE: show source if requested
if show_source:
ret[prop_data["property"]] = prop_data
del ret[prop_data["property"]]["property"]
else:
ret[prop_data["property"]] = prop_data["value"]
return ret
def status(zpool=None):
"""
Return the status of the named zpool
zpool : string
optional name of storage pool
.. versionadded:: 2016.3.0
CLI Example:
.. code-block:: bash
salt '*' zpool.status myzpool
"""
ret = OrderedDict()
## collect status output
res = __salt__["cmd.run_all"](
__utils__["zfs.zpool_command"]("status", target=zpool),
python_shell=False,
)
if res["retcode"] != 0:
return __utils__["zfs.parse_command_result"](res)
# NOTE: command output for reference
# =====================================================================
# pool: data
# state: ONLINE
# scan: scrub repaired 0 in 2h27m with 0 errors on Mon Jan 8 03:27:25 2018
# config:
#
# NAME STATE READ WRITE CKSUM
# data ONLINE 0 0 0
# mirror-0 ONLINE 0 0 0
# c0tXXXXCXXXXXXXXXXXd0 ONLINE 0 0 0
# c0tXXXXCXXXXXXXXXXXd0 ONLINE 0 0 0
# c0tXXXXCXXXXXXXXXXXd0 ONLINE 0 0 0
#
# errors: No known data errors
# =====================================================================
## parse status output
# NOTE: output is 'key: value' except for the 'config' key.
# multiple pools will repeat the output, so if switch pools if
# we see 'pool:'
current_pool = None
current_prop = None
for zpd in res["stdout"].splitlines():
if zpd.strip() == "":
continue
if ":" in zpd and zpd[0] != "\t":
# NOTE: line is 'key: value' format, we just update a dict
prop = zpd.split(":")[0].strip()
value = ":".join(zpd.split(":")[1:]).strip()
if prop == "pool" and current_pool != value:
current_pool = value
ret[current_pool] = OrderedDict()
if prop != "pool":
ret[current_pool][prop] = value
current_prop = prop
else:
# NOTE: we append the line output to the last property
# this should only happens once we hit the config
# section
ret[current_pool][current_prop] = "{}\n{}".format(
ret[current_pool][current_prop], zpd)
## parse config property for each pool
# NOTE: the config property has some structured data
# sadly this data is in a different format than
# the rest and it needs further processing
for pool in ret:
if "config" not in ret[pool]:
continue
header = None
root_vdev = None
vdev = None
dev = None
rdev = None
config = ret[pool]["config"]
config_data = OrderedDict()
for line in config.splitlines():
# NOTE: the first line is the header
# we grab all the none whitespace values
if not header:
header = line.strip().lower()
header = [x for x in header.split(" ") if x not in [""]]
continue
# NOTE: data is indented by 1 tab, then multiples of 2 spaces
# to differential root vdev, vdev, and dev
#
# we just strip the initial tab (can't use .strip() here)
if line[0] == "\t":
line = line[1:]
# NOTE: transform data into dict
stat_data = OrderedDict(
list(
zip(
header,
[x for x in line.strip().split(" ") if x not in [""]],
)))
# NOTE: decode the zfs values properly
stat_data = __utils__["zfs.from_auto_dict"](stat_data)
# NOTE: store stat_data in the proper location
if line.startswith(" " * 6):
rdev = stat_data["name"]
config_data[root_vdev][vdev][dev][rdev] = stat_data
elif line.startswith(" " * 4):
rdev = None
dev = stat_data["name"]
config_data[root_vdev][vdev][dev] = stat_data
elif line.startswith(" " * 2):
rdev = dev = None
vdev = stat_data["name"]
config_data[root_vdev][vdev] = stat_data
else:
rdev = dev = vdev = None
root_vdev = stat_data["name"]
config_data[root_vdev] = stat_data
# NOTE: name already used as identifier, drop duplicate data
del stat_data["name"]
ret[pool]["config"] = config_data
return ret
import salt.exceptions
from salt.utils.odict import OrderedDict
from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin
log = logging.getLogger(__name__)
EXT_NAME_MAPPINGS = OrderedDict([
('basicConstraints', 'X509v3 Basic Constraints'),
('keyUsage', 'X509v3 Key Usage'),
('extendedKeyUsage', 'X509v3 Extended Key Usage'),
('subjectKeyIdentifier', 'X509v3 Subject Key Identifier'),
('authorityKeyIdentifier', 'X509v3 Authority Key Identifier'),
('issuserAltName', 'X509v3 Issuer Alternative Name'),
('authorityInfoAccess', 'X509v3 Authority Info Access'),
('subjectAltName', 'X509v3 Subject Alternative Name'),
('crlDistributionPoints', 'X509v3 CRL Distribution Points'),
('issuingDistributionPoint', 'X509v3 Issuing Distribution Point'),
('certificatePolicies', 'X509v3 Certificate Policies'),
('policyConstraints', 'X509v3 Policy Constraints'),
('inhibitAnyPolicy', 'X509v3 Inhibit Any Policy'),
('nameConstraints', 'X509v3 Name Constraints'),
('noCheck', 'X509v3 OCSP No Check'),
('nsComment', 'Netscape Comment'),
('nsCertType', 'Netscape Certificate Type'),
])
CERT_DEFAULTS = {
'days_valid': 365,
'version': 3,
'serial_bits': 64,
'algorithm': 'sha256'
def handle_func(self, func, data):
'''
Execute this method in a multiprocess or thread
'''
if salt.utils.is_windows():
self.functions = salt.loader.minion_mods(self.opts)
self.returners = salt.loader.returners(self.opts, self.functions)
ret = {'id': self.opts.get('id', 'master'),
'fun': func,
'jid': '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())}
proc_fn = os.path.join(
salt.minion.get_proc_dir(self.opts['cachedir']),
ret['jid']
)
# Check to see if there are other jobs with this
# signature running. If there are more than maxrunning
# jobs present then don't start another.
# If jid_include is False for this job we can ignore all this
# NOTE--jid_include defaults to True, thus if it is missing from the data
# dict we treat it like it was there and is True
if 'jid_include' not in data or data['jid_include']:
jobcount = 0
for basefilename in os.listdir(salt.minion.get_proc_dir(self.opts['cachedir'])):
fn = os.path.join(salt.minion.get_proc_dir(self.opts['cachedir']), basefilename)
with salt.utils.fopen(fn, 'r') as fp_:
job = salt.payload.Serial(self.opts).load(fp_)
log.debug('schedule.handle_func: Checking job against '
'fun {0}: {1}'.format(ret['fun'], job))
if ret['fun'] == job['fun'] and os_is_running(job['pid']):
jobcount += 1
log.debug(
'schedule.handle_func: Incrementing jobcount, now '
'{0}, maxrunning is {1}'.format(
jobcount, data['maxrunning']))
if jobcount >= data['maxrunning']:
log.debug(
'schedule.handle_func: The scheduled job {0} '
'was not started, {1} already running'.format(
func, data['maxrunning']))
return False
salt.utils.daemonize_if(self.opts)
ret['pid'] = os.getpid()
if 'jid_include' not in data or data['jid_include']:
log.debug('schedule.handle_func: adding this job to the jobcache '
'with data {0}'.format(ret))
# write this to /var/cache/salt/minion/proc
with salt.utils.fopen(proc_fn, 'w+') as fp_:
fp_.write(salt.payload.Serial(self.opts).dumps(ret))
args = None
if 'args' in data:
args = data['args']
kwargs = None
if 'kwargs' in data:
kwargs = data['kwargs']
try:
if args and kwargs:
ret['return'] = self.functions[func](*args, **kwargs)
if args and not kwargs:
ret['return'] = self.functions[func](*args)
if kwargs and not args:
ret['return'] = self.functions[func](**kwargs)
if not kwargs and not args:
ret['return'] = self.functions[func]()
data_returner = data.get('returner', None)
if data_returner or self.schedule_returner:
rets = []
for returner in [data_returner, self.schedule_returner]:
if isinstance(returner, str):
rets.append(returner)
elif isinstance(returner, list):
rets.extend(returner)
# simple de-duplication with order retained
rets = OrderedDict.fromkeys(rets).keys()
for returner in rets:
ret_str = '{0}.returner'.format(returner)
if ret_str in self.returners:
ret['success'] = True
self.returners[ret_str](ret)
else:
log.info(
'Job {0} using invalid returner: {1} Ignoring.'.format(
func, returner
)
)
except Exception:
log.exception("Unhandled exception running {0}".format(ret['fun']))
# Although catch-all exception handlers are bad, the exception here
# is to let the exception bubble up to the top of the thread context,
# where the thread will die silently, which is worse.
finally:
try:
os.unlink(proc_fn)
except OSError as e:
if e.errno == errno.EEXIST:
# EEXIST is OK because the file is gone and that's what
# we wanted
pass
else:
log.error("Failed to delete '{0}': {1}".format(proc_fn, e.errno))
# Otherwise, failing to delete this file is not something
# we can cleanly handle.
raise
def read_certificate(certificate):
'''
Returns a dict containing details of a certificate. Input can be a PEM string or file path.
certificate:
The certificate to be read. Can be a path to a certificate file, or a string containing
the PEM formatted text of the certificate.
CLI Example:
.. code-block:: bash
salt '*' x509.read_certificate /etc/pki/mycert.crt
'''
if isinstance(certificate, M2Crypto.X509.X509):
cert = certificate
else:
cert = _get_certificate_obj(certificate)
ret = {
# X509 Verison 3 has a value of 2 in the field.
# Version 2 has a value of 1.
# https://tools.ietf.org/html/rfc5280#section-4.1.2.1
'Version':
cert.get_version() + 1,
# Get size returns in bytes. The world thinks of key sizes in bits.
'Key Size':
cert.get_pubkey().size() * 8,
'Serial Number':
_dec2hex(cert.get_serial_number()),
'SHA-256 Finger Print':
_pretty_hex(cert.get_fingerprint(md='sha256')),
'MD5 Finger Print':
_pretty_hex(cert.get_fingerprint(md='md5')),
'SHA1 Finger Print':
_pretty_hex(cert.get_fingerprint(md='sha1')),
'Subject':
_parse_subject(cert.get_subject()),
'Subject Hash':
_dec2hex(cert.get_subject().as_hash()),
'Issuer':
_parse_subject(cert.get_issuer()),
'Issuer Hash':
_dec2hex(cert.get_issuer().as_hash()),
'Not Before':
cert.get_not_before().get_datetime().strftime('%Y-%m-%d %H:%M:%S'),
'Not After':
cert.get_not_after().get_datetime().strftime('%Y-%m-%d %H:%M:%S'),
'Public Key':
get_public_key(cert.as_pem())
}
exts = OrderedDict()
for ext_index in range(0, cert.get_ext_count()):
ext = cert.get_ext_at(ext_index)
name = ext.get_name()
val = ext.get_value()
if ext.get_critical():
val = 'critical ' + val
exts[name] = val
if exts:
ret['X509v3 Extensions'] = exts
return ret
def history(zpool=None, internal=False, verbose=False):
"""
.. versionadded:: 2016.3.0
Displays the command history of the specified pools, or all pools if no
pool is specified
zpool : string
Optional storage pool
internal : boolean
Toggle display of internally logged ZFS events
verbose : boolean
Toggle display of the user name, the hostname, and the zone in which
the operation was performed
CLI Example:
.. code-block:: bash
salt '*' zpool.upgrade myzpool
"""
ret = OrderedDict()
## Configure pool
# NOTE: initialize the defaults
flags = []
# NOTE: set extra config
if verbose:
flags.append("-l")
if internal:
flags.append("-i")
## Lookup history
res = __salt__["cmd.run_all"](
__utils__["zfs.zpool_command"](
command="history",
flags=flags,
target=zpool,
),
python_shell=False,
)
if res["retcode"] != 0:
return __utils__["zfs.parse_command_result"](res)
else:
pool = "unknown"
for line in res["stdout"].splitlines():
if line.startswith("History for"):
pool = line[13:-2]
ret[pool] = OrderedDict()
else:
if line == "":
continue
log_timestamp = line[0:19]
log_command = line[20:]
ret[pool][log_timestamp] = log_command
return ret
def test_include(self):
ret = self.render(include_template)
self.assertEqual(ret, OrderedDict([
('include', ['http']),
]))
def iostat(zpool=None, sample_time=5, parsable=True):
"""
Display I/O statistics for the given pools
zpool : string
optional name of storage pool
sample_time : int
seconds to capture data before output
default a sample of 5 seconds is used
parsable : boolean
display data in pythonc values (True, False, Bytes,...)
.. versionadded:: 2016.3.0
.. versionchanged:: 2018.3.1
Added ```parsable``` parameter that defaults to True
CLI Example:
.. code-block:: bash
salt '*' zpool.iostat myzpool
"""
ret = OrderedDict()
## get iostat output
res = __salt__["cmd.run_all"](
__utils__["zfs.zpool_command"](command="iostat",
flags=["-v"],
target=[zpool, sample_time, 2]),
python_shell=False,
)
if res["retcode"] != 0:
return __utils__["zfs.parse_command_result"](res)
# NOTE: command output for reference
# =====================================================================
# capacity operations bandwidth
# pool alloc free read write read write
# ------------------------- ----- ----- ----- ----- ----- -----
# mypool 648G 1.18T 10 6 1.30M 817K
# mirror 648G 1.18T 10 6 1.30M 817K
# c0tXXXXCXXXXXXXXXXXd0 - - 9 5 1.29M 817K
# c0tXXXXCXXXXXXXXXXXd0 - - 9 5 1.29M 817K
# c0tXXXXCXXXXXXXXXXXd0 - - 9 5 1.29M 817K
# ------------------------- ----- ----- ----- ----- ----- -----
# =====================================================================
## parse iostat output
# NOTE: hardcode the header
# the double header line is hard to parse, we opt to
# hardcode the header fields
header = [
"name",
"capacity-alloc",
"capacity-free",
"operations-read",
"operations-write",
"bandwidth-read",
"bandwidth-write",
]
root_vdev = None
vdev = None
dev = None
current_data = OrderedDict()
for line in res["stdout"].splitlines():
# NOTE: skip header
if line.strip() == "" or line.strip().split()[-1] in [
"write", "bandwidth"
]:
continue
# NOTE: reset pool on line separator
if line.startswith("-") and line.endswith("-"):
ret.update(current_data)
current_data = OrderedDict()
continue
# NOTE: transform data into dict
io_data = OrderedDict(
list(
zip(
header,
[x for x in line.strip().split(" ") if x not in [""]],
)))
# NOTE: normalize values
if parsable:
# NOTE: raw numbers and pythonic types
io_data = __utils__["zfs.from_auto_dict"](io_data)
else:
# NOTE: human readable zfs types
io_data = __utils__["zfs.to_auto_dict"](io_data)
# NOTE: store io_data in the proper location
if line.startswith(" " * 4):
dev = io_data["name"]
current_data[root_vdev][vdev][dev] = io_data
elif line.startswith(" " * 2):
dev = None
vdev = io_data["name"]
current_data[root_vdev][vdev] = io_data
else:
dev = vdev = None
root_vdev = io_data["name"]
current_data[root_vdev] = io_data
# NOTE: name already used as identifier, drop duplicate data
del io_data["name"]
return ret
def list_(properties="size,alloc,free,cap,frag,health",
zpool=None,
parsable=True):
"""
.. versionadded:: 2015.5.0
Return information about (all) storage pools
zpool : string
optional name of storage pool
properties : string
comma-separated list of properties to list
parsable : boolean
display numbers in parsable (exact) values
.. versionadded:: 2018.3.0
.. note::
The ``name`` property will always be included, while the ``frag``
property will get removed if not available
zpool : string
optional zpool
.. note::
Multiple storage pool can be provided as a space separated list
CLI Example:
.. code-block:: bash
salt '*' zpool.list
salt '*' zpool.list zpool=tank
salt '*' zpool.list 'size,free'
salt '*' zpool.list 'size,free' tank
"""
ret = OrderedDict()
## update properties
# NOTE: properties should be a list
if not isinstance(properties, list):
properties = properties.split(",")
# NOTE: name should be first property
while "name" in properties:
properties.remove("name")
properties.insert(0, "name")
# NOTE: remove 'frags' if we don't have feature flags
if not __utils__["zfs.has_feature_flags"]():
while "frag" in properties:
properties.remove("frag")
## collect list output
res = __salt__["cmd.run_all"](
__utils__["zfs.zpool_command"](
command="list",
flags=["-H"],
opts={
"-o": ",".join(properties)
},
target=zpool,
),
python_shell=False,
)
if res["retcode"] != 0:
return __utils__["zfs.parse_command_result"](res)
# NOTE: command output for reference
# ========================================================================
# data 1992864825344 695955501056 1296909324288 34 11% ONLINE
# =========================================================================
## parse list output
for line in res["stdout"].splitlines():
# NOTE: transform data into dict
zpool_data = OrderedDict(
list(zip(
properties,
line.strip().split("\t"),
)))
# NOTE: normalize values
if parsable:
# NOTE: raw numbers and pythonic types
zpool_data = __utils__["zfs.from_auto_dict"](zpool_data)
else:
# NOTE: human readable zfs types
zpool_data = __utils__["zfs.to_auto_dict"](zpool_data)
ret[zpool_data["name"]] = zpool_data
del ret[zpool_data["name"]]["name"]
return ret
def process_results(self, rows):
'''
This function takes a list of database results and iterates over,
merging them in to a dict form.
'''
listify = OrderedDict()
listify_dicts = OrderedDict()
for ret in rows:
# crd is the Current Return Data level, to make this non-recursive.
crd = self.focus
# Walk and create dicts above the final layer
for i in range(0, self.depth-1):
# At the end we'll use listify to find values to make a list of
if i+1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[i] not in listify[id(crd)]:
listify[id(crd)].append(ret[i])
if ret[i] not in crd:
# Key missing
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# Check type of collision
ty = type(crd[ret[i]])
if ty is list:
# Already made list
temp = {}
crd[ret[i]].append(temp)
crd = temp
elif ty is not dict:
# Not a list, not a dict
if self.as_list:
# Make list
temp = {}
crd[ret[i]] = [crd[ret[i]], temp]
crd = temp
else:
# Overwrite
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# dict, descend.
crd = crd[ret[i]]
# If this test is true, the penultimate field is the key
if self.depth == self.num_fields - 1:
nk = self.num_fields-2 # Aka, self.depth-1
# Should we and will we have a list at the end?
if ((self.as_list and (ret[nk] in crd)) or
(nk+1 in self.with_lists)):
if ret[nk] in crd:
if type(crd[ret[nk]]) is not list:
crd[ret[nk]] = [crd[ret[nk]]]
# if it's already a list, do nothing
else:
crd[ret[nk]] = []
crd[ret[nk]].append(ret[self.num_fields-1])
else:
# No clobber checks then
crd[ret[nk]] = ret[self.num_fields-1]
else:
# Otherwise, the field name is the key but we have a spare.
# The spare results because of {c: d} vs {c: {"d": d, "e": e }}
# So, make that last dict
if ret[self.depth-1] not in crd:
crd[ret[self.depth-1]] = {}
# This bit doesn't escape listify
if self.depth in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[self.depth-1] not in listify[id(crd)]:
listify[id(crd)].append(ret[self.depth-1])
crd = crd[ret[self.depth-1]]
# Now for the remaining keys, we put them in to the dict
for i in range(self.depth, self.num_fields):
nk = self.field_names[i]
# Listify
if i+1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if nk not in listify[id(crd)]:
listify[id(crd)].append(nk)
# Collision detection
if self.as_list and (nk in crd):
# Same as before...
if type(crd[nk]) is list:
crd[nk].append(ret[i])
else:
crd[nk] = [crd[nk], ret[i]]
else:
crd[nk] = ret[i]
# Get key list and work backwards. This is inner-out processing
ks = listify_dicts.keys()
ks.reverse()
for i in ks:
d = listify_dicts[i]
for k in listify[i]:
if type(d[k]) is dict:
d[k] = d[k].values()
elif type(d[k]) is not list:
d[k] = [d[k]]
def empty(self):
self.states = OrderedDict()
self.requisites = []
self.includes = []
self.extends = OrderedDict()
