def status(environment):
'''
Print a list of all the expected minions and their state - True (up) or false
CLI Example:
.. code-block:: bash
salt-run alchemy.status dev
:param environment:
Specify the pillar environment for which you want to get the list
:return:
A simple dict the minion id as key and the status as value
'''
_pillar = cpillar.fake_pillar(None, environment, '1nc', __opts__)
domain = cpillar.dig(_pillar, 'defaults:network:manage:domain', '1nc')
nodes = _pillar.get('hosts', {})
nodes.update(_pillar.get('containers', {}))
client = salt.client.LocalClient(__opts__['conf_file'])
minions_status = {}
for node, nodedata in nodes.iteritems():
minions_status[nodedata.get('fqdn')] = False
minions_found = client.cmd('*.' + domain, 'test.ping', timeout=1)
for minion in sorted(minions_found):
minions_status[minion] = True
return minions_status
def _get_master_pillar(full_path, default=None):
'''
This function instantiates a masterminion and compiles its
pillar data. Returns either the pillar or parts of it.
Remember pillar is compiled based on the pillar top file. So
you might not get the full pillar back - the restriction is that
the salt id of the master will be fqdn.of.host_master. Your
targets in the top file need to match that!
:param full_path:
Path to the specific pillar item you want to be returned
:param default:
What to return if the data at path is not found
:return:
Returns a dict with the requested pillar data
'''
opts = salt.config.master_config('/etc/salt/master')
opts['quiet'] = True
opts['file_client'] = 'local'
minion = salt.minion.MasterMinion(opts)
_pillar = minion.functions['pillar.items']()
data = cpillar.dig(_pillar, full_path, default)
# Channel through json to get clean dict instead of odict
import json
data_clean = json.loads(json.dumps(data))
return data_clean
def look(environment, pillar=""):
'''
Lookup the pillar for a specific pillar environment
CLI Example:
.. code-block:: bash
salt-run test.look dev defaults:network
:param environment:
The name of the pillar environment to get
:param pillar:
The pillar path to return in colon notation
:return:
The pillar dict found or none
'''
_pillar = cpillar.fake_pillar(None, environment, '1nc', __opts__)
leaf = cpillar.dig(_pillar, pillar)
res = leaf
tree = pillar.split(':')
for branch in tree:
res = {branch: res}
return res
def hosts_destroy(environment, name=None):
'''
Destroy a single host or all hosts in a salt environment. This includes completely erasing
their data - definition and historical - from the MaaS database.
CLI Example:
.. code-block:: bash
salt-run alchemy.hosts_destroy dev
salt-run alchemy.hosts_destroy dev node-01.dev.1nc
:param environment:
Name of the pillar environment to work on
:param name:
Optional name of a known host in the working environment
:return:
The data structure returned by the client call
'''
params = {}
res = {}
if name is not None:
params = 'name={}'.format(name)
glob = name
else:
_pillar = cpillar.fake_pillar(None, environment, '1nc', __opts__)
glob = '*.{}'.format(
cpillar.dig(_pillar, 'defaults:network:manage:domain'))
zone = cpillar.dig(_pillar, 'defaults:maas:zone')
params = 'name={}'.format(zone)
# delete all matched keys - it is either name or all
_get_wheel().call_func('key.delete', match=glob)
opt = {'timeout': 1, 'expr_form': 'compound'}
target = 'G@roles:maas and *.cde.1nc'
client = salt.client.LocalClient(__opts__['conf_file'], __opts__)
res['release'] = client.cmd(target, 'maas.node_release', [params], **opt)
res['delete'] = client.cmd(target, 'maas.node_delete', [params], **opt)
return res
def _merge_defaults_for_group(defaults, group):
'''
Merge top level defaults into a groups defaults. Currently this will merge defaults:network
into defaults:[group], but may be extended to do more in the future
:param defaults:
The full dict of pillar:defaults
:param group:
The name of the group as string
:return:
The dict for pillar:defaults:[group] with defaults:[xxx] merged into it
'''
# what we have in our tummy - the original defaults for the given group
group_defaults = cpillar.dig(defaults, group, {})
# the real network defaults
master_defaults = {"network": cpillar.dig(defaults, "network", {})}
# Merge but prioritise group values over default values
return udict.merge_recurse(master_defaults, group_defaults)
def masterlist(environment,
name=None,
target=None,
role=None,
active_only=True):
import cpillar
callerrole = __opts__.get('__role', None)
if callerrole is None or callerrole == 'minion':
log.error("Masterlist called with callerrole %s!", callerrole)
raise CommandExecutionError(
'Func masterlist not usable in minion context! Use list instead!')
log.warning("Masterlist called with callerrole %s!", callerrole)
_pillar = cpillar.fake_pillar(None, environment, "1nc", __opts__)
container_pillar = cpillar.dig(_pillar, 'containers')
return _getcontainerlist(name, target, role, active_only, container_pillar)
def gen_dns(environment, top="1nc", test=False):
'''
Function that creates the dns zone data for an environment and passes it to the corresponding state
on the dns servers. The state will create the zone files and include them into the server coniguration
CLI Example:
.. code-block:: bash
salt-run nodes.gen_dns dev
:param environment:
The environment you want the dns zone files be created for
:param top:
The top level domain if none is given in the pillar configuration for the environment/network
:param test:
If true, only return the data - do not call the states
:return:
If just for test, returns the data created. If run, it returns the state results
'''
_pillar = cpillar.fake_pillar(None, environment, top, __opts__)
_nodes = _get_nodes(_pillar)
# ensure that we have the same soa for all changed files
# soa = {'serial': time.strftime('%Y%m%d%H%M%S')}
serial = time.strftime('%s')
# add static dns records first
_dns_zone_default = _get_static_dns_records(
cpillar.dig(_pillar, "defaults:dns:records", {})).get('default')
_zones = _get_static_dns_records(
cpillar.dig(_pillar, "defaults:dns:records", {}), False,
_dns_zone_default)
_soa = {}
for id, node in _nodes.iteritems():
# outer loop: iterate all networks
for network, netdata in node.get('network', {}).iteritems():
# dont handle that network at all if no domain is set ...
if 'domain' not in netdata:
continue
domain = netdata.get('domain')
arpa = _get_arpa(netdata.get('ip4'))
if domain not in _zones:
_zones[domain] = list(_dns_zone_default)
_zones[domain].append('{:20} IN {:10} {}'.format(
id, "A", netdata.get('ip4')))
for cname in netdata.get('cnames', []):
_zones[domain].append('{:20} IN {:10} {}'.format(
cname, "CNAME", id))
if arpa not in _zones:
_zones[arpa] = list(_dns_zone_default)
_zones[arpa].append('{:<20} IN {:10} {}.'.format(
node.get('ip4'), "PTR", netdata.get('fqdn')))
for zone in _zones.keys():
if zone.endswith('arpa'):
_soa[zone] = {'type': 'arpa', 'serial': serial}
else:
_soa[zone] = {'type': 'zone', 'serial': serial}
_statepillar = {"zones": _zones, "soa": _soa}
if test:
return _statepillar
client = salt.client.LocalClient(__opts__['conf_file'], __opts__)
opt = {'expr_form': 'compound'}
_t = 'micros-a?.cde.1nc and G@roles:dns'
staterun = client.cmd(_t,
'state.sls', ['bind.zones'],
kwarg={'pillar': _statepillar},
**opt)
return staterun
def gen_dhcp(environment, top="1nc", test=False):
'''
Function that creates the dhcp subnet data for an environment and passes it to the corresponding state
on the dhcp servers. The state will create the subnet files and include them into the server configuration.
CLI Example:
.. code-block:: bash
salt-run nodes.gen_dhcp dev
:param environment:
The environment you want the dhcp subnet files be created for
:param top:
The top level domain if none is given in the pillar configuration for the environment/network
:param test:
If true, only return the data - do not call the states
:return:
If just for test, returns the data created. If run, it returns the state results
'''
_pillar = cpillar.fake_pillar(None, environment, top, __opts__)
_defaults = _pillar.get('defaults', {})
_nodes = _pillar.get("hosts", {})
_network = cpillar.dig(_pillar, "defaults:network:manage", {})
_schema = cpillar.dig(_pillar, "defaults:network:schema", None)
nameservers = ", ".join(
cpillar.dig(_defaults, "dns:servers", ['127.10.10.1']))
if _schema == 'racked':
_ipnetwork = _network.get('ip4net', {
'rack01': '127.10.10.{0}/24'
}).get('rack01').format('0')
else:
_ipnetwork = _network.get('ip4net', '127.10.10.{0}/24').format('0')
_subnet = {}
_subnet['group'] = environment
_subnet['address'] = ipaddr.IPv4Network(_ipnetwork).network.__str__()
_subnet['netmask'] = ipaddr.IPv4Network(_ipnetwork).netmask.__str__()
_subnet['interface'] = _network.get('phys', 'eth0')
# TODO: define currently hardcoded options in config or deduct from config
_options = {}
_options["subnet-mask"] = _subnet['netmask']
_options["broadcast-address"] = ipaddr.IPv4Network(
_ipnetwork).broadcast.__str__()
_options["domain-name-servers"] = nameservers
_options["ntp-servers"] = ", ".join(
cpillar.dig(_defaults, "ntp-servers:internal", {
'localhost': '127.10.10.1'
}).values())
_options["domain-search"] = '"' + _network.get("domain", "1nc") + '"'
_options["domain-name"] = '"' + _network.get("domain", "1nc") + '"'
_options["routers"] = _network.get('gateway', '127.10.10.1')
# host loop
_hosts = []
for id, node in _nodes.iteritems():
# if the node does not have the network we need, hopp over
if "manage" not in node['network']:
continue
_host = {}
_host['name'] = id
_host['ip'] = cpillar.dig(node, "network:manage:ip4", "")
_host['fqdn'] = cpillar.dig(node, "network:manage:fqdn", "")
_host['mac'] = cpillar.dig(node, "network:manage:mac", "")
_hosts.append(_host)
_hosts.sort(key=itemgetter('ip'))
_statepillar = {
'dhcpd': {
'subnet': _subnet,
'options': _options,
'hosts': _hosts
}
}
if test:
return _statepillar
client = salt.client.LocalClient(__opts__['conf_file'], __opts__)
_t = 'micros-a?.cde.1nc and G@roles:dhcp'
staterun = client.cmd(_t,
'state.sls', ['dhcpd.subnet'],
expr_form="compound",
kwarg={'pillar': _statepillar})
return staterun
def enlist(environment, test=False, force=False, execute=True, commission=True):
'''
Create a bash script on the maas server that does the enlisting of all machines belonging to
the named salt environment.
:param environment:
Salt pillar environment that we want to enlist the machines for
:param test:
Dry run - only tell what would be done
:param force:
If a node exists already, force it to be deleted and recreated
:param execute:
Execute the script once it has been generated
:param commission:
Start commissioning of the machines once added
:return:
A state result dict
'''
_pillar = cpillar.fake_pillar(None, environment, '1nc', __opts__)
defaults = _pillar.get('defaults', {})
maas = defaults.get('maas', {})
nodes = _pillar.get('hosts', {})
cluster = maas.get('cluster', 'cde')
domain = cpillar.dig(defaults, 'network:manage:domain', 'localnet')
res = []
for id, node in nodes.items():
_node = {}
_node['name'] = cpillar.dig(node, 'network:manage:fqdn')
_node['mac'] = cpillar.dig(node, 'network:manage:mac')
_node['arch'] = defaults.get('arch', 'amd64')
_node['sub'] = maas.get('sub', '')
_node['powertype'] = maas.get('powertype', 'ipmi')
if _node['powertype'] == 'virsh':
_node['poweraddress'] = cpillar.dig(node, 'network:console:poweraddress', cpillar.dig(node, 'network:console:ip4'))
else:
_node['poweraddress'] = cpillar.dig(node, 'network:console:ip4')
_node['powerpass'] = maas.get('powerpass', '')
_node['powerid'] = id
_node['partitions'] = node.get('partitions', '')
_node['zone'] = maas.get('zone', '')
_node['maas'] = node.get('no_maas', False) is False
res.append(_node)
_state_pillar = {
'enlist': {'nodes': res, 'domain': domain, 'cluster': cluster,
'commission': commission,
'execute': execute,
'force_nodes': force}}
if test:
return _state_pillar
target = 'maas-a1.cde.1nc'
client = salt.client.LocalClient(__opts__['conf_file'], __opts__)
staterun = client.cmd(target, 'state.sls', ['maas.enlist'], kwarg={'pillar': _state_pillar})
ret = {'data': {target: staterun}}
ret['data']['retcode'] = 0 if salt.utils.check_state_result(ret['data']) else 1
return ret
def hosts_deploy(environment, nodes=None):
'''
Deploy a selected OS to all hosts of a named salt envrionment. This creates a public
and a private key for the minion on the master, stores the public key in the masters
accepted keys folder and sends both keys to the new host.
As the private key is only stored on the minion itself, the master has no way to restore
it if the host needs to be redeployed. Thus a new keypair will be created each time
this function is run.
CLI Example:
.. code-block:: bash
salt-run alchemy.hosts_deploy dev
salt-run alchemy.hosts_deploy dev node-01.dev.1nc
:param environment:
Name of the salt pillar environment
:param nodes:
Optional name of a known host in the working environment
:return:
The data structure returned by the client call
'''
import time
wheel = _get_wheel()
_pillar = cpillar.fake_pillar(None, environment, '1nc', __opts__)
_nodes = _pillar.get('hosts', {})
_zone = cpillar.dig(_pillar, 'defaults:maas:zone')
opt = {'timeout': 1, 'expr_form': 'compound'}
target = 'G@roles:maas and *.cde.1nc'
client = salt.client.LocalClient(__opts__['conf_file'], __opts__)
if nodes is not None:
nodes = nodes.split(',')
res = {}
for nid, node in _nodes.iteritems():
fqdn = node.get('fqdn')
if nodes is not None and fqdn not in nodes:
log.debug("Skipping node %s", fqdn)
continue
log.info("Deploying node %s", nid)
keys = wheel.call_func('key.gen_accept',
args=[fqdn],
kwargs={'force': True})
userdata = _minion_key_wrapper(keys)
actions = {}
actions['acquired'] = client.cmd(target, 'maas.node_acquire', [fqdn],
**opt)
time.sleep(5)
actions['deploy'] = client.cmd(target, 'maas.node_power_on',
[fqdn, userdata], **opt)
res[nid] = actions
return res
def minion_assemble(target, caller='cmd', test=False, force=False):
'''
This function is called by the assemble reactor when a minion comes up. It needs to
inspect the current hints of the minion and decide what to do next
:param target:
The minion id taken from the start event
:param caller:
Assume this function is called from the commandline and return state output. If you
call it from somewhere else - like a reactor - set this value to something else to
suppress any return values
:param test:
:param force:
:return:
Whatever result is returnable.
'''
client = salt.client.LocalClient(__opts__['conf_file'], __opts__)
hints = client.cmd(target, 'grains.get', ['hints', {}]).get(target)
if hints is None:
message = 'minon_started: called by a minion that is unknown or not responding! {}'.format(
target)
log.critical(message)
if caller == 'cmd':
return {'Error': message}
return
log.info('minion_assemble: %s with hints=%s', target, hints)
# compile the pillar for the target - we get it here with all roles reflected
_pillar = cpillar.minion_pillar(target)
if 'pillar' in hints:
_pillar = salt.utils.dictupdate.merge_recurse(_pillar,
hints.get('pillar'), {})
# If we are not forced to run in any case check the pillar config if assemblies are enabled
if not force and not cpillar.dig(_pillar, 'defaults:assemble', True):
log.info(
'minion_assemble: Assembly reactor disabled by pillar configuration.'
)
return True
# get the list of available assemblies from the pillar (created from the files in base by external pillar)
assembly = _pillar.get('assembly', {})
# get a list assemblies requested to run
# we want to use the local roles, but we try hints:agenda first
roles = cpillar.dig(_pillar, 'local:roles', [])
agenda = list(hints.get('agenda', roles))
log.info('Agenda for target %s: %s', target, agenda)
# retrieve the targets pillar environment from the fresh pillar
pillarenv = _pillar.get('pillarenv', 'base')
# add ready as last topic on the agenda - in the future we will stop there
if 'ready' not in agenda:
agenda.append('ready')
# forward until we have a runnable topic - some might just be roles or not implemented, so we ignore them
topic, short_topic = _get_fqtopic(target, pillarenv, hints, agenda,
assembly)
if topic is None:
hint_update(target, hints, client)
client.cmd(
target, 'event.send',
['salt/minion/{}/topic/{}/success'.format(target, short_topic)])
return True
if topic is False:
return False
# if we run in test mode get out here and provide test data
if test:
data = {'module': topic, 'pillarenv': pillarenv, 'kwargs': hints}
if __opts__.get('log_level', 'info') == 'info':
data['roles'] = agenda
data['assemblies'] = assembly
return {target: data}
# Run the assembly orchestration package
log.warning('Running assemble on target %s for topic %s', target, topic)
res = assemble(target, topic, **hints)
# log.warning('Result of the call is %s', res)
retcode = res['data'].get('retcode', 'X')
# If the call was successful set the hints for the minion to the next topic and fire the proceed event
if retcode == 0:
log.debug('minion_assemble: Topic %s applied successfully.', topic)
hint_update(target, hints, client)
client.cmd(target, 'event.send',
['salt/minion/{}/topic/{}/success'.format(target, topic)])
client.cmd(target, 'event.send',
['salt/minion/{}/assemble'.format(target)])
if caller != 'cmd':
return True
else:
return res
client.cmd(target, 'event.send',
['salt/minion/{}/topic/{}/fail'.format(target, topic)])
log.error('minion_assemble: Topic %s:%s caused an error (retcode is %s)',
target, topic, retcode)
return res
def ext_pillar(minion_id, pillar, *args, **kwargs):
'''
Enrich the simple pillar by preparing proper structures for either hosts or containers.
This is the central place to do all merge and calculation procedures needed to get the
full configuration information for a node, which is either a host or a container
:param minion_id:
The id of the minion that is requesting this pillar
:param pillar:
The compiled pillar for the requesting minion - also processes the top file
:param args:
Unnamed positional args
:param kwargs:
Named args given
:return:
A dict witch will be merged with the original pillar, overwriting the existing elements.
In the case of this external pillar will be
{'hosts': [hostconfiguration], 'containers': [containerconfiguration] }
'''
nodebase_files = cpillar.dig(pillar, 'defaults:nodebase', [])
nodebase_data = nodebase.data(nodebase_files, __opts__['file_buffer_size'])
_nodes = {}
for group in ["hosts", "containers"]:
_nodebase_groupnodes = dict(nodebase_data.get(group, {}))
_newgroupnodes = {}
_groupnodes = cpillar.dig(pillar, group, {})
_groupdefaults = _merge_defaults_for_group(pillar.get("defaults", {}),
group)
_netschema = cpillar.dig(_groupdefaults, 'network:schema', 'None')
for nid, node_config in _groupnodes.iteritems():
# merge the nodes config with the one from the nodebase
config = udict.merge_recurse(_nodebase_groupnodes.get(nid, {}),
node_config)
# if we use racked schema, add rackid from node name
if _netschema == 'racked' and 'rackid' not in config:
config['rackid'] = 'rack{:0>2}'.format(
nid.split('-').pop()[1:])
_node = udict.merge_recurse(_groupdefaults, config)
_node['network'] = _merge_networks_for_node(
nid, config, _groupdefaults)
_node['fqdn'] = cpillar.dig(_node, 'network:manage:fqdn')
_node['hostname'] = nid
_node['nature'] = group
# TODO: remove when using 2015.8+
# take special care about lists
for mergelist in ['roles', 'packages']:
if mergelist in config or mergelist in _groupdefaults:
_node[mergelist] = _merge_lists_for_node(
mergelist, config, _groupdefaults)
if 'target' in _node:
_node['target'] = '{0}.{1}'.format(
_node['target'], cpillar.dig(_node,
'network:manage:domain'))
if 'mole' not in _node:
_node['mole'] = "hosts"
if 'mount' in _node:
_node['mount'] = _expand_mounts_for_node(
_node.get('fqdn'), _node.get("mount", {}))
_newgroupnodes[nid] = _node
if _node['fqdn'] == minion_id:
_nodes['local'] = _node
_nodes[group] = _newgroupnodes
return _nodes
def list_nodes(environment, groups="hosts", top="1nc"):
'''
Function that lists out all nodes in an environment - for manual configuration checks
CLI Example:
.. code-block:: bash
salt-run test.list_nodes dev
salt-run test.list_nodes dev containers
:param environment:
The saltenv you want the dns zone files be created for
:param top:
The top level domain if none is given in the pillar configuration for the environment/network
:return:
If just for test, returns the data created. If run, it returns the state results
'''
_pillar = cpillar.fake_pillar(None, environment, top, __opts__)
_networks = cpillar.dig(_pillar, "defaults:network", {})
_ignorelist = ['common', 'schema']
_nodes = {}
groups = groups.split(',')
for group in groups:
partial = cpillar.dig(_pillar, group, {})
_nodes.update(partial)
snodelist = sorted(_nodes.keys())
headers = ['id', 'nature', 'ip', 'target', 'rackid']
for network in _networks.keys():
if network in _ignorelist:
continue
headers.append(network + ':fqdn')
headers.append(network + ':ip4')
headers.append(network + ':mac')
_infonodes = []
for id in snodelist:
node = _nodes.get(id, {})
if node.get('nature') == 'hosts':
node['target'] = cpillar.dig(node, 'network:manage:fqdn', id)
_nodenetworks = [
id,
node.get('nature', ''),
node.get('ip4').__str__(),
node.get('target', ''),
node.get('rackid', '')
]
for network in _networks.keys():
if network in _ignorelist:
continue
_ndata = node.get('network', {}).get(network, {})
_nodenetworks.append(_ndata.get('fqdn', ''))
_nodenetworks.append(_ndata.get('ip4', ''))
_nodenetworks.append(_ndata.get('mac', ''))
_infonodes.append(','.join(_nodenetworks))
_infonodes.insert(0, ','.join(headers))
return '\n'.join(_infonodes)
