def add_reclass_parameter(paths, key, value, verbose=False, merge=False):
"""Add a value to the specified key to all the files in the paths
if merge=False (default):
- new value replaces previous key content completely.
if merge=True:
- if the specified key type is list, then value will be appended
to the list. Value examples:
'1000'
'new_lab_name'
'cluster.virtual_cluster_name.infra'
'http://archive.ubuntu.com'
'[a, b, c]' # a list in the list
'{a:1, b:2, c:3}' # a dict in the list
- if the specified key type is an existing dict, then the dict
will be extended with the dict in the value. Value example:
'{address: 192.168.1.1, netmask: 255.255.255.0}'
- If the specified key type is string/int/bool - it will replace previous
value
"""
add_key = key.split('.')
for path in paths:
for fyml in walkfiles(path, verbose=verbose):
if fyml.fname.endswith('.yml'):
model = helpers.yaml_read(fyml.fname)
if model is not None:
nested_key = helpers.get_nested_key(model, add_key)
if nested_key is not None:
if merge is False:
nested_key = value
else:
if type(nested_key) is list:
nested_key.append(value)
elif type(nested_key) is dict:
nested_key.update(value)
else:
nested_key = value
else:
nested_key = value
helpers.create_nested_key(model, path=add_key,
value=nested_key)
with open(fyml.fname, 'w') as f:
f.write(
yaml.dump(
model, default_flow_style=False, width=255
)
)
def vcp_list(domain=None, inventory=None):
"""List VCP node names
Scan all nodes for the object salt.control.cluster.internal.node.XXX.name
Return set of tuples ((nodename1, domain), (nodename2, domain), ...)
"""
inventory = inventory or inventory_list(domain=domain)
vcp_path = 'parameters.salt.control.cluster.internal.node'.split('.')
domain_path = 'parameters._param.cluster_domain'.split('.')
vcp_node_names = set()
for node_name, node in inventory.items():
vcp_nodes = helpers.get_nested_key(node, path=vcp_path)
if vcp_nodes is not None:
for vcp_node_name, vcp_node in vcp_nodes.items():
vcp_node_names.add((vcp_node['name'],
helpers.get_nested_key(node,
path=domain_path)))
return vcp_node_names
def _recurse_entity(self,
entity,
merge_base=None,
seen=None,
nodename=None):
def _new_merge_dict(self, cur, new, path):
try:
return orig_merge_dict(self, cur, new, path)
except TypeError as e:
if "Current value:" not in e.message:
e.message += "\nValue path: {}\nCurrent value: {}\nNew value: {}\n".format(
path, cur, new)
raise TypeError(e.message)
if seen is None:
seen = {}
if '__visited' not in seen:
seen['__visited'] = []
orig_visited = copy.deepcopy(seen['__visited'])
seen['__visited'].append(entity.name)
orig_merge_dict = reclass_parameters.Parameters._merge_dict
with mock.patch.object(reclass_parameters.Parameters,
'_merge_dict',
new=_new_merge_dict):
try:
result = super(ReclassCore,
self)._recurse_entity(entity, merge_base, seen,
nodename)
except Exception:
print("### Interpolation failed in the class: " +
' < '.join(seen['__visited']))
raise
if self.track_key_path:
key = helpers.get_nested_key(entity.parameters.as_dict(),
path=self.track_key_path)
if key is not None:
print("# " + ' < '.join(seen['__visited']))
out_dict = {}
helpers.create_nested_key(out_dict,
['parameters'] + self.track_key_path,
key)
print(yaml.dump(out_dict, default_flow_style=False, width=255))
# Reset the data collected by child entries
seen['__visited'] = orig_visited
return result
def remove_reclass_parameter(paths, key,
verbose=False,
pretend=False):
"""Removes specified key from parameters from all reclass models
:param key: string with point-separated nested objects, for
example: parameters.linux.network.interface
:rtype dict: { 'file path': {nested_key}, ...}
"""
remove_key = key.split('.')
# found_keys = {}
for path in paths:
for fyml in walkfiles(path, verbose=verbose):
if fyml.fname.endswith('.yml'):
try:
model = helpers.yaml_read(fyml.fname)
except yaml.scanner.ScannerError as e:
print(e, file=sys.stderr)
continue
if model is not None:
# Clear linux.network.interfaces
nested_key = helpers.get_nested_key(model, remove_key)
if nested_key is not None:
# found_keys[fyml.fname] = copy.deepcopy(nested_key)
if pretend:
print("\n---\n# Found {0} in {1}"
.format('.'.join(remove_key), fyml.fname))
print(yaml.dump(nested_key,
default_flow_style=False,
width=255))
else:
print("\n---\n# Removing {0} from {1}"
.format('.'.join(remove_key), fyml.fname))
print(yaml.dump(nested_key,
default_flow_style=False,
width=255))
helpers.remove_nested_key(model, remove_key)
with open(fyml.fname, 'w') as f:
f.write(
yaml.dump(
model, default_flow_style=False,
width=255
)
)
def reclass_storage(domain=None, inventory=None):
"""List VCP node names
Scan all nodes for the object salt.control.cluster.internal.node.XXX.name
"""
inventory = inventory or inventory_list(domain=domain)
storage_path = 'parameters.reclass.storage.node'.split('.')
res = dict()
for node_name, node in inventory.items():
storage_nodes = helpers.get_nested_key(node, path=storage_path)
if storage_nodes is not None:
for storage_node_name, storage_node in storage_nodes.items():
if storage_node['domain'] not in res:
res[storage_node['domain']] = dict()
res[storage_node['domain']][storage_node_name] = storage_node
return res
def _nodeinfo(self, nodename):
if self.track_key_path:
print("\n" + nodename)
print("-" * len(nodename))
result = super(ReclassCore, self)._nodeinfo(nodename)
if self.track_key_path:
key = helpers.get_nested_key(result.parameters.as_dict(),
path=self.track_key_path)
if key is not None:
print("### Final result after interpolation: ###")
out_dict = {}
helpers.create_nested_key(out_dict,
['parameters'] + self.track_key_path,
key)
print(yaml.dump(out_dict, default_flow_style=False, width=255))
return result
def get_all_reclass_params(paths, verbose=False):
"""Return dict with all used values for each param"""
_params = dict()
for path in paths:
for log in walkfiles(path, verbose):
if log.fname.endswith('.yml'):
model = helpers.yaml_read(log.fname)
if model is not None:
# Collect all params from the models
_param = helpers.get_nested_key(
model,
['parameters', '_param'])
if _param:
for key, val in _param.items():
if key in _params:
# Keep list values sorted
_params[key].append(val)
_params[key] = sorted(_params[key])
else:
_params[key] = [val]
return _params
def create_inventory_context(domain=None, keys=None):
"""Dumps the current inventory per domain
Example of context:
<global_settings>: # only if required
...
current_clusters:
<cluster_names>:
# here are cluster settings if required
nodes:
<node_names>:
name: ctl01
reclass_storage_name: openstack_control_node01
roles:
- vcp # 'vcp' or None
parameters: # specified keys to dump, for example
# parameters.linux.network.interface below:
linux:
network:
interfaces:
..
"""
inventory = reclass_models.inventory_list(domain=domain)
vcp_list = reclass_models.vcp_list(domain=domain, inventory=inventory)
reclass_storage = reclass_models.reclass_storage(domain=domain,
inventory=inventory)
if domain is None:
sys.exit("Error: please specify a domain name from: \n{}".format(
'\n'.join(reclass_storage.keys())))
for storage_domain, storage_nodes in reclass_storage.items():
if storage_domain != domain:
continue
current_cluster_nodes = {}
for storage_node_name, storage_node in storage_nodes.items():
inventory_node_name = "{0}.{1}".format(storage_node['name'],
storage_node['domain'])
current_cluster_nodes[inventory_node_name] = {
'name': storage_node['name'],
'reclass_storage_name': storage_node_name,
'roles': list(),
'parameters': dict(),
}
if (storage_node['name'], storage_node['domain']) in vcp_list:
# Add role 'vcp' to mark the VM nodes.
current_cluster_nodes[inventory_node_name]['roles'].append(
'vcp')
if keys:
# Dump specified parameters for the node
# Will fail with KeyError if 'inventory_node_name' doesn't
# exists in reclass inventory
# (wasn't generated with reclass.storage yet, for example)
node = inventory[inventory_node_name]
for key in keys:
key_path = key.split('.')
reclass_key = helpers.get_nested_key(node, path=key_path)
if reclass_key:
helpers.create_nested_key(
current_cluster_nodes[inventory_node_name],
path=key_path,
value=reclass_key)
current_underlay_context = {
'cookiecutter': {
'cluster_name': storage_domain,
'nodes': current_cluster_nodes,
}
}
return current_underlay_context