def finalize(self):
LOG.info('%s()' % KenLog.fcn())
allocated_addresses = CloudModel.get(self._cloud_internal,
'address_allocations', {})
host_aliases = CloudModel.get(self._cloud_internal, 'host_aliases', {})
address_data = {}
for group_name, group in allocated_addresses.iteritems():
if group_name not in address_data:
address_data[group_name] = {}
for network_name, network in group.iteritems():
if network_name not in address_data[group_name]:
address_data[group_name][network_name] = {}
for addr in sorted(network):
aliases = host_aliases.get(group_name, {}).get(
network_name, {}).get(addr, [])
address_data[group_name][network_name][addr] = []
for name in aliases:
address_data[group_name][network_name][addr].append(name)
filename = os.path.join(self._file_path, 'address_info.yml')
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(address_data, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
file_name = os.path.join(self._file_path, 'hosts.hf')
self.add_artifact(file_name, ArtifactMode.CREATED)
allocated_addresses = CloudModel.get(self._cloud_internal, 'address_allocations')
host_aliases = CloudModel.get(self._cloud_internal, 'host_aliases')
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
with open(file_name, 'w') as fp:
fp.write("# Cloud: %s\n" % (cloud_name))
fp.write("\n")
fp.write("# Localhost Information\n")
fp.write("127.0.0.1 localhost\n")
fp.write("\n")
for group_name, group in allocated_addresses.iteritems():
fp.write("#\n")
fp.write("# Network Group: %s\n" % (group_name))
fp.write("#\n")
for network_name, network in group.iteritems():
fp.write("# Network: %s\n" % (network_name))
ips = []
for addr in network:
aliases = host_aliases.get(group_name,
{}).get(network_name,
{}).get(addr, [])
for name in aliases:
# Expand the address to a string with leading spaces
# in each quad so that it sorts by version
ips.append(["%3s.%3s.%3s.%3s" % tuple(addr.split(".")), name])
for ip in sorted(ips, key=itemgetter(0)):
fp.write("%-16s %s\n" % (ip[0].replace(" ", ""), ip[1]))
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
# If we have an error in an earlier generator we may not have
# control_planes in the internal model
if not control_planes:
return
routes = {}
for cp_name, cp in control_planes.iteritems():
load_balancers = cp.get('load-balancers', {})
for cluster in cp['clusters']:
for server in cluster.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers,
routes)
for r_name, r in cp.get('resources', {}).iteritems():
for server in r.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers,
routes)
#
default_routes = {}
# Warn about any routes using the "default" route
for src_net, net_routes in routes.iteritems():
for dest_net, route_data in net_routes.iteritems():
if route_data['default']:
hosts = set()
if src_net not in default_routes:
default_routes[src_net] = []
for src, src_data in route_data['used_by'].iteritems():
for dest, host_list in src_data.iteritems():
for host in host_list:
hosts.add(host)
default_routes[src_net].append({
'net': dest_net,
'hosts': hosts
})
if default_routes:
msg = (
"Default routing used between networks\n"
"The following networks are using a 'default' route rule. To remove this warning\n"
"either add an explict route in the source network group or force the network to\n"
"attach in the interface model used by the servers.\n")
for src in sorted(default_routes):
dest_list = default_routes[src]
for dest_data in dest_list:
msg += " %s to %s\n" % (src, dest_data['net'])
for host in sorted(dest_data['hosts']):
msg += " %s\n" % (host)
self.add_warning(msg)
CloudModel.put(cloud_internal, 'routes', routes)
def _generate_ring_specifications_info(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
ring_specifications_config = CloudModel.get(cloud_version,
'ring-specifications', [])
cloud_internal = CloudModel.internal(self._models['CloudModel'])
CloudModel.put(cloud_internal, 'ring-specifications',
ring_specifications_config)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
components = CloudModel.get(self._cloud_internal, 'components')
components_by_mnemonic = CloudModel.get(self._cloud_internal, 'components_by_mnemonic')
for cp_name, cp in control_planes.iteritems():
self._build_ansible_group_vars(cloud_name, cp, components, components_by_mnemonic)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
# If we have an error in an earlier generator we may not have
# control_planes in the internal model
if not control_planes:
return
routes = {}
for cp_name, cp in control_planes.iteritems():
load_balancers = cp.get('load-balancers', {})
for cluster in cp['clusters']:
for server in cluster.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
for r_name, r in cp.get('resources', {}).iteritems():
for server in r.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
#
default_routes = {}
# Warn about any routes using the "default" route
for src_net, net_routes in routes.iteritems():
for dest_net, route_data in net_routes.iteritems():
if route_data['default']:
hosts = set()
if src_net not in default_routes:
default_routes[src_net] = []
for src, src_data in route_data['used_by'].iteritems():
for dest, host_list in src_data.iteritems():
for host in host_list:
hosts.add(host)
default_routes[src_net].append({'net': dest_net, 'hosts': hosts})
if default_routes:
msg = ("Default routing used between networks\n"
"The following networks are using a 'default' route rule. To remove this warning\n"
"either add an explict route in the source network group or force the network to\n"
"attach in the interface model used by the servers.\n")
for src in sorted(default_routes):
dest_list = default_routes[src]
for dest_data in dest_list:
msg += " %s to %s\n" % (src, dest_data['net'])
for host in sorted(dest_data['hosts']):
msg += " %s\n" % (host)
self.add_warning(msg)
CloudModel.put(cloud_internal, 'routes', routes)
def _generate_ring_specifications_info(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(
self._models['CloudModel'], self._version)
ring_specifications_config = CloudModel.get(
cloud_version, 'ring-specifications', [])
cloud_internal = CloudModel.internal(self._models['CloudModel'])
CloudModel.put(cloud_internal,
'ring-specifications',
ring_specifications_config)
def build(self):
LOG.info('%s()' % KenLog.fcn())
firewall_settings = CloudModel.get(self._cloud_internal,
'firewall_settings')
print(firewall_settings)
cloud_firewall = CloudModel.get(self._cloud_internal, 'cloud-firewall')
# Convert from per server to per network
firewall = {}
for server_name, data in cloud_firewall.iteritems():
for addr, rules in data.get('rules', {}).iteritems():
for rule in rules:
net_group = rule['chain']
component = rule['component']
if net_group not in firewall:
firewall[net_group] = {}
min_port = rule['port-range-min']
max_port = rule['port-range-max']
if min_port == max_port:
port = str(min_port)
else:
port = "%s:%s" % (min_port, max_port)
if port not in firewall[net_group]:
firewall[net_group][port] = {
'port': port,
'protocol': rule['protocol'],
'components': [],
'addresses': []
}
if component not in firewall[net_group][port][
'components']:
firewall[net_group][port]['components'].append(
component)
if addr not in firewall[net_group][port]['addresses']:
firewall[net_group][port]['addresses'].append(addr)
# rebuild get a list with the ports sorted
sorted_firewall = {}
for net_grp, data in firewall.iteritems():
sorted_firewall[net_grp] = []
for port in sorted(data, key=lambda x: int(x.split(":")[0])):
sorted_firewall[net_grp].append(data[port])
filename = "%s/info/firewall_info.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(sorted_firewall, fp, default_flow_style=False, indent=4)
def __init__(self, instructions, models, controllers):
super(RouteInfoBuilder, self).__init__(
2.0, instructions, models, controllers,
'route-info-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions, self.cloud_desc)
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model, self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
def __init__(self, instructions, models, controllers):
super(AnsibleAllVarsBuilder, self).__init__(
2.0, instructions, models, controllers,
'ansible-all-vars-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions, self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'ansible')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model, self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
def __init__(self, instructions, models, controllers):
super(AnsibleHostsBuilder, self).__init__(
2.0, instructions, models, controllers,
'ansible-hosts-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions, self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'ansible')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model, self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
def __init__(self, instructions, models, controllers):
super(NetworkInfoBuilder, self).__init__(2.0, instructions, models,
controllers, 'net-info-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions,
self.cloud_desc)
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
def build(self):
LOG.info('%s()' % KenLog.fcn())
firewall_settings = CloudModel.get(self._cloud_internal, 'firewall_settings')
print (firewall_settings)
cloud_firewall = CloudModel.get(self._cloud_internal, 'cloud-firewall')
# Convert from per server to per network
firewall = {}
for server_name, data in cloud_firewall.iteritems():
for addr, rules in data.get('rules', {}).iteritems():
for rule in rules:
net_group = rule['chain']
component = rule['component']
if net_group not in firewall:
firewall[net_group] = {}
min_port = rule['port-range-min']
max_port = rule['port-range-max']
if min_port == max_port:
port = str(min_port)
else:
port = "%s:%s" % (min_port, max_port)
if port not in firewall[net_group]:
firewall[net_group][port] = {'port': port,
'protocol': rule['protocol'],
'components': [],
'addresses': []}
if component not in firewall[net_group][port]['components']:
firewall[net_group][port]['components'].append(component)
if addr not in firewall[net_group][port]['addresses']:
firewall[net_group][port]['addresses'].append(addr)
# rebuild get a list with the ports sorted
sorted_firewall = {}
for net_grp, data in firewall.iteritems():
sorted_firewall[net_grp] = []
for port in sorted(data, key=lambda x: int(x.split(":")[0])):
sorted_firewall[net_grp].append(data[port])
filename = "%s/info/firewall_info.yml" % (
self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(sorted_firewall, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
servers = CloudModel.get(self._cloud_internal, 'servers')
server_info = {}
for server in servers:
server_info[server['id']] = {}
data = server_info[server['id']]
data['state'] = server['state']
if server['state'] == ServerState.ALLOCATED:
data['hostname'] = server['hostname']
data['failure-zone'] = server['failure-zone']
net_info = {}
for if_name, if_data in server['interfaces'].iteritems():
net_info[if_name] = {}
for net_name, net_data in if_data['networks'].iteritems():
net_info[if_name][net_name] = \
{'addr': net_data.get('addr'),
'vlan-id': net_data.get('vlanid'),
'tagged-vlan': net_data.get('tagged-vlan', True)}
data['net_data'] = net_info
filename = "%s/info/server_info.yml" % (
self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(server_info, fp, default_flow_style=False, indent=4)
def __init__(self, instructions, models, controllers, config_files):
super(ServiceViewFinalizer, self).__init__(
2.0, instructions, models, controllers, config_files,
'service-view-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'info')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
def __init__(self, instructions, models, controllers, config_files):
super(ServiceViewFinalizer,
self).__init__(2.0, instructions, models, controllers,
config_files, 'service-view-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'info')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
def finalize(self):
LOG.info('%s()' % KenLog.fcn())
service_view = CloudModel.get(self._cloud_internal, 'service_view', {})
filename = os.path.join(self._file_path, 'service_info.yml')
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(service_view.get('by_region', {}), fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
net_info = {}
# Service IPs
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
if 'service-ips' in cluster:
if 'service_ips' not in net_info:
net_info['service_ips'] = {}
for name, net_data in cluster['service-ips'].iteritems():
if name not in net_info['service_ips']:
net_info['service_ips'][name] = []
for net_name, data in net_data.iteritems():
info = {
'control_plane': cp_name,
'cluster': cluster['name'],
'network': net_name,
'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {})
}
net_info['service_ips'][name].append(info)
if 'resources' in cp:
for res_name, resources in cp['resources'].iteritems():
if 'service-ips' in resources:
if 'service_ips' not in net_info:
net_info['service_ips'] = {}
for name, net_data in resources[
'service-ips'].iteritems():
if name not in net_info['service_ips']:
net_info['service_ips'][name] = []
for net_name, data in net_data.iteritems():
info = {
'control_plane': cp_name,
'cluster': res_name,
'network': net_name,
'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {})
}
net_info['service_ips'][name].append(info)
filename = "%s/info/net_info.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(net_info, fp, default_flow_style=False, indent=4)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
components = CloudModel.get(cloud_internal, 'components', [])
# If we have an error in an earlier generator we may not have
# components in the internal model
if not components:
return
components_by_mnemonic = CloudModel.get(cloud_internal, 'components_by_mnemonic')
control_planes = CloudModel.get(cloud_internal, 'control-planes')
for cp_name, cp in control_planes.iteritems():
for comp_name, comp_data in cp.get('components', []).iteritems():
comp_data['consumes'] = self._get_consumes(comp_name,
components,
components_by_mnemonic,
cp)
def __init__(self, instructions, models, controllers):
super(DiagramBuilder, self).__init__(2.0, instructions, models,
controllers, 'diagram-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'info')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
self._padding_x = 2
self._padding_y = 2
self._server_width = 50
self._line_width = 136
def build(self):
LOG.info('%s()' % KenLog.fcn())
routes = CloudModel.get(self._cloud_internal, 'routes')
filename = "%s/info/route_info.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(routes, fp, default_flow_style=False, indent=4)
def __init__(self, instructions, models, controllers):
super(DiagramBuilder, self).__init__(
2.0, instructions, models, controllers,
'diagram-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = HlmPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'info')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
HlmPaths.make_path(self._file_path)
self._padding_x = 2
self._padding_y = 2
self._server_width = 50
self._line_width = 136
def build(self):
LOG.info('%s()' % KenLog.fcn())
file_name = os.path.join(self._file_path, 'hosts.hf')
self.add_artifact(file_name, ArtifactMode.CREATED)
allocated_addresses = CloudModel.get(self._cloud_internal,
'address_allocations')
host_aliases = CloudModel.get(self._cloud_internal, 'host_aliases')
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
with open(file_name, 'w') as fp:
fp.write("# Cloud: %s\n" % (cloud_name))
fp.write("\n")
fp.write("# Localhost Information\n")
fp.write("127.0.0.1 localhost\n")
fp.write("\n")
for group_name, group in allocated_addresses.iteritems():
fp.write("#\n")
fp.write("# Network Group: %s\n" % (group_name))
fp.write("#\n")
for network_name, network in group.iteritems():
fp.write("# Network: %s\n" % (network_name))
ips = []
for addr in network:
aliases = host_aliases.get(group_name,
{}).get(network_name,
{}).get(addr, [])
for name in aliases:
# Expand the address to a string with leading spaces
# in each quad so that it sorts by version
ips.append([
"%3s.%3s.%3s.%3s" % tuple(addr.split(".")),
name
])
for ip in sorted(ips, key=itemgetter(0)):
fp.write("%-16s %s\n" %
(ip[0].replace(" ", ""), ip[1]))
def build(self):
LOG.info('%s()' % KenLog.fcn())
routes = CloudModel.get(self._cloud_internal, 'routes')
filename = "%s/info/route_info.yml" % (
self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(routes, fp, default_flow_style=False, indent=4)
def finalize(self):
LOG.info('%s()' % KenLog.fcn())
service_view = CloudModel.get(self._cloud_internal, 'service_view', {})
filename = os.path.join(self._file_path, 'service_info.yml')
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(service_view.get('by_region', {}),
fp,
default_flow_style=False,
indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
net_info = {}
# Service IPs
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
if 'service-ips' in cluster:
if 'service_ips' not in net_info:
net_info['service_ips'] = {}
for name, net_data in cluster['service-ips'].iteritems():
if name not in net_info['service_ips']:
net_info['service_ips'][name] = []
for net_name, data in net_data.iteritems():
info = {'control_plane': cp_name,
'cluster': cluster['name'],
'network': net_name,
'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {})}
net_info['service_ips'][name].append(info)
if 'resources' in cp:
for res_name, resources in cp['resources'].iteritems():
if 'service-ips' in resources:
if 'service_ips' not in net_info:
net_info['service_ips'] = {}
for name, net_data in resources['service-ips'].iteritems():
if name not in net_info['service_ips']:
net_info['service_ips'][name] = []
for net_name, data in net_data.iteritems():
info = {'control_plane': cp_name,
'cluster': res_name,
'network': net_name,
'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {})}
net_info['service_ips'][name].append(info)
filename = "%s/info/net_info.yml" % (
self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(net_info, fp, default_flow_style=False, indent=4)
def load_versioned_model(self, version, config_key, optional=False, alias=None):
LOG.info('%s()' % KenLog.fcn())
v = Version.normalize(version)
config_value = self._get_config_value(v, config_key)
if not config_value and alias:
LOG.warn("Use of %s is deprecated, use %s instead" %
(alias, config_key))
config_value = self._get_config_value(v, alias)
if config_value:
cloud_model = CloudModel.version(self._models['CloudModel'], v)
cloud_model[config_key] = config_value
elif not optional:
raise Exception('Could not load model key "%s"' % config_key)
def load_versioned_model(self,
version,
config_key,
optional=False,
alias=None):
LOG.info('%s()' % KenLog.fcn())
v = Version.normalize(version)
config_value = self._get_config_value(v, config_key)
if not config_value and alias:
LOG.warn("Use of %s is deprecated, use %s instead" %
(alias, config_key))
config_value = self._get_config_value(v, alias)
if config_value:
cloud_model = CloudModel.version(self._models['CloudModel'], v)
cloud_model[config_key] = config_value
elif not optional:
raise Exception('Could not load model key "%s"' % config_key)
def _render_control_planes(self, fp):
LOG.info('%s()' % KenLog.fcn())
control_planes = CloudModel.get(self._cloud_version, 'control-planes')
for elem_r in control_planes:
(r_w, r_h) = self._determine_size_for_control_plane(elem_r)
r_box = Box(r_w, r_h)
r_type = ControlPlane.name(elem_r)
r_type_name = ControlPlane.region_name(elem_r)
title = 'ControlPlane: %s (%s)' % (r_type_name, r_type)
r_box.set_title(title)
tier_y = 2
tot_box_w = 0
for elem_c in ControlPlane.clusters(elem_r):
(t_w, t_h) = self._determine_size_for_cluster(elem_r, elem_c)
t_box = Box(t_w, t_h)
if tot_box_w == 0:
tot_box_w = t_w
title = 'Cluster %s (%s)' % (Cluster.name(elem_c),
Cluster.id(elem_c))
t_box.set_title(title)
r_box.add_layer(t_box, 2, tier_y)
tier_y += t_h + (self._padding_y / 2)
member_x = 2
for elem_s in Cluster.servers(elem_c):
(m_w, m_h) = self._determine_size_for_server(
elem_r, elem_c, elem_s)
m_box = Box(m_w, m_h)
ip = Server.address(elem_s)
if ip:
title = '%s (%s)' % (Server.name(elem_s), ip)
else:
title = '%s' % Server.name(elem_s)
m_box.set_title(title)
t_box.add_layer(m_box, member_x, 2)
member_x += (self._server_width + self._padding_x)
service_y = 2
services = Server.services(elem_s)
for elem_c in sorted(services):
m_box.add_string_absolute(elem_c, 2, service_y)
service_y += 1
for elem_comp in sorted(services[elem_c]):
m_box.add_string_absolute(elem_comp, 4, service_y)
service_y += 1
service_y += 1
sep = "-" * m_w
m_box.add_string_absolute(sep, 2, service_y)
service_y += 2
interfaces = Server.interfaces(elem_s)
routes = Server.routes(elem_s)
for i in sorted(interfaces):
elem_i = interfaces[i]
device = elem_i['device']['name']
if 'bond-data' in elem_i:
device += " ("
first = True
for bond_dev in elem_i['bond-data'].get(
'devices', []):
if not first:
device += ", "
first = False
device += "%s" % bond_dev['name']
device += ")"
m_box.add_string_absolute(device, 2, service_y)
service_y += 1
networks = Interface.networks(elem_i)
for n in sorted(networks):
elem_n = networks[n]
name = Network.name(elem_n)
if 'addr' in elem_n:
name += " (%s)" % elem_n['addr']
m_box.add_string_absolute(name, 4, service_y)
service_y += 1
net_routes = routes.get(n, {})
for r in sorted(net_routes):
elem_route = net_routes[r]
r_name = "-> %s " % r
if elem_route['default']:
r_name += "(default)"
m_box.add_string_absolute(r_name, 6, service_y)
service_y += 1
r_y = tier_y
self._render_resource_nodes(r_box, r_y, elem_r, tot_box_w)
r_box.display(fp)
def _render_control_planes(self, fp):
LOG.info('%s()' % KenLog.fcn())
control_planes = CloudModel.get(self._cloud_version, 'control-planes')
for elem_r in control_planes:
(r_w, r_h) = self._determine_size_for_control_plane(elem_r)
r_box = Box(r_w, r_h)
r_type = ControlPlane.name(elem_r)
r_type_name = ControlPlane.region_name(elem_r)
title = 'ControlPlane: %s (%s)' % (r_type_name, r_type)
r_box.set_title(title)
tier_y = 2
tot_box_w = 0
for elem_c in ControlPlane.clusters(elem_r):
(t_w, t_h) = self._determine_size_for_cluster(
elem_r, elem_c)
t_box = Box(t_w, t_h)
if tot_box_w == 0:
tot_box_w = t_w
title = 'Cluster %s (%s)' % (
Cluster.name(elem_c), Cluster.id(elem_c))
t_box.set_title(title)
r_box.add_layer(t_box, 2, tier_y)
tier_y += t_h + (self._padding_y / 2)
member_x = 2
for elem_s in Cluster.servers(elem_c):
(m_w, m_h) = self._determine_size_for_server(
elem_r, elem_c, elem_s)
m_box = Box(m_w, m_h)
ip = Server.address(elem_s)
if ip:
title = '%s (%s)' % (Server.name(elem_s), ip)
else:
title = '%s' % Server.name(elem_s)
m_box.set_title(title)
t_box.add_layer(m_box, member_x, 2)
member_x += (self._server_width + self._padding_x)
service_y = 2
services = Server.services(elem_s)
for elem_c in sorted(services):
m_box.add_string_absolute(elem_c, 2, service_y)
service_y += 1
for elem_comp in sorted(services[elem_c]):
m_box.add_string_absolute(elem_comp, 4, service_y)
service_y += 1
service_y += 1
sep = "-" * m_w
m_box.add_string_absolute(sep, 2, service_y)
service_y += 2
interfaces = Server.interfaces(elem_s)
routes = Server.routes(elem_s)
for i in sorted(interfaces):
elem_i = interfaces[i]
device = elem_i['device']['name']
if 'bond-data' in elem_i:
device += " ("
first = True
for bond_dev in elem_i['bond-data'].get('devices', []):
if not first:
device += ", "
first = False
device += "%s" % bond_dev['name']
device += ")"
m_box.add_string_absolute(device, 2, service_y)
service_y += 1
networks = Interface.networks(elem_i)
for n in sorted(networks):
elem_n = networks[n]
name = Network.name(elem_n)
if 'addr' in elem_n:
name += " (%s)" % elem_n['addr']
m_box.add_string_absolute(name, 4, service_y)
service_y += 1
net_routes = routes.get(n, {})
for r in sorted(net_routes):
elem_route = net_routes[r]
r_name = "-> %s " % r
if elem_route['default']:
r_name += "(default)"
m_box.add_string_absolute(r_name, 6, service_y)
service_y += 1
r_y = tier_y
self._render_resource_nodes(r_box, r_y, elem_r, tot_box_w)
r_box.display(fp)
def build(self):
LOG.info('%s()' % KenLog.fcn())
filename = "%s/group_vars/all" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
global_vars = {'global': {'ansible_vars': [],
'all_servers': []}}
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
service_view = CloudModel.get(self._cloud_internal, 'service_view')
service_view = service_view['by_region']
servers = CloudModel.get(self._cloud_internal, 'servers')
ring_specifications = CloudModel.get(self._cloud_internal,
'ring-specifications', [])
pass_through = CloudModel.get(self._cloud_internal, 'pass_through')
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
ntp_servers = self.cloud_desc['ntp-servers']
if ntp_servers:
global_vars['global']['ntp_servers'] = ntp_servers
#
# Add a list of all vips
#
vips = set()
for cp_name, cp in control_planes.iteritems():
for ep_name, ep_data in cp['endpoints'].iteritems():
for role, role_data in ep_data.iteritems():
if role not in ['internal', 'admin']:
continue
for data in role_data:
access = data.get('access', {})
if 'hostname' in access:
vips.add(data['access']['hostname'])
for host_data in access.get('members', []):
vips.add(host_data['hostname'])
if ring_specifications:
ring_specifications = DataTransformer(
ring_specifications).all_output('-', '_')
global_vars['global']['all_ring_specifications'] =\
ring_specifications
else:
global_vars['global']['all_ring_specifications'] = []
if pass_through:
global_vars['global']['pass_through'] = pass_through['global']
global_vars['global']['vips'] = sorted(vips)
global_vars['topology'] = {'cloud_name': cloud_name,
'control_planes': []}
for cp_name in sorted(service_view):
cp = service_view[cp_name]
cp_data = {'name': cp_name,
'services': []}
for service_name in sorted(cp):
components = cp[service_name]
service_data = {'name': service_name,
'components': []}
for component_name in sorted(components):
hosts = components[component_name]
component_data = {'name': component_name,
'hosts': sorted(hosts)}
service_data['components'].append(component_data)
cp_data['services'].append(service_data)
global_vars['topology']['control_planes'].append(cp_data)
#
# Include disk details of all servers for Swift
#
for server in servers:
if server['state'] == ServerState.ALLOCATED:
disk_model_out = DataTransformer(
server['disk-model']).all_output('-', '_')
server_info = {'name': server['hostname'],
'rack': server.get('rack', None),
'region': server.get('region', None),
'disk_model': disk_model_out}
global_vars['global']['all_servers'].append(server_info)
network_names = []
for if_name, if_data in server['interfaces'].iteritems():
for net_name, net_data in if_data['networks'].iteritems():
if 'hostname' in net_data:
network_names.append(net_data['hostname'])
server_info['network_names'] = network_names
with open(filename, 'w') as fp:
yaml.dump(global_vars, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
filename = "%s/group_vars/all" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
global_vars = {'global': {'ansible_vars': [], 'all_servers': []}}
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
service_view = CloudModel.get(self._cloud_internal, 'service_view')
service_view = service_view['by_region']
servers = CloudModel.get(self._cloud_internal, 'servers')
ring_specifications = CloudModel.get(self._cloud_internal,
'ring-specifications', [])
pass_through = CloudModel.get(self._cloud_internal, 'pass_through')
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
ntp_servers = self.cloud_desc['ntp-servers']
if ntp_servers:
global_vars['global']['ntp_servers'] = ntp_servers
#
# Add a list of all vips
#
vips = set()
for cp_name, cp in control_planes.iteritems():
for ep_name, ep_data in cp['endpoints'].iteritems():
for role, role_data in ep_data.iteritems():
if role not in ['internal', 'admin']:
continue
for data in role_data:
access = data.get('access', {})
if 'hostname' in access:
vips.add(data['access']['hostname'])
for host_data in access.get('members', []):
vips.add(host_data['hostname'])
if ring_specifications:
ring_specifications = DataTransformer(
ring_specifications).all_output('-', '_')
global_vars['global']['all_ring_specifications'] =\
ring_specifications
else:
global_vars['global']['all_ring_specifications'] = []
if pass_through:
global_vars['global']['pass_through'] = pass_through['global']
global_vars['global']['vips'] = sorted(vips)
global_vars['topology'] = {
'cloud_name': cloud_name,
'control_planes': []
}
for cp_name in sorted(service_view):
cp = service_view[cp_name]
cp_data = {'name': cp_name, 'services': []}
for service_name in sorted(cp):
components = cp[service_name]
service_data = {'name': service_name, 'components': []}
for component_name in sorted(components):
hosts = components[component_name]
component_data = {
'name': component_name,
'hosts': sorted(hosts)
}
service_data['components'].append(component_data)
cp_data['services'].append(service_data)
global_vars['topology']['control_planes'].append(cp_data)
#
# Include disk details of all servers for Swift
#
for server in servers:
if server['state'] == ServerState.ALLOCATED:
disk_model_out = DataTransformer(
server['disk-model']).all_output('-', '_')
server_info = {
'name': server['hostname'],
'rack': server.get('rack', None),
'region': server.get('region', None),
'disk_model': disk_model_out
}
global_vars['global']['all_servers'].append(server_info)
network_names = []
for if_name, if_data in server['interfaces'].iteritems():
for net_name, net_data in if_data['networks'].iteritems():
if 'hostname' in net_data:
network_names.append(net_data['hostname'])
server_info['network_names'] = network_names
with open(filename, 'w') as fp:
yaml.dump(global_vars, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
ntp_servers = CloudModel.get(self._cloud_internal, 'ntp_servers')
dns_settings = CloudModel.get(self._cloud_internal, 'dns_settings')
smtp_settings = CloudModel.get(self._cloud_internal, 'smtp_settings')
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
net_group_firewall = CloudModel.get(self._cloud_internal, 'net-group-firewall')
firewall_settings = CloudModel.get(self._cloud_internal, 'firewall_settings')
pass_through = CloudModel.get(self._cloud_internal, 'pass_through')
components = CloudModel.get(self._cloud_internal, 'components')
services = CloudModel.get(self._cloud_internal, 'services')
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
for s in cluster['servers']:
self._build_ansible_host_vars(cloud_name, s, cp['endpoints'],
cp, cluster['name'],
ntp_servers, dns_settings, smtp_settings,
pass_through, components, services,
net_group_firewall, firewall_settings)
for r_name, resources in cp.get('resources', {}).iteritems():
for s in resources['servers']:
self._build_ansible_host_vars(cloud_name, s, cp['endpoints'],
cp, resources['name'],
ntp_servers, dns_settings, smtp_settings,
pass_through, components, services,
net_group_firewall, firewall_settings)
CloudModel.put(self._cloud_internal, 'cloud-firewall', self._cloud_firewall)
def _build_ansible_host_vars(self, cloud_name, server, cp_endpoints, cp, cluster_name,
ntp_servers=[], dns_settings={}, smtp_settings={}, pass_through={},
components={}, services={},
net_group_firewall={}, firewall_settings={}):
LOG.info('%s()' % KenLog.fcn())
components = CloudModel.get(self._cloud_internal, 'components')
components_by_mnemonic = CloudModel.get(self._cloud_internal, 'components_by_mnemonic')
filename = "%s/host_vars/%s" % (self._file_path, server['hostname'])
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
for if_name, if_data in server['interfaces'].iteritems():
for net_name, net_data in if_data['networks'].iteritems():
if net_data.get('hostname') == server['hostname']:
my_addr = net_data['addr']
host_vars = {
'host': {
'vars': {
'member_id': server.get('member_id', 'cpn'),
'my_network_address': my_addr,
'my_network_name': server['hostname'],
'my_network_interfaces': {}
},
'bind': {},
'tls_in': [],
'tls_out': [],
'my_id': server['id'],
'pass_through': pass_through['servers'].get(server['id'], {}),
'my_service_ips': {},
'failure_zone': server.get('failure-zone'),
'role': server.get('role'),
'my_dimensions': {
'cloud_name': cloud_name,
'hostname': server['hostname'],
'cluster': cluster_name,
'control_plane': cp['name']
},
'my_logical_volumes': {},
'my_device_groups': {}
},
'ntp_servers': ntp_servers,
'dns': dns_settings,
'smtp': smtp_settings,
}
nic_mapping = {}
if server.get('nic_map', {}):
nic_mapping['nic_mappings'] = []
for dev in server['nic_map']['physical-ports']:
nic_map = {'logical_name': dev['logical-name'],
'bus_address': dev['bus-address'],
'type': dev['type']}
if 'port-attributes' in dev:
nic_map['port_attributes'] = {}
for key, val in dev['port-attributes'].iteritems():
new_key = key.replace('-', '_')
nic_map['port_attributes'][new_key] = str(val)
nic_mapping['nic_mappings'].append(nic_map)
#
# Add per-service ips if they exist
#
if 'service-ips' in server:
host_vars['host']['my_service_ips'] = server['service-ips']
#
# Add list of bind addresses
#
for component_name, endpoint_data in cp_endpoints.iteritems():
if component_name in server['components']:
mnemonic = components[component_name]['mnemonic'].replace('-', '_')
if mnemonic not in host_vars['host']['bind']:
host_vars['host']['bind'][mnemonic] = {}
for role, role_data in endpoint_data.iteritems():
for data in role_data:
if 'address' in data['bind']:
bind_address = data['bind']['address']
else:
# May have to map to a network
for if_name, if_data in server['interfaces'].iteritems():
for net_name, net_data in if_data['networks'].iteritems():
if data['bind']['network_group'] == net_data['network-group']:
bind_address = net_data['addr']
break
bind_port = data['bind']['port']
host_vars['host']['bind'][mnemonic][role] = {'ip_address': bind_address,
'port': bind_port}
#
# Add list of tls terminations
#
for component_name, endpoint_data in cp_endpoints.iteritems():
if component_name in server['components']:
for role, role_data in endpoint_data.iteritems():
for data in role_data:
if 'tls-term'in data:
# Find the addesss in the right group
accept_addr = None
for if_name, if_data in server['interfaces'].iteritems():
for net_name, net_data in if_data['networks'].iteritems():
if data['tls-term']['network_group'] == net_data['network-group']:
accept_addr = net_data['addr']
break
if not accept_addr:
msg = ("Can't find address in Net Group %s on %s when "
"configuring tls for %s port %s" %
(data['tls-term']['network_group'], server['name'],
component_name, data['tls-term']['port']))
self.add_error(msg)
accept = {'ip_address': accept_addr,
'port': data['tls-term']['port']}
connect = {'ip_address': data['bind']['address'],
'port': data['bind']['port']}
term = {'name': component_name,
'role': role,
'accept': accept,
'connect': connect}
host_vars['host']['tls_in'].append(term)
#
# Add a list of tls initiations
#
# Build a list of all consumed services from this host
consumed = set()
for component_name in server['components']:
if component_name not in components:
print "Warning: No data for %s when buiding tls list" % component_name
continue
component_data = components[component_name]
for consumes in component_data.get('consumes-services', []):
consumed.add(consumes['service-name'])
for consumed_service in consumed:
service_name = components_by_mnemonic[consumed_service]['name']
if service_name in cp_endpoints:
endpoint_data = cp_endpoints[service_name]
for role, role_data in endpoint_data.iteritems():
for data in role_data:
if 'tls-init'in data:
accept = {'ip_address': data['access']['address'],
'host': data['access']['hostname'],
'port': data['access']['port']}
connect = {'ip_address': data['tls-init']['address'],
'host': data['tls-init']['hostname'],
'port': data['tls-init']['port']}
init = {'name': service_name,
'role': role,
'accept': accept,
'connect': connect}
host_vars['host']['tls_out'].append(init)
#
# Add Disk info
#
disk_model = DataTransformer(server['disk-model']).all_output('-', '_')
host_vars['host']['my_disk_models'] = disk_model
#
# Add a list of logical volumes by consuming component. Makes it
# possible for a service to find its mountpoint
#
for vg in disk_model.get('volume_groups', []):
for lv in vg.get('logical_volumes', []):
if 'consumer' in lv:
component_name = lv['consumer'].get('name')
if not component_name:
msg = ("Consumer attribute on %s:%s in "
"disk-model %s does not have a 'name' value ." %
(vg['name'], lv['name'], disk_model['name']))
self.add_error(msg)
continue
elif component_name not in components:
# Make this a warning, as it could be the customer is passing this
# to some playbook we don't know about
msg = ("Unknown component '%s'as consumer of "
"%s:%s in disk-model %s." %
(component_name, vg['name'], lv['name'], disk_model['name']))
self.add_warning(msg)
mnemonic = component_name
else:
if component_name not in server['components']:
msg = ("Server %s (%s) uses disk-model %s which includes a logical "
"volume to be consumed by '%s', but that component does not "
"run on this server." %
(server['id'], server['hostname'], disk_model['name'],
component_name))
self.add_warning(msg)
mnemonic = components[component_name]['mnemonic'].replace('-', '_')
if mnemonic not in host_vars['host']['my_logical_volumes']:
host_vars['host']['my_logical_volumes'][mnemonic] = []
host_vars['host']['my_logical_volumes'][mnemonic].append(deepcopy(lv))
#
# Add a list of device-groups volumes by consumer
#
for device_group in disk_model.get('device_groups', []):
if 'consumer' in device_group:
consumer_name = device_group['consumer'].get('name')
if not consumer_name:
msg = ("Consumer attribute on device-group %s in "
"disk-model %s does not have a 'name' value ." %
(device_group['name'], disk_model['name']))
self.add_error(msg)
continue
elif consumer_name not in services and consumer_name not in components:
msg = ("Unknown consumer '%s' of device-group %s"
"in disk-model %s." %
(consumer_name, device_group['name'], disk_model['name']))
self.add_warning(msg)
elif consumer_name not in server['services'] and consumer_name not in server['components']:
msg = ("Server %s (%s) uses disk-model %s which includes a device-group "
"be consumed by '%s', but there are no components of that service "
"on this server." %
(server['id'], server['hostname'], disk_model['name'],
consumer_name))
self.add_warning(msg)
if consumer_name not in host_vars['host']['my_device_groups']:
host_vars['host']['my_device_groups'][consumer_name] = []
host_vars['host']['my_device_groups'][consumer_name].append(deepcopy(device_group))
#
# Generate os-config network data
#
# create network_interface role compatible host_vars
(service_tags, ovs_bridge_host_vars, vlan_host_vars,
bond_host_vars, ether_host_vars, net_group_dict) = self._build_network_host_vars(server)
host_vars['host']['my_network_tags'] = service_tags
#
# Generate a structure to tell keepalived what device to bind vips to.
# Although this is really a group thing, we have to do this in here as
# we don't have the device info anywhere else
#
net_iface_vars = {}
for vip_net_name, vip_net_data in cp['vip_networks'].iteritems():
#
# Build a list of all VIPs on this network
#
vips = set()
for vip_data in vip_net_data:
if vip_data['provider'] in server['components']:
vips.add(vip_data['address'])
# Find the device for this network from the networkd groups
device = None
for ng_name, ng_data in net_group_dict.items():
for net in ng_data.get('networks', []):
if net['name'] == vip_net_name:
device = ng_data['device']
break
if device:
for vip in vips:
device_data = {'device': device,
'interface': vip_net_name,
'vip_address': vip}
if 'network_interfaces' not in net_iface_vars:
net_iface_vars['network_interfaces'] = []
net_iface_vars['network_interfaces'].append(device_data)
self.add_vips_to_network_host_vars(net_iface_vars, bond_host_vars,
ether_host_vars, ovs_bridge_host_vars,
vlan_host_vars)
# Get server firewall settings
host_vars['firewall'] = self.getFirewall(server, cp, net_group_firewall, firewall_settings,
net_group_dict)
# Save the firewall settings for this server
self._cloud_firewall[server['name']] = host_vars['firewall']
with open(filename, 'w') as fp:
yaml.dump(host_vars, fp, default_flow_style=False, indent=4)
if ovs_bridge_host_vars['ovs_bridge_interfaces']:
yaml.dump(ovs_bridge_host_vars, fp, default_flow_style=False, indent=4)
if vlan_host_vars['network_vlan_interfaces']:
yaml.dump(vlan_host_vars, fp, default_flow_style=False, indent=4)
if bond_host_vars['network_bond_interfaces']:
yaml.dump(bond_host_vars, fp, default_flow_style=False, indent=4)
if ether_host_vars['network_ether_interfaces']:
yaml.dump(ether_host_vars, fp, default_flow_style=False, indent=4)
if nic_mapping:
# Have to set default_style to ensure the bus address is in quotes
yaml.dump(nic_mapping, fp, default_style='\'', default_flow_style=False, indent=4)
if net_iface_vars:
yaml.dump(net_iface_vars, fp, default_flow_style=False, indent=4)
def _build_ansible_host_vars(self,
cloud_name,
server,
cp_endpoints,
cp,
cluster_name,
ntp_servers=[],
dns_settings={},
smtp_settings={},
pass_through={},
components={},
services={},
net_group_firewall={},
firewall_settings={}):
LOG.info('%s()' % KenLog.fcn())
components = CloudModel.get(self._cloud_internal, 'components')
components_by_mnemonic = CloudModel.get(self._cloud_internal,
'components_by_mnemonic')
filename = "%s/host_vars/%s" % (self._file_path, server['hostname'])
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
for if_name, if_data in server['interfaces'].iteritems():
for net_name, net_data in if_data['networks'].iteritems():
if net_data.get('hostname') == server['hostname']:
my_addr = net_data['addr']
host_vars = {
'host': {
'vars': {
'member_id': server.get('member_id', 'cpn'),
'my_network_address': my_addr,
'my_network_name': server['hostname'],
'my_network_interfaces': {}
},
'bind': {},
'tls_in': [],
'tls_out': [],
'my_id': server['id'],
'pass_through': pass_through['servers'].get(server['id'], {}),
'my_service_ips': {},
'failure_zone': server.get('failure-zone'),
'role': server.get('role'),
'my_dimensions': {
'cloud_name': cloud_name,
'hostname': server['hostname'],
'cluster': cluster_name,
'control_plane': cp['name']
},
'my_logical_volumes': {},
'my_device_groups': {}
},
'ntp_servers': ntp_servers,
'dns': dns_settings,
'smtp': smtp_settings,
}
nic_mapping = {}
if server.get('nic_map', {}):
nic_mapping['nic_mappings'] = []
for dev in server['nic_map']['physical-ports']:
nic_map = {
'logical_name': dev['logical-name'],
'bus_address': dev['bus-address'],
'type': dev['type']
}
if 'port-attributes' in dev:
nic_map['port_attributes'] = {}
for key, val in dev['port-attributes'].iteritems():
new_key = key.replace('-', '_')
nic_map['port_attributes'][new_key] = str(val)
nic_mapping['nic_mappings'].append(nic_map)
#
# Add per-service ips if they exist
#
if 'service-ips' in server:
host_vars['host']['my_service_ips'] = server['service-ips']
#
# Add list of bind addresses
#
for component_name, endpoint_data in cp_endpoints.iteritems():
if component_name in server['components']:
mnemonic = components[component_name]['mnemonic'].replace(
'-', '_')
if mnemonic not in host_vars['host']['bind']:
host_vars['host']['bind'][mnemonic] = {}
for role, role_data in endpoint_data.iteritems():
for data in role_data:
if 'address' in data['bind']:
bind_address = data['bind']['address']
else:
# May have to map to a network
for if_name, if_data in server[
'interfaces'].iteritems():
for net_name, net_data in if_data[
'networks'].iteritems():
if data['bind'][
'network_group'] == net_data[
'network-group']:
bind_address = net_data['addr']
break
bind_port = data['bind']['port']
host_vars['host']['bind'][mnemonic][role] = {
'ip_address': bind_address,
'port': bind_port
}
#
# Add list of tls terminations
#
for component_name, endpoint_data in cp_endpoints.iteritems():
if component_name in server['components']:
for role, role_data in endpoint_data.iteritems():
for data in role_data:
if 'tls-term' in data:
# Find the addesss in the right group
accept_addr = None
for if_name, if_data in server[
'interfaces'].iteritems():
for net_name, net_data in if_data[
'networks'].iteritems():
if data['tls-term'][
'network_group'] == net_data[
'network-group']:
accept_addr = net_data['addr']
break
if not accept_addr:
msg = (
"Can't find address in Net Group %s on %s when "
"configuring tls for %s port %s" %
(data['tls-term']['network_group'],
server['name'], component_name,
data['tls-term']['port']))
self.add_error(msg)
accept = {
'ip_address': accept_addr,
'port': data['tls-term']['port']
}
connect = {
'ip_address': data['bind']['address'],
'port': data['bind']['port']
}
term = {
'name': component_name,
'role': role,
'accept': accept,
'connect': connect
}
host_vars['host']['tls_in'].append(term)
#
# Add a list of tls initiations
#
# Build a list of all consumed services from this host
consumed = set()
for component_name in server['components']:
if component_name not in components:
print "Warning: No data for %s when buiding tls list" % component_name
continue
component_data = components[component_name]
for consumes in component_data.get('consumes-services', []):
consumed.add(consumes['service-name'])
for consumed_service in consumed:
service_name = components_by_mnemonic[consumed_service]['name']
if service_name in cp_endpoints:
endpoint_data = cp_endpoints[service_name]
for role, role_data in endpoint_data.iteritems():
for data in role_data:
if 'tls-init' in data:
accept = {
'ip_address': data['access']['address'],
'host': data['access']['hostname'],
'port': data['access']['port']
}
connect = {
'ip_address': data['tls-init']['address'],
'host': data['tls-init']['hostname'],
'port': data['tls-init']['port']
}
init = {
'name': service_name,
'role': role,
'accept': accept,
'connect': connect
}
host_vars['host']['tls_out'].append(init)
#
# Add Disk info
#
disk_model = DataTransformer(server['disk-model']).all_output('-', '_')
host_vars['host']['my_disk_models'] = disk_model
#
# Add a list of logical volumes by consuming component. Makes it
# possible for a service to find its mountpoint
#
for vg in disk_model.get('volume_groups', []):
for lv in vg.get('logical_volumes', []):
if 'consumer' in lv:
component_name = lv['consumer'].get('name')
if not component_name:
msg = ("Consumer attribute on %s:%s in "
"disk-model %s does not have a 'name' value ." %
(vg['name'], lv['name'], disk_model['name']))
self.add_error(msg)
continue
elif component_name not in components:
# Make this a warning, as it could be the customer is passing this
# to some playbook we don't know about
msg = ("Unknown component '%s'as consumer of "
"%s:%s in disk-model %s." %
(component_name, vg['name'], lv['name'],
disk_model['name']))
self.add_warning(msg)
mnemonic = component_name
else:
if component_name not in server['components']:
msg = (
"Server %s (%s) uses disk-model %s which includes a logical "
"volume to be consumed by '%s', but that component does not "
"run on this server." %
(server['id'], server['hostname'],
disk_model['name'], component_name))
self.add_warning(msg)
mnemonic = components[component_name][
'mnemonic'].replace('-', '_')
if mnemonic not in host_vars['host']['my_logical_volumes']:
host_vars['host']['my_logical_volumes'][mnemonic] = []
host_vars['host']['my_logical_volumes'][mnemonic].append(
deepcopy(lv))
#
# Add a list of device-groups volumes by consumer
#
for device_group in disk_model.get('device_groups', []):
if 'consumer' in device_group:
consumer_name = device_group['consumer'].get('name')
if not consumer_name:
msg = ("Consumer attribute on device-group %s in "
"disk-model %s does not have a 'name' value ." %
(device_group['name'], disk_model['name']))
self.add_error(msg)
continue
elif consumer_name not in services and consumer_name not in components:
msg = ("Unknown consumer '%s' of device-group %s"
"in disk-model %s." %
(consumer_name, device_group['name'],
disk_model['name']))
self.add_warning(msg)
elif consumer_name not in server[
'services'] and consumer_name not in server[
'components']:
msg = (
"Server %s (%s) uses disk-model %s which includes a device-group "
"be consumed by '%s', but there are no components of that service "
"on this server." %
(server['id'], server['hostname'], disk_model['name'],
consumer_name))
self.add_warning(msg)
if consumer_name not in host_vars['host']['my_device_groups']:
host_vars['host']['my_device_groups'][consumer_name] = []
host_vars['host']['my_device_groups'][consumer_name].append(
deepcopy(device_group))
#
# Generate os-config network data
#
# create network_interface role compatible host_vars
(service_tags, ovs_bridge_host_vars, vlan_host_vars, bond_host_vars,
ether_host_vars,
net_group_dict) = self._build_network_host_vars(server)
host_vars['host']['my_network_tags'] = service_tags
#
# Generate a structure to tell keepalived what device to bind vips to.
# Although this is really a group thing, we have to do this in here as
# we don't have the device info anywhere else
#
net_iface_vars = {}
for vip_net_name, vip_net_data in cp['vip_networks'].iteritems():
#
# Build a list of all VIPs on this network
#
vips = set()
for vip_data in vip_net_data:
if vip_data['provider'] in server['components']:
vips.add(vip_data['address'])
# Find the device for this network from the networkd groups
device = None
for ng_name, ng_data in net_group_dict.items():
for net in ng_data.get('networks', []):
if net['name'] == vip_net_name:
device = ng_data['device']
break
if device:
for vip in vips:
device_data = {
'device': device,
'interface': vip_net_name,
'vip_address': vip
}
if 'network_interfaces' not in net_iface_vars:
net_iface_vars['network_interfaces'] = []
net_iface_vars['network_interfaces'].append(device_data)
self.add_vips_to_network_host_vars(net_iface_vars, bond_host_vars,
ether_host_vars,
ovs_bridge_host_vars,
vlan_host_vars)
# Get server firewall settings
host_vars['firewall'] = self.getFirewall(server, cp,
net_group_firewall,
firewall_settings,
net_group_dict)
# Save the firewall settings for this server
self._cloud_firewall[server['name']] = host_vars['firewall']
with open(filename, 'w') as fp:
yaml.dump(host_vars, fp, default_flow_style=False, indent=4)
if ovs_bridge_host_vars['ovs_bridge_interfaces']:
yaml.dump(ovs_bridge_host_vars,
fp,
default_flow_style=False,
indent=4)
if vlan_host_vars['network_vlan_interfaces']:
yaml.dump(vlan_host_vars,
fp,
default_flow_style=False,
indent=4)
if bond_host_vars['network_bond_interfaces']:
yaml.dump(bond_host_vars,
fp,
default_flow_style=False,
indent=4)
if ether_host_vars['network_ether_interfaces']:
yaml.dump(ether_host_vars,
fp,
default_flow_style=False,
indent=4)
if nic_mapping:
# Have to set default_style to ensure the bus address is in quotes
yaml.dump(nic_mapping,
fp,
default_style='\'',
default_flow_style=False,
indent=4)
if net_iface_vars:
yaml.dump(net_iface_vars,
fp,
default_flow_style=False,
indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
ntp_servers = CloudModel.get(self._cloud_internal, 'ntp_servers')
dns_settings = CloudModel.get(self._cloud_internal, 'dns_settings')
smtp_settings = CloudModel.get(self._cloud_internal, 'smtp_settings')
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
net_group_firewall = CloudModel.get(self._cloud_internal,
'net-group-firewall')
firewall_settings = CloudModel.get(self._cloud_internal,
'firewall_settings')
pass_through = CloudModel.get(self._cloud_internal, 'pass_through')
components = CloudModel.get(self._cloud_internal, 'components')
services = CloudModel.get(self._cloud_internal, 'services')
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
for s in cluster['servers']:
self._build_ansible_host_vars(
cloud_name, s, cp['endpoints'], cp, cluster['name'],
ntp_servers, dns_settings, smtp_settings, pass_through,
components, services, net_group_firewall,
firewall_settings)
for r_name, resources in cp.get('resources', {}).iteritems():
for s in resources['servers']:
self._build_ansible_host_vars(
cloud_name, s, cp['endpoints'], cp, resources['name'],
ntp_servers, dns_settings, smtp_settings, pass_through,
components, services, net_group_firewall,
firewall_settings)
CloudModel.put(self._cloud_internal, 'cloud-firewall',
self._cloud_firewall)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
components = CloudModel.get(self._cloud_internal, 'components')
server_groups = CloudModel.get(self._cloud_internal, 'server-groups')
filename = "%s/hosts/localhost" % (self._file_path)
self.add_artifact(filename, ArtifactMode.CREATED)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write("localhost\n")
filename = "%s/hosts/verb_hosts" % (self._file_path)
self.add_artifact(filename, ArtifactMode.CREATED)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write("[localhost]\n")
f.write("localhost\n")
f.write("\n")
f.write("[resources:children]\n")
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
for cluster in cp['clusters']:
for server in cluster['servers']:
f.write("%s\n" % server['hostname'])
for resource_group_name, resource_group in cp.get('resources', {}).iteritems():
for server in resource_group['servers']:
f.write("%s\n" % server['hostname'])
f.write("\n")
# Build a list of all control_planes
f.write("[%s:children]\n" % (cloud_name))
for cp_name in sorted(control_planes):
f.write("%s-%s\n" % (cloud_name, cp_name))
f.write("\n")
# List all clusters and resource in a control plane
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
f.write("[%s-%s:children]\n" % (cloud_name, cp_name))
for cluster in cp['clusters']:
f.write("%s-%s-%s\n" % (cloud_name, cp_name, cluster['name']))
for resource_group_name in cp.get('resources', []):
f.write("%s-%s-%s\n" % (cloud_name, cp_name, resource_group_name))
f.write("\n")
# List all members of each clusters in a cp
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
for cluster in cp['clusters']:
f.write("[%s-%s-%s:children]\n" % (cloud_name, cp_name, cluster['name']))
for server in cluster['servers']:
f.write("%s\n" % server['hostname'])
f.write("\n")
for server in cluster['servers']:
f.write("[%s]\n" % server['hostname'])
f.write("%s ansible_ssh_host=%s\n" % (server['hostname'], server['addr']))
f.write("\n")
for resource_group_name, resource_group in cp.get('resources', {}).iteritems():
f.write("[%s-%s-%s:children]\n" % (cloud_name, cp_name, resource_group_name))
for server in resource_group['servers']:
f.write("%s\n" % server['hostname'])
f.write("\n")
for server in resource_group['servers']:
f.write("[%s]\n" % server['hostname'])
f.write("%s ansible_ssh_host=%s\n" % (server['hostname'], server['addr']))
f.write("\n")
# Build list of hosts by component accross all cps
component_list = {}
for cp_name, cp in control_planes.iteritems():
for component_name, component_data in cp['components'].iteritems():
if component_name not in components:
print "Warning: No data for %s when building host_vars" % component_name
continue
component_mnemonic = components[component_name]['mnemonic']
if component_mnemonic not in component_list:
component_list[component_mnemonic] = {}
if cp_name not in component_list[component_mnemonic]:
component_list[component_mnemonic][cp_name] = {}
for cluster in cp['clusters']:
if (component_name in cluster['service-components'] or
component_name in cp.get('common-service-components', [])):
if cluster['name'] not in component_list[component_mnemonic][cp_name]:
component_list[component_mnemonic][cp_name][cluster['name']] = []
host_list = component_list[component_mnemonic][cp_name][cluster['name']]
for server in cluster['servers']:
host_list.append(server['hostname'])
if 'resources' in cp:
for r_name, resources in cp['resources'].iteritems():
if (component_name in resources['service-components'] or
component_name in cp.get('common-service-components', [])):
if r_name not in component_list[component_mnemonic][cp_name]:
component_list[component_mnemonic][cp_name][r_name] = []
host_list = component_list[component_mnemonic][cp_name][r_name]
for server in resources['servers']:
host_list.append(server['hostname'])
for component_name in sorted(component_list):
component_data = component_list[component_name]
f.write("[%s:children]\n" % (component_name))
for cp_name in sorted(component_data):
f.write("%s-%s\n" % (component_name, cp_name))
f.write("\n")
for cp_name in sorted(component_data):
f.write("[%s-%s:children]\n" % (component_name, cp_name))
cluster_data = component_data[cp_name]
for cluster in sorted(cluster_data):
f.write("%s-%s-%s\n" % (component_name, cp_name, cluster))
f.write("\n")
for cluster in sorted(cluster_data):
f.write("[%s-%s-%s:children]\n" % (component_name, cp_name, cluster))
hosts = cluster_data[cluster]
for host in sorted(hosts):
f.write("%s\n" % host)
f.write("\n")
# Build list of server groups
for sg_name, sg in server_groups.iteritems():
f.write("[%s:children]\n" % (sg_name))
for child in sg.get('server-groups', []):
f.write("%s\n" % child)
for server in sg.get('servers', []):
if server['state'] == ServerState.ALLOCATED:
f.write("%s\n" % server['hostname'])
f.write("\n")
def _generate_firewall_rules(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
firewall_rules = CloudModel.get(cloud_version, 'firewall-rules', [])
network_groups = CloudModel.get(cloud_version, 'network-groups')
components = CloudModel.get(cloud_internal, 'components', [])
# If component is not set then it means an earlier generator failed
if not components:
return
# Initialise the firewall structure
net_group_firewall = {}
for net_group in network_groups:
net_group_firewall[net_group['name']] = {
'user': [],
'component': {}
}
#
# Add ports from components.
#
explicit_components = set()
for net_group in network_groups:
for comp in self._get_netgroup_components(net_group,
exclude_default=True):
explicit_components.add(comp)
for net_group in network_groups:
net_group_components = self._get_netgroup_components(net_group)
for comp_name, comp_data in components.iteritems():
if (comp_name in net_group_components
or ('default' in net_group_components
and comp_name not in explicit_components)):
comp_rules = []
for endpoint in comp_data.get('endpoints', []):
if ':' in str(endpoint['port']):
ports = str.split(endpoint['port'], ':')
else:
ports = [endpoint['port'], endpoint['port']]
rule = {
'type': 'allow',
'remote-ip-prefix': '0.0.0.0/0',
'port-range-min': ports[0],
'port-range-max': ports[1],
'protocol': endpoint.get('protocol', 'tcp')
}
comp_rules.append(rule)
if comp_rules:
net_group_firewall[net_group['name']]['component'][
comp_name] = comp_rules
# Annotate the rules so we can trace where they came from
for rule_group in firewall_rules:
for rule in rule_group.get('rules', []):
rule['component'] = "user-%s" % rule_group['name']
#
# Add user defined rules to network groups
#
for rule in firewall_rules:
if rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(
rule['rules'])
#
# Add user defined final rules to network groups
#
for rule in firewall_rules:
if not rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(
rule['rules'])
CloudModel.put(cloud_internal, 'net-group-firewall',
net_group_firewall)
def _generate_firewall_rules(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(
self._models['CloudModel'], self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
firewall_rules = CloudModel.get(cloud_version, 'firewall-rules', [])
network_groups = CloudModel.get(cloud_version, 'network-groups')
components = CloudModel.get(cloud_internal, 'components', [])
# If component is not set then it means an earlier generator failed
if not components:
return
# Initialise the firewall structure
net_group_firewall = {}
for net_group in network_groups:
net_group_firewall[net_group['name']] = {'user': [],
'component': {}}
#
# Add ports from components.
#
explicit_components = set()
for net_group in network_groups:
for comp in self._get_netgroup_components(net_group, exclude_default=True):
explicit_components.add(comp)
for net_group in network_groups:
net_group_components = self._get_netgroup_components(net_group)
for comp_name, comp_data in components.iteritems():
if (comp_name in net_group_components
or ('default' in net_group_components
and comp_name not in explicit_components)):
comp_rules = []
for endpoint in comp_data.get('endpoints', []):
if ':' in str(endpoint['port']):
ports = str.split(endpoint['port'], ':')
else:
ports = [endpoint['port'], endpoint['port']]
rule = {'type': 'allow',
'remote-ip-prefix': '0.0.0.0/0',
'port-range-min': ports[0],
'port-range-max': ports[1],
'protocol': endpoint.get('protocol', 'tcp')}
comp_rules.append(rule)
if comp_rules:
net_group_firewall[net_group['name']]['component'][comp_name] = comp_rules
# Annotate the rules so we can trace where they came from
for rule_group in firewall_rules:
for rule in rule_group.get('rules', []):
rule['component'] = "user-%s" % rule_group['name']
#
# Add user defined rules to network groups
#
for rule in firewall_rules:
if rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(rule['rules'])
#
# Add user defined final rules to network groups
#
for rule in firewall_rules:
if not rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(rule['rules'])
CloudModel.put(cloud_internal, 'net-group-firewall', net_group_firewall)
