def finalize(self):
LOG.info('%s()' % KenLog.fcn())
allocated_addresses = CloudModel.get(self._cloud_internal,
'address_allocations', {})
host_aliases = CloudModel.get(self._cloud_internal, 'host_aliases', {})
address_data = {}
for group_name, group in allocated_addresses.iteritems():
if group_name not in address_data:
address_data[group_name] = {}
for network_name, network in group.iteritems():
if network_name not in address_data[group_name]:
address_data[group_name][network_name] = {}
for addr in sorted(network):
aliases = host_aliases.get(group_name,
{}).get(network_name,
{}).get(addr, [])
address_data[group_name][network_name][addr] = []
for name in sorted(aliases):
address_data[group_name][network_name][addr].append(
name)
filename = os.path.join(self._file_path, 'address_info.yml')
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(address_data, fp, default_flow_style=False, indent=4)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_internal = CloudModel.internal(self._models['CloudModel'])
# Control Planes
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Network Groups
network_groups = CloudModel.get(cloud_internal, 'network-groups')
# components
components = CloudModel.get(cloud_internal, 'components')
# services
services = CloudModel.get(cloud_internal, 'services')
#
# Find which tags have been deprecated or replaced
#
replaced_tags = {}
deprecated_tags = {}
for component_name, component in components.iteritems():
for comp_tag in component.get('network-tags', []):
for alias in comp_tag.get('aliases', []):
replaced_tags[alias] = comp_tag['name']
if 'deprecated' in comp_tag:
deprecated_tags[comp_tag['name']] = comp_tag['deprecated']
# In previous versions of Ardana OpenStack, any tags defined on a network group are
# treated as applying to all control planes
netgroup_tags = self._get_netgroup_tags(network_groups)
# Process the tags for each cluster / resource group
tag_found = {}
for cp_name, cp in control_planes.iteritems():
cp_components = set()
for cluster in cp['clusters']:
for comp_name in cluster['service-components']:
cp_components.add(comp_name)
for r in cp.get('resources', []):
for comp_name in r['service-components']:
cp_components.add(comp_name)
cp['network-tags'] = self._process_network_tags(
cp['config-data-network-tags'], netgroup_tags, cp_components,
components, services, replaced_tags, deprecated_tags,
tag_found)
self._validate_network_tags(cp_name, cp['network-tags'],
cp['neutron-provider-nets'])
for tag_name, found in tag_found.iteritems():
if not found:
msg = ("Tag '%s' is not defined by any service" % tag_name)
self.add_warning(msg)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
services = CloudModel.get(cloud_internal, 'services', [])
config_data = {}
for data in CloudModel.get(cloud_version, 'configuration-data', []):
config_data[data['name']] = data
control_planes = CloudModel.get(cloud_internal, 'control-planes')
for cp_name, cp in control_planes.iteritems():
# Get any config data at the CP level
cp_config_data = self._expand_config(
cp.get('configuration-data', []), config_data, services)
# Extract all of the network tags defined in
# configuration data for this control plane
cp['config-data-network-tags'] = self._get_network_tags(
cp['name'], cp.get('configuration-data', []), config_data)
# Extract the list of physnets that have been defined
# in neutron configuration data for this control plane
# so they can be validated in the NetworkTagGenerator
cp['neutron-provider-nets'] = self._get_provider_nets(
cp['name'], cp.get('configuration-data', []), config_data)
for cluster in cp['clusters']:
cluster_config_data = self._expand_config(
cluster.get('configuration-data', []), config_data,
services)
context = "%s:%s" % (cp_name, cluster['name'])
cluster['config-data'] = self._merge(cp_config_data,
cluster_config_data,
context)
self._check_no_network_tags(
context, cluster.get('configuration-data', []),
config_data)
for r in cp.get('resources', []):
r_config_data = self._expand_config(
r.get('configuration-data', []), config_data, services)
context = "%s:%s" % (cp_name, r['name'])
r['config-data'] = self._merge(cp_config_data, r_config_data,
context)
self._check_no_network_tags(context,
r.get('configuration-data', []),
config_data)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
components = CloudModel.get(cloud_internal, 'components', [])
# If we have an error in an earlier generator we may not have
# components in the internal model
if not components:
return
components_by_mnemonic = CloudModel.get(cloud_internal, 'components_by_mnemonic')
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Resolve the service level consumes for all control planes before we
# do the component level consumes - as a component may need to find
# the service relationship in a parent control plane
#
for cp_name, cp in control_planes.iteritems():
for component_name, endpoints in cp.get('advertises', {}).iteritems():
component = components[component_name]
for adv_to in component.get('advertises-to-services', []):
adv_to_name = adv_to['service-name']
if adv_to_name in components:
adv_to_mnemonic = components[adv_to_name]['mnemonic']
else:
# We have some cases where a service advertises to a dummy
# component
if adv_to_name not in components_by_mnemonic:
continue
adv_to_mnemonic = adv_to_name
adv_to_name = components_by_mnemonic[adv_to_mnemonic]['name']
for entry in adv_to.get('entries', []):
adv = {'service_name': entry['service-name'],
'service_type': entry['service-type'],
'service_description': entry.get('service-description'),
'regions': [],
'component_name': component_name,
'from_cp_name': cp_name}
for role in ['public', 'internal', 'admin']:
if role in endpoints:
adv[role + '_url'] = endpoints[role]['url'] + entry.get('url-suffix', '')
for region_name in sorted(cp.get('regions', {})):
region_services = cp['regions'][region_name]
service = component.get('service', 'foundation')
if service in region_services:
adv['regions'].append(region_name)
self._advertised(adv_to_name, cp, adv, control_planes)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles'):
server_roles[role['name']] = role
memory_models = {}
for memory_model in CloudModel.get(cloud_version, 'memory-models', []):
memory_models[memory_model['name']] = memory_model
for server in servers:
if server['state'] != 'allocated':
continue
server['memory-model'] = {}
server_role = server_roles[server['role']]
if 'memory-model' not in server_role:
continue
memory_model = memory_models[server_role['memory-model']]
my_memory_model = {'name': memory_model['name']}
non_numa_pages = []
numa_pages = []
for page in memory_model.get('huge-pages', {}):
page_info = {
'size': page['size'],
'count': page['count'],
'size_in_k': self._page_size(page['size'])
}
if 'numa-node' in page:
page_info['numa_node'] = page['numa-node']
numa_pages.append(page_info)
else:
non_numa_pages.append(page_info)
if memory_model.get('vm-size', None):
my_memory_model['vm_ram_size_in_k'] = self._memory_size(
memory_model['vm-size']['ram'])
my_memory_model['numa_huge_pages'] = numa_pages
my_memory_model['non_numa_huge_pages'] = non_numa_pages
if 'default-huge-page-size' in memory_model:
my_memory_model['default_huge_page_size'] = memory_model[
'default-huge-page-size']
server['memory-model'] = my_memory_model
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_data = self._models['CloudDescription']['cloud']
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
audit_settings = {
'audit-dir': '/var/audit',
'default': 'disabled',
'enabled-services': [],
'disabled-services': []
}
audit_settings.update(cloud_data.get('audit-settings', {}))
for cp_name, cp in control_planes.iteritems():
cp_services = cp.get('services', {})
for service_name, service_data in cp_services.iteritems():
service_data['audit-settings'] = {
'dir': audit_settings['audit-dir'],
'enabled': audit_settings['default'] == 'enabled'
}
if service_name in audit_settings['enabled-services']:
service_data['audit-settings']['enabled'] = True
elif service_name in audit_settings['disabled-services']:
service_data['audit-settings']['enabled'] = False
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
self.services = CloudModel.get(cloud_internal, 'services', [])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
components = CloudModel.get(cloud_internal, 'components')
# Resolve the service data relationships.
for cp_name, cp in control_planes.iteritems():
for comp_name, comp_data in cp.get('components', []).iteritems():
provider = components[comp_name]
provider_service = self.services.get(provider['service'], {})
self._set_provided_data(provider, provider_service,
comp_data, cp, control_planes)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
services = CloudModel.get(self._cloud_internal, 'services')
components = CloudModel.get(self._cloud_internal, 'components')
components_by_mnemonic = CloudModel.get(self._cloud_internal, 'components_by_mnemonic')
pass_through = CloudModel.get(self._cloud_internal, 'pass_through')
# Create the group vars for each Control Plane
group_vars = {}
for cp_name, cp in control_planes.iteritems():
group_vars[cp_name] = self._build_ansible_group_vars(cloud_name, cp,
services, components,
components_by_mnemonic)
#
# Include disk details of all servers for Swift
#
group_vars[cp_name]['control_plane_servers'] = self._build_server_list(cp,
pass_through)
# Add any global vars - hope to remove this at some stage
global_vars = {}
for cp_name, cp in control_planes.iteritems():
for component_name, config_set in cp.get('config-sets', {}).iteritems():
if 'global_vars' in config_set:
mnemonic = components[component_name]['mnemonic'].replace('-', '_')
if mnemonic not in global_vars:
global_vars[mnemonic] = {'vars': {}}
global_vars[mnemonic]['vars'].update(config_set['global_vars'])
for component_name in cp['components']:
if components[component_name].get('advertises_global', False):
mnemonic = components[component_name]['mnemonic'].replace('-', '_')
if mnemonic not in global_vars:
global_vars[mnemonic] = {}
global_vars[mnemonic]['advertises'] = group_vars[cp_name][mnemonic]['advertises']
for cp_name, cp in control_planes.iteritems():
for component, values in global_vars.iteritems():
if component not in group_vars[cp_name]:
group_vars[cp_name][component] = values
for cp_name, vars in group_vars.iteritems():
self._write_cp_group_vars(cloud_name, cp_name, vars)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
networks = {}
network_groups = {}
for group in CloudModel.get(cloud_version, 'network-groups'):
network_groups[group['name']] = group
network_groups[group['name']]['networks'] = []
for net in CloudModel.get(cloud_version, 'networks'):
networks[net['name']] = net
# add networks into their respective network groups
for net_name, net in networks.iteritems():
if net.get('network-group', None):
network_groups[net['network-group']]['networks'].append(net)
# Inject a network group for each provider network, so we can support
# routes to them
for config_data in CloudModel.get(cloud_version, 'configuration-data',
[]):
if 'neutron' in config_data.get('services', []):
for net in config_data.get('data',
{}).get('neutron_provider_networks',
[]):
provider_net = {
'name': net['name'],
'cidr': net['cidr'],
'network-group': net['name'],
'neutron_network': True
}
group = {'name': net['name'], 'networks': [provider_net]}
networks[net['name']] = provider_net
network_groups[net['name']] = group
self._generate_default_network_mtu(network_groups)
CloudModel.put(cloud_internal, 'networks', networks)
CloudModel.put(cloud_internal, 'network-groups', network_groups)
def build(self):
LOG.info('%s()' % KenLog.fcn())
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
for cp_name, cp in control_planes.iteritems():
cert_data = cp['lb-cert-data']
for lb_provider, certs in cert_data.iteritems():
for cert_name, cert_details in certs.iteritems():
self._write_cert_req_info(cert_name, cert_details)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
# If we have an error in an earlier generator we may not have
# control_planes in the internal model
if not control_planes:
return
routes = {}
for cp_name, cp in control_planes.iteritems():
load_balancers = cp.get('load-balancer-config', {})
for cluster in cp['clusters']:
for server in cluster.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
for r_name, r in cp.get('resources', {}).iteritems():
for server in r.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
#
default_routes = {}
# Warn about any routes using the "default" route
for src_net, net_routes in routes.iteritems():
for dest_net, route_data in net_routes.iteritems():
if route_data['default']:
hosts = set()
if src_net not in default_routes:
default_routes[src_net] = []
for src, src_data in route_data['used_by'].iteritems():
for dest, host_list in src_data.iteritems():
for host in host_list:
hosts.add(host)
default_routes[src_net].append({'net': dest_net, 'hosts': hosts})
if default_routes:
msg = ("Default routing used between networks\n"
"The following networks are using a 'default' route rule. To remove this warning\n"
"either add an explicit route in the source network group or force the network to\n"
"attach in the interface model used by the servers.\n")
for src in sorted(default_routes):
dest_list = default_routes[src]
for dest_data in dest_list:
msg += " %s to %s\n" % (src, dest_data['net'])
for host in sorted(dest_data['hosts']):
msg += " %s\n" % (host)
self.add_warning(msg)
CloudModel.put(cloud_internal, 'routes', routes)
def _generate_ring_specifications_info(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(
self._models['CloudModel'], self._version)
ring_specifications_config = CloudModel.get(
cloud_version, 'ring-specifications', [])
cloud_internal = CloudModel.internal(self._models['CloudModel'])
CloudModel.put(cloud_internal,
'ring-specifications',
ring_specifications_config)
def build(self):
LOG.info('%s()' % KenLog.fcn())
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
net_info = {}
# Service IPs
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
if 'service-ips' in cluster:
if 'service_ips' not in net_info:
net_info['service_ips'] = {}
for name, net_data in cluster['service-ips'].iteritems():
if name not in net_info['service_ips']:
net_info['service_ips'][name] = []
for net_name, data in net_data.iteritems():
info = {
'control_plane': cp_name,
'cluster': cluster['name'],
'network': net_name,
'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {})
}
net_info['service_ips'][name].append(info)
if 'resources' in cp:
for res_name, resources in cp['resources'].iteritems():
if 'service-ips' in resources:
if 'service_ips' not in net_info:
net_info['service_ips'] = {}
for name, net_data in resources[
'service-ips'].iteritems():
if name not in net_info['service_ips']:
net_info['service_ips'][name] = []
for net_name, data in net_data.iteritems():
info = {
'control_plane': cp_name,
'cluster': res_name,
'network': net_name,
'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {})
}
net_info['service_ips'][name].append(info)
filename = "%s/info/net_info.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(net_info, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
routes = CloudModel.get(self._cloud_internal, 'routes')
filename = "%s/info/route_info.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(routes, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cp_topology = CloudModel.get(self._cloud_internal, 'cp-topology')
filename = "%s/info/control_plane_topology.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(cp_topology, fp, default_flow_style=False, indent=4)
region_topology = CloudModel.get(self._cloud_internal,
'region-topology')
filename = "%s/info/region_topology.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(region_topology, fp, default_flow_style=False, indent=4)
service_topology = CloudModel.get(self._cloud_internal,
'service-topology')
filename = "%s/info/service_topology.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(service_topology, fp, default_flow_style=False, indent=4)
network_topology = CloudModel.get(self._cloud_internal,
'network-topology')
filename = "%s/info/network_topology.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(network_topology, fp, default_flow_style=False, indent=4)
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_firewall = CloudModel.get(self._cloud_internal, 'cloud-firewall')
# Convert from per server to per network
firewall = {}
for server_name, data in cloud_firewall.iteritems():
for addr, rules in data.get('rules', {}).iteritems():
for rule in rules:
net_group = rule['chain']
component = rule['component']
if net_group not in firewall:
firewall[net_group] = {}
min_port = rule['port-range-min']
max_port = rule['port-range-max']
if min_port == max_port:
port = str(min_port)
else:
port = "%s:%s" % (min_port, max_port)
if port not in firewall[net_group]:
firewall[net_group][port] = {
'port': port,
'protocol': rule['protocol'],
'components': [],
'addresses': []
}
if component not in firewall[net_group][port][
'components']:
firewall[net_group][port]['components'].append(
component)
if addr not in firewall[net_group][port]['addresses']:
firewall[net_group][port]['addresses'].append(addr)
# rebuild get a list with the ports sorted
sorted_firewall = {}
for net_grp, data in firewall.iteritems():
sorted_firewall[net_grp] = []
for port in sorted(data, key=lambda x: int(x.split(":")[0])):
sorted_firewall[net_grp].append(data[port])
filename = "%s/info/firewall_info.yml" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(sorted_firewall, fp, default_flow_style=False, indent=4)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
networks = CloudModel.get(cloud_internal, 'networks')
network_groups = CloudModel.get(cloud_internal, 'network-groups')
components = CloudModel.get(cloud_internal, 'components')
cp_topology = self._generate_cp_topology(control_planes, components)
CloudModel.put(cloud_internal, 'cp-topology', cp_topology)
CloudModel.put(cloud_internal, 'region-topology',
self._generate_region_topology(control_planes))
CloudModel.put(cloud_internal, 'service-topology',
self._generate_service_topology(cp_topology))
CloudModel.put(
cloud_internal, 'network-topology',
self._generate_network_topology(network_groups, networks,
control_planes))
def _build_service_ips(self, group_vars, group, components):
if 'service-ips' in group:
networks = CloudModel.get(self._cloud_internal, 'networks')
if 'service_ips' not in group_vars:
group_vars['service_ips'] = {}
for name, net_data in group['service-ips'].iteritems():
mnemonic = components[name]['mnemonic'].replace('-', '_')
if mnemonic not in group_vars['service_ips']:
group_vars['service_ips'][mnemonic] = []
for net_name, data in net_data.iteritems():
info = {'hosts': data.get('hosts', []),
'cluster_ip': data.get('cluster-ip', {}),
'cidr': networks[net_name].get('cidr')}
group_vars['service_ips'][mnemonic].append(info)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', [])
for cp_name, cp in control_planes.iteritems():
deleted_components = []
for cluster in cp['clusters']:
for server in cluster['servers']:
self._add_deleted_components(deleted_components, server)
for r_name, resources in cp.get('resources', {}).iteritems():
for server in resources.get('servers', []):
self._add_deleted_components(deleted_components, server)
cp['deleted-components'] = deleted_components
CloudModel.put(cloud_internal, 'control-planes', control_planes)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_internal = CloudModel.internal(self._models['CloudModel'])
servers = CloudModel.get(cloud_internal, 'servers', {})
self._add_vm_info_to_factory(servers)
def generate(self):
def _get_bitmask(cpus):
bitmask = 0
for x in cpus:
bitmask += (1 << x)
return bitmask
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
components = CloudModel.get(cloud_internal, 'components', {})
cpu_models = {}
for model in CloudModel.get(cloud_version, 'cpu-models', []):
cpu_models[model['name']] = model
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles', []):
server_roles[role['name']] = role
for s in servers:
if s['state'] != 'allocated':
continue
server_role = server_roles[s['role']]
s['cpu-assignments'] = {}
if 'cpu-model' in server_role:
server_cpus = []
cpu_model = cpu_models[server_role['cpu-model']]
if cpu_model.get('vm-size', {}):
s['vm_no_of_vcpus'] = cpu_model.get('vm-size')['vcpus']
for service_data in cpu_model.get('assignments', []):
comp_on_server = False
for comp_name in service_data['components']:
if comp_name in s['components']:
comp_on_server = True
break
if not comp_on_server:
msg = (
"CPU Assignment for components %s in '%s' "
"has not been applied to server '%s' (id: %s role: %s) "
"because it does not host any of the listed components."
%
(str(service_data['components']).strip('[]'),
cpu_model['name'], s['name'], s['id'], s['role']))
self.add_warning(msg)
continue
assignment_data = {}
for cpu_data in service_data.get('cpu', []):
cpu_list = cpu_data['processor-ids']
if cpu_data.get('isolate', True):
server_cpus.extend(cpu_list)
assignment_data[cpu_data['role']] = \
{'processor_ids': cpu_list,
'bitmask': "0x%04x" % _get_bitmask(cpu_list),
'processor_list': str(sorted(cpu_list)).strip('[]').replace(' ', '')}
for comp_name in service_data['components']:
mnemonic = components[comp_name]['mnemonic'].replace(
'-', '_')
s['cpu-assignments'][mnemonic] = deepcopy(
assignment_data)
# Add Server level data
s['cpu-assignments']['server'] = {}
if server_cpus:
s['cpu-assignments']['server'] = \
{'processor_ids': server_cpus,
'bitmask': "0x%04x" % _get_bitmask(server_cpus),
'processor_list': str(sorted(server_cpus)).strip('[]').replace(' ', '')}
def build(self):
LOG.info('%s()' % KenLog.fcn())
file_name = os.path.join(self._file_path, 'hosts.hf')
self.add_artifact(file_name, ArtifactMode.CREATED)
allocated_addresses = CloudModel.get(self._cloud_internal,
'address_allocations')
host_aliases = CloudModel.get(self._cloud_internal, 'host_aliases')
host_names = CloudModel.get(self._cloud_internal, 'host_names')
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
with open(file_name, 'w') as fp:
fp.write("# Cloud: %s\n" % (cloud_name))
fp.write("\n")
fp.write("# Localhost Information\n")
fp.write("127.0.0.1 localhost\n")
fp.write("\n")
fp.write(
"# The following lines are desirable for IPv6 capable hosts\n")
fp.write("::1 localhost ip6-localhost ip6-loopback\n")
fp.write("fe00::0 ip6-localnet\n")
fp.write("ff00::0 ip6-mcastprefix\n")
fp.write("ff02::1 ip6-allnodes\n")
fp.write("ff02::2 ip6-allrouters\n")
fp.write("\n")
for group_name, group in allocated_addresses.iteritems():
fp.write("#\n")
fp.write("# Network Group: %s\n" % (group_name))
fp.write("#\n")
for network_name, network in group.iteritems():
fp.write("# Network: %s\n" % (network_name))
ips = []
for addr in network:
aliases = host_aliases.get(group_name,
{}).get(network_name,
{}).get(addr, [])
hostnames = host_names.get(group_name,
{}).get(network_name, {})
# Need to find out if there is a hostname in the list of aliases,
# and if there are alaias for that hostname
hostname = ""
has_alias = False
for name in aliases:
if name in hostnames:
hostname = name
else:
has_alias = True
ip_version = IPAddress(addr).version
for name in aliases:
# If we have a hostname and one or more aliases then
# we want to have these as aliases in the hosts file
# so that any fqdn lookup will get the hostname not
# the alias
if hostname and has_alias:
if hostname == name:
continue
else:
name_string = "%s %s" % (hostname, name)
else:
name_string = name
if ip_version == 4:
# Expand the address to a string with leading
# spaces in each quad so that it sorts by
# version
ips.append([
"%3s.%3s.%3s.%3s" % tuple(addr.split(".")),
name_string
])
else:
ips.append([addr, name_string])
for ip in sorted(ips, key=itemgetter(0)):
# Also add lowercase hostnames for eventlet-0.20.0, which treats /etc/hosts
# as case-sensitive
fp.write(
"%-16s %s %s\n" %
(ip[0].replace(" ", ""), ip[1], ip[1].lower()))
def build(self):
LOG.info('%s()' % KenLog.fcn())
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
control_planes = deepcopy(
CloudModel.get(self._cloud_internal, 'control-planes'))
components = CloudModel.get(self._cloud_internal, 'components')
server_groups = CloudModel.get(self._cloud_internal, 'server-groups')
servers = CloudModel.get(self._cloud_internal, 'servers')
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
if cluster.get('use-localhost', False):
local_name = "%s-%s-local" % (
cp_name, cluster.get('cluster-prefix'))
cluster['servers'].append({
'hostname': local_name,
'ardana_ansible_host': local_name,
'addr': "127.0.0.1"
})
filename = "%s/hosts/localhost" % (self._file_path)
self.add_artifact(filename, ArtifactMode.CREATED)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write("localhost\n")
filename = "%s/hosts/verb_hosts" % (self._file_path)
self.add_artifact(filename, ArtifactMode.CREATED)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write("[localhost]\n")
f.write("localhost\n")
f.write("\n")
f.write("[resources:children]\n")
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
for cluster in cp['clusters']:
for server in cluster['servers']:
if (server['hostname'] != "%s-%s-local" %
(cp_name, cluster.get('cluster-prefix'))):
f.write("%s\n" % server['ardana_ansible_host'])
for resource_group_name, resource_group in cp.get(
'resources', {}).iteritems():
for server in resource_group['servers']:
f.write("%s\n" % server['ardana_ansible_host'])
f.write("\n")
f.write("[ardana-hypervisors:children]\n")
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
for cluster in cp['clusters']:
for server in cluster['servers']:
if server.get('vm-factory', False):
f.write("%s\n" % server['hostname'])
for resource_group_name, resource_group in cp.get(
'resources', {}).iteritems():
for server in resource_group['servers']:
if server.get('vm-factory', False):
f.write("%s\n" % server['hostname'])
f.write("\n")
# Build a host group for vms on a vmfactory
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
for cluster in cp['clusters']:
for server in cluster['servers']:
if server.get('vm-factory', False):
f.write("[%s-vms:children]\n" %
server['ardana_ansible_host'])
for my_vm in server.get('vms'):
f.write("%s\n" % my_vm['ardana_ansible_host'])
f.write("\n")
f.write("[%s-all:children]\n" %
server['ardana_ansible_host'])
f.write("%s\n" % server['ardana_ansible_host'])
f.write("%s-vms\n" % server['ardana_ansible_host'])
f.write("\n")
for resource_group_name, resource_group in cp.get(
'resources', {}).iteritems():
for server in resource_group['servers']:
if server.get('vm-factory', False):
f.write("[%s-vms:children]\n" %
server['ardana_ansible_host'])
for my_vm in server.get('vms'):
f.write("%s\n" % my_vm['ardana_ansible_host'])
f.write("\n")
f.write("[%s-all:children]\n" %
server['ardana_ansible_host'])
f.write("%s\n" % server['ardana_ansible_host'])
f.write("%s-vms\n" % server['ardana_ansible_host'])
f.write("\n")
# Build a list of all control_planes
f.write("[%s:children]\n" % (cloud_name))
for cp_name in sorted(control_planes):
f.write("%s-%s\n" % (cloud_name, cp_name))
f.write("\n")
# List all clusters and resource in a control plane
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
f.write("[%s-%s:children]\n" % (cloud_name, cp_name))
for cluster in cp['clusters']:
f.write("%s-%s-%s\n" %
(cloud_name, cp_name, cluster['name']))
for resource_group_name in cp.get('resources', []):
f.write("%s-%s-%s\n" %
(cloud_name, cp_name, resource_group_name))
f.write("\n")
# List all members of each clusters in a cp
for cp_name in sorted(control_planes):
cp = control_planes[cp_name]
for cluster in cp['clusters']:
f.write("[%s-%s-%s:children]\n" %
(cloud_name, cp_name, cluster['name']))
for server in sorted(
cluster['servers'],
key=lambda s: s['ardana_ansible_host']):
f.write("%s\n" % server['ardana_ansible_host'])
f.write("\n")
for server in sorted(
cluster['servers'],
key=lambda s: s['ardana_ansible_host']):
f.write("[%s]\n" % server['ardana_ansible_host'])
f.write(
"%s ansible_ssh_host=%s" %
(server['ardana_ansible_host'], server['addr']))
if server.get('ansible_options'):
f.write(" %s" % server['ansible_options'])
f.write("\n\n")
for resource_group_name, resource_group in cp.get(
'resources', {}).iteritems():
f.write("[%s-%s-%s:children]\n" %
(cloud_name, cp_name, resource_group_name))
for server in sorted(
resource_group['servers'],
key=lambda s: s['ardana_ansible_host']):
f.write("%s\n" % server['ardana_ansible_host'])
f.write("\n")
for server in sorted(
resource_group['servers'],
key=lambda s: s['ardana_ansible_host']):
f.write("[%s]\n" % server['ardana_ansible_host'])
f.write(
"%s ansible_ssh_host=%s" %
(server['ardana_ansible_host'], server['addr']))
if server.get('ansible_options'):
f.write(" %s" % server['ansible_options'])
f.write("\n\n")
# Build list of hosts by component accross all cps
component_list = {}
for cp_name, cp in control_planes.iteritems():
for component_name, component_data in cp[
'components'].iteritems():
if component_name not in components:
print "Warning: No data for %s when building host_vars" % component_name
continue
component_mnemonic = components[component_name]['mnemonic']
if component_mnemonic not in component_list:
component_list[component_mnemonic] = {}
if cp_name not in component_list[component_mnemonic]:
component_list[component_mnemonic][cp_name] = {}
for cluster in cp['clusters']:
if (component_name in cluster['service-components']):
if cluster['name'] not in component_list[
component_mnemonic][cp_name]:
component_list[component_mnemonic][cp_name][
cluster['name']] = []
host_list = component_list[component_mnemonic][
cp_name][cluster['name']]
for server in cluster['servers']:
host_list.append(server['ardana_ansible_host'])
if 'resources' in cp:
for r_name, resources in cp['resources'].iteritems():
if (component_name
in resources['service-components']):
if r_name not in component_list[
component_mnemonic][cp_name]:
component_list[component_mnemonic][
cp_name][r_name] = []
host_list = component_list[component_mnemonic][
cp_name][r_name]
for server in resources['servers']:
host_list.append(
server['ardana_ansible_host'])
# Build a list of the first host of each component in each control plane
first_member_list = {}
for name in sorted(component_list):
if name not in first_member_list:
first_member_list[name] = []
for cp_name, clusters in component_list[name].iteritems():
hosts = []
for cluster_name, cluster_hosts in clusters.iteritems():
hosts.extend(cluster_hosts)
if len(hosts) > 0:
first_member_list[name].append(sorted(hosts)[0])
# Add in any deleted components
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
for server in cluster['servers']:
self._add_deleted_components(component_list,
components, server,
cp_name, cluster['name'])
for r_name, resources in cp.get('resources', {}).iteritems():
for server in resources.get('servers', []):
self._add_deleted_components(component_list,
components, server,
cp_name, r_name)
for component_name in sorted(component_list):
component_data = component_list[component_name]
# Check there is at least one host
host_count = 0
for cp_name, cp_data in component_data.iteritems():
for cluster_name, hosts in cp_data.iteritems():
host_count += len(hosts)
if host_count == 0:
continue
f.write("[%s:children]\n" % (component_name))
for cp_name in sorted(component_data):
f.write("%s-%s\n" % (component_name, cp_name))
f.write("\n")
for cp_name in sorted(component_data):
f.write("[%s-%s:children]\n" % (component_name, cp_name))
cluster_data = component_data[cp_name]
for cluster in sorted(cluster_data):
f.write("%s-%s-%s\n" %
(component_name, cp_name, cluster))
f.write("\n")
for cluster in sorted(cluster_data):
f.write("[%s-%s-%s:children]\n" %
(component_name, cp_name, cluster))
hosts = cluster_data[cluster]
for host in sorted(hosts):
f.write("%s\n" % host)
f.write("\n")
if component_name in first_member_list:
f.write("[%s--first-member:children]\n" % (component_name))
for host in sorted(first_member_list[component_name]):
f.write("%s\n" % host)
f.write("\n")
# Build list of server groups
for sg_name, sg in server_groups.iteritems():
f.write("[%s:children]\n" % (sg_name))
for child in sg.get('server-groups', []):
f.write("%s\n" % child)
for server in sg.get('servers', []):
if server['state'] == ServerState.ALLOCATED:
f.write("%s\n" % server['ardana_ansible_host'])
f.write("\n")
# Build a list of host aliases
for server in servers:
if server['state'] != ServerState.ALLOCATED:
continue
if server['hostname'] != server['ardana_ansible_host']:
f.write("[%s:children]\n" % server['hostname'])
f.write("%s\n\n" % server['ardana_ansible_host'])
for iface, iface_data in server['interfaces'].iteritems():
for net_name, net in iface_data.get('networks',
{}).iteritems():
if 'addr' in net:
if (net['hostname'] !=
server['ardana_ansible_host']
and net['hostname'] != server['hostname']):
f.write("[%s:children]\n" % net['hostname'])
f.write("%s\n\n" %
server['ardana_ansible_host'])
def build(self):
LOG.info('%s()' % KenLog.fcn())
filename = "%s/group_vars/all" % (self._file_path)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.add_artifact(filename, ArtifactMode.CREATED)
global_vars = {'global': {'ansible_vars': [], 'all_servers': []}}
control_planes = CloudModel.get(self._cloud_internal, 'control-planes')
components = CloudModel.get(self._cloud_internal, 'components')
service_view = CloudModel.get(self._cloud_internal, 'service_view')
service_view = service_view['by_region']
ring_specifications = CloudModel.get(self._cloud_internal,
'ring-specifications', [])
pass_through = CloudModel.get(self._cloud_internal, 'pass_through')
cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
ntp_servers = self.cloud_desc.get('ntp-servers', {})
dns_settings = self.cloud_desc.get('dns-settings', {})
if ntp_servers:
global_vars['global']['ntp_servers'] = ntp_servers
if dns_settings:
global_vars['global']['dns_nameservers'] = dns_settings[
'nameservers']
global_vars['global']['control_planes'] = []
for cp_name in control_planes:
global_vars['global']['control_planes'].append(
"%s-%s" % (cloud_name, cp_name))
#
# Add a list of all vips
#
vips = set()
vip_addresses = set()
for cp_name, cp in control_planes.iteritems():
for ep_name, ep_data in cp['endpoints'].iteritems():
for role, role_data in ep_data.iteritems():
if role not in ['internal', 'admin']:
continue
for data in role_data:
access = data.get('access', {})
if 'hostname' in access:
vips.add(data['access']['hostname'])
vip_addresses.add(data['access']['address'])
for host_data in access.get('members', []):
vips.add(host_data['hostname'])
if ring_specifications:
ring_specifications = DataTransformer(
ring_specifications).all_output('-', '_')
global_vars['global']['all_ring_specifications'] =\
ring_specifications
else:
global_vars['global']['all_ring_specifications'] = []
if pass_through:
global_vars['global']['pass_through'] = pass_through['global']
global_vars['global']['vips'] = sorted(vips)
global_vars['global']['vip_addresses'] = sorted(vip_addresses)
global_vars['topology'] = {
'cloud_name': cloud_name,
'control_planes': []
}
for cp_name in sorted(service_view):
cp = service_view[cp_name]
cp_data = {'name': cp_name, 'services': []}
for service_name in sorted(cp):
service_components = cp[service_name]
service_data = {'name': service_name, 'components': []}
for component_name in sorted(service_components):
hosts = service_components[component_name]
component_data = {
'name': component_name,
'hosts': sorted(hosts)
}
service_data['components'].append(component_data)
cp_data['services'].append(service_data)
global_vars['topology']['control_planes'].append(cp_data)
#
# Add region and service topologies
#
rt = global_vars['region_topology'] = CloudModel.get(
self._cloud_internal, 'region-topology')
global_vars['service_topology'] = CloudModel.get(
self._cloud_internal, 'service-topology')
#
# BUG 4227: provide a view onto the region topology suitable for use by Ansible to generate
# multiple slices for different Tempest configurations
#
region_by_cp_by_service = set()
for region_name, region in rt['regions'].iteritems():
for cp_name, control_plane in region['control_planes'].iteritems():
for service_name, scs in control_plane['services'].iteritems():
region_by_cp_by_service.update(
(region_name, cp_name, service_name, sc) for sc in scs)
global_vars['region_by_cp_by_service'] = [{
'region': r,
'control_plane': c,
'service': s,
'service_component': sc,
} for r, c, s, sc in sorted(region_by_cp_by_service)]
#
# Add Cert data
#
global_vars['cert_data'] = {'hosts': [], 'services': {}}
for cp_name, cp in control_planes.iteritems():
cp_cert_data = cp['cert-data']
for host_cert in cp_cert_data['hosts']:
global_vars['cert_data']['hosts'].append(host_cert)
for component_name, certs in cp_cert_data['services'].iteritems():
mnemonic = components[component_name]['mnemonic'].replace(
'-', '_')
if mnemonic not in global_vars['cert_data']['services']:
global_vars['cert_data']['services'][mnemonic] = []
global_vars['cert_data']['services'][mnemonic].extend(certs)
global_vars['cp_tempest_data'] = self._get_tempest_vars(control_planes)
global_vars['cp_tempest_test_plans'] = self._get_tempest_test_plans(
control_planes)
global_vars['deployer_media_legacy_layout'] = False
with open(filename, 'w') as fp:
yaml.dump(global_vars, fp, default_flow_style=False, indent=4)
def _generate_firewall_rules(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
firewall_rules = CloudModel.get(cloud_version, 'firewall-rules', [])
network_groups = CloudModel.get(cloud_version, 'network-groups')
components = CloudModel.get(cloud_internal, 'components', [])
# If component is not set then it means an earlier generator failed
if not components:
return
# Initialise the firewall structure
net_group_firewall = {}
for net_group in network_groups:
net_group_firewall[net_group['name']] = {
'user': [],
'component': {}
}
#
# Add ports from components.
#
explicit_components = set()
for net_group in network_groups:
for comp in self._get_netgroup_components(net_group,
exclude_default=True):
explicit_components.add(comp)
for net_group in network_groups:
net_group_components = self._get_netgroup_components(net_group)
for comp_name, comp_data in components.iteritems():
if (comp_name in net_group_components
or ('default' in net_group_components
and comp_name not in explicit_components)):
comp_rules = []
for endpoint in comp_data.get('endpoints', []):
if (comp_name in net_group.get(
'tls-component-endpoints', [])
or 'default' in net_group.get(
'tls-component-endpoints', [])):
port_str = endpoint.get('tls-port',
endpoint['port'])
else:
port_str = endpoint['port']
if ':' in str(port_str):
ports = str.split(port_str, ':')
else:
ports = [port_str, port_str]
rule = {
'type': 'allow',
'remote-ip-prefix': '0.0.0.0/0',
'port-range-min': ports[0],
'port-range-max': ports[1],
'protocol': endpoint.get('protocol', 'tcp')
}
comp_rules.append(rule)
if comp_rules:
net_group_firewall[net_group['name']]['component'][
comp_name] = comp_rules
# Annotate the rules so we can trace where they came from
for rule_group in firewall_rules:
for rule in rule_group.get('rules', []):
rule['component'] = "user-%s" % rule_group['name']
#
# Add user defined rules to network groups
#
for rule in firewall_rules:
if rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(
rule['rules'])
#
# Add user defined final rules to network groups
#
for rule in firewall_rules:
if not rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(
rule['rules'])
CloudModel.put(cloud_internal, 'net-group-firewall',
net_group_firewall)
def __init__(self, instructions, models, controllers):
super(HTMLDiagramBuilder,
self).__init__(2.0, instructions, models, controllers,
'html-diagram-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = ArdanaPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'html')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
self.cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
self.control_planes = CloudModel.get(self._cloud_internal,
'control-planes')
self.iface_models = CloudModel.get(self._cloud_version,
'interface-models')
self.server_roles = CloudModel.get(self._cloud_version, 'server-roles')
self.disk_models = CloudModel.get(self._cloud_version, 'disk-models',
[])
self.memory_models = CloudModel.get(self._cloud_version,
'memory-models', [])
self.cpu_models = CloudModel.get(self._cloud_version, 'cpu-models', [])
self.components = CloudModel.get(self._cloud_internal, 'components')
self.services = CloudModel.get(self._cloud_internal, 'services')
self.servers = CloudModel.get(self._cloud_internal, 'servers')
self.server_groups = CloudModel.get(self._cloud_internal,
'server-groups')
self.network_groups = CloudModel.get(self._cloud_internal,
'network-groups')
self.region_topology = CloudModel.get(self._cloud_internal,
'region-topology')
self.service_topology = CloudModel.get(self._cloud_internal,
'service-topology')
self.cp_topology = CloudModel.get(self._cloud_internal, 'cp-topology')
self.network_topology = CloudModel.get(self._cloud_internal,
'network-topology')
self.components_by_mnemonic = {}
for comp_name, comp_data in self.components.iteritems():
if 'alias-for' not in comp_data:
self.components_by_mnemonic[comp_data['mnemonic']] = comp_name
ArdanaPaths.make_path(self._file_path)
for subdir in ['Servers']:
ArdanaPaths.make_path(self._file_path + "/%s" % subdir)
# Create a mapping from hostname to server id
self.server_by_hostname = {}
self.server_by_name = {}
for s in self.servers:
if 'hostname' in s:
self.server_by_hostname[s['hostname']] = s
if 'name' in s:
self.server_by_name[s['name']] = s
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_data = self._models['CloudDescription']['cloud']
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
services = {}
components = {}
components_by_mnemonic = {}
for service in CloudModel.get(cloud_version, 'services'):
services[service['name']] = service
for component in CloudModel.get(cloud_version, 'service-components'):
components[component['name']] = component
components_by_mnemonic[component['mnemonic']] = component
control_planes = {}
iface_models = {}
disk_models = {}
nic_mappings = {}
nic_device_types = {}
nic_device_families = {}
server_roles = {}
server_groups = {}
bm_servers = []
pass_through = {'global': {}, 'servers': {}}
# Servers
if 'servers' in cloud_version:
bm_servers.extend(cloud_version['servers'])
# Build a list of server addresses so we can
# reserve them
server_addresses = {}
for s in bm_servers:
server_addresses[s['ip-addr']] = s['id']
# Control Planes
for cp in CloudModel.get(cloud_version, 'control-planes'):
control_planes[cp['name']] = dict(cp)
# Network Groups
network_groups = CloudModel.get(cloud_internal, 'network-groups')
# Networks
networks = CloudModel.get(cloud_internal, 'networks')
# Interface Models
for iface in CloudModel.get(cloud_version, 'interface-models'):
iface_models[iface['name']] = iface
# Disk Models
for disk_model in CloudModel.get(cloud_version, 'disk-models'):
disk_models[disk_model['name']] = disk_model
# NIC Device Families
for nic_family in CloudModel.get(cloud_version, 'nic-device-families',
[]):
nic_device_families[nic_family['name']] = nic_family
# NIC Device Types
for nic_dev in CloudModel.get(cloud_version, 'nic-device-types', []):
nic_device_types[nic_dev['name']] = nic_dev
nic_device_types[nic_dev['name']]['family_data'] = \
nic_device_families[nic_dev['family']]
# NIC Mapping
for nic_map in CloudModel.get(cloud_version, 'nic-mappings', []):
for port in nic_map.get('physical-ports', []):
# Add in any data from the NIC device type
if 'nic-device-type' in port:
port['nic-device-type'] = nic_device_types[
port['nic-device-type']]
nic_mappings[nic_map['name']] = nic_map
# Server Roles
for role in CloudModel.get(cloud_version, 'server-roles'):
server_roles[role['name']] = role
# Server Groups
for group in CloudModel.get(cloud_version, 'server-groups', []):
server_groups[group['name']] = group
# Pass Through
pt_data = CloudModel.get(cloud_version, 'pass-through', [])
####################################
#
# End of reading input data
#
###################################
# Combine pass through data which maybe in multiple
# input files into a single dict structured by server and
# control plane.
for pt in pt_data:
for key in pt.get('global', {}):
if key in pass_through['global']:
msg = ("Key %s is defined more than once in global "
"pass-through data" % (key, ))
self.add_error(msg)
pass_through['global'][key] = pt['global'][key]
for server in pt.get('servers', []):
if server['id'] not in pass_through['servers']:
pass_through['servers'][server['id']] = {}
server_data = pass_through['servers'][server['id']]
for key in server.get('data'):
if key in server_data:
msg = (
"Key %s is defined more than once for server %s "
"in pass-through data" % (key, server['id']))
self.add_error(msg)
server_data[key] = server['data'][key]
# Add proxy relationships
for component_name, component in components.iteritems():
for container_data in component.get('has-container', []):
container_name = container_data['service-name']
if container_name in components_by_mnemonic:
container_name = components_by_mnemonic[container_name][
'name']
if 'contains' not in components[container_name]:
components[container_name]['contains'] = {}
components[container_name]['contains'][component_name] = {
'name': component['mnemonic'].replace('-', '_'),
'data': container_data
}
component['container-name'] = container_name
# Can't do any more if we have errors in the network groups
if self._errors:
return
# Create a default interface-model for any roles that don't define one
default_net_iface = [{
'name': 'default_iface',
'network-groups': [x for x in network_groups],
'ports': ['ethX']
}]
# Create a deafult server group to hold any networks and servers not
# specificially assigned
default_server_group = {}
# Fix up relationships between server groups
for group_name, group in server_groups.iteritems():
for child in ServerGroup.server_groups(group):
ServerGroup.add_group(group, server_groups[child])
# Map networks to net_groups in server groups
networks_in_a_group = set()
for group_name, group in server_groups.iteritems():
for net in ServerGroup.networks(group):
ServerGroup.add_network(group, net,
networks[net]['network-group'])
networks_in_a_group.add(net)
# Add any unassinged networks to the default server group
for net_name, net in networks.iteritems():
if net_name not in networks_in_a_group and not net.get(
'neutron_network', False):
if server_groups:
self.add_warning("Network %s is not listed in a server "
"group." % (net_name))
ServerGroup.add_network(default_server_group, net_name,
net['network-group'])
# Establish the min and max size of each cluster and resource group
for cp_name, cp in control_planes.iteritems():
for cluster in cp.get('clusters', []):
if 'member-count' in cluster:
cluster['min-count'] = cluster['member-count']
cluster['max-count'] = cluster['member-count']
else:
cluster['min-count'] = cluster.get('min-count', 1)
for rgroup in cp.get('resources', []):
if 'member-count' in rgroup:
rgroup['min-count'] = rgroup['member-count']
rgroup['max-count'] = rgroup['member-count']
else:
rgroup['min-count'] = rgroup.get('min-count', 0)
# Create a list of servers with the network details for each resolved
servers = []
for s in bm_servers:
server_role = server_roles[s['role']]
# resolve the networking
# Find the interface model, and take a copy of the interfaces, as we may only use part of the model
# If there is no interface-model in the server role then all networks map to the existing NIC
fcoe_interfaces = []
if 'interface-model' in server_role:
iface_model = iface_models[server_role['interface-model']]
server_interfaces = deepcopy(iface_model['network-interfaces'])
if 'fcoe-interfaces' in iface_model:
fcoe_interfaces = iface_model['fcoe-interfaces']
else:
server_interfaces = deepcopy(default_net_iface)
# Find the disk model, and take a copy
if 'disk-model' in server_role:
disk_model = deepcopy(disk_models[server_role['disk-model']])
else:
disk_model = {'drives': {}}
# Translate network groups to the specific networks for this server
# Note: At this stage we have all possible networks groups defined
# by the interface model. We will reduce that to just those
# needed once we have assinged the server to a particular role
for iface in server_interfaces:
iface['networks'] = {}
iface_net_groups = (
iface.get('network-groups', []) +
iface.get('forced-network-groups', []) +
iface.get('passthrough-network-groups', []))
for net_group in iface_net_groups:
# Find network in the group for this server
if 'server-group' in s:
server_group = server_groups[s['server-group']]
else:
server_group = None
net_name = ServerGroup.find_network(
server_group, net_group, default_server_group)
if net_name:
network = networks[net_name]
iface['networks'][network['name']] = deepcopy(network)
if net_group in iface.get('forced-network-groups', []):
iface['networks'][network['name']]['forced'] = True
else:
iface['networks'][
network['name']]['forced'] = False
# Marked the network as passthrough if its network group is
# in the list of passthought network groups.
if net_group not in iface.get(
'passthrough-network-groups', []):
iface['networks'][
network['name']]['passthrough'] = False
iface['networks'][
network['name']]['passthrough-only'] = False
else:
iface['networks'][
network['name']]['passthrough'] = True
# Mark the network as passthrough-only if its network group is
# not listed in network-groups or forced-network-groups
if net_group not in (
iface.get('network-groups', []) +
iface.get('forced-network-groups', [])):
iface['networks'][
network['name']]['passthrough-only'] = True
else:
iface['networks'][network['name']][
'passthrough-only'] = False
server = {
'id': s['id'],
'hostname': s.get('hostname', None),
'hypervisor-id': s.get('hypervisor-id', None),
'role': s['role'],
'server_group': s.get('server-group'),
'rack': s.get('rack'),
'addr': s['ip-addr'],
'if-model': server_role.get('interface-model', 'default'),
'disk-model': disk_model,
'interfaces': server_interfaces,
'fcoe_interfaces': fcoe_interfaces,
'nic_map': nic_mappings.get(s.get('nic-mapping', 'none')),
'ansible_options': s.get('ansible-options'),
'state': None,
'vm-factory': s.get('vm-factory', False),
'port-groups': s.get('port-groups', []),
'previous_config': {}
}
servers.append(server)
# Add servers to ServerGroups
if 'server-group' in s:
sgrp = server_groups[s['server-group']]
ServerGroup.add_server(sgrp, server)
server['server-group-list'] = ServerGroup.get_group_list(sgrp)
else:
# If there are server groups defined it would be odd to have
# a server which isn't a member of a group
if server_groups:
self.add_warning("Server %s is not a member of a server "
"group." % (s['ip-addr']))
ServerGroup.add_server(default_server_group, server)
server['server-group-list'] = []
# Roles for a cluster/resource can be either a string or a list, so change to
# always be a list
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
if isinstance(cluster['server-role'], basestring):
cluster['server-role'] = [cluster['server-role']]
for r in cp.get('resources', []):
if isinstance(r['server-role'], basestring):
r['server-role'] = [r['server-role']]
# Add a list of all services to each Control Plane
for cp_name, cp in control_planes.iteritems():
cp['services'] = {}
for cluster in cp['clusters']:
cluster['services'] = {}
for comp_name in cluster['service-components']:
service_name = components[comp_name]['service']
if service_name not in cluster['services']:
cluster['services'][service_name] = []
cluster['services'][service_name].append(comp_name)
cp['services'][service_name] = {}
for r in cp.get('resources', []):
r['services'] = {}
for comp_name in r['service-components']:
service_name = components[comp_name]['service']
if service_name not in r['services']:
r['services'][service_name] = []
r['services'][service_name].append(comp_name)
cp['services'][service_name] = {}
ntp_servers = cloud_data.get('ntp-servers', [])
dns_settings = cloud_data.get('dns-settings', {})
smtp_settings = cloud_data.get('smtp-settings', {})
firewall_settings = cloud_data.get('firewall-settings', {})
CloudModel.put(cloud_internal, 'control-planes', control_planes)
CloudModel.put(cloud_internal, 'networks', networks)
CloudModel.put(cloud_internal, 'servers', servers)
CloudModel.put(cloud_internal, 'server-groups', server_groups)
CloudModel.put(cloud_internal, 'default-server-group',
default_server_group)
CloudModel.put(cloud_internal, 'services', services)
CloudModel.put(cloud_internal, 'components', components)
CloudModel.put(cloud_internal, 'components_by_mnemonic',
components_by_mnemonic)
CloudModel.put(cloud_internal, 'ntp_servers', ntp_servers)
CloudModel.put(cloud_internal, 'dns_settings', dns_settings)
CloudModel.put(cloud_internal, 'smtp_settings', smtp_settings)
CloudModel.put(cloud_internal, 'firewall_settings', firewall_settings)
CloudModel.put(cloud_internal, 'pass_through', pass_through)
CloudModel.put(cloud_internal, 'iface_models', iface_models)
CloudModel.put(cloud_internal, 'disk_models', disk_models)
CloudModel.put(cloud_internal, 'nic_mappings', nic_mappings)
CloudModel.put(cloud_internal, 'server_roles', server_roles)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
components = CloudModel.get(cloud_internal, 'components', [])
# If we have an error in an earlier generator we may not have
# components in the internal model
if not components:
return
components_by_mnemonic = CloudModel.get(cloud_internal, 'components_by_mnemonic')
self.services = CloudModel.get(cloud_internal, 'services', [])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Resolve the service level consumes for all control planes before we
# do the component level consumes - as a component may need to find
# the service relationship in a parent control plane
#
for cp_name, cp in control_planes.iteritems():
for service_name, service_data in cp.get('services', []).iteritems():
if service_name not in self.services:
continue
service_data['consumes'] = self._get_consumes(self.services[service_name],
components,
components_by_mnemonic,
cp,
control_planes)
# Resolve the component level consumes relationships. If there is a
# service level consumes relationship then any values from that override
# any component level vaules (Normally if there is a service level relationship
# then only that has relationship vars)
#
for cp_name, cp in control_planes.iteritems():
for comp_name, comp_data in cp.get('components', []).iteritems():
consumer = components[comp_name]
comp_data['consumes'] = self._get_consumes(components[comp_name],
components,
components_by_mnemonic,
cp,
control_planes)
# Add any values from a service level relationship
consumer_service = consumer.get('service', 'foundation')
for consumed_component in comp_data['consumes']:
service_consumes = self._get_service_consumes(consumer_service,
consumed_component, cp)
if service_consumes:
self._update_no_overwrite(comp_data['consumes'][consumed_component],
deepcopy(service_consumes))
# Process the config set for this component
if 'config-sets' not in cp:
cp['config-sets'] = {}
context = {'cp': cp['name'],
'consuming-cp': cp['name'],
'component': comp_name,
'clusters': self._get_clusters_for_component(cp, comp_name)}
cp['config-sets'][comp_name] = self._process_config_set(components[comp_name],
context)
self._validate_multi_consumers(control_planes, components)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
self._valid = True
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'], self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
components = CloudModel.get(cloud_internal, 'components', {})
interface_models = {}
for model in CloudModel.get(cloud_version, 'interface-models', []):
interface_models[model['name']] = model
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles', []):
server_roles[role['name']] = role
for s in servers:
if s['state'] != 'allocated':
continue
server_role = server_roles[s['role']]
interface_model = interface_models[server_role['interface-model']]
# Create a structure indexed by device name
iface_devices = {}
for iface, iface_data in s['interfaces'].iteritems():
iface_devices[iface_data['device']['name']] = iface_data
# Find out which devices we should configure based on the
# components on this server
server_dpdk_data = {}
dpdk_devices = {}
for dpdk_data in interface_model.get('dpdk-devices', []):
# Build a list of the devices so we can check we match
# each only once based on the components
for device in dpdk_data['devices']:
if device['name'] not in dpdk_devices:
dpdk_devices[device['name']] = {'possible': [],
'matched': []}
if 'driver' in device:
dpdk_devices[device['name']]['driver'] = device['driver']
for component in dpdk_data['components']:
for device in dpdk_data['devices']:
dpdk_devices[device['name']]['possible'].append(component)
if component in s['components']:
mnemonic = components[component]['mnemonic'].replace('-', '_')
# Anything other than devices is a server level attribute
if mnemonic not in server_dpdk_data:
server_dpdk_data[mnemonic] = {'devices': []}
for k in dpdk_data:
if k in ['components', 'devices']:
continue
server_dpdk_data[mnemonic][str(k).replace('-', '_')] = dpdk_data[k]
for device in dpdk_data['devices']:
dpdk_devices[device['name']]['matched'].append(component)
# Check we have one component for each device
for device_name, dpdk_data in dpdk_devices.iteritems():
# Multiple dpdk devices can be grouped into a collection and there
# are several dpdk applications that can use a dpdk collection.
# The only dpdk collection currently understood is an Open vSwitch bond
# Check to see if this device is part of a dpdk bond
dpdk_collection = False
for idev, idev_data in iface_devices.iteritems():
if 'bond-data' in idev_data:
for dpdk_dev in idev_data['bond-data']['devices']:
if dpdk_dev['name'] == device_name:
dpdk_collection = True
dpdk_collection_name = idev
if device_name not in iface_devices and not dpdk_collection:
if len(dpdk_data['matched']) > 0:
msg = ("DPDK data for device '%s' has not been applied to "
"server '%s' (id: %s role: %s) because the device is "
"not used on the server although at least one of the "
"associated components is on the server: %s" %
(device_name, s['name'], s['id'], s['role'],
str(dpdk_data['matched']).strip('[]')))
self.add_warning(msg)
else:
if len(dpdk_data['matched']) > 1:
msg = ("DPDK data for device '%s' on server '%s' (id: %s role: %s) "
"matches more than one component: %s." %
(device_name, s['name'], s['id'], s['role'],
str(dpdk_data['matched']).strip('[]')))
self.add_error(msg)
self._valid = False
elif len(dpdk_data['matched']) == 0:
msg = ("DPDK data for device '%s' has not been applied to "
"server '%s' (id: %s role: %s) because none of the "
"components are on the server: %s." %
(device_name, s['name'], s['id'], s['role'],
str(dpdk_data['possible']).strip('[]')))
self.add_warning(msg)
else:
check_name = dpdk_collection_name if dpdk_collection else device_name
# Check we're not trying to configure DPDK on the server's config
# interface
for net_name, net_data in iface_devices[check_name]['networks'].iteritems():
if 'addr' in net_data and net_data['addr'] == s['addr']:
msg = ("Device '%s' on server '%s' (id: %s role: %s) "
"cannot be used for DPDK as it is also the "
"interface used by the lifecycle manager." %
(check_name, s['name'], s['id'], s['role']))
self.add_error(msg)
self._valid = False
break
else:
mnemonic = components[dpdk_data['matched'][0]]['mnemonic'].replace('-', '_')
iface_dpdk_data = {'component': mnemonic, 'composite': False}
if not dpdk_collection:
if 'driver' in dpdk_data:
iface_dpdk_data['driver'] = dpdk_data['driver']
else:
# get the current data as it may already have devices
iface_dpdk_data = iface_devices[check_name].get('dpdk-data', iface_dpdk_data)
iface_dpdk_data['composite'] = True
if 'devices' not in iface_dpdk_data:
iface_dpdk_data['devices'] = []
dev_data = {'name': device_name}
if 'driver' in dpdk_data:
dev_data['driver'] = dpdk_data['driver']
iface_dpdk_data['devices'].append(dev_data)
iface_devices[check_name]['dpdk-data'] = iface_dpdk_data
s['dpdk-data'] = server_dpdk_data