def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_data = self._models['CloudDescription']['cloud']
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
audit_settings = {
'audit-dir': '/var/audit',
'default': 'disabled',
'enabled-services': [],
'disabled-services': []
}
audit_settings.update(cloud_data.get('audit-settings', {}))
for cp_name, cp in control_planes.iteritems():
cp_services = cp.get('services', {})
for service_name, service_data in cp_services.iteritems():
service_data['audit-settings'] = {
'dir': audit_settings['audit-dir'],
'enabled': audit_settings['default'] == 'enabled'
}
if service_name in audit_settings['enabled-services']:
service_data['audit-settings']['enabled'] = True
elif service_name in audit_settings['disabled-services']:
service_data['audit-settings']['enabled'] = False
def __init__(self, instructions, models, controllers):
super(HTMLDiagramBuilder,
self).__init__(2.0, instructions, models, controllers,
'html-diagram-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = ArdanaPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'html')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
self.cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
self.control_planes = CloudModel.get(self._cloud_internal,
'control-planes')
self.iface_models = CloudModel.get(self._cloud_version,
'interface-models')
self.server_roles = CloudModel.get(self._cloud_version, 'server-roles')
self.disk_models = CloudModel.get(self._cloud_version, 'disk-models',
[])
self.memory_models = CloudModel.get(self._cloud_version,
'memory-models', [])
self.cpu_models = CloudModel.get(self._cloud_version, 'cpu-models', [])
self.components = CloudModel.get(self._cloud_internal, 'components')
self.services = CloudModel.get(self._cloud_internal, 'services')
self.servers = CloudModel.get(self._cloud_internal, 'servers')
self.server_groups = CloudModel.get(self._cloud_internal,
'server-groups')
self.network_groups = CloudModel.get(self._cloud_internal,
'network-groups')
self.region_topology = CloudModel.get(self._cloud_internal,
'region-topology')
self.service_topology = CloudModel.get(self._cloud_internal,
'service-topology')
self.cp_topology = CloudModel.get(self._cloud_internal, 'cp-topology')
self.network_topology = CloudModel.get(self._cloud_internal,
'network-topology')
self.components_by_mnemonic = {}
for comp_name, comp_data in self.components.iteritems():
if 'alias-for' not in comp_data:
self.components_by_mnemonic[comp_data['mnemonic']] = comp_name
ArdanaPaths.make_path(self._file_path)
for subdir in ['Servers']:
ArdanaPaths.make_path(self._file_path + "/%s" % subdir)
# Create a mapping from hostname to server id
self.server_by_hostname = {}
self.server_by_name = {}
for s in self.servers:
if 'hostname' in s:
self.server_by_hostname[s['hostname']] = s
if 'name' in s:
self.server_by_name[s['name']] = s
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
services = CloudModel.get(cloud_internal, 'services', [])
config_data = {}
for data in CloudModel.get(cloud_version, 'configuration-data', []):
config_data[data['name']] = data
control_planes = CloudModel.get(cloud_internal, 'control-planes')
for cp_name, cp in control_planes.iteritems():
# Get any config data at the CP level
cp_config_data = self._expand_config(
cp.get('configuration-data', []), config_data, services)
# Extract all of the network tags defined in
# configuration data for this control plane
cp['config-data-network-tags'] = self._get_network_tags(
cp['name'], cp.get('configuration-data', []), config_data)
# Extract the list of physnets that have been defined
# in neutron configuration data for this control plane
# so they can be validated in the NetworkTagGenerator
cp['neutron-provider-nets'] = self._get_provider_nets(
cp['name'], cp.get('configuration-data', []), config_data)
for cluster in cp['clusters']:
cluster_config_data = self._expand_config(
cluster.get('configuration-data', []), config_data,
services)
context = "%s:%s" % (cp_name, cluster['name'])
cluster['config-data'] = self._merge(cp_config_data,
cluster_config_data,
context)
self._check_no_network_tags(
context, cluster.get('configuration-data', []),
config_data)
for r in cp.get('resources', []):
r_config_data = self._expand_config(
r.get('configuration-data', []), config_data, services)
context = "%s:%s" % (cp_name, r['name'])
r['config-data'] = self._merge(cp_config_data, r_config_data,
context)
self._check_no_network_tags(context,
r.get('configuration-data', []),
config_data)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_internal = CloudModel.internal(self._models['CloudModel'])
# Control Planes
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Network Groups
network_groups = CloudModel.get(cloud_internal, 'network-groups')
# components
components = CloudModel.get(cloud_internal, 'components')
# services
services = CloudModel.get(cloud_internal, 'services')
#
# Find which tags have been deprecated or replaced
#
replaced_tags = {}
deprecated_tags = {}
for component_name, component in components.iteritems():
for comp_tag in component.get('network-tags', []):
for alias in comp_tag.get('aliases', []):
replaced_tags[alias] = comp_tag['name']
if 'deprecated' in comp_tag:
deprecated_tags[comp_tag['name']] = comp_tag['deprecated']
# In previous versions of Ardana OpenStack, any tags defined on a network group are
# treated as applying to all control planes
netgroup_tags = self._get_netgroup_tags(network_groups)
# Process the tags for each cluster / resource group
tag_found = {}
for cp_name, cp in control_planes.iteritems():
cp_components = set()
for cluster in cp['clusters']:
for comp_name in cluster['service-components']:
cp_components.add(comp_name)
for r in cp.get('resources', []):
for comp_name in r['service-components']:
cp_components.add(comp_name)
cp['network-tags'] = self._process_network_tags(
cp['config-data-network-tags'], netgroup_tags, cp_components,
components, services, replaced_tags, deprecated_tags,
tag_found)
self._validate_network_tags(cp_name, cp['network-tags'],
cp['neutron-provider-nets'])
for tag_name, found in tag_found.iteritems():
if not found:
msg = ("Tag '%s' is not defined by any service" % tag_name)
self.add_warning(msg)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles'):
server_roles[role['name']] = role
memory_models = {}
for memory_model in CloudModel.get(cloud_version, 'memory-models', []):
memory_models[memory_model['name']] = memory_model
for server in servers:
if server['state'] != 'allocated':
continue
server['memory-model'] = {}
server_role = server_roles[server['role']]
if 'memory-model' not in server_role:
continue
memory_model = memory_models[server_role['memory-model']]
my_memory_model = {'name': memory_model['name']}
non_numa_pages = []
numa_pages = []
for page in memory_model.get('huge-pages', {}):
page_info = {
'size': page['size'],
'count': page['count'],
'size_in_k': self._page_size(page['size'])
}
if 'numa-node' in page:
page_info['numa_node'] = page['numa-node']
numa_pages.append(page_info)
else:
non_numa_pages.append(page_info)
if memory_model.get('vm-size', None):
my_memory_model['vm_ram_size_in_k'] = self._memory_size(
memory_model['vm-size']['ram'])
my_memory_model['numa_huge_pages'] = numa_pages
my_memory_model['non_numa_huge_pages'] = non_numa_pages
if 'default-huge-page-size' in memory_model:
my_memory_model['default_huge_page_size'] = memory_model[
'default-huge-page-size']
server['memory-model'] = my_memory_model
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
components = CloudModel.get(cloud_internal, 'components', [])
# If we have an error in an earlier generator we may not have
# components in the internal model
if not components:
return
components_by_mnemonic = CloudModel.get(cloud_internal, 'components_by_mnemonic')
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Resolve the service level consumes for all control planes before we
# do the component level consumes - as a component may need to find
# the service relationship in a parent control plane
#
for cp_name, cp in control_planes.iteritems():
for component_name, endpoints in cp.get('advertises', {}).iteritems():
component = components[component_name]
for adv_to in component.get('advertises-to-services', []):
adv_to_name = adv_to['service-name']
if adv_to_name in components:
adv_to_mnemonic = components[adv_to_name]['mnemonic']
else:
# We have some cases where a service advertises to a dummy
# component
if adv_to_name not in components_by_mnemonic:
continue
adv_to_mnemonic = adv_to_name
adv_to_name = components_by_mnemonic[adv_to_mnemonic]['name']
for entry in adv_to.get('entries', []):
adv = {'service_name': entry['service-name'],
'service_type': entry['service-type'],
'service_description': entry.get('service-description'),
'regions': [],
'component_name': component_name,
'from_cp_name': cp_name}
for role in ['public', 'internal', 'admin']:
if role in endpoints:
adv[role + '_url'] = endpoints[role]['url'] + entry.get('url-suffix', '')
for region_name in sorted(cp.get('regions', {})):
region_services = cp['regions'][region_name]
service = component.get('service', 'foundation')
if service in region_services:
adv['regions'].append(region_name)
self._advertised(adv_to_name, cp, adv, control_planes)
def _generate_ring_specifications_info(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(
self._models['CloudModel'], self._version)
ring_specifications_config = CloudModel.get(
cloud_version, 'ring-specifications', [])
cloud_internal = CloudModel.internal(self._models['CloudModel'])
CloudModel.put(cloud_internal,
'ring-specifications',
ring_specifications_config)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
# If we have an error in an earlier generator we may not have
# control_planes in the internal model
if not control_planes:
return
routes = {}
for cp_name, cp in control_planes.iteritems():
load_balancers = cp.get('load-balancer-config', {})
for cluster in cp['clusters']:
for server in cluster.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
for r_name, r in cp.get('resources', {}).iteritems():
for server in r.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
#
default_routes = {}
# Warn about any routes using the "default" route
for src_net, net_routes in routes.iteritems():
for dest_net, route_data in net_routes.iteritems():
if route_data['default']:
hosts = set()
if src_net not in default_routes:
default_routes[src_net] = []
for src, src_data in route_data['used_by'].iteritems():
for dest, host_list in src_data.iteritems():
for host in host_list:
hosts.add(host)
default_routes[src_net].append({'net': dest_net, 'hosts': hosts})
if default_routes:
msg = ("Default routing used between networks\n"
"The following networks are using a 'default' route rule. To remove this warning\n"
"either add an explicit route in the source network group or force the network to\n"
"attach in the interface model used by the servers.\n")
for src in sorted(default_routes):
dest_list = default_routes[src]
for dest_data in dest_list:
msg += " %s to %s\n" % (src, dest_data['net'])
for host in sorted(dest_data['hosts']):
msg += " %s\n" % (host)
self.add_warning(msg)
CloudModel.put(cloud_internal, 'routes', routes)
def __init__(self, instructions, models, controllers):
super(AnsibleAllVarsBuilder, self).__init__(
2.0, instructions, models, controllers,
'ansible-all-vars-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = ArdanaPaths.get_output_path(self._instructions, self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'ansible')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model, self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
ArdanaPaths.make_path(self._file_path)
def __init__(self, instructions, models, controllers):
super(RouteInfoBuilder, self).__init__(2.0, instructions, models,
controllers, 'route-info-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = ArdanaPaths.get_output_path(self._instructions,
self.cloud_desc)
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
ArdanaPaths.make_path(self._file_path)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
self.services = CloudModel.get(cloud_internal, 'services', [])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
components = CloudModel.get(cloud_internal, 'components')
# Resolve the service data relationships.
for cp_name, cp in control_planes.iteritems():
for comp_name, comp_data in cp.get('components', []).iteritems():
provider = components[comp_name]
provider_service = self.services.get(provider['service'], {})
self._set_provided_data(provider, provider_service,
comp_data, cp, control_planes)
def __init__(self, instructions, models, controllers, config_files):
super(AddressAllocationFinalizer,
self).__init__(2.0, instructions, models, controllers,
config_files, 'address-allocation-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = ArdanaPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'info')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
ArdanaPaths.make_path(self._file_path)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
networks = {}
network_groups = {}
for group in CloudModel.get(cloud_version, 'network-groups'):
network_groups[group['name']] = group
network_groups[group['name']]['networks'] = []
for net in CloudModel.get(cloud_version, 'networks'):
networks[net['name']] = net
# add networks into their respective network groups
for net_name, net in networks.iteritems():
if net.get('network-group', None):
network_groups[net['network-group']]['networks'].append(net)
# Inject a network group for each provider network, so we can support
# routes to them
for config_data in CloudModel.get(cloud_version, 'configuration-data',
[]):
if 'neutron' in config_data.get('services', []):
for net in config_data.get('data',
{}).get('neutron_provider_networks',
[]):
provider_net = {
'name': net['name'],
'cidr': net['cidr'],
'network-group': net['name'],
'neutron_network': True
}
group = {'name': net['name'], 'networks': [provider_net]}
networks[net['name']] = provider_net
network_groups[net['name']] = group
self._generate_default_network_mtu(network_groups)
CloudModel.put(cloud_internal, 'networks', networks)
CloudModel.put(cloud_internal, 'network-groups', network_groups)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', [])
for cp_name, cp in control_planes.iteritems():
deleted_components = []
for cluster in cp['clusters']:
for server in cluster['servers']:
self._add_deleted_components(deleted_components, server)
for r_name, resources in cp.get('resources', {}).iteritems():
for server in resources.get('servers', []):
self._add_deleted_components(deleted_components, server)
cp['deleted-components'] = deleted_components
CloudModel.put(cloud_internal, 'control-planes', control_planes)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
networks = CloudModel.get(cloud_internal, 'networks')
network_groups = CloudModel.get(cloud_internal, 'network-groups')
components = CloudModel.get(cloud_internal, 'components')
cp_topology = self._generate_cp_topology(control_planes, components)
CloudModel.put(cloud_internal, 'cp-topology', cp_topology)
CloudModel.put(cloud_internal, 'region-topology',
self._generate_region_topology(control_planes))
CloudModel.put(cloud_internal, 'service-topology',
self._generate_service_topology(cp_topology))
CloudModel.put(
cloud_internal, 'network-topology',
self._generate_network_topology(network_groups, networks,
control_planes))
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
self._valid = True
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'], self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
components = CloudModel.get(cloud_internal, 'components', {})
interface_models = {}
for model in CloudModel.get(cloud_version, 'interface-models', []):
interface_models[model['name']] = model
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles', []):
server_roles[role['name']] = role
for s in servers:
if s['state'] != 'allocated':
continue
server_role = server_roles[s['role']]
interface_model = interface_models[server_role['interface-model']]
# Create a structure indexed by device name
iface_devices = {}
for iface, iface_data in s['interfaces'].iteritems():
iface_devices[iface_data['device']['name']] = iface_data
# Find out which devices we should configure based on the
# components on this server
server_dpdk_data = {}
dpdk_devices = {}
for dpdk_data in interface_model.get('dpdk-devices', []):
# Build a list of the devices so we can check we match
# each only once based on the components
for device in dpdk_data['devices']:
if device['name'] not in dpdk_devices:
dpdk_devices[device['name']] = {'possible': [],
'matched': []}
if 'driver' in device:
dpdk_devices[device['name']]['driver'] = device['driver']
for component in dpdk_data['components']:
for device in dpdk_data['devices']:
dpdk_devices[device['name']]['possible'].append(component)
if component in s['components']:
mnemonic = components[component]['mnemonic'].replace('-', '_')
# Anything other than devices is a server level attribute
if mnemonic not in server_dpdk_data:
server_dpdk_data[mnemonic] = {'devices': []}
for k in dpdk_data:
if k in ['components', 'devices']:
continue
server_dpdk_data[mnemonic][str(k).replace('-', '_')] = dpdk_data[k]
for device in dpdk_data['devices']:
dpdk_devices[device['name']]['matched'].append(component)
# Check we have one component for each device
for device_name, dpdk_data in dpdk_devices.iteritems():
# Multiple dpdk devices can be grouped into a collection and there
# are several dpdk applications that can use a dpdk collection.
# The only dpdk collection currently understood is an Open vSwitch bond
# Check to see if this device is part of a dpdk bond
dpdk_collection = False
for idev, idev_data in iface_devices.iteritems():
if 'bond-data' in idev_data:
for dpdk_dev in idev_data['bond-data']['devices']:
if dpdk_dev['name'] == device_name:
dpdk_collection = True
dpdk_collection_name = idev
if device_name not in iface_devices and not dpdk_collection:
if len(dpdk_data['matched']) > 0:
msg = ("DPDK data for device '%s' has not been applied to "
"server '%s' (id: %s role: %s) because the device is "
"not used on the server although at least one of the "
"associated components is on the server: %s" %
(device_name, s['name'], s['id'], s['role'],
str(dpdk_data['matched']).strip('[]')))
self.add_warning(msg)
else:
if len(dpdk_data['matched']) > 1:
msg = ("DPDK data for device '%s' on server '%s' (id: %s role: %s) "
"matches more than one component: %s." %
(device_name, s['name'], s['id'], s['role'],
str(dpdk_data['matched']).strip('[]')))
self.add_error(msg)
self._valid = False
elif len(dpdk_data['matched']) == 0:
msg = ("DPDK data for device '%s' has not been applied to "
"server '%s' (id: %s role: %s) because none of the "
"components are on the server: %s." %
(device_name, s['name'], s['id'], s['role'],
str(dpdk_data['possible']).strip('[]')))
self.add_warning(msg)
else:
check_name = dpdk_collection_name if dpdk_collection else device_name
# Check we're not trying to configure DPDK on the server's config
# interface
for net_name, net_data in iface_devices[check_name]['networks'].iteritems():
if 'addr' in net_data and net_data['addr'] == s['addr']:
msg = ("Device '%s' on server '%s' (id: %s role: %s) "
"cannot be used for DPDK as it is also the "
"interface used by the lifecycle manager." %
(check_name, s['name'], s['id'], s['role']))
self.add_error(msg)
self._valid = False
break
else:
mnemonic = components[dpdk_data['matched'][0]]['mnemonic'].replace('-', '_')
iface_dpdk_data = {'component': mnemonic, 'composite': False}
if not dpdk_collection:
if 'driver' in dpdk_data:
iface_dpdk_data['driver'] = dpdk_data['driver']
else:
# get the current data as it may already have devices
iface_dpdk_data = iface_devices[check_name].get('dpdk-data', iface_dpdk_data)
iface_dpdk_data['composite'] = True
if 'devices' not in iface_dpdk_data:
iface_dpdk_data['devices'] = []
dev_data = {'name': device_name}
if 'driver' in dpdk_data:
dev_data['driver'] = dpdk_data['driver']
iface_dpdk_data['devices'].append(dev_data)
iface_devices[check_name]['dpdk-data'] = iface_dpdk_data
s['dpdk-data'] = server_dpdk_data
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
components = CloudModel.get(cloud_internal, 'components', [])
# If we have an error in an earlier generator we may not have
# components in the internal model
if not components:
return
components_by_mnemonic = CloudModel.get(cloud_internal, 'components_by_mnemonic')
self.services = CloudModel.get(cloud_internal, 'services', [])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Resolve the service level consumes for all control planes before we
# do the component level consumes - as a component may need to find
# the service relationship in a parent control plane
#
for cp_name, cp in control_planes.iteritems():
for service_name, service_data in cp.get('services', []).iteritems():
if service_name not in self.services:
continue
service_data['consumes'] = self._get_consumes(self.services[service_name],
components,
components_by_mnemonic,
cp,
control_planes)
# Resolve the component level consumes relationships. If there is a
# service level consumes relationship then any values from that override
# any component level vaules (Normally if there is a service level relationship
# then only that has relationship vars)
#
for cp_name, cp in control_planes.iteritems():
for comp_name, comp_data in cp.get('components', []).iteritems():
consumer = components[comp_name]
comp_data['consumes'] = self._get_consumes(components[comp_name],
components,
components_by_mnemonic,
cp,
control_planes)
# Add any values from a service level relationship
consumer_service = consumer.get('service', 'foundation')
for consumed_component in comp_data['consumes']:
service_consumes = self._get_service_consumes(consumer_service,
consumed_component, cp)
if service_consumes:
self._update_no_overwrite(comp_data['consumes'][consumed_component],
deepcopy(service_consumes))
# Process the config set for this component
if 'config-sets' not in cp:
cp['config-sets'] = {}
context = {'cp': cp['name'],
'consuming-cp': cp['name'],
'component': comp_name,
'clusters': self._get_clusters_for_component(cp, comp_name)}
cp['config-sets'][comp_name] = self._process_config_set(components[comp_name],
context)
self._validate_multi_consumers(control_planes, components)
def generate(self):
def _get_bitmask(cpus):
bitmask = 0
for x in cpus:
bitmask += (1 << x)
return bitmask
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
components = CloudModel.get(cloud_internal, 'components', {})
cpu_models = {}
for model in CloudModel.get(cloud_version, 'cpu-models', []):
cpu_models[model['name']] = model
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles', []):
server_roles[role['name']] = role
for s in servers:
if s['state'] != 'allocated':
continue
server_role = server_roles[s['role']]
s['cpu-assignments'] = {}
if 'cpu-model' in server_role:
server_cpus = []
cpu_model = cpu_models[server_role['cpu-model']]
if cpu_model.get('vm-size', {}):
s['vm_no_of_vcpus'] = cpu_model.get('vm-size')['vcpus']
for service_data in cpu_model.get('assignments', []):
comp_on_server = False
for comp_name in service_data['components']:
if comp_name in s['components']:
comp_on_server = True
break
if not comp_on_server:
msg = (
"CPU Assignment for components %s in '%s' "
"has not been applied to server '%s' (id: %s role: %s) "
"because it does not host any of the listed components."
%
(str(service_data['components']).strip('[]'),
cpu_model['name'], s['name'], s['id'], s['role']))
self.add_warning(msg)
continue
assignment_data = {}
for cpu_data in service_data.get('cpu', []):
cpu_list = cpu_data['processor-ids']
if cpu_data.get('isolate', True):
server_cpus.extend(cpu_list)
assignment_data[cpu_data['role']] = \
{'processor_ids': cpu_list,
'bitmask': "0x%04x" % _get_bitmask(cpu_list),
'processor_list': str(sorted(cpu_list)).strip('[]').replace(' ', '')}
for comp_name in service_data['components']:
mnemonic = components[comp_name]['mnemonic'].replace(
'-', '_')
s['cpu-assignments'][mnemonic] = deepcopy(
assignment_data)
# Add Server level data
s['cpu-assignments']['server'] = {}
if server_cpus:
s['cpu-assignments']['server'] = \
{'processor_ids': server_cpus,
'bitmask': "0x%04x" % _get_bitmask(server_cpus),
'processor_list': str(sorted(server_cpus)).strip('[]').replace(' ', '')}
def _generate_firewall_rules(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
firewall_rules = CloudModel.get(cloud_version, 'firewall-rules', [])
network_groups = CloudModel.get(cloud_version, 'network-groups')
components = CloudModel.get(cloud_internal, 'components', [])
# If component is not set then it means an earlier generator failed
if not components:
return
# Initialise the firewall structure
net_group_firewall = {}
for net_group in network_groups:
net_group_firewall[net_group['name']] = {
'user': [],
'component': {}
}
#
# Add ports from components.
#
explicit_components = set()
for net_group in network_groups:
for comp in self._get_netgroup_components(net_group,
exclude_default=True):
explicit_components.add(comp)
for net_group in network_groups:
net_group_components = self._get_netgroup_components(net_group)
for comp_name, comp_data in components.iteritems():
if (comp_name in net_group_components
or ('default' in net_group_components
and comp_name not in explicit_components)):
comp_rules = []
for endpoint in comp_data.get('endpoints', []):
if (comp_name in net_group.get(
'tls-component-endpoints', [])
or 'default' in net_group.get(
'tls-component-endpoints', [])):
port_str = endpoint.get('tls-port',
endpoint['port'])
else:
port_str = endpoint['port']
if ':' in str(port_str):
ports = str.split(port_str, ':')
else:
ports = [port_str, port_str]
rule = {
'type': 'allow',
'remote-ip-prefix': '0.0.0.0/0',
'port-range-min': ports[0],
'port-range-max': ports[1],
'protocol': endpoint.get('protocol', 'tcp')
}
comp_rules.append(rule)
if comp_rules:
net_group_firewall[net_group['name']]['component'][
comp_name] = comp_rules
# Annotate the rules so we can trace where they came from
for rule_group in firewall_rules:
for rule in rule_group.get('rules', []):
rule['component'] = "user-%s" % rule_group['name']
#
# Add user defined rules to network groups
#
for rule in firewall_rules:
if rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(
rule['rules'])
#
# Add user defined final rules to network groups
#
for rule in firewall_rules:
if not rule.get('final'):
continue
for firewall_netgroup in rule['network-groups']:
for net_group in network_groups:
if (firewall_netgroup == "all"
or firewall_netgroup == net_group['name']):
net_group_firewall[net_group['name']]['user'].extend(
rule['rules'])
CloudModel.put(cloud_internal, 'net-group-firewall',
net_group_firewall)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_internal = CloudModel.internal(self._models['CloudModel'])
servers = CloudModel.get(cloud_internal, 'servers', {})
self._add_vm_info_to_factory(servers)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
cloud_data = self._models['CloudDescription']['cloud']
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
services = {}
components = {}
components_by_mnemonic = {}
for service in CloudModel.get(cloud_version, 'services'):
services[service['name']] = service
for component in CloudModel.get(cloud_version, 'service-components'):
components[component['name']] = component
components_by_mnemonic[component['mnemonic']] = component
control_planes = {}
iface_models = {}
disk_models = {}
nic_mappings = {}
nic_device_types = {}
nic_device_families = {}
server_roles = {}
server_groups = {}
bm_servers = []
pass_through = {'global': {}, 'servers': {}}
# Servers
if 'servers' in cloud_version:
bm_servers.extend(cloud_version['servers'])
# Build a list of server addresses so we can
# reserve them
server_addresses = {}
for s in bm_servers:
server_addresses[s['ip-addr']] = s['id']
# Control Planes
for cp in CloudModel.get(cloud_version, 'control-planes'):
control_planes[cp['name']] = dict(cp)
# Network Groups
network_groups = CloudModel.get(cloud_internal, 'network-groups')
# Networks
networks = CloudModel.get(cloud_internal, 'networks')
# Interface Models
for iface in CloudModel.get(cloud_version, 'interface-models'):
iface_models[iface['name']] = iface
# Disk Models
for disk_model in CloudModel.get(cloud_version, 'disk-models'):
disk_models[disk_model['name']] = disk_model
# NIC Device Families
for nic_family in CloudModel.get(cloud_version, 'nic-device-families',
[]):
nic_device_families[nic_family['name']] = nic_family
# NIC Device Types
for nic_dev in CloudModel.get(cloud_version, 'nic-device-types', []):
nic_device_types[nic_dev['name']] = nic_dev
nic_device_types[nic_dev['name']]['family_data'] = \
nic_device_families[nic_dev['family']]
# NIC Mapping
for nic_map in CloudModel.get(cloud_version, 'nic-mappings', []):
for port in nic_map.get('physical-ports', []):
# Add in any data from the NIC device type
if 'nic-device-type' in port:
port['nic-device-type'] = nic_device_types[
port['nic-device-type']]
nic_mappings[nic_map['name']] = nic_map
# Server Roles
for role in CloudModel.get(cloud_version, 'server-roles'):
server_roles[role['name']] = role
# Server Groups
for group in CloudModel.get(cloud_version, 'server-groups', []):
server_groups[group['name']] = group
# Pass Through
pt_data = CloudModel.get(cloud_version, 'pass-through', [])
####################################
#
# End of reading input data
#
###################################
# Combine pass through data which maybe in multiple
# input files into a single dict structured by server and
# control plane.
for pt in pt_data:
for key in pt.get('global', {}):
if key in pass_through['global']:
msg = ("Key %s is defined more than once in global "
"pass-through data" % (key, ))
self.add_error(msg)
pass_through['global'][key] = pt['global'][key]
for server in pt.get('servers', []):
if server['id'] not in pass_through['servers']:
pass_through['servers'][server['id']] = {}
server_data = pass_through['servers'][server['id']]
for key in server.get('data'):
if key in server_data:
msg = (
"Key %s is defined more than once for server %s "
"in pass-through data" % (key, server['id']))
self.add_error(msg)
server_data[key] = server['data'][key]
# Add proxy relationships
for component_name, component in components.iteritems():
for container_data in component.get('has-container', []):
container_name = container_data['service-name']
if container_name in components_by_mnemonic:
container_name = components_by_mnemonic[container_name][
'name']
if 'contains' not in components[container_name]:
components[container_name]['contains'] = {}
components[container_name]['contains'][component_name] = {
'name': component['mnemonic'].replace('-', '_'),
'data': container_data
}
component['container-name'] = container_name
# Can't do any more if we have errors in the network groups
if self._errors:
return
# Create a default interface-model for any roles that don't define one
default_net_iface = [{
'name': 'default_iface',
'network-groups': [x for x in network_groups],
'ports': ['ethX']
}]
# Create a deafult server group to hold any networks and servers not
# specificially assigned
default_server_group = {}
# Fix up relationships between server groups
for group_name, group in server_groups.iteritems():
for child in ServerGroup.server_groups(group):
ServerGroup.add_group(group, server_groups[child])
# Map networks to net_groups in server groups
networks_in_a_group = set()
for group_name, group in server_groups.iteritems():
for net in ServerGroup.networks(group):
ServerGroup.add_network(group, net,
networks[net]['network-group'])
networks_in_a_group.add(net)
# Add any unassinged networks to the default server group
for net_name, net in networks.iteritems():
if net_name not in networks_in_a_group and not net.get(
'neutron_network', False):
if server_groups:
self.add_warning("Network %s is not listed in a server "
"group." % (net_name))
ServerGroup.add_network(default_server_group, net_name,
net['network-group'])
# Establish the min and max size of each cluster and resource group
for cp_name, cp in control_planes.iteritems():
for cluster in cp.get('clusters', []):
if 'member-count' in cluster:
cluster['min-count'] = cluster['member-count']
cluster['max-count'] = cluster['member-count']
else:
cluster['min-count'] = cluster.get('min-count', 1)
for rgroup in cp.get('resources', []):
if 'member-count' in rgroup:
rgroup['min-count'] = rgroup['member-count']
rgroup['max-count'] = rgroup['member-count']
else:
rgroup['min-count'] = rgroup.get('min-count', 0)
# Create a list of servers with the network details for each resolved
servers = []
for s in bm_servers:
server_role = server_roles[s['role']]
# resolve the networking
# Find the interface model, and take a copy of the interfaces, as we may only use part of the model
# If there is no interface-model in the server role then all networks map to the existing NIC
fcoe_interfaces = []
if 'interface-model' in server_role:
iface_model = iface_models[server_role['interface-model']]
server_interfaces = deepcopy(iface_model['network-interfaces'])
if 'fcoe-interfaces' in iface_model:
fcoe_interfaces = iface_model['fcoe-interfaces']
else:
server_interfaces = deepcopy(default_net_iface)
# Find the disk model, and take a copy
if 'disk-model' in server_role:
disk_model = deepcopy(disk_models[server_role['disk-model']])
else:
disk_model = {'drives': {}}
# Translate network groups to the specific networks for this server
# Note: At this stage we have all possible networks groups defined
# by the interface model. We will reduce that to just those
# needed once we have assinged the server to a particular role
for iface in server_interfaces:
iface['networks'] = {}
iface_net_groups = (
iface.get('network-groups', []) +
iface.get('forced-network-groups', []) +
iface.get('passthrough-network-groups', []))
for net_group in iface_net_groups:
# Find network in the group for this server
if 'server-group' in s:
server_group = server_groups[s['server-group']]
else:
server_group = None
net_name = ServerGroup.find_network(
server_group, net_group, default_server_group)
if net_name:
network = networks[net_name]
iface['networks'][network['name']] = deepcopy(network)
if net_group in iface.get('forced-network-groups', []):
iface['networks'][network['name']]['forced'] = True
else:
iface['networks'][
network['name']]['forced'] = False
# Marked the network as passthrough if its network group is
# in the list of passthought network groups.
if net_group not in iface.get(
'passthrough-network-groups', []):
iface['networks'][
network['name']]['passthrough'] = False
iface['networks'][
network['name']]['passthrough-only'] = False
else:
iface['networks'][
network['name']]['passthrough'] = True
# Mark the network as passthrough-only if its network group is
# not listed in network-groups or forced-network-groups
if net_group not in (
iface.get('network-groups', []) +
iface.get('forced-network-groups', [])):
iface['networks'][
network['name']]['passthrough-only'] = True
else:
iface['networks'][network['name']][
'passthrough-only'] = False
server = {
'id': s['id'],
'hostname': s.get('hostname', None),
'hypervisor-id': s.get('hypervisor-id', None),
'role': s['role'],
'server_group': s.get('server-group'),
'rack': s.get('rack'),
'addr': s['ip-addr'],
'if-model': server_role.get('interface-model', 'default'),
'disk-model': disk_model,
'interfaces': server_interfaces,
'fcoe_interfaces': fcoe_interfaces,
'nic_map': nic_mappings.get(s.get('nic-mapping', 'none')),
'ansible_options': s.get('ansible-options'),
'state': None,
'vm-factory': s.get('vm-factory', False),
'port-groups': s.get('port-groups', []),
'previous_config': {}
}
servers.append(server)
# Add servers to ServerGroups
if 'server-group' in s:
sgrp = server_groups[s['server-group']]
ServerGroup.add_server(sgrp, server)
server['server-group-list'] = ServerGroup.get_group_list(sgrp)
else:
# If there are server groups defined it would be odd to have
# a server which isn't a member of a group
if server_groups:
self.add_warning("Server %s is not a member of a server "
"group." % (s['ip-addr']))
ServerGroup.add_server(default_server_group, server)
server['server-group-list'] = []
# Roles for a cluster/resource can be either a string or a list, so change to
# always be a list
for cp_name, cp in control_planes.iteritems():
for cluster in cp['clusters']:
if isinstance(cluster['server-role'], basestring):
cluster['server-role'] = [cluster['server-role']]
for r in cp.get('resources', []):
if isinstance(r['server-role'], basestring):
r['server-role'] = [r['server-role']]
# Add a list of all services to each Control Plane
for cp_name, cp in control_planes.iteritems():
cp['services'] = {}
for cluster in cp['clusters']:
cluster['services'] = {}
for comp_name in cluster['service-components']:
service_name = components[comp_name]['service']
if service_name not in cluster['services']:
cluster['services'][service_name] = []
cluster['services'][service_name].append(comp_name)
cp['services'][service_name] = {}
for r in cp.get('resources', []):
r['services'] = {}
for comp_name in r['service-components']:
service_name = components[comp_name]['service']
if service_name not in r['services']:
r['services'][service_name] = []
r['services'][service_name].append(comp_name)
cp['services'][service_name] = {}
ntp_servers = cloud_data.get('ntp-servers', [])
dns_settings = cloud_data.get('dns-settings', {})
smtp_settings = cloud_data.get('smtp-settings', {})
firewall_settings = cloud_data.get('firewall-settings', {})
CloudModel.put(cloud_internal, 'control-planes', control_planes)
CloudModel.put(cloud_internal, 'networks', networks)
CloudModel.put(cloud_internal, 'servers', servers)
CloudModel.put(cloud_internal, 'server-groups', server_groups)
CloudModel.put(cloud_internal, 'default-server-group',
default_server_group)
CloudModel.put(cloud_internal, 'services', services)
CloudModel.put(cloud_internal, 'components', components)
CloudModel.put(cloud_internal, 'components_by_mnemonic',
components_by_mnemonic)
CloudModel.put(cloud_internal, 'ntp_servers', ntp_servers)
CloudModel.put(cloud_internal, 'dns_settings', dns_settings)
CloudModel.put(cloud_internal, 'smtp_settings', smtp_settings)
CloudModel.put(cloud_internal, 'firewall_settings', firewall_settings)
CloudModel.put(cloud_internal, 'pass_through', pass_through)
CloudModel.put(cloud_internal, 'iface_models', iface_models)
CloudModel.put(cloud_internal, 'disk_models', disk_models)
CloudModel.put(cloud_internal, 'nic_mappings', nic_mappings)
CloudModel.put(cloud_internal, 'server_roles', server_roles)