def _calculate(self):
LOG.info('%s()' % KenLog.fcn())
value = self._current_value['current-value']
self._current_value['current-value'] += 1
LOG.debug('%s() -> %s' % (KenLog.fcn(), value))
return value
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_data = self._models['CloudDescription']['cloud']
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
audit_settings = {
'audit-dir': '/var/audit',
'default': 'disabled',
'enabled-services': [],
'disabled-services': []
}
audit_settings.update(cloud_data.get('audit-settings', {}))
for cp_name, cp in control_planes.iteritems():
cp_services = cp.get('services', {})
for service_name, service_data in cp_services.iteritems():
service_data['audit-settings'] = {
'dir': audit_settings['audit-dir'],
'enabled': audit_settings['default'] == 'enabled'
}
if service_name in audit_settings['enabled-services']:
service_data['audit-settings']['enabled'] = True
elif service_name in audit_settings['disabled-services']:
service_data['audit-settings']['enabled'] = False
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "networks")
self._valid = self.validate_schema(input, "network")
if self._valid:
networks = input.get('networks', [])
self._get_ip_version(networks)
self._validate_names(networks)
for net in networks:
self._validate_vlans(net)
self._validate_cidr(net)
self._validate_vxlan_net_has_cidr(net)
self._validate_addresses_in_cidr(net)
self._validate_no_network_overlaps()
self._validate_gateways(networks)
if self._valid:
networks = input.get('networks', [])
for net in networks:
self._validate_vips_in_net(net)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "configuration-data")
# Configuration data is optional
if not input:
return
schema_valid = self.validate_schema(input, "configuration-data")
if schema_valid:
config_data = input.get('configuration-data', [])
services = self._get_dict_from_config_value(version, 'services')
self._networks = self._get_dict_from_config_value(
version, 'networks')
self._network_groups = self._get_dict_from_config_value(
version, 'network-groups')
self._validate_names(config_data)
for data in config_data:
self._validate_services(data, services)
for service_name in data.get('services', []):
validator = "_validate_%s_data" % service_name
if hasattr(self, validator):
getattr(self, validator)(data, config_data)
LOG.info('%s()' % KenLog.fcn())
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "nic-mappings")
# Nic mappings are optional
if not input:
return True
nic_device_types = self._get_dict_from_config_value(
version, 'nic-device-types')
if not nic_device_types:
nic_device_types = {}
self._valid = self.validate_schema(input, "nic_mapping")
if self._valid:
nic_mappings = input['nic-mappings']
self._validate_names(nic_mappings)
for mapping in nic_mappings:
self._validate_logical_names_and_addresses(
mapping, nic_device_types)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "pass-through")
if input:
return self.validate_schema(input, "pass_through")
LOG.info('%s()' % KenLog.fcn())
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
cloud_internal = CloudModel.internal(self._models['CloudModel'])
services = CloudModel.get(cloud_internal, 'services', [])
config_data = {}
for data in CloudModel.get(cloud_version, 'configuration-data', []):
config_data[data['name']] = data
control_planes = CloudModel.get(cloud_internal, 'control-planes')
for cp_name, cp in control_planes.iteritems():
# Get any config data at the CP level
cp_config_data = self._expand_config(
cp.get('configuration-data', []), config_data, services)
# Extract all of the network tags defined in
# configuration data for this control plane
cp['config-data-network-tags'] = self._get_network_tags(
cp['name'], cp.get('configuration-data', []), config_data)
# Extract the list of physnets that have been defined
# in neutron configuration data for this control plane
# so they can be validated in the NetworkTagGenerator
cp['neutron-provider-nets'] = self._get_provider_nets(
cp['name'], cp.get('configuration-data', []), config_data)
for cluster in cp['clusters']:
cluster_config_data = self._expand_config(
cluster.get('configuration-data', []), config_data,
services)
context = "%s:%s" % (cp_name, cluster['name'])
cluster['config-data'] = self._merge(cp_config_data,
cluster_config_data,
context)
self._check_no_network_tags(
context, cluster.get('configuration-data', []),
config_data)
for r in cp.get('resources', []):
r_config_data = self._expand_config(
r.get('configuration-data', []), config_data, services)
context = "%s:%s" % (cp_name, r['name'])
r['config-data'] = self._merge(cp_config_data, r_config_data,
context)
self._check_no_network_tags(context,
r.get('configuration-data', []),
config_data)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
components = CloudModel.get(cloud_internal, 'components', [])
# If we have an error in an earlier generator we may not have
# components in the internal model
if not components:
return
components_by_mnemonic = CloudModel.get(cloud_internal, 'components_by_mnemonic')
control_planes = CloudModel.get(cloud_internal, 'control-planes')
# Resolve the service level consumes for all control planes before we
# do the component level consumes - as a component may need to find
# the service relationship in a parent control plane
#
for cp_name, cp in control_planes.iteritems():
for component_name, endpoints in cp.get('advertises', {}).iteritems():
component = components[component_name]
for adv_to in component.get('advertises-to-services', []):
adv_to_name = adv_to['service-name']
if adv_to_name in components:
adv_to_mnemonic = components[adv_to_name]['mnemonic']
else:
# We have some cases where a service advertises to a dummy
# component
if adv_to_name not in components_by_mnemonic:
continue
adv_to_mnemonic = adv_to_name
adv_to_name = components_by_mnemonic[adv_to_mnemonic]['name']
for entry in adv_to.get('entries', []):
adv = {'service_name': entry['service-name'],
'service_type': entry['service-type'],
'service_description': entry.get('service-description'),
'regions': [],
'component_name': component_name,
'from_cp_name': cp_name}
for role in ['public', 'internal', 'admin']:
if role in endpoints:
adv[role + '_url'] = endpoints[role]['url'] + entry.get('url-suffix', '')
for region_name in sorted(cp.get('regions', {})):
region_services = cp['regions'][region_name]
service = component.get('service', 'foundation')
if service in region_services:
adv['regions'].append(region_name)
self._advertised(adv_to_name, cp, adv, control_planes)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
cloud_version = CloudModel.version(self._models['CloudModel'],
self._version)
servers = CloudModel.get(cloud_internal, 'servers', {})
server_roles = {}
for role in CloudModel.get(cloud_version, 'server-roles'):
server_roles[role['name']] = role
memory_models = {}
for memory_model in CloudModel.get(cloud_version, 'memory-models', []):
memory_models[memory_model['name']] = memory_model
for server in servers:
if server['state'] != 'allocated':
continue
server['memory-model'] = {}
server_role = server_roles[server['role']]
if 'memory-model' not in server_role:
continue
memory_model = memory_models[server_role['memory-model']]
my_memory_model = {'name': memory_model['name']}
non_numa_pages = []
numa_pages = []
for page in memory_model.get('huge-pages', {}):
page_info = {
'size': page['size'],
'count': page['count'],
'size_in_k': self._page_size(page['size'])
}
if 'numa-node' in page:
page_info['numa_node'] = page['numa-node']
numa_pages.append(page_info)
else:
non_numa_pages.append(page_info)
if memory_model.get('vm-size', None):
my_memory_model['vm_ram_size_in_k'] = self._memory_size(
memory_model['vm-size']['ram'])
my_memory_model['numa_huge_pages'] = numa_pages
my_memory_model['non_numa_huge_pages'] = non_numa_pages
if 'default-huge-page-size' in memory_model:
my_memory_model['default_huge_page_size'] = memory_model[
'default-huge-page-size']
server['memory-model'] = my_memory_model
def _generate_ring_specifications_info(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_version = CloudModel.version(
self._models['CloudModel'], self._version)
ring_specifications_config = CloudModel.get(
cloud_version, 'ring-specifications', [])
cloud_internal = CloudModel.internal(self._models['CloudModel'])
CloudModel.put(cloud_internal,
'ring-specifications',
ring_specifications_config)
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
control_planes = CloudModel.get(cloud_internal, 'control-planes', {})
# If we have an error in an earlier generator we may not have
# control_planes in the internal model
if not control_planes:
return
routes = {}
for cp_name, cp in control_planes.iteritems():
load_balancers = cp.get('load-balancer-config', {})
for cluster in cp['clusters']:
for server in cluster.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
for r_name, r in cp.get('resources', {}).iteritems():
for server in r.get('servers', []):
self._add_routes_from_server(cp, server, load_balancers, routes)
#
default_routes = {}
# Warn about any routes using the "default" route
for src_net, net_routes in routes.iteritems():
for dest_net, route_data in net_routes.iteritems():
if route_data['default']:
hosts = set()
if src_net not in default_routes:
default_routes[src_net] = []
for src, src_data in route_data['used_by'].iteritems():
for dest, host_list in src_data.iteritems():
for host in host_list:
hosts.add(host)
default_routes[src_net].append({'net': dest_net, 'hosts': hosts})
if default_routes:
msg = ("Default routing used between networks\n"
"The following networks are using a 'default' route rule. To remove this warning\n"
"either add an explicit route in the source network group or force the network to\n"
"attach in the interface model used by the servers.\n")
for src in sorted(default_routes):
dest_list = default_routes[src]
for dest_data in dest_list:
msg += " %s to %s\n" % (src, dest_data['net'])
for host in sorted(dest_data['hosts']):
msg += " %s\n" % (host)
self.add_warning(msg)
CloudModel.put(cloud_internal, 'routes', routes)
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "servers")
schema_valid = self.validate_schema(input, "server")
if schema_valid:
servers = input.get('servers', [])
baremetal = {}
try:
baremetal = self._create_content(version,
'baremetal')['baremetal'][0]
except TypeError:
try:
# baremetal_networks is still suppoted for backwards compatibility
baremetal = self._create_content(
version, 'baremetal_network')['baremetal_network'][0]
except TypeError:
# Possible to have servers with no baremetal section if not using
# cobbler to deploy
pass
nic_mappings = self._get_dict_from_config_value(
version, 'nic-mappings')
if not nic_mappings:
nic_mappings = {}
iface_models = self._get_dict_from_config_value(
version, 'interface-models')
server_roles = self._get_dict_from_config_value(
version, 'server-roles')
server_groups = self._get_dict_from_config_value(
version, 'server-groups')
nic_device_types = self._get_dict_from_config_value(
version, 'nic-device-types')
if not nic_device_types:
nic_device_types = {}
nic_device_families = self._get_dict_from_config_value(
version, 'nic-device-families')
if not nic_device_families:
nic_device_families = {}
if baremetal:
self._validate_baremetal_net(baremetal)
self._validate_unique_ids(servers)
self._validate_ip_addresses(servers)
self._validate_mac_addresses(servers)
self._validate_server_groups(servers, server_groups)
self._validate_net_devices(servers, nic_mappings, server_roles,
iface_models, nic_device_types,
nic_device_families)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "service-components")
self._valid = self.validate_schema(input, "service_component")
if not self._valid:
return self._valid
component_list = input['service-components']
# Add any aliases. Alias allow us to change the name of a component
# without breaking compatibility with any existing input models
aliased_components = []
for component in component_list:
for alias in component.get('aliases', []):
a_comp = deepcopy(component)
a_comp['name'] = alias
a_comp['alias-for'] = component['name']
del a_comp['aliases']
aliased_components.append(a_comp)
component_list.extend(aliased_components)
service_list = self._get_config_value(version, 'services')
services = {}
for service in service_list:
services[service['name']] = service
self._validate_names(services, component_list)
components = {}
components_by_mnemonic = {}
for comp in component_list:
components[comp['name']] = comp
if 'alias-for' not in comp:
components_by_mnemonic[comp['mnemonic']] = comp
self._validate_services(services, components)
for c_name, component in components.iteritems():
self._check_endpoint_roles(component)
self._check_port_range(component)
self._check_port_attributes(component)
self._check_supported_resource_allocations(component)
self._check_config_set(component)
self._check_relationship_vars(component)
self._validate_consumers(component, services)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "services")
self._valid = self.validate_schema(input, "service")
if not self._valid:
return self._valid
service_list = self._get_config_value(version, 'services')
self._validate_names(service_list)
self._validate_relationship_vars(service_list)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def __init__(self, instructions, config_files):
super(CloudConfigValidator,
self).__init__(2.0, instructions, config_files,
'cloudconfig-2.0')
LOG.info('%s()' % KenLog.fcn())
self._is_built = False
self._path = instructions['cloud_input_path']
def finalize(self):
LOG.info('%s()' % KenLog.fcn())
allocated_addresses = CloudModel.get(self._cloud_internal,
'address_allocations', {})
host_aliases = CloudModel.get(self._cloud_internal, 'host_aliases', {})
address_data = {}
for group_name, group in allocated_addresses.iteritems():
if group_name not in address_data:
address_data[group_name] = {}
for network_name, network in group.iteritems():
if network_name not in address_data[group_name]:
address_data[group_name][network_name] = {}
for addr in sorted(network):
aliases = host_aliases.get(group_name,
{}).get(network_name,
{}).get(addr, [])
address_data[group_name][network_name][addr] = []
for name in sorted(aliases):
address_data[group_name][network_name][addr].append(
name)
filename = os.path.join(self._file_path, 'address_info.yml')
self.add_artifact(filename, ArtifactMode.CREATED)
with open(filename, 'w') as fp:
yaml.dump(address_data, fp, default_flow_style=False, indent=4)
def validate(self):
LOG.info('%s()' % KenLog.fcn())
if 'encryption_key' in self._instructions:
path = self._instructions['cloud_input_path']
status, messages = CPSecurity.validate(
path, self._instructions['encryption_key'])
if not status:
message = 'The Encryption Key does not meet the following ' \
'requirement(s):\n# %s' % \
'\n# '.join(messages)
self.add_error(message)
return False
score, msg = CPSecurity.calculate_complexity(
self._instructions['encryption_key'])
print('\n\nThe encryption key has a complexity score of %d ('
'%s)\n\n' % (score, msg))
if ('previous_encryption_key' in self._instructions and
'encryption_key' in self._instructions):
if (self._instructions['encryption_key'] ==
self._instructions['previous_encryption_key']):
message = 'The New Encryption Key and the Previous ' \
'Encryption Key must be different.'
self.add_error(message)
return False
return True
def _determine_height_for_cp_networks(self, elem_cp):
LOG.info('%s()' % KenLog.fcn())
height = 0
interfaces = dict()
nt = elem_cp['network-topology']
for k, v in six.iteritems(nt):
(intf, vlan_tag) = self._get_interface_info(elem_cp['type'], k)
index = '%s-%s' % (intf, vlan_tag)
if index not in interfaces:
interfaces[index] = dict()
interfaces[index]['interface'] = intf
interfaces[index]['vlan-tag'] = vlan_tag
interfaces[index]['mnemonics'] = []
if k not in interfaces[index]['mnemonics']:
interfaces[index]['mnemonics'].append(k)
for k, v in six.iteritems(interfaces):
height += self._determine_height_for_cp_interface(v)
height += 1 # Separation
return height
def __init__(self, instructions, config_files):
super(NetworksValidator, self).__init__(2.0, instructions,
config_files, 'networks-2.0')
self._valid = False
self._valid_cidr = True
self._ipsets = {}
LOG.info('%s()' % KenLog.fcn())
def __init__(self, instructions, models, controllers):
super(HTMLDiagramBuilder,
self).__init__(2.0, instructions, models, controllers,
'html-diagram-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
self._file_path = ArdanaPaths.get_output_path(self._instructions,
self.cloud_desc)
self._file_path = os.path.join(self._file_path, 'html')
self._cloud_model = self._models['CloudModel']
self._cloud_version = CloudModel.version(self._cloud_model,
self._version)
self._cloud_internal = CloudModel.internal(self._cloud_model)
self.cloud_name = CloudDescription.get_cloud_name(self.cloud_desc)
self.control_planes = CloudModel.get(self._cloud_internal,
'control-planes')
self.iface_models = CloudModel.get(self._cloud_version,
'interface-models')
self.server_roles = CloudModel.get(self._cloud_version, 'server-roles')
self.disk_models = CloudModel.get(self._cloud_version, 'disk-models',
[])
self.memory_models = CloudModel.get(self._cloud_version,
'memory-models', [])
self.cpu_models = CloudModel.get(self._cloud_version, 'cpu-models', [])
self.components = CloudModel.get(self._cloud_internal, 'components')
self.services = CloudModel.get(self._cloud_internal, 'services')
self.servers = CloudModel.get(self._cloud_internal, 'servers')
self.server_groups = CloudModel.get(self._cloud_internal,
'server-groups')
self.network_groups = CloudModel.get(self._cloud_internal,
'network-groups')
self.region_topology = CloudModel.get(self._cloud_internal,
'region-topology')
self.service_topology = CloudModel.get(self._cloud_internal,
'service-topology')
self.cp_topology = CloudModel.get(self._cloud_internal, 'cp-topology')
self.network_topology = CloudModel.get(self._cloud_internal,
'network-topology')
self.components_by_mnemonic = {}
for comp_name, comp_data in self.components.iteritems():
if 'alias-for' not in comp_data:
self.components_by_mnemonic[comp_data['mnemonic']] = comp_name
ArdanaPaths.make_path(self._file_path)
for subdir in ['Servers']:
ArdanaPaths.make_path(self._file_path + "/%s" % subdir)
# Create a mapping from hostname to server id
self.server_by_hostname = {}
self.server_by_name = {}
for s in self.servers:
if 'hostname' in s:
self.server_by_hostname[s['hostname']] = s
if 'name' in s:
self.server_by_name[s['name']] = s
def __init__(self, instructions, config_files):
super(ServiceComponentValidator,
self).__init__(2.0, instructions, config_files,
'service-components-2.0')
LOG.info('%s()' % KenLog.fcn())
self._valid = True
def __init__(self, instructions, models, controllers):
super(RandomPasswordVariable,
self).__init__(2.0, instructions, models, controllers,
'random-password-2.0')
LOG.info('%s()' % KenLog.fcn())
random.seed()
def calculate(self, payload=None):
LOG.info('%s()' % KenLog.fcn())
if not payload:
payload = dict()
if 'algorithm' not in payload:
payload['algorithm'] = 'RSA'
elif payload['algorithm'] not in (
'RSA',
'DSA',
):
self.add_error('algorithm must be one of RSA or DSA')
return None
if 'length' not in payload:
payload['length'] = 2048
if 'comment' not in payload:
payload['comment'] = None
if 'passphrase' not in payload:
payload['passphrase'] = None
return self._calculate(payload)
def _determine_size_for_control_plane(self, elem_cp):
LOG.info('%s(elem_cp="%s")' %
(KenLog.fcn(), ControlPlane.get_name(elem_cp)))
width = 0
height = self._padding_y
for elem_t in elem_cp['tiers']:
if not Tier.is_active_or_empty(elem_t):
continue
(t_w, t_h) = self._determine_size_for_tier(elem_cp, elem_t)
if t_w > width:
width = t_w
height += t_h
width += (self._padding_x * 2)
height += (self._padding_y * 2)
height += self._determine_height_for_resource_nodes(elem_cp)
height += self._determine_height_for_cp_networks(elem_cp)
return width, height
def checkpoint(self):
LOG.info('%s()' % KenLog.fcn())
path = self._instructions['cloud_input_path']
cloud_name, nickname = CloudNameController.get_cloud_names(path)
self.prepare_filesystem(cloud_name, 'config')
return self._checkpoint()
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "nic-device-families")
# Nic device families are optional
if not input:
return True
self._valid = self.validate_schema(input, "nic_device_family")
if self._valid:
nic_dev_families = input['nic-device-families']
self._validate_names(nic_dev_families)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def generate(self):
LOG.info('%s()' % KenLog.fcn())
self._action = KenLog.fcn()
cloud_internal = CloudModel.internal(self._models['CloudModel'])
self.services = CloudModel.get(cloud_internal, 'services', [])
control_planes = CloudModel.get(cloud_internal, 'control-planes')
components = CloudModel.get(cloud_internal, 'components')
# Resolve the service data relationships.
for cp_name, cp in control_planes.iteritems():
for comp_name, comp_data in cp.get('components', []).iteritems():
provider = components[comp_name]
provider_service = self.services.get(provider['service'], {})
self._set_provided_data(provider, provider_service,
comp_data, cp, control_planes)
def validate(self):
LOG.info('%s()' % KenLog.fcn())
version = float(self.version())
input = self._create_content(version, "network-groups")
self._valid = self.validate_schema(input, "network_group")
if self._valid:
components = self._get_dict_from_config_value(
version, 'service-components')
network_groups = input['network-groups']
self._validate_names(network_groups)
self._validate_unique_hash_prefix(network_groups)
self._validate_components(network_groups, components)
LOG.info('%s()' % KenLog.fcn())
return self._valid
def __init__(self, instructions, models, controllers):
super(InternalModelGenerator,
self).__init__(2.0, instructions, models, controllers,
'internal-model-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']
def __init__(self, instructions, models, controllers):
super(NetworkTagGenerator,
self).__init__(2.0, instructions, models, controllers,
'network-tag-generator-2.0')
LOG.info('%s()' % KenLog.fcn())
self.cloud_desc = self._models['CloudDescription']['cloud']