def test_vm_role_hardcoded(yaml_file):
"""
Validate vm_role value when hardcoded in the template
"""
heat = Heat(filepath=yaml_file)
servers = heat.get_resource_by_type("OS::Nova::Server")
errors = []
for r_id, server in servers.items():
props = server.get("properties") or {}
metadata = props.get("metadata") or {}
if "vm_role" not in metadata:
continue
vm_role_value = metadata["vm_role"]
if isinstance(vm_role_value, dict):
continue # Likely using get_param - validate separately
if not re.match(r"^\w+$", vm_role_value):
errors.append(
"OS::Nova::Server {} vm_role = {}".format(r_id, vm_role_value)
)
msg = (
"vm_role's value must only contain alphanumerics and underscores. "
+ "Invalid vm_role's detected: "
+ ". ".join(errors)
)
assert not errors, msg
def test_detected_volume_module_follows_naming_convention(template_dir):
all_files = [os.path.join(template_dir, f) for f in os.listdir(template_dir)]
yaml_files = [f for f in all_files if f.endswith(".yaml") or f.endswith(".yml")]
errors = []
for yaml_file in non_nested_files(yaml_files):
heat = Heat(filepath=yaml_file)
if not heat.resources:
continue
base_dir, filename = os.path.split(yaml_file)
resources = heat.get_all_resources(base_dir)
non_nested_ids = {
r_id
for r_id, r_data in resources.items()
if not Resource(r_id, r_data).is_nested()
}
volume_ids = {
r_id
for r_id, r_data in resources.items()
if Resource(r_id, r_data).resource_type == "OS::Cinder::Volume"
}
non_volume_ids = non_nested_ids.difference(volume_ids)
if non_volume_ids:
continue # Not a volume module
base_name, ext = os.path.splitext(filename)
if not base_name.endswith("_volume") or ext not in (".yaml", ".yml"):
errors.append(yaml_file)
msg = (
"Volume modules detected, but they do not follow the expected "
+ " naming convention {{module_name}}_volume.[yaml|yml]: {}"
).format(", ".join(errors))
assert not errors, msg
def test_server_and_port_vmtype_indices_match(yaml_file):
# NOTE: This test is only going to validate that the index values
# match between the between the ports and server names. Other
# tests already cover the other aspects of this requirement
heat = Heat(filepath=yaml_file)
servers = heat.get_resource_by_type("OS::Nova::Server")
errors = []
for r_id, server in servers.items():
match = SERVER_ID_PATTERN.match(r_id)
if not match:
continue # other tests cover valid server ID format
server_index = match.group(1)
ports = get_ports(server)
for port in ports:
port_match = PORT_ID_PATTERN.match(port)
if port_match:
port_vm_index = port_match.group(1)
if port_vm_index != server_index:
errors.append(
("{{vm-type_index}} ({}) in port ID ({}) " +
"does not match the {{index}} ({}) in the " +
"servers resource ID ({})").format(
port_vm_index, port, server_index, r_id))
assert not errors, ". ".join(errors)
def test_port_connected_to_multiple_servers(yaml_file):
"""
SDC will throw an error if a single port is connected to more than
one server. This test detects that condition and logs a test failure.
"""
heat = Heat(yaml_file)
if not heat.resources:
pytest.skip("No resources")
port_to_server = defaultdict(list)
for server_id, server_data in heat.get_resource_by_type(
"OS::Nova::Server").items():
server = Resource(server_id, server_data)
ports = server.properties.get("networks", [])
for port in ports:
port_val = port.get("port")
if isinstance(port_val, dict) and "get_resource" in port_val:
port_id = port_val["get_resource"]
port_to_server[port_id].append(server_id)
errors = []
for port, servers in port_to_server.items():
if len(servers) > 1:
errors.append("Port {} is connected to {}".format(
port, ", ".join(servers)))
msg = "A port cannot be connected to more than 1 server: {}".format(
". ".join(errors))
assert not errors, msg
def test_contrail_incremental_module_internal_subnet_usage(yaml_files):
base_path = get_base_template_from_yaml_files(yaml_files)
if not base_path:
pytest.skip("No base module detected to check")
base_outputs = Heat(filepath=base_path).outputs
incremental_modules = get_incremental_modules(yaml_files)
errors = []
for module in incremental_modules:
heat = Heat(filepath=module)
ips = heat.get_resource_by_type(
ContrailV2InstanceIpProcessor.resource_type)
internal_ips = ((r_id, props) for r_id, props in ips.items()
if "_int_" in r_id)
for r_id, ip in internal_ips:
subnet_uuid = (ip.get("properties") or {}).get("subnet_uuid")
subnet_param = get_param(subnet_uuid)
if not subnet_param:
continue
if subnet_param not in base_outputs:
errors.append(
("Resource ({}) is designated as an internal IP, but its "
"subnet_uuid parameter ({}) does not refer to subnet in "
"this template nor is it defined in the output section "
"of the base module ({})").format(
r_id, subnet_param, os.path.basename(base_path)))
assert not errors, ". ".join(errors)
def get_nesting(yaml_files):
"""return bad, files, heat, depths
bad - list of error messages.
files - dict: key is filename, value is dict of nested files.
This is the tree.
heat - dict,: key is filename, value is Heat instance.
depths - dict: key is filename, value is a depth tuple
level: 0 1 2 3
file: template -> nested -> nested -> nested
depth: 3 2 1 0
"""
bad = []
files = {}
heat = {}
depths = {}
for yaml_file in yaml_files:
dirname, basename = path.split(yaml_file)
h = Heat(filepath=yaml_file)
heat[basename] = h
files[basename] = get_dict_of_nested_files(h.yml, dirname)
for filename in files:
depths[filename] = _get_nesting_depth_start(0, filename, files, [])
for depth in depths[filename]:
if depth[0] > MAX_DEPTH:
bad.append("{} {}".format(filename, str(depth[1])))
return bad, files, heat, depths
def test_availability_zones_start_at_0(heat_template):
if nested_files.file_is_a_nested_template(heat_template):
pytest.skip("Test does not apply to nested files")
params = Heat(heat_template).parameters
invalid_params = check_indices(AZ_PATTERN, params,
"Availability Zone Parameters")
assert not invalid_params, ". ".join(invalid_params)
def test_ips_start_at_0(yaml_file):
heat = Heat(filepath=yaml_file)
ports = heat.get_resource_by_type("OS::Neutron::Port")
ip_parameters = []
for rid, resource in ports.items():
fips = nested_dict.get(resource, "properties", "fixed_ips", default={})
for fip in fips:
ip_address = fip.get("ip_address", {})
param = ip_address.get("get_param")
if isinstance(param, list):
param = param[0]
if isinstance(param, str):
ip_parameters.append(param)
invalid_params = check_indices(IP_PARAM_PATTERN, ip_parameters,
"IP Parameters")
assert not invalid_params, ". ".join(invalid_params)
def test_contrail_vmi_aap_does_not_exist_in_environment_file(yaml_file):
# This test needs to check a more complex structure. Rather than try to force
# that into the existing run_check_resource_parameter logic we'll just check it
# directly
pairs = get_environment_pair(yaml_file)
if not pairs:
pytest.skip("No matching env file found")
heat = Heat(filepath=yaml_file)
env_parameters = pairs["eyml"].get("parameters") or {}
vmis = heat.get_resource_by_type("OS::ContrailV2::VirtualMachineInterface")
external_vmis = {
rid: data
for rid, data in vmis.items() if "_int_" not in rid
}
invalid_params = []
for r_id, vmi in external_vmis.items():
aap_value = nested_dict.get(
vmi,
"properties",
"virtual_machine_interface_allowed_address_pairs",
"virtual_machine_interface_allowed_address_pairs_allowed_address_pair",
)
if not aap_value or not isinstance(aap_value, list):
# Skip if aap not used or is not a list.
continue
for pair_ip in aap_value:
if not isinstance(pair_ip, dict):
continue # Invalid Heat will be detected by another test
settings = (pair_ip.get("virtual_machine_interface_allowed_address"
"_pairs_allowed_address_pair_ip") or {})
if isinstance(settings, dict):
ip_prefix = (settings.get(
"virtual_machine_interface_allowed_address"
"_pairs_allowed_address_pair_ip_ip_prefix") or {})
ip_prefix_param = get_param(ip_prefix)
if ip_prefix_param and ip_prefix_param in env_parameters:
invalid_params.append(ip_prefix_param)
msg = ("OS::ContrailV2::VirtualMachineInterface "
"virtual_machine_interface_allowed_address_pairs"
"_allowed_address_pair_ip_ip_prefix "
"parameters found in environment file {}: {}").format(
pairs.get("name"), ", ".join(invalid_params))
assert not invalid_params, msg
def test_external_network_parameter(heat_template):
heat = Heat(filepath=heat_template)
errors = []
for rid, port in heat.neutron_port_resources.items():
rid_match = EXTERNAL_PORT.match(rid)
if not rid_match:
continue # only test external ports
network = (port.get("properties") or {}).get("network") or {}
if not isinstance(network, dict) or "get_param" not in network:
errors.append(
("The external port ({}) must assign the network property "
"using get_param. If this port is for an internal network, "
"then change the resource ID format to the external format."
).format(rid))
continue
param = get_param(network)
if not param:
errors.append(
("The get_param function on the network property of port ({}) "
"must only take a single, string parameter.").format(rid))
continue
param_match = EXTERNAL_NAME_PATTERN.match(
param) or EXTERNAL_UUID_PATTERN.match(param)
if not param_match:
errors.append((
"The network parameter ({}) on port ({}) does not match one of "
"{{network-role}}_net_id or {{network-role}}_net_name."
).format(param, rid))
continue
rid_network_role = rid_match.groupdict()["network_role"]
param_network_role = param_match.groupdict()["network_role"]
if rid_network_role.lower() != param_network_role.lower():
errors.append(
("The network role ({}) extracted from the resource ID ({}) "
"does not match network role ({}) extracted from the "
"network parameter ({})").format(rid_network_role, rid,
param_network_role, param))
assert not errors, ". ".join(errors)
def test_neutron_port_internal_fixed_ips_subnet_in_base(yaml_files):
base_path = get_base_template_from_yaml_files(yaml_files)
if not base_path:
pytest.skip("No base module detected")
base_heat = load_yaml(base_path)
base_outputs = base_heat.get("outputs") or {}
nested_template_paths = get_nested_files(yaml_files)
errors = []
for yaml_file in yaml_files:
if yaml_file == base_path or yaml_file in nested_template_paths:
continue # Only applies to incremental modules
heat = Heat(filepath=yaml_file)
internal_ports = {
r_id: p
for r_id, p in heat.neutron_port_resources.items()
if get_network_type_from_port(p) == "internal"
}
for r_id, port in internal_ports.items():
props = port.get("properties") or {}
fip_list = props.get("fixed_ips") or []
if not isinstance(fip_list, list):
continue
for ip in fip_list:
subnet = ip.get("subnet")
if not subnet:
continue
if "get_param" not in subnet:
continue
param = subnet.get("get_param")
if param not in base_outputs:
errors.append((
"Internal fixed_ips/subnet parameter {} is attached to "
"port {}, but the subnet parameter "
"is not defined as an output in the base module ({})."
).format(param, r_id, base_path))
assert not errors, " ".join(errors)
def test_internal_network_parameters(yaml_files):
base_path = get_base_template_from_yaml_files(yaml_files)
if not base_path:
pytest.skip("No base module found")
base_heat = Heat(filepath=base_path)
nested_paths = get_nested_files(yaml_files)
incremental_modules = [
f for f in yaml_files
if is_incremental_module(f, base_path, nested_paths)
]
errors = []
for module in incremental_modules:
heat = Heat(filepath=module)
for rid, port in heat.neutron_port_resources.items():
rid_match = INTERNAL_PORT.match(rid)
if not rid_match:
continue
network = (port.get("properties") or {}).get("network") or {}
if isinstance(network, dict) and ("get_resource" in network
or "get_attr" in network):
continue
param = get_param(network)
if not param:
errors.append(
("The internal port ({}) must either connect to a network "
"in the base module using get_param or to a network "
"created in this module ({})").format(
rid,
os.path.split(module)[1]))
continue
param_match = INTERNAL_UUID_PATTERN.match(
param) or INTERNAL_NAME_PATTERN.match(param)
if not param_match:
errors.append((
"The internal port ({}) network parameter ({}) does not "
"match one of the required naming conventions of "
"int_{{network-role}}_net_id or "
"int_{{network-role}}_net_name "
"for connecting to an internal network. "
"If this is not an internal port, then change the resource "
"ID to adhere to the external port naming convention."
).format(rid, param))
continue
if param not in base_heat.yml.get("outputs", {}):
base_module = os.path.split(base_path)[1]
errors.append((
"The internal network parameter ({}) attached to port ({}) "
"must be defined in the output section of the base module ({})."
).format(param, rid, base_module))
continue
param_network_role = param_match.groupdict().get("network_role")
rid_network_role = rid_match.groupdict().get("network_role")
if param_network_role.lower() != rid_network_role.lower():
errors.append((
"The network role ({}) extracted from the resource ID ({}) "
"does not match network role ({}) extracted from the "
"network parameter ({})").format(rid_network_role, rid,
param_network_role,
param))
resources = base_heat.get_all_resources(
os.path.split(base_path)[0])
networks = {
rid: resource
for rid, resource in resources.items() if resource.get("type")
in {"OS::Neutron::Net", "OS::ContrailV2::VirtualNetwork"}
}
matches = (INTERNAL_NETWORK_PATTERN.match(n) for n in networks)
roles = {
m.groupdict()["network_role"].lower()
for m in matches if m
}
if param_network_role.lower() not in roles:
errors.append(
("No internal network with a network role of {} was "
"found in the base modules networks: {}").format(
param_network_role, ", ".join(networks)))
assert not errors, ". ".join(errors)
def check_parameter_format(yaml_file,
regx,
intext,
resource_processor,
*properties,
exemptions_allowed=False):
"""
yaml_file: input file to check
regx: dictionary containing the regex to use to validate parameter
intext: internal or external
resource_processor: resource type specific helper, defined in structures.py
properties: arg list of property that is being checked
exemptions_allowed: If True, then parameters in the aap_exempt list are allowed to
not follow the rules
"""
invalid_parameters = []
heat = Heat(filepath=yaml_file)
resource_type = resource_processor.resource_type
resources = heat.get_resource_by_type(resource_type)
heat_parameters = heat.parameters
for rid, resource in resources.items():
resource_intext, port_match = resource_processor.get_rid_match_tuple(
rid)
if not port_match:
continue # port resource ID not formatted correctely
if (resource_intext !=
intext): # skipping if type (internal/external) doesn't match
continue
for param in prop_iterator(resource, *properties):
if (param and isinstance(param, dict)
and "get_resource" not in param
and "get_attr" not in param):
# checking parameter uses get_param
parameter = param.get("get_param")
if not parameter:
msg = (
"Unexpected parameter format for {} {} property {}: {}. "
"Please consult the heat guidelines documentation for details."
).format(resource_type, rid, properties, param)
invalid_parameters.append(msg) # should this be a failure?
continue
# getting parameter if the get_param uses list, and getting official
# HEAT parameter type
parameter_type = parameter_type_to_heat_type(parameter)
if parameter_type == "comma_delimited_list":
parameter = parameter[0]
elif parameter_type != "string":
continue
# checking parameter format = parameter type defined in parameters
# section
heat_parameter_type = nested_dict.get(heat_parameters,
parameter, "type")
if not heat_parameter_type or heat_parameter_type != parameter_type:
msg = ("{} {} parameter {} defined as type {} " +
"is being used as type {} in the heat template"
).format(
resource_type,
properties,
parameter,
heat_parameter_type,
parameter_type,
)
invalid_parameters.append(
msg) # should this actually be an error?
continue
if exemptions_allowed and parameter in get_aap_exemptions(
resource):
continue
# if parameter type is not in regx dict, then it is not supported
# by automation
regx_dict = regx[resource_intext].get(parameter_type)
if not regx_dict:
msg = (
"{} {} {} parameter {} defined as type {} "
"which is required by platform data model for proper "
"assignment and inventory.").format(
resource_type, rid, properties, parameter,
parameter_type)
if exemptions_allowed:
msg = "WARNING: {} {}".format(msg, AAP_EXEMPT_CAVEAT)
invalid_parameters.append(msg)
continue
# checking if param adheres to guidelines format
regexp = regx[resource_intext][parameter_type]["machine"]
readable_format = regx[resource_intext][parameter_type][
"readable"]
match = regexp.match(parameter)
if not match:
msg = (
"{} {} property {} parameter {} does not follow {} "
"format {} which is required by platform data model for proper "
"assignment and inventory.").format(
resource_type,
rid,
properties,
parameter,
resource_intext,
readable_format,
)
if exemptions_allowed:
msg = "WARNING: {} {}".format(msg, AAP_EXEMPT_CAVEAT)
invalid_parameters.append(msg)
continue
# checking that parameter includes correct vm_type/network_role
parameter_checks = regx.get(
"parameter_to_resource_comparisons", [])
for check in parameter_checks:
resource_match = port_match.group(check)
if (resource_match
and not parameter.startswith(resource_match)
and parameter.find(
"_{}_".format(resource_match)) == -1):
msg = ("{0} {1} property {2} parameter "
"{3} {4} does match resource {4} {5}").format(
resource_type,
rid,
properties,
parameter,
check,
resource_match,
)
invalid_parameters.append(msg)
continue
assert not invalid_parameters, "%s" % "\n".join(invalid_parameters)
def find_output_param(param, templates):
templates = (t for t in templates if param in Heat(t).outputs)
return [os.path.basename(t) for t in templates]
def test_nova_server_name_parameter_starts_at(yaml_file):
params = Heat(yaml_file).parameters
invalid_params = check_indices(
SERVER_NAME_PARAM, params, "OS::Nova::Server Name Parameters"
)
assert not invalid_params, ". ".join(invalid_params)