def get_node_type_priority(self, node_type_definitions, node_type_name):
"""
:param node_type_definitions:
:param node_type_name:
:return:
"""
(_, _, type_short) = utils.tosca_type_parse(node_type_name)
if type_short == ROOT:
return 0
requirement_definitions_list = []
tmp = node_type_name
while tmp != None:
definition = node_type_definitions.get(tmp, {})
requirement_definitions_list.extend(definition.get(REQUIREMENTS, []))
tmp = definition.get(DERIVED_FROM, None)
requirement_definitions = dict()
for i in range(len(requirement_definitions_list), 0, -1):
requirement_definitions.update(requirement_definitions_list[i-1])
max_dependency_priority = 0
for k, v in requirement_definitions.items():
req_node_type = v.get(NODE)
(_, _, req_short_type) = utils.tosca_type_parse(req_node_type)
if not req_short_type == ROOT and not req_node_type == node_type_name:
p = self.get_node_type_priority(node_type_definitions, req_node_type)
if p > max_dependency_priority or max_dependency_priority == 0:
max_dependency_priority = p + 1
if max_dependency_priority >= self.MAX_NUM_PRIORITIES:
ProviderResource.MAX_NUM_PRIORITIES = max_dependency_priority + 1
return max_dependency_priority
def resolve_in_template_dependencies(self):
"""
TODO think through the logic to replace mentions by id
Changes all mentions of node_templates by name in requirements, places dictionary with node_filter instead
:return:
"""
for node in self.node_templates:
for req in node.requirements:
for k, v in req.items():
if type(v) is str:
# The case when the requirement is a name of node or relationship template
# Store the dependencies of nodes
self.add_template_dependency(node.name, v)
nodetemplate = node.templates.get(v)
node_filter = dict()
properties = nodetemplate.get(PROPERTIES)
props_list = []
if properties:
for prop_name, prop in properties.items():
props_list.append({prop_name: prop})
capabilities = nodetemplate.get(CAPABILITIES)
caps_list = []
if capabilities:
for cap_name, cap in capabilities.items():
cap_props = cap.get(PROPERTIES, {})
cap_props_list = []
for prop_name, prop in cap_props.items():
cap_props_list.append({prop_name, prop})
caps_list.append({PROPERTIES: cap_props_list})
if properties:
node_filter[PROPERTIES] = props_list
if capabilities:
node_filter[CAPABILITIES] = caps_list
req[k] = dict(
node_filter=node_filter
)
else:
# The case when requirement has parameters.
# Valid keys are ('node', 'node_filter', 'relationship', 'capability', 'occurrences')
# Only node and relationship might be a template name or a type
req_relationship = req[k].get(RELATIONSHIP)
req_node = req[k].get(NODE)
if req_relationship is not None:
_, _, type_name = utils.tosca_type_parse(req_relationship)
if type_name is None:
self.add_template_dependency(node.name, req_relationship)
if req_node is not None:
_, _, type_name = utils.tosca_type_parse(req_node)
if type_name is None:
self.add_template_dependency(node.name, req_node)
def translate(tosca_elements_map_to_provider, topology_template, provider):
"""
Main function of this file, the only which is used outside the file
:param tosca_elements_map_to_provider: dict from provider specific file
tosca_elements_map_to_<provider>.yaml
:param node_templates: input node_templates
:return: list of new node templates and relationship templates and
list of artifacts to be used for generating scripts
"""
node_templates = topology_template.nodetemplates
relationship_templates = topology_template.relationship_templates
element_templates = node_templates + relationship_templates
new_element_templates = {}
artifacts = []
conditions = []
extra = dict()
self = dict()
self[ARTIFACTS] = []
self[EXTRA] = dict()
for element in element_templates:
(namespace, _, _) = utils.tosca_type_parse(element.type)
self[NAME] = element.name
self[KEYNAME] = element.name
self[BUFFER] = {}
if namespace != provider:
restructured_mapping = restructure_mapping(
tosca_elements_map_to_provider, element, self)
restructured_mapping, extra_mappings, new_conditions = restructure_mapping_facts(
restructured_mapping)
restructured_mapping.extend(extra_mappings)
conditions.extend(new_conditions)
tpl_structure = translate_node_from_tosca(restructured_mapping,
element.name, self)
for tpl_name, temp_tpl in tpl_structure.items():
for node_type, tpl in temp_tpl.items():
(_, element_type, _) = utils.tosca_type_parse(node_type)
tpl[TYPE] = node_type
new_element_templates[
element_type] = new_element_templates.get(
element_type, {})
new_element_templates[element_type].update(
{tpl_name: copy.deepcopy(tpl)})
else:
new_element = translate_element_from_provider(element)
new_element_templates = utils.deep_update_dict(
new_element_templates, new_element)
conditions = set(conditions)
return new_element_templates, self[ARTIFACTS], conditions, self[EXTRA]
def translate(service_tmpl):
"""
Main function of this file, the only which is used outside the file
:param tosca_elements_map_to_provider: dict from provider specific file
tosca_elements_map_to_<provider>.yaml
:param node_templates: input node_templates
:return: list of new node templates and relationship templates and
list of artifacts to be used for generating scripts
"""
element_templates = copy.copy(service_tmpl.node_templates)
element_templates.update(copy.copy(service_tmpl.relationship_templates))
new_element_templates = {}
template_mapping = {}
self = dict()
self[ARTIFACTS] = []
self[EXTRA] = dict()
self[PUBLIC_KEY] = service_tmpl.public_key_path
for tmpl_name, element in element_templates.items():
(namespace, _, _) = utils.tosca_type_parse(element[TYPE])
self[NAME] = tmpl_name
self[KEYNAME] = tmpl_name
self[BUFFER] = {}
if namespace != service_tmpl.provider:
restructured_mapping = restructure_mapping(service_tmpl, element, tmpl_name, self)
restructured_mapping = sort_host_ip_parameter(restructured_mapping, service_tmpl.host_ip_parameter)
restructured_mapping = restructure_mapping_buffer(restructured_mapping, self)
restructured_mapping, extra_mappings = restructure_mapping_facts(restructured_mapping, self, service_tmpl.is_delete, service_tmpl.cluster_name)
restructured_mapping.extend(extra_mappings)
restructured_mapping = restructure_get_attribute(restructured_mapping, service_tmpl, self)
tpl_structure = translate_node_from_tosca(restructured_mapping, tmpl_name, self)
for tpl_name, temp_tpl in tpl_structure.items():
for node_type, tpl in temp_tpl.items():
(_, element_type, _) = utils.tosca_type_parse(node_type)
tpl[TYPE] = node_type
if tmpl_name not in template_mapping:
template_mapping[tmpl_name] = {tpl_name}
else:
template_mapping[tmpl_name].add(tpl_name)
new_element_templates[element_type] = new_element_templates.get(element_type, {})
new_element_templates[element_type].update({tpl_name: copy.deepcopy(tpl)})
else:
new_element = translate_element_from_provider(tmpl_name, element)
new_element_templates = utils.deep_update_dict(new_element_templates, new_element)
self_extra = utils.replace_brackets(self[EXTRA], False)
self_artifacts = utils.replace_brackets(self[ARTIFACTS], False)
execute(self_artifacts, service_tmpl.is_delete, service_tmpl.cluster_name)
return new_element_templates, self_extra, template_mapping
def translate_to_provider(self):
new_element_templates, new_extra, template_mapping = translate_to_provider(self)
dict_tpl = {}
self.template_mapping = template_mapping
if new_element_templates.get(NODES):
dict_tpl[NODE_TEMPLATES] = new_element_templates[NODES]
if new_element_templates.get(RELATIONSHIPS):
dict_tpl[RELATIONSHIP_TEMPLATES] = new_element_templates[RELATIONSHIPS]
if new_element_templates.get(OUTPUTS):
dict_tpl[OUTPUTS] = new_element_templates[OUTPUTS]
if self.inputs:
dict_tpl[INPUTS] = self.inputs
rel_types = {}
for k, v in self.definitions.items():
(_, element_type, _) = utils.tosca_type_parse(k)
if element_type == RELATIONSHIPS:
rel_types[k] = v
logging.debug("TOSCA template with non normative types for provider %s was generated: \n%s"
% (self.provider, yaml.dump(dict_tpl)))
try:
topology_tpl = TopologyTemplate(dict_tpl, self.definitions, rel_types=rel_types)
except:
logging.exception("Failed to parse intermidiate non-normative TOSCA template with OpenStack tosca-parser")
sys.exit(1)
self.extra_configuration_tool_params = utils.deep_update_dict(self.extra_configuration_tool_params, new_extra)
self.node_templates = new_element_templates.get(NODES, {})
self.relationship_templates = new_element_templates.get(RELATIONSHIPS, {})
self.outputs = new_element_templates.get(OUTPUTS, {})
def resolve_in_template_dependencies(self):
"""
TODO think through the logic to replace mentions by id
Changes all mentions of node_templates by name in requirements, places dictionary with node_filter instead
:return:
"""
for node_name, node in self.node_templates.items():
for req in node.get(REQUIREMENTS, []):
for req_name, req_body in req.items():
# Valid keys are ('node', 'node_filter', 'relationship', 'capability', 'occurrences')
# Only node and relationship might be a template name or a type
req_relationship = req_body.get(RELATIONSHIP)
req_node = req_body.get(NODE)
if req_relationship is not None:
(_, _, type_name) = utils.tosca_type_parse(req_relationship)
if type_name is None:
self.add_template_dependency(node_name, req_relationship)
self._relation_target_source[req_relationship] = {
'source': node_name,
'target': req_node
}
if req_node is not None:
(_, _, type_name) = utils.tosca_type_parse(req_node)
if type_name is None:
self.add_template_dependency(node_name, req_node)
node_types_from_requirements = set()
req_definitions = self.definitions[node[TYPE]].get(REQUIREMENTS, [])
for req in req_definitions:
for req_name, req_def in req.items():
if req_def.get(NODE, None) is not None:
if req_def[NODE] != node[TYPE]:
node_types_from_requirements.add(req_def[NODE])
for req_node_name, req_node_tmpl in self.node_templates.items():
if req_node_tmpl[TYPE] in node_types_from_requirements:
self.add_template_dependency(node_name, req_node_name)
# Search in all nodes for get function mentions and get its target name
for node_name, node_tmpl in self.node_templates.items():
self.search_get_function(node_name, node_tmpl)
for rel_name, rel_tmpl in self.relationship_templates.items():
self.search_get_function(rel_name, rel_tmpl)
def translate_element_from_provider(node):
(_, element_type, _) = utils.tosca_type_parse(node.type)
node_templates = {
element_type: {
node.name: copy.deepcopy(node.entity_tpl)
}
}
return node_templates
def translate_element_from_provider(tmpl_name, node_tmpl):
(_, element_type, _) = utils.tosca_type_parse(node_tmpl[TYPE])
node_templates = {
element_type: {
tmpl_name: copy.deepcopy(node_tmpl)
}
}
return node_templates
def compute_node_priorities(self, node_type_definitions):
"""
Use node type definitions to count priority of the node_type
:param node_type_definitions: dict of node type definitions
:return:
"""
ProviderResource.MAX_NUM_PRIORITIES = 1
node_priorities_by_type = {}
for node_type_name, node_type_def in node_type_definitions.items():
(namespace, element_type, type_short) = utils.tosca_type_parse(node_type_name)
if type_short != ROOT and element_type == NODES and namespace != TOSCA:
node_priorities_by_type[node_type_name] = self.get_node_type_priority(node_type_definitions,
node_type_name)
return node_priorities_by_type
def _provider_nodes(self):
"""
Create a list of ProviderResource classes to represent a node in TOSCA
:return: list of class objects inherited from ProviderResource
"""
provider_nodes = list()
for node in self.node_templates:
(namespace, category, type_name) = utils.tosca_type_parse(node.type)
is_software_component = node.name in self.software_component_names
if namespace != self.provider and not is_software_component or category != NODES:
ExceptionCollector.appendException(Exception('Unexpected values'))
provider_node_instance = ProviderResource(self.provider, node, self.relationship_templates,
is_software_component)
provider_nodes.append(provider_node_instance)
return provider_nodes
def init_queue(self, nodes_relationships_queue):
elements_queue = []
software_queue = []
for v in nodes_relationships_queue:
self.gather_global_operations(v)
for op_name, op in self.global_operations_info.items():
self.global_operations_info[op_name] = self.replace_all_get_functions(op)
for v in nodes_relationships_queue:
(_, element_type, _) = utils.tosca_type_parse(v.type)
if element_type == NODES:
new_conf_args = self.replace_all_get_functions(v.configuration_args)
v.configuration_args = new_conf_args
if v.is_software_component:
software_queue.append(v)
else:
elements_queue.append(v)
return elements_queue, software_queue
def __init__(self, requirement_definitions, provider):
"""
Get the requirements of type list from requirement definitions
:param requirement_definitions: list of requirement definitions with name added
"""
self.provider = provider
self.requirement_definitions = dict()
for req in requirement_definitions:
self.requirement_definitions[req[NAME]] = req
# self.requirement_definitions = requirement_definitions
self.requirement_names_of_type_list = set()
self._node_name_by_requirement_name = dict()
# NOTE generate the dictionary, where the keys are the name of requirement and
# the values are the node_types of requirement
for req_name, req_def in self.requirement_definitions.items():
# req_name = req_def[NAME]
req_node = req_def.get(NODE)
if req_node:
(_, _, type_name) = utils.tosca_type_parse(req_node)
node_name = utils.snake_case(type_name)
if node_name == 'root':
continue
temp_req_val = self._node_name_by_requirement_name.get(req_name)
if temp_req_val is None:
temp_req_val = node_name
elif isinstance(temp_req_val, str):
temp_req_val = (temp_req_val, node_name)
else:
temp_req_val = temp_req_val + (node_name)
self._node_name_by_requirement_name[req_name] = temp_req_val
# NOTE set the list required requirements and the list of multiple requirements (of type list)
self.required_requirement_keys = set()
for req_name, req_def in self.requirement_definitions.items():
occurrences = req_def.get(OCCURRENCES, [0, 'UNBOUNDED']) # list
min_ocs = occurrences[0]
max_ocs = occurrences[1]
if int(min_ocs) > 0:
self.required_requirement_keys.add(req_name)
if str(max_ocs) == 'UNBOUNDED':
self.requirement_names_of_type_list.add(req_name)
elif int(max_ocs) > 1:
self.requirement_names_of_type_list.add(req_name)
def _provider_nodes(self):
"""
Create a list of ProviderResource classes to represent a node in TOSCA
:return: list of class objects inherited from ProviderResource
"""
provider_nodes = dict()
for node_name, node in self.node_templates.items():
(namespace, category, type_name) = utils.tosca_type_parse(node[TYPE])
is_software_component = node[TYPE] in self.software_types
if namespace != self.provider and not is_software_component or category != NODES:
logging.error('Unexpected values: node \'%s\' not a software component and has a provider \'%s\'. '
'Node will be ignored' % (node.name, namespace))
else:
provider_node_instance = ProviderResource(self.provider, self.is_delete, self.cluster_name, self.configuration_tool, node,
node_name,
self.host_ip_parameter, self.definitions[node[TYPE]],
is_software_component=is_software_component)
provider_nodes[node_name] = provider_node_instance
return provider_nodes
def translate_node_from_tosca(restructured_mapping, tpl_name, self):
"""
Translator from TOSCA definitions in provider definitions using rules from element_map_to_provider
:param restructured_mapping: list of dicts(parameter, map, value)
:param tpl_name: str
:return: entity_tpl as dict
"""
resulted_structure = {}
for item in restructured_mapping:
ExceptionCollector.start()
self[PARAMETER] = item[PARAMETER]
self[VALUE] = item[VALUE]
mapped_param = restructure_value(mapping_value=item[MAP_KEY],
self=self)
ExceptionCollector.stop()
if ExceptionCollector.exceptionsCaught():
raise ValidationError(
message='\nTranslating to provider failed: '.join(
ExceptionCollector.getExceptionsReport()))
structures, keyname = get_structure_of_mapped_param(
mapped_param, item[VALUE])
for structure in structures:
r = retrieve_node_templates(structure)
for node_type, tpl in r.items():
if not keyname:
(_, _, type_name) = utils.tosca_type_parse(node_type)
if not type_name:
ExceptionCollector.appendException()
keyname = self[KEYNAME] + "_" + utils.snake_case(type_name)
node_tpl_with_name = {keyname: {node_type: tpl}}
resulted_structure = utils.deep_update_dict(
resulted_structure, node_tpl_with_name)
for keyname, node in resulted_structure.items():
for node_type, tpl in node.items():
if tpl.get(REQUIREMENTS):
reqs = []
for req_name, req in tpl[REQUIREMENTS].items():
reqs.append({req_name: req})
resulted_structure[keyname][node_type][REQUIREMENTS] = reqs
return resulted_structure
def init_graph(self, operations_graph):
ts = TopologicalSorter(operations_graph)
elements_queue = [*ts.static_order()]
for v in elements_queue:
self.gather_global_operations(v)
for op_name, op in self.global_operations_info.items():
self.global_operations_info[
op_name] = self.replace_all_get_functions(op)
for v in elements_queue:
(_, element_type, _) = utils.tosca_type_parse(v.type)
if element_type == NODES or element_type == RELATIONSHIPS:
new_conf_args = self.replace_all_get_functions(
v.configuration_args)
v.configuration_args = new_conf_args
else:
del operations_graph[v]
for key in operations_graph:
if v in operations_graph[key]:
operations_graph[key].remove(v)
return operations_graph
def translate_to_provider(self):
new_element_templates, new_artifacts, conditions_set, new_extra = translate_to_provider(
self.tosca_elements_map_to_provider(), self.tosca_topology_template, self.provider)
self.used_conditions_set = conditions_set
dict_tpl = copy.deepcopy(self.tosca_topology_template.tpl)
if new_element_templates.get(NODES):
dict_tpl[NODE_TEMPLATES] = new_element_templates[NODES]
if new_element_templates.get(RELATIONSHIPS):
dict_tpl[RELATIONSHIP_TEMPLATES] = new_element_templates[RELATIONSHIPS]
rel_types = []
for k, v in self.provider_defs.items():
(_, element_type, _) = utils.tosca_type_parse(k)
if element_type == RELATIONSHIP_TYPES:
rel_types.append(v)
topology_tpl = TopologyTemplate(dict_tpl, self.full_provider_defs, rel_types)
self.artifacts.extend(new_artifacts)
self.extra_configuration_tool_params = utils.deep_update_dict(self.extra_configuration_tool_params, new_extra)
return topology_tpl
def _get_full_defintion(self, definition, def_type, ready_set):
if def_type in ready_set:
return definition, def_type in self.software_types
(_, _, def_type_short) = utils.tosca_type_parse(def_type)
is_software_type = def_type_short == 'SoftwareComponent'
is_software_parent = False
parent_def_name = definition.get(DERIVED_FROM, None)
if parent_def_name is not None:
if def_type == parent_def_name:
logging.critical("Invalid type \'%s\' is derived from itself" % def_type)
sys.exit(1)
if parent_def_name in ready_set:
parent_definition = self.definitions[parent_def_name]
is_software_parent = parent_def_name in self.software_types
else:
parent_definition, is_software_parent = \
self._get_full_defintion(self.definitions[parent_def_name], parent_def_name, ready_set)
parent_definition = copy.deepcopy(parent_definition)
definition = utils.deep_update_dict(parent_definition, definition)
if is_software_type or is_software_parent:
self.software_types.add(def_type)
ready_set.add(def_type)
return definition, def_type in self.software_types
def sort_nodes_and_operations_by_graph_dependency(self):
"""
This method generates dict fith ProviderTemplates with operation, sorted by
dependencies from normative and provider TOSCA templates
"""
nodes = set(self.provider_nodes.keys())
nodes = nodes.union(set(self.provider_relations.keys()))
dependencies = {}
lifecycle = ['configure', 'start', 'stop', 'delete']
reversed_full_lifecycle = lifecycle[::-1] + ['create']
# generate only dependencies from nodes
for templ_name in nodes:
set_intersection = nodes.intersection(self.template_dependencies.get(templ_name, set()))
templ = self.provider_nodes.get(templ_name, self.provider_relations.get(templ_name))
(_, element_type, _) = utils.tosca_type_parse(templ.type)
if element_type == NODES:
if 'interfaces' in templ.tmpl and 'Standard' in templ.tmpl['interfaces']:
new_operations = ['create']
# operation create always exists
for elem in lifecycle:
if elem in templ.tmpl['interfaces']['Standard']:
new_operations.append(elem)
# if there is any other operations - add ti new_operations and translate to dict
# in format {node.op: {node1, node2}}
# node requieres node1 and node2
if len(new_operations) == 1:
utils.deep_update_dict(dependencies, {templ_name + SEPARATOR + 'create': set_intersection})
else:
for i in range(1, len(new_operations)):
utils.deep_update_dict(dependencies, {
templ_name + SEPARATOR + new_operations[i]: {
templ_name + SEPARATOR + new_operations[i - 1]}})
utils.deep_update_dict(dependencies,
{templ_name + SEPARATOR + new_operations[0]: set_intersection})
else:
utils.deep_update_dict(dependencies, {templ_name + SEPARATOR + 'create': set_intersection})
new_normative_graph = {}
# getting dependencies for create operaions of nodes, translated from 1 normative node
for key, value in self.normative_nodes_graph.items():
for elem in value:
for op in reversed_full_lifecycle:
new_oper = elem + SEPARATOR + op
if new_oper in dependencies:
if key + SEPARATOR + 'create' in new_normative_graph:
new_normative_graph[key + SEPARATOR + 'create'].add(new_oper)
else:
new_normative_graph[key + SEPARATOR + 'create'] = {new_oper}
break
else:
logging.error("Operation create not found")
sys.exit(1)
# update dependencies
dependencies = utils.deep_update_dict(dependencies, new_normative_graph)
new_dependencies = {}
# new_dependencies is needed for updating set operations
# dict must be in format {node.op: {node1, node2}}
for key, value in dependencies.items():
new_set = set()
for elem in value:
for oper in reversed_full_lifecycle:
if elem + SEPARATOR + oper in dependencies:
new_set.add(elem + SEPARATOR + oper)
break
elif elem in dependencies:
new_set.add(elem)
break
new_dependencies[key] = new_set
# adding relationships operations pre_configure_source after create source node
# pre_configure_target after create target node
# add_source in parallel with pre_configure_source but in will be executed on target
# post_configure_target after configure target node (if not configure then create - in parallel
# with pre_configure_target)
# post_configure_source after configure target node (if not configure then create - in parallel
# with pre_configure_source)
# other - not supported!
for templ_name in nodes:
templ = self.provider_nodes.get(templ_name, self.provider_relations.get(templ_name))
(_, element_type, _) = utils.tosca_type_parse(templ.type)
if element_type == RELATIONSHIPS:
if 'interfaces' in templ.tmpl and 'Configure' in templ.tmpl['interfaces']:
if 'pre_configure_source' in templ.tmpl['interfaces']['Configure']:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.source,
'pre_configure_source', 'create', ['add_source'])
if 'pre_configure_target' in templ.tmpl['interfaces']['Configure']:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.target,
'pre_configure_target', 'create')
if 'post_configure_source' in templ.tmpl['interfaces']['Configure']:
if templ.source + SEPARATOR + 'configure' in new_dependencies:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.source,
'post_configure_source', 'configure')
else:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.source,
'post_configure_source', 'create')
if 'post_configure_target' in templ.tmpl['interfaces']['Configure']:
if templ.target + SEPARATOR + 'configure' in new_dependencies:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.target,
'post_configure_target', 'configure')
else:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.target,
'post_configure_target', 'create')
if 'add_source' in templ.tmpl['interfaces']['Configure']:
new_dependencies = self.update_relationships(new_dependencies, templ.name, templ.source,
'add_source', 'create', ['pre_configure_source'])
if 'add_target' in templ.tmpl['interfaces']['Configure']:
logging.warning('Operation add_target not supported, it will be skipped')
if 'target_changed' in templ.tmpl['interfaces']['Configure']:
logging.warning('Operation target_changed not supported, it will be skipped')
if 'remove_target' in templ.tmpl['interfaces']['Configure']:
logging.warning('Operation remove_target not supported, it will be skipped')
# mapping strings 'node.op' to provider template of this node with this operation
templ_mappling = {}
for elem in new_dependencies:
templ_name = elem.split(SEPARATOR)[0]
templ = copy.deepcopy(self.provider_nodes.get(templ_name, self.provider_relations.get(templ_name)))
templ.operation = elem.split(SEPARATOR)[1]
templ_mappling[elem] = templ
templ_dependencies = {}
reversed_templ_dependencies = {}
# create dict where all elements will be replaced with provider template from templ_mappling
# reversed_templ_dependencies needed for delete - it just a reversed version of graph
for key, value in new_dependencies.items():
new_set = set()
for elem in value:
new_set.add(templ_mappling[elem])
if templ_mappling[elem] not in reversed_templ_dependencies:
reversed_templ_dependencies[templ_mappling[elem]] = {templ_mappling[key]}
elif templ_mappling[key] not in reversed_templ_dependencies[templ_mappling[elem]]:
reversed_templ_dependencies[templ_mappling[elem]].add(templ_mappling[key])
templ_dependencies[templ_mappling[key]] = new_set
return templ_dependencies, reversed_templ_dependencies
def get_keyname_from_type(keyname, node_type):
(_, _, type_name) = utils.tosca_type_parse(node_type)
if not type_name:
logging.critical("Unable to parse type name: %s" % json.dumps(node_type))
sys.exit(1)
return keyname + "_" + utils.snake_case(type_name)
def __init__(self,
provider,
is_delete,
cluster_name,
configuration_tool,
tmpl,
node_name,
host_ip_parameter,
node_type,
is_software_component=False,
is_relationship=False,
relation_target_source=dict()):
"""
:param provider:
:param tmpl:
:param node_name:
:param node_type:
:param is_software_component:
:param is_relationship:
"""
self.provider = provider
self.tmpl = tmpl
self.cluster_name = cluster_name
self.name = node_name
self.type = tmpl[TYPE]
(_, _, type_name) = utils.tosca_type_parse(self.type)
self.type_name = type_name
self.type_definition = node_type
self.is_software_component = is_software_component
self.host = None
self.self = node_name
self.target = None
self.source = None
self.operation = None
self.is_delete = is_delete
self.set_defaults()
# NOTE: Get the parameters from template using provider definition
self.configuration_args = dict()
# NOTE: node is NodeTemplate instance
for key in self.type_definition.get(PROPERTIES, {}).keys():
value = self.tmpl.get(PROPERTIES, {}).get(key, None)
if value is not None:
self.configuration_args[key] = value
# NOTE: filling the parameters from openstack definition to parse from input template
if is_relationship:
self.source = relation_target_source.get(self.name,
{}).get('source')
self.target = relation_target_source.get(self.name,
{}).get('target')
else:
capability_defs = self.type_definition.get(CAPABILITIES, {})
for cap_key, cap_def in capability_defs.items():
properties = self.tmpl.get(CAPABILITIES, {}).get(cap_key)
definition_property_keys = cap_def.get(PROPERTIES, {}).keys()
if properties:
for def_prop_key in definition_property_keys:
value = properties.get_property_value(def_prop_key)
if value:
self.configuration_args[def_prop_key] = value
for key, value in self.tmpl.get(ARTIFACTS, []):
self.configuration_args[key] = value
provider_requirements = ProviderRequirements(
self.requirement_definitions, self.provider)
self.requirements = provider_requirements.get_requirements(tmpl)
for req_name, reqs in self.requirements.items():
if isinstance(reqs, list):
iter_reqs = reqs
else:
iter_reqs = [reqs]
for req in iter_reqs:
relationship = req.definition[RELATIONSHIP]
(_, _, type_name) = utils.tosca_type_parse(relationship)
if type_name == 'HostedOn':
if self.host is not None:
logging.critical(
"Node \'\' can be hosted only on one node" %
self.name)
sys.exit(1)
if host_ip_parameter not in ('public_address',
'private_address'):
host_ip_parameter = 'private_address'
self.host = req.value + '_' + host_ip_parameter
self.node_filter_artifacts = []
for key, req in self.requirements.items():
if type(req) is list:
self.configuration_args[key] = list(v.get_value()
for v in req)
else:
self.configuration_args[key] = req.get_value()
if configuration_tool == 'ansible':
provider_config = ProviderConfiguration(provider)
node_filter_config = provider_config.get_subsection(
ANSIBLE, NODE_FILTER)
if not node_filter_config:
node_filter_config = {}
node_filter_source_prefix = node_filter_config.get(
'node_filter_source_prefix', '')
node_filter_source_postfix = node_filter_config.get(
'node_filter_source_postfix', '')
node_filter_exceptions = node_filter_config.get(
'node_filter_exceptions', '')
node_filter_inner_variable = node_filter_config.get(
'node_filter_inner_variable')
for arg_key, arg in self.configuration_args.items():
if isinstance(arg, dict):
node_filter_key = arg.get(SOURCE, {}).get(NODE_FILTER)
node_filter_value = arg.get(VALUE)
node_filter_params = arg.get(PARAMETERS)
if node_filter_key and node_filter_value and node_filter_params:
node_filter_source = node_filter_source_prefix + node_filter_key + node_filter_source_postfix
if node_filter_exceptions.get(node_filter_key):
node_filter_source = node_filter_exceptions[
node_filter_key]
NODE_FILTER_FACTS = 'node_filter_facts'
NODE_FILTER_FACTS_REGISTER = NODE_FILTER_FACTS + '_raw'
NODE_FILTER_FACTS_VALUE = NODE_FILTER_FACTS_REGISTER
if node_filter_inner_variable:
if isinstance(node_filter_inner_variable,
dict):
node_filter_inner_variable = node_filter_inner_variable.get(
node_filter_key, '')
if isinstance(node_filter_inner_variable,
six.string_types):
node_filter_inner_variable = [
node_filter_inner_variable
]
if isinstance(node_filter_inner_variable,
list):
for v in node_filter_inner_variable:
NODE_FILTER_FACTS_VALUE += '[\"' + v + '\"]'
else:
logging.error(
"Provider configuration parameter "
"\'ansible.node_filter: node_filter_inner_variable\' is missing "
"or has unsupported value \'%s\'" %
node_filter_inner_variable)
sys.exit(1)
tmp_ansible_tasks = [{
SOURCE: node_filter_source,
VALUE: NODE_FILTER_FACTS_REGISTER,
EXECUTOR: configuration_tool,
PARAMETERS: {}
}, {
SOURCE: SET_FACT_SOURCE,
PARAMETERS: {
"target_objects":
"\\{\\{ " + NODE_FILTER_FACTS_VALUE +
" \\}\\}"
},
VALUE: "tmp_value",
EXECUTOR: configuration_tool
}, {
SOURCE: 'debug',
PARAMETERS: {
'var': 'target_objects'
},
VALUE: "tmp_value",
EXECUTOR: configuration_tool
}, {
SOURCE: SET_FACT_SOURCE,
PARAMETERS: {
"input_facts": '{{ target_objects }}'
},
EXECUTOR: configuration_tool,
VALUE: "tmp_value"
}, {
SOURCE: SET_FACT_SOURCE,
PARAMETERS: {
"input_args": node_filter_params
},
EXECUTOR: configuration_tool,
VALUE: "tmp_value"
}, {
SOURCE: IMPORT_TASKS_MODULE,
PARAMETERS: "artifacts/equals.yaml",
EXECUTOR: configuration_tool,
VALUE: "tmp_value"
}, {
SOURCE: SET_FACT_SOURCE,
PARAMETERS: {
node_filter_value:
"\{\{ matched_object[\"" +
node_filter_value + "\"] \}\}"
},
VALUE: "tmp_value",
EXECUTOR: configuration_tool
}]
arg = str(
execute(tmp_ansible_tasks, self.is_delete,
self.cluster_name, node_filter_value))
self.configuration_args[arg_key] = arg
def get_ansible_tasks_from_interface(self,
element_object,
target_directory,
is_delete,
operation,
cluster_name,
additional_args=None):
if additional_args is None:
additional_args = {}
else:
additional_args_global = copy.deepcopy(
additional_args.get('global', {}))
additional_args_element = copy.deepcopy(
additional_args.get(element_object.name, {}))
additional_args = utils.deep_update_dict(additional_args_global,
additional_args_element)
ansible_tasks = []
scripts = []
primary = False
for interface_name, interface in self.get_interfaces_from_node(
element_object).items():
interface_operation = interface.get(operation, {})
if isinstance(interface_operation, six.string_types):
implementations = interface_operation
primary = True
else:
implementations = interface_operation.get(IMPLEMENTATION)
(_, element_type, _) = utils.tosca_type_parse(element_object.type)
if (interface_name == 'Standard' and element_type == NODES
or interface_name == 'Configure' and element_type
== RELATIONSHIPS) and implementations is not None:
if isinstance(implementations, six.string_types):
implementations = [implementations]
if isinstance(
implementations,
dict) and 'primary' in implementations and isinstance(
implementations['primary'], six.string_types):
implementations = [implementations['primary']]
primary = True
scripts.extend(implementations)
for script in implementations:
target_filename = os.path.join(utils.get_tmp_clouni_dir(),
cluster_name,
target_directory, script)
os.makedirs(os.path.dirname(target_filename),
exist_ok=True)
script_filename_1 = os.path.join(os.getcwd(), script)
script_filename_2 = os.path.join(
self.get_ansible_artifacts_directory(), script)
if os.path.isfile(script_filename_1):
copyfile(script_filename_1, target_filename)
elif os.path.isfile(script_filename_2):
copyfile(script_filename_2, target_filename)
else:
logging.error(
"Artifact filename %s was not found in %s or %s" %
(script, script_filename_1, script_filename_2))
if not primary and interface_operation.get(
INPUTS) is not None:
for input_name, input_value in interface_operation[
INPUTS].items():
ansible_tasks.append(
{SET_FACT: {
input_name: input_value
}})
new_ansible_task = {IMPORT_TASKS_MODULE: target_filename}
new_ansible_task.update(additional_args)
ansible_tasks.append(new_ansible_task)
return ansible_tasks
def __init__(self, provider, node, relationship_templates, is_software_component=False):
"""
:param node: class NodeTemplate from toscaparser
:param relationship_templates: list of RelationshipTemplate from toscaparser
"""
# NOTE: added as a parameter in toscatranslator.providers.common.tosca_template:ProviderToscaTemplate
self.provider = provider
self.nodetemplate = node
self.name = node.name
self.type = node.type
(_, _, type_name) = utils.tosca_type_parse(self.type)
self.type_name = type_name
# NOTE: filling the parameters from openstack definition to parse from input template
node_type = node.type_definition # toscaparser.elements.nodetype.NodeType
self.definitions_by_name = None
self.requirement_definitions = None
self.set_definitions_by_name(node_type)
self.attribute_keys = list(self.attribute_definitions_by_name().keys())
self.property_keys = list(self.property_definitions_by_name().keys())
self.requirement_keys = list(self.requirement_definitions_by_name().keys())
self.artifact_keys = list(self.artifact_definitions_by_name().keys())
self.capability_keys = list(self.capability_definitions_by_name().keys())
self.relationship_templates = list()
self.dependency_order = 0
self.is_software_component = is_software_component
self.host = None
# NOTE: Get the parameters from template using provider definition
self.configuration_args = dict()
# NOTE: node is NodeTemplate instance
for key in self.property_keys:
value = node.get_property_value(key)
if value is not None:
self.configuration_args[key] = value
for key in self.attribute_keys:
value = node.entity_tpl.get(ATTRIBUTES, {}).get(key)
if value is not None:
self.configuration_args[key] = value
capability_defs = self.capability_definitions_by_name()
for cap_key, cap_def in capability_defs.items():
properties = node.get_capabilities().get(cap_key)
definition_property_keys = cap_def.get(PROPERTIES, {}).keys()
if properties:
for def_prop_key in definition_property_keys:
value = properties.get_property_value(def_prop_key)
if value:
self.configuration_args[def_prop_key] = value
if hasattr(node, ARTIFACTS):
# TODO: oneliner
for key, value in node.artifacts:
self.configuration_args[key] = value
relationship_template_names = set()
provider_requirements = ProviderRequirements(self.requirement_definitions, self.provider)
self.requirements = provider_requirements.get_requirements(node)
if self.requirements.get('host', None) != None:
self.host = self.requirements['host'][0].value
self.node_filter_artifacts = []
for key, req in self.requirements.items():
if type(req) is list:
self.configuration_args[key] = list(v.get_value() for v in req)
temp_req = req
else:
self.configuration_args[key] = req.get_value()
temp_req = [req]
for v in temp_req:
relation = v.relationship
if relation is not None:
_, _, type_name = utils.tosca_type_parse(relation)
if type_name is None:
relationship_template_names.add(relation)
for relation in relationship_templates:
if relation.name in relationship_template_names:
self.relationship_templates.append(relation)
self.node_priority = self.compute_node_priorities(node.custom_def)
def gather_global_operations(self, element_object):
"""
:param element_object:
:return:
"""
interfaces = []
element_template_name = None
(_, element_type, _) = utils.tosca_type_parse(element_object.type)
if element_type == NODES:
interfaces = self.get_interfaces_from_node(element_object)
element_template_name = element_object.name
op_required = self.list_get_operation_outputs(element_object.tmpl)
self.manage_operation_output(op_required, element_template_name)
elif element_type == RELATIONSHIPS:
# NOTE interfaces can't be used as it contains the error ()
interfaces = self.get_interfaces_from_relationship(element_object)
element_template_name = element_object.name
if not element_template_name:
return
operations = {}
for interface_name, ops in interfaces.items():
for operation_name, operation_data in ops.items():
operations['_'.join([interface_name.lower(),
operation_name])] = operation_data
# Sort operations by dependency
prev_len = len(operations) + 1
required_operations = {}
for op_name, op in operations.items():
if isinstance(op, six.string_types):
operations[op_name] = {IMPLEMENTATION: op}
op_required = self.list_get_operation_outputs(op)
required_operations[op_name] = op_required
self.manage_operation_output(op_required, element_template_name)
while len(operations) > 0 and prev_len > len(operations):
ops_for_iter = copy.deepcopy(operations)
prev_len = len(operations)
for op_name, op in ops_for_iter.items():
op_required = required_operations[op_name]
if_executable_now = True
for i in op_required:
if i[0] == SELF:
i[0] = element_template_name
temp_op_name = '_'.join(i[:3]).lower()
if temp_op_name not in self.global_operations_queue:
if_executable_now = False
break
if if_executable_now:
temp_op_name = '_'.join([element_template_name,
op_name]).lower()
self.global_operations_queue.append(temp_op_name)
updating_op_info = {temp_op_name: op}
utils.deep_update_dict(self.global_operations_info,
updating_op_info)
operations.pop(op_name)
if len(operations) > 0:
logging.critical(
"Resolving dependencies in template failed on element \'%s\'" %
element_template_name)
sys.exit(1)
def to_dsl(self,
provider,
operations_graph,
reversed_operations_graph,
cluster_name,
is_delete,
artifacts=None,
target_directory=None,
inputs=None,
outputs=None,
extra=None,
debug=False):
if artifacts is None:
artifacts = []
if target_directory is None:
target_directory = self.initial_artifacts_directory
self.artifacts = {}
for art in artifacts:
self.artifacts[art[NAME]] = art
provider_config = ProviderConfiguration(provider)
ansible_config = provider_config.get_section(ANSIBLE)
node_filter_config = provider_config.get_subsection(
ANSIBLE, NODE_FILTER)
ids_file_path = os.getcwd(
) + '/id_vars_' + cluster_name + self.get_artifact_extension()
self.init_global_variables(inputs)
operations_graph = self.init_graph(operations_graph)
# the graph of operations at the moment is a dictionary of copies of ProviderTemplatre objects,
# of the form Node/Relationship: {the set of opers of Nodes/Relationships on which it depends}
elements = TopologicalSorter(operations_graph)
# use TopologicalSorter for creating graph
if is_delete:
reversed_operations_graph = self.init_graph(
reversed_operations_graph)
elements = TopologicalSorter(reversed_operations_graph)
elements.prepare()
# first operations from on top of the graph in state 'ready'
ansible_playbook = []
if not debug:
self.prepare_for_run()
# function for initializing tmp clouni directory
q = Queue()
# queue for node names + operations
active = []
# list of parallel active operations
first = True
while elements.is_active():
node_name = None
# try to get new finished operation from queue and find it in list of active
# if get - mark done this operation (but before it find in graph)
# if ready operations exists - get it and execute, remove from active
try:
node_name = q.get_nowait()
except:
time.sleep(1)
if node_name is not None:
for node in active:
if node.name == node_name.split(
SEPARATOR
)[0] and node.operation == node_name.split(SEPARATOR)[1]:
active.remove(node)
elements.done(node)
for v in elements.get_ready():
# in delete mode we skip all operations exept delete and create operation transforms to delete
if is_delete:
if v.operation == 'create':
v.operation = 'delete'
else:
elements.done(v)
continue
logging.debug("Creating ansible play from operation: %s" %
v.name + ':' + v.operation)
extra_tasks_for_delete = self.get_extra_tasks_for_delete(
v.type, v.name.replace('-', '_'), ids_file_path)
description_prefix, module_prefix = self.get_module_prefixes(
is_delete, ansible_config)
description_by_type = self.ansible_description_by_type(
v.type_name, description_prefix)
module_by_type = self.ansible_module_by_type(
v.type_name, module_prefix)
ansible_play_for_elem = dict(name=description_prefix + ' ' +
provider + ' cluster: ' + v.name +
':' + v.operation,
hosts=self.default_host,
tasks=[])
# reload id_vars file
if not is_delete and first:
first = False
ansible_play_for_elem['tasks'].append(
copy.deepcopy(
{FILE: {
PATH: ids_file_path,
STATE: 'absent'
}}))
ansible_play_for_elem['tasks'].append(
copy.deepcopy(
{FILE: {
PATH: ids_file_path,
STATE: 'touch'
}}))
# create playbook for every operation
if v.operation == 'delete':
if not v.is_software_component:
ansible_play_for_elem['tasks'].append(
copy.deepcopy({'include_vars': ids_file_path}))
ansible_tasks = self.get_ansible_tasks_for_delete(
v,
description_by_type,
module_by_type,
additional_args=extra)
ansible_tasks.extend(
self.get_ansible_tasks_from_interface(
v,
target_directory,
is_delete,
v.operation,
cluster_name,
additional_args=extra))
if not any(item == module_by_type
for item in ansible_config.get(
'modules_skipping_delete', [])):
ansible_play_for_elem['tasks'].extend(
copy.deepcopy(ansible_tasks))
elif v.operation == 'create':
if not v.is_software_component:
ansible_play_for_elem['tasks'].extend(
copy.deepcopy(
self.get_ansible_tasks_for_inputs(inputs)))
ansible_tasks = self.get_ansible_tasks_for_create(
v,
target_directory,
node_filter_config,
description_by_type,
module_by_type,
additional_args=extra)
ansible_tasks.extend(
self.get_ansible_tasks_from_interface(
v,
target_directory,
is_delete,
v.operation,
cluster_name,
additional_args=extra))
ansible_play_for_elem['tasks'].extend(
copy.deepcopy(ansible_tasks))
ansible_play_for_elem['tasks'].extend(
copy.deepcopy(extra_tasks_for_delete))
else:
ansible_play_for_elem['hosts'] = v.host
ansible_play_for_elem['tasks'].extend(
copy.deepcopy(
self.get_ansible_tasks_from_interface(
v,
target_directory,
is_delete,
v.operation,
cluster_name,
additional_args=extra)))
else:
(_, element_type, _) = utils.tosca_type_parse(v.type)
if element_type == NODES:
if v.is_software_component:
ansible_play_for_elem['hosts'] = v.host
# operations for relationships executes on target/source host depends on operation
elif element_type == RELATIONSHIPS:
if v.operation == 'pre_configure_target' or v.operation == 'post_configure_target' or v.operation == 'add_source':
for elem in operations_graph:
if elem.name == v.target:
if elem.is_software_component:
ansible_play_for_elem[
'hosts'] = elem.host
break
elif v.operation == 'pre_configure_source' or v.operation == 'post_configure_source':
for elem in operations_graph:
if elem.name == v.source:
if elem.is_software_component:
ansible_play_for_elem[
'hosts'] = elem.host
break
else:
logging.error(
"Unsupported operation for relationship in operation graph"
)
sys.exit(1)
else:
logging.error(
"Unsupported element type in operation graph")
sys.exit(1)
ansible_play_for_elem['tasks'].extend(
copy.deepcopy(
self.get_ansible_tasks_from_interface(
v,
target_directory,
is_delete,
v.operation,
cluster_name,
additional_args=extra)))
ansible_playbook.append(ansible_play_for_elem)
# run playbooks
if not debug:
self.parallel_run([ansible_play_for_elem], v.name,
v.operation, q, cluster_name)
# add element to active list
active.append(v)
else:
elements.done(v)
if is_delete:
last_play = dict(name='Renew id_vars_example.yaml',
hosts=self.default_host,
tasks=[])
last_play['tasks'].append(
copy.deepcopy({FILE: {
PATH: ids_file_path,
STATE: 'absent'
}}))
if not debug:
self.parallel_run([last_play], None, None, q, cluster_name)
done = q.get()
if done != 'Done':
logging.error("Something wrong with multiprocessing queue")
sys.exit(1)
ansible_playbook.append(last_play)
# delete dir with cluster_name in tmp clouni dir
if not debug:
rmtree(os.path.join(utils.get_tmp_clouni_dir(), cluster_name))
return yaml.dump(ansible_playbook,
default_flow_style=False,
sort_keys=False)
