def clean_domain_from_dov(self, domain): """ Clean domain by removing initiated NFs and flowrules related to BiSBiS nodes of the given domain :param domain: domain name :type domain: str :return: updated Dov :rtype: :class:`NFFG` """ if self.__global_nffg.is_empty(): log.debug("DoV is empty! Skip cleanup domain: %s" % domain) return self.__global_nffg if self.__global_nffg.is_bare(): log.debug("No initiated service has been detected in DoV! " "Skip cleanup domain: %s" % domain) return self.__global_nffg NFFGToolBox.clear_domain(base=self.__global_nffg, domain=domain, log=log) log.debug("DoV stat:\n%s" % self.__global_nffg.get_stat()) log.log(VERBOSE, "Cleaned Dov:\n%s" % self.__global_nffg.dump()) self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.CHANGE) return self.__global_nffg
def __filter_external_domains(nffg): """ Filter out domains detected by external DomainManagers. :param nffg: filtered NFFG :return: :class:`NFFG` """ log.debug("Filtering domains detected from external DomainManagers...") # Get External DomainManager names ext_mgr = CONFIG.get_external_managers() # Copy NFFG filtered_nffg = nffg.copy() # Remove the detected domains by External DomainManagers for ext in ext_mgr: # Get all the domains domains = NFFGToolBox.detect_domains(nffg=filtered_nffg) # Get domains detected and initiated by the External DomainManager ext_domains = [d for d in domains if ext in d] # Remove collected domains from NFFG for domain in ext_domains: log.debug( "Remove domain: %s originated from external DomainManager: %s" % (domain, ext)) NFFGToolBox.remove_domain(base=filtered_nffg, domain=domain, log=log) filtered_nffg.name += "-filtered" return filtered_nffg
def remerge_domain_in_dov(self, domain, nffg): """ Update the existing domain in the merged Global view with explicit domain remove and re-add. :param nffg: changed infrastructure info :type nffg: :class:`NFFG` :param domain: name of the merging domain :type domain: str :return: updated Dov :rtype: :class:`NFFG` """ NFFGToolBox.remove_domain(base=self.__global_nffg, domain=domain, log=log) # log.log(VERBOSE, "Reduced Dov:\n%s" % self.__global_nffg.dump()) NFFGToolBox.merge_new_domain(base=self.__global_nffg, nffg=nffg, log=log) log.debug("DoV stat:\n%s" % self.__global_nffg.get_stat()) log.log(VERBOSE, "Re-merged DoV:\n%s" % self.__global_nffg.dump()) if self.__global_nffg.is_empty(): log.warning( "No Node had been remained after updating the domain part: " "%s! DoV is empty!" % domain) # Raise event for observing Virtualizers about topology change self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.CHANGE) return self.__global_nffg
def remove_deployed_elements(self): """ Remove all the NFs, flowrules and dynamic ports from DoV. :return: updated Dov :rtype: :class:`NFFG` """ if self.__global_nffg.is_empty(): log.debug("DoV is empty! Skip DoV cleanup") return self.__global_nffg NFFGToolBox.remove_deployed_services(nffg=self.__global_nffg, log=log) log.debug("DoV stat:\n%s" % self.__global_nffg.get_stat()) log.log(VERBOSE, "Cleared Dov:\n%s" % self.__global_nffg.dump()) self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.CHANGE) return self.__global_nffg
def _proceed_to_instantiate_NFFG(self, mapped_nffg): """ Send NFFG to Resource Orchestration Sublayer in an implementation-specific way. General function which is used from microtask and Python thread also. This function contains the last steps before the mapped NFFG will be sent to the next layer. :param mapped_nffg: mapped Service Graph :type mapped_nffg: :class:`NFFG` :return: None """ # Rebind requirement link fragments for lower layer mapping mapped_nffg = NFFGToolBox.rebind_e2e_req_links(nffg=mapped_nffg, log=log) # Log verbose mapping result in unified way (threaded/non-threaded) log.log(VERBOSE, "Mapping result of Service Layer:\n%s" % mapped_nffg.dump()) # Notify remote visualizer about the mapping result if it's needed # notify_remote_visualizer(data=mapped_nffg, id=LAYER_NAME) sas_res = self.__get_sas_resource_view().get_resource_info() # Sending mapped SG / NF-FG to Orchestration layer as an Event # Exceptions in event handlers are caught by default in a non-blocking way self.raiseEventNoErrors(InstantiateNFFGEvent, mapped_nffg, sas_res) log.getChild('API').info( "Generated NF-FG: %s has been sent to Orchestration..." % mapped_nffg)
def _acquire_resource(self): """ Compute and return with the Single BiS-BiS view based on the global view. :return: single BiSBiS representation of the global view :rtype: :class:`NFFG` """ dov = self.global_view.get_resource_info() if dov.is_empty(): # DoV is not initialized yet! Probably only just remote Mgrs has been # enabled! return with the default empty DoV log.warning( "Requested global resource view is empty! Return the default empty " "topology!") return dov else: if str(self.sbb_id).startswith('$'): if str(self.sbb_id)[1:] in os.environ: self.sbb_id = os.environ.get(str(self.sbb_id)[1:]) log.debug("Detected SBB id from environment variable: %s" % self.sbb_id) # Generate the Single BiSBiS representation sbb = NFFGToolBox.generate_SBB_representation(nffg=dov, sbb_id=self.sbb_id, log=log) log.log(VERBOSE, "Generated SBB:\n%s" % sbb.dump()) return sbb
def merge_new_domain_into_dov(self, nffg): """ Add a newly detected domain to DoV. Based on the feature: escape.util.nffg.NFFGToolBox#merge_domains :param nffg: NFFG object need to be merged into DoV :type nffg: :any:`NFFG` :return: updated Dov :rtype: :any:`NFFG` """ # Using general merging function from NFFGToolBox and return the updated # NFFG NFFGToolBox.merge_new_domain(base=self.__global_nffg, nffg=nffg, log=log) # Raise event for observing Virtualizers about topology change log.log(VERBOSE, "Merged Dov:\n%s" % self.__global_nffg.dump()) self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.EXTEND) return self.__global_nffg
def remove_domain_from_dov(self, domain): """ Remove the nodes and edges with the given from Global view. :param domain: domain name :type domain: str :return: updated Dov :rtype: :any:`NFFG` """ NFFGToolBox.remove_domain(base=self.__global_nffg, domain=domain, log=log) if self.__global_nffg.is_empty(): log.warning( "No Node had been remained after updating the domain part: " "%s! DoV is empty!" % domain) log.log(VERBOSE, "Reduced Dov:\n%s" % self.__global_nffg.dump()) # Raise event for observing Virtualizers about topology change self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.REDUCE) return self.__global_nffg
def set_global_view(self, nffg): """ Replace the global view with the given topology. :param nffg: new global topology :type nffg: :any:`NFFG` :return: None """ log.debug("Update the whole Global view (DoV) with the NFFG: %s..." % nffg) self.dov.update_full_global_view(nffg=nffg) self.__tracked_domains.clear() self.__tracked_domains.update(NFFGToolBox.detect_domains(nffg))
def update_domain_status_in_dov(self, domain, nffg): """ Set status of initiated NFs and flowrules related to BiSBiS nodes of the given domain. :param domain: domain name :type domain: str :param nffg: changed infrastructure info :type nffg: :class:`NFFG` :return: updated Dov :rtype: :class:`NFFG` """ if self.__global_nffg.is_empty(): log.debug("DoV is empty! Skip cleanup domain: %s" % domain) return self.__global_nffg NFFGToolBox.update_status_info(nffg=nffg, status=NFFG.STATUS_DEPLOY) NFFGToolBox.update_nffg_by_status(base=self.__global_nffg, updated=nffg, log=log) log.log(VERBOSE, "Updated Dov:\n%s" % self.__global_nffg.dump()) self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.CHANGE) return self.__global_nffg
def update_domain_in_dov(self, domain, nffg): """ Update the existing domain in the merged Global view. :param nffg: NFFG object need to be updated with :type nffg: :any:`NFFG` :param domain: name of the merging domain :type domain: str :return: updated Dov :rtype: :any:`NFFG` """ NFFGToolBox.update_domain(base=self.__global_nffg, updated=nffg, log=log) if self.__global_nffg.is_empty(): log.warning( "No Node had been remained after updating the domain part: " "%s! DoV is empty!" % domain) log.log(VERBOSE, "Updated DoV:\n%s" % self.__global_nffg.dump()) # Raise event for observing Virtualizers about topology change self.raiseEventNoErrors(DoVChangedEvent, cause=DoVChangedEvent.TYPE.CHANGE) return self.__global_nffg
def _acquire_resource(self): """ Compute and return with the Single BiS-BiS view based on the global view. :return: single BiSBiS representation of the global view :rtype: :any:`NFFG` """ dov = self.global_view.get_resource_info() if dov.is_empty(): # DoV is not initialized yet! Probably only just remote Mgrs has been # enabled! return with the default empty DoV log.warning( "Requested global resource view is empty! Return the default empty " "topology!") return dov else: # Generate the Single BiSBiS representation return NFFGToolBox.generate_SBB_representation(nffg=dov, log=log)
def _acquire_resource(self): """ Compute and return with the Single BiS-BiS view based on the global view. :return: single BiSBiS representation of the global view :rtype: :class:`NFFG` """ dov = self.global_view.get_resource_info() if dov.is_empty(): # DoV is not initialized yet! Probably only just remote Mgrs has been # enabled! return with the default empty DoV log.warning( "Requested global resource view is empty! Return the default empty " "topology!") return dov else: filtered_dov = self.__filter_external_domains(nffg=dov) # Generate the Single BiSBiS representation sbb = NFFGToolBox.generate_SBB_representation(nffg=filtered_dov, log=log) log.log(VERBOSE, "Generated SBB:\n%s" % sbb.dump()) return sbb
def __is_changed(self, new_data): """ Return True if the given ``new_data`` is different compared to cached ``last_topo``. :param new_data: received new data :type new_data: :class:`NFFG` :return: changed or not :rtype: bool """ # If got error before, mark domain as unchanged by default if new_data is None: log.error("Missing new topology!") return False # Calculate differences # No need to recreate SG hop in this case, no SG is received in # Virtualizer format add_nffg, del_nffg = NFFGToolBox.generate_difference_of_nffgs( old=self.last_topo, new=new_data) # If both NFFG are empty --> no difference if add_nffg.is_empty() and del_nffg.is_empty(): return False else: return True
def convert_mip_solution_to_nffg (reqs, net, file_inputs=False, full_remap=False): if file_inputs: request_seq = [] for reqfile in reqs: with open(reqfile, "r") as f: req = NFFG.parse(f.read()) request_seq.append(req) with open(net, "r") as g: net = NFFG.parse(g.read()) else: request_seq = reqs # all input NFFG-s are obtained somehow ###################################################################### ##### This is taken from the MappingAlgorithms.MAP() function #### ###################################################################### request = request_seq[0] # batch together all nffgs for r in request_seq[1:]: request = NFFGToolBox.merge_nffgs (request, r) chainlist = [] cid = 1 edgereqlist = [] for req in request.reqs: edgereqlist.append(req) request.del_edge(req.src, req.dst, req.id) # construct chains from EdgeReqs for req in edgereqlist: if len(req.sg_path) == 1: # then add it as linklocal req instead of E2E req log.info("Interpreting one SGHop long EdgeReq (id: %s) as link " "requirement on SGHop: %s."%(req.id, req.sg_path[0])) reqlink = None for sg_link in request.sg_hops: if sg_link.id == req.sg_path[0]: reqlink = sg_link break if reqlink is None: log.warn("EdgeSGLink object not found for EdgeSGLink ID %s! " "(maybe ID-s stored in EdgeReq.sg_path are not the " "same type as EdgeSGLink ID-s?)") if req.delay is not None: setattr(reqlink, 'delay', req.delay) if req.bandwidth is not None: setattr(reqlink, 'bandwidth', req.bandwidth) elif len(req.sg_path) == 0: raise uet.BadInputException( "If EdgeReq is given, it should specify which SGHop path does it " "apply to", "Empty SGHop path was given to %s EdgeReq!" % req.id) else: try: chain = {'id': cid, 'link_ids': req.sg_path, 'bandwidth': req.bandwidth if req.bandwidth is not None else 0, 'delay': req.delay if req.delay is not None else float("inf")} except AttributeError: raise uet.BadInputException( "EdgeReq attributes are: sg_path, bandwidth, delay", "Missing attribute of EdgeReq") # reconstruct NF path from EdgeSGLink path nf_chain = [] for reqlinkid in req.sg_path: # find EdgeSGLink object of 'reqlinkid' reqlink = None for sg_link in request.sg_hops: if sg_link.id == reqlinkid: reqlink = sg_link break else: raise uet.BadInputException( "Elements of EdgeReq.sg_path should be EdgeSGLink.id-s.", "SG link %s couldn't be found in input request NFFG" % reqlinkid) # add the source node id of the EdgeSGLink to NF path nf_chain.append(reqlink.src.node.id) # add the destination node id of the last EdgeSGLink to NF path if reqlinkid == req.sg_path[-1]: if reqlink.dst.node.id != req.dst.node.id: raise uet.BadInputException( "EdgeReq.sg_path should select a path between its two ends", "Last NF (%s) of EdgeReq.sg_path and destination of EdgeReq (" "%s) are not the same!" % (reqlink.dst.node.id, req.dst.node.id)) nf_chain.append(reqlink.dst.node.id) # validate EdgeReq ends. if reqlinkid == req.sg_path[0] and \ reqlink.src.node.id != req.src.node.id: raise uet.BadInputException( "EdgeReq.sg_path should select a path between its two ends", "First NF (%s) of EdgeReq.sg_path and source of EdgeReq (%s) are " "not the same!" % (reqlink.src.node.id, req.src.node.id)) chain['chain'] = nf_chain cid += 1 chainlist.append(chain) # if some resource value is not set (is None) then be permissive and set it # to a comportable value. for respar in ('cpu', 'mem', 'storage', 'delay', 'bandwidth'): for n in net.infras: if n.resources[respar] is None: if respar == 'delay': log.warn("Resource parameter %s is not given in %s, " "substituting with 0!"%(respar, n.id)) n.resources[respar] = 0 else: log.warn("Resource parameter %s is not given in %s, " "substituting with infinity!"%(respar, n.id)) n.resources[respar] = float("inf") # If link res is None or doesn't exist, replace it with a neutral value. for i, j, d in net.network.edges_iter(data=True): if d.type == 'STATIC': if getattr(d, 'delay', None) is None: if d.src.node.type != 'SAP' and d.dst.node.type != 'SAP': log.warn("Resource parameter delay is not given in link %s " "substituting with zero!"%d.id) setattr(d, 'delay', 0) if getattr(d, 'bandwidth', None) is None: if d.src.node.type != 'SAP' and d.dst.node.type != 'SAP': log.warn("Resource parameter bandwidth is not given in link %s " "substituting with infinity!"%d.id) setattr(d, 'bandwidth', float("inf")) # create the class of the algorithm # unnecessary preprocessing is executed ############################################################################ # HACK: We only want to use the algorithm class to generate an NFFG, we will # fill the mapping struct with the one found by MIP alg = CoreAlgorithm(net, request, chainlist, full_remap, False) # move 'availres' and 'availbandwidth' values of the network to maxres, # because the MIP solution takes them as availabel resource. net = alg.bare_infrastucture_nffg for n in net.infras: n.resources = n.availres for d in net.links: # there shouldn't be any Dynamic links by now. d.bandwidth = d.availbandwidth mapping_of_reqs = get_MIP_solution(request_seq, net) mappedNFFG = NFFG(id="MILP-mapped") for transformed_req in mapping_of_reqs: if mapping_of_reqs[transformed_req].is_embedded: alg.manager.vnf_mapping = [] alg.manager.link_mapping = nx.MultiDiGraph() for n, vlist in mapping_of_reqs[transformed_req].\ snode_to_hosted_vnodes.items(): for v in vlist: alg.manager.vnf_mapping.append((v, n)) trans_link_mapping = mapping_of_reqs[transformed_req].vedge_to_spath for trans_sghop in trans_link_mapping: vnf1 = trans_sghop[0] vnf2 = trans_sghop[3] reqlid = get_edge_id(alg.req, vnf1, trans_sghop[1], trans_sghop[2], vnf2) mapped_path = [] path_link_ids = [] for trans_link in trans_link_mapping[trans_sghop]: n1 = trans_link[0] n2 = trans_link[3] lid = get_edge_id(alg.net, n1, trans_link[1], trans_link[2], n2) mapped_path.append(n1) path_link_ids.append(lid) if len(trans_link_mapping[trans_sghop]) == 0: mapped_path.append(alg.manager.getIdOfChainEnd_fromNetwork(vnf1)) else: mapped_path.append(n2) alg.manager.link_mapping.add_edge(vnf1, vnf2, key=reqlid, mapped_to=mapped_path, path_link_ids=path_link_ids) oneNFFG = alg.constructOutputNFFG() mappedNFFG = NFFGToolBox.merge_nffgs(mappedNFFG, oneNFFG) else: print "MILP didn't produce a mapping for request %s"%transformed_req return None # replace Infinity values MappingAlgorithms._purgeNFFGFromInfinityValues(mappedNFFG) # print mappedNFFG.dump() return mappedNFFG
def __proceed_instantiation(self, nffg, resource_nffg): """ Helper function to instantiate the NFFG mapping from different source. :param nffg: pre-mapped service request :type nffg: :class:`NFFG` :return: None """ self.log.info("Invoke instantiation on %s with NF-FG: %s" % (self.__class__.__name__, nffg.name)) stats.add_measurement_start_entry(type=stats.TYPE_ORCHESTRATION, info=LAYER_NAME) # Get shown topology view if resource_nffg is None: log.error("Missing resource for difference calculation!") return log.debug("Got resource view for difference calculation: %s" % resource_nffg) self.log.debug("Store received NFFG request info...") msg_id = self.api_mgr.request_cache.cache_request_by_nffg(nffg=nffg) if msg_id is not None: self.api_mgr.request_cache.set_in_progress(id=msg_id) self.log.debug("Request is stored with id: %s" % msg_id) else: self.log.debug("No request info detected.") # Check if mapping mode is set globally in CONFIG mapper_params = CONFIG.get_mapping_config(layer=LAYER_NAME) if 'mode' in mapper_params and mapper_params['mode'] is not None: mapping_mode = mapper_params['mode'] log.info("Detected mapping mode from configuration: %s" % mapping_mode) elif nffg.mode is not None: mapping_mode = nffg.mode log.info("Detected mapping mode from NFFG: %s" % mapping_mode) else: mapping_mode = None log.info("No mapping mode was defined explicitly!") if not CONFIG.get_mapping_enabled(layer=LAYER_NAME): log.warning("Mapping is disabled! Skip difference calculation...") elif nffg.status == NFFG.MAP_STATUS_SKIPPED: log.debug("Detected NFFG map status: %s! " "Skip difference calculation and " "proceed with original request..." % nffg.status) elif mapping_mode != NFFG.MODE_REMAP: # Calculated ADD-DELETE difference log.debug("Calculate ADD - DELETE difference with mapping mode...") # Recreate SG-hops for diff calc. log.debug("Recreate SG hops for difference calculation...") NFFGToolBox.recreate_all_sghops(nffg=nffg) NFFGToolBox.recreate_all_sghops(nffg=resource_nffg) log.log(VERBOSE, "New NFFG:\n%s" % nffg.dump()) log.log(VERBOSE, "Resource NFFG:\n%s" % resource_nffg.dump()) # Calculate difference add_nffg, del_nffg = NFFGToolBox.generate_difference_of_nffgs( old=resource_nffg, new=nffg, ignore_infras=True) log.log(VERBOSE, "Calculated ADD NFFG:\n%s" % add_nffg.dump()) log.log(VERBOSE, "Calculated DEL NFFG:\n%s" % del_nffg.dump()) if not add_nffg.is_bare() and del_nffg.is_bare(): nffg = add_nffg log.info("DEL NFFG is bare! Calculated mapping mode: %s" % nffg.mode) elif add_nffg.is_bare() and not del_nffg.is_bare(): nffg = del_nffg log.info("ADD NFFG is bare! Calculated mapping mode: %s" % nffg.mode) elif not add_nffg.is_bare() and not del_nffg.is_bare(): log.warning("Both ADD / DEL mode is not supported currently") self.__process_mapping_result(nffg_id=nffg.id, fail=True) stats.add_measurement_end_entry(type=stats.TYPE_ORCHESTRATION, info=LAYER_NAME + "-FAILED") self.raiseEventNoErrors( InstantiationFinishedEvent, id=nffg.id, result=InstantiationFinishedEvent.ABORTED) return else: log.debug("Difference calculation resulted empty subNFFGs!") log.warning( "No change has been detected in request! Skip mapping...") self.log.debug("Invoked instantiation on %s is finished!" % self.__class__.__name__) self.__process_mapping_result(nffg_id=nffg.id, fail=False) stats.add_measurement_end_entry(type=stats.TYPE_ORCHESTRATION, info=LAYER_NAME + "-SKIPPED") return else: log.debug( "Mode: %s detected from config! Skip difference calculation..." % mapping_mode) try: if CONFIG.get_mapping_enabled(layer=LAYER_NAME): # Initiate request mapping mapped_nffg = self.orchestrator.instantiate_nffg(nffg=nffg) else: log.warning("Mapping is disabled! Skip instantiation step...") mapped_nffg = nffg mapped_nffg.status = NFFG.MAP_STATUS_SKIPPED log.debug("Mark NFFG status: %s!" % mapped_nffg.status) # Rewrite REMAP mode for backward compatibility if mapped_nffg is not None and mapping_mode == NFFG.MODE_REMAP: mapped_nffg.mode = mapping_mode log.debug("Rewrite mapping mode: %s into mapped NFFG..." % mapped_nffg.mode) else: log.debug("Skip mapping mode rewriting! Mode remained: %s" % mapping_mode) self.log.debug("Invoked instantiate_nffg on %s is finished!" % self.__class__.__name__) # If mapping is not threaded and finished with OK if mapped_nffg is not None and not self.orchestrator.mapper.threaded: self._proceed_to_install_NFFG(mapped_nffg=mapped_nffg, original_request=nffg) else: log.warning( "Something went wrong in service request instantiation: " "mapped service request is missing!") self.__process_mapping_result(nffg_id=nffg.id, fail=True) stats.add_measurement_end_entry(type=stats.TYPE_ORCHESTRATION, info=LAYER_NAME + "-FAILED") self.raiseEventNoErrors( InstantiationFinishedEvent, id=nffg.id, result=InstantiationFinishedEvent.MAPPING_ERROR) except ProcessorError as e: self.__process_mapping_result(nffg_id=nffg.id, fail=True) stats.add_measurement_end_entry(type=stats.TYPE_ORCHESTRATION, info=LAYER_NAME + "-DENIED") self.raiseEventNoErrors( InstantiationFinishedEvent, id=nffg.id, result=InstantiationFinishedEvent.REFUSED_BY_VERIFICATION, error=e)
def MAP(request, network, full_remap=False, enable_shortest_path_cache=False, bw_factor=1, res_factor=1, lat_factor=1, shortest_paths=None, return_dist=False, bt_limit=6, bt_branching_factor=3): """ The parameters are NFFG classes. Calculates service chain requirements from EdgeReq classes. enable_shortest_path_cache: whether we should store the calculated shortest paths in a file for later usage. full_remap: whether the resources of the VNF-s contained in the resource NFFG be subtracted and deleted or just deleted from the resource NFFG before mapping. """ sg_hops_given = True try: # if there is at least ONE SGHop in the graph, we don't do SGHop retrieval. next(request.sg_hops) except StopIteration: # retrieve the SGHops from the TAG values of the flow rules, in case they # are cannot be found in the request graph and can only be deduced from the # flows helper.log.warn( "No SGHops were given in the Service Graph, retrieving them" " based on the flowrules...") sg_hops_given = False sg_hop_info = NFFGToolBox.retrieve_all_SGHops(request) helper.log.log(5, "Retrieved SG hops:\n" + pformat(sg_hop_info)) if len(sg_hop_info) == 0: raise uet.BadInputException( "If SGHops are not given, flowrules should be" " in the NFFG", "No SGHop could be retrieved based on the " "flowrules of the NFFG.") for k, v in sg_hop_info.iteritems(): # VNF ports are given to the function request.add_sglink(v[0], v[1], flowclass=v[2], bandwidth=v[3], delay=v[4], id=k[2]) chainlist = [] cid = 1 edgereqlist = [] # a delay value which is assumed to be infinity in terms of connection RTT # or latency requirement (set it to 100s = 100 000ms) overall_highest_delay = 100000 for req in request.reqs: edgereqlist.append(req) request.del_edge(req.src, req.dst, req.id) if len(edgereqlist) != 0 and not sg_hops_given: helper.log.warn( "EdgeReqs were given, but the SGHops (which the EdgeReqs " "refer to by id) are retrieved based on the flowrules of " "infrastructure. This can cause error later if the " "flowrules was malformed...") # construct chains from EdgeReqs for req in edgereqlist: if len(req.sg_path) == 1: # then add it as linklocal req instead of E2E req helper.log.info( "Interpreting one SGHop long EdgeReq (id: %s) as link " "requirement on SGHop: %s." % (req.id, req.sg_path[0])) reqlink = None for sg_link in request.sg_hops: if sg_link.id == req.sg_path[0]: reqlink = sg_link break if reqlink is None: helper.log.warn( "EdgeSGLink object not found for EdgeSGLink ID %s! " "(maybe ID-s stored in EdgeReq.sg_path are not the " "same type as EdgeSGLink ID-s?)") if req.delay is not None: setattr(reqlink, 'delay', req.delay) if req.bandwidth is not None: setattr(reqlink, 'bandwidth', req.bandwidth) elif len(req.sg_path) == 0: raise uet.BadInputException( "If EdgeReq is given, it should specify which SGHop path does it " "apply to", "Empty SGHop path was given to %s EdgeReq!" % req.id) else: try: chain = {'id': cid, 'link_ids': req.sg_path, 'bandwidth': req.bandwidth if req.bandwidth is not None else 0, 'delay': req.delay if req.delay is not None \ else overall_highest_delay} except AttributeError: raise uet.BadInputException( "EdgeReq attributes are: sg_path, bandwidth, delay", "Missing attribute of EdgeReq") # reconstruct NF path from EdgeSGLink path nf_chain = [] for reqlinkid in req.sg_path: # find EdgeSGLink object of 'reqlinkid' reqlink = None for sg_link in request.sg_hops: if sg_link.id == reqlinkid: reqlink = sg_link break else: raise uet.BadInputException( "Elements of EdgeReq.sg_path should be EdgeSGLink.id-s.", "SG link %s couldn't be found in input request NFFG" % reqlinkid) # add the source node id of the EdgeSGLink to NF path nf_chain.append(reqlink.src.node.id) # add the destination node id of the last EdgeSGLink to NF path if reqlinkid == req.sg_path[-1]: if reqlink.dst.node.id != req.dst.node.id: raise uet.BadInputException( "EdgeReq.sg_path should select a path between its two ends", "Last NF (%s) of EdgeReq.sg_path and destination of EdgeReq (" "%s) are not the same!" % (reqlink.dst.node.id, req.dst.node.id)) nf_chain.append(reqlink.dst.node.id) # validate EdgeReq ends. if reqlinkid == req.sg_path[0] and \ reqlink.src.node.id != req.src.node.id: raise uet.BadInputException( "EdgeReq.sg_path should select a path between its two ends", "First NF (%s) of EdgeReq.sg_path and source of EdgeReq (%s) are " "not the same!" % (reqlink.src.node.id, req.src.node.id)) chain['chain'] = nf_chain cid += 1 chainlist.append(chain) # if some resource value is not set (is None) then be permissive and set it # to a comportable value. for respar in ('cpu', 'mem', 'storage', 'delay', 'bandwidth'): for n in network.infras: if n.resources[respar] is None: if respar == 'delay': helper.log.warn( "Resource parameter %s is not given in %s, " "substituting with 0!" % (respar, n.id)) n.resources[respar] = 0 else: helper.log.warn( "Resource parameter %s is not given in %s, " "substituting with infinity!" % (respar, n.id)) n.resources[respar] = float("inf") # If link res is None or doesn't exist, replace it with a neutral value. for i, j, d in network.network.edges_iter(data=True): if d.type == 'STATIC': if getattr(d, 'delay', None) is None: if d.src.node.type != 'SAP' and d.dst.node.type != 'SAP': helper.log.warn( "Resource parameter delay is not given in link %s " "substituting with zero!" % d.id) setattr(d, 'delay', 0) if getattr(d, 'bandwidth', None) is None: if d.src.node.type != 'SAP' and d.dst.node.type != 'SAP': helper.log.warn( "Resource parameter bandwidth is not given in link %s " "substituting with infinity!" % d.id) setattr(d, 'bandwidth', float("inf")) # create the class of the algorithm alg = CoreAlgorithm(network, request, chainlist, full_remap, enable_shortest_path_cache, overall_highest_delay, bw_factor=bw_factor, res_factor=res_factor, lat_factor=lat_factor, shortest_paths=shortest_paths) alg.setBacktrackParameters(bt_limit, bt_branching_factor) mappedNFFG = alg.start() # replace Infinity values _purgeNFFGFromInfinityValues(mappedNFFG) # print mappedNFFG.dump() # The printed format is vnfs: (vnf_id, node_id) and links: MultiDiGraph, edge # data is the paths (with link ID-s) where the request links are mapped. helper.log.info("The VNF mappings are (vnf_id, node_id): \n%s" % pformat(alg.manager.vnf_mapping)) helper.log.debug( "The link mappings are: \n%s" % pformat(alg.manager.link_mapping.edges(data=True, keys=True))) if return_dist: return mappedNFFG, alg.preprocessor.shortest_paths else: return mappedNFFG
def install_nffg(self, mapped_nffg): """ Start NF-FG installation. Process given :any:`NFFG`, slice information self.__global_nffg on domains and invoke DomainManagers to install domain specific parts. :param mapped_nffg: mapped NF-FG instance which need to be installed :type mapped_nffg: NFFG :return: mapping result :rtype: bool """ log.debug("Invoke %s to install NF-FG(%s)" % (self.__class__.__name__, mapped_nffg.name)) # # Notify remote visualizer about the deployable NFFG if it's needed # notify_remote_visualizer(data=mapped_nffg, id=LAYER_NAME) slices = NFFGToolBox.split_into_domains(nffg=mapped_nffg, log=log) if slices is None: log.warning("Given mapped NFFG: %s can not be sliced! " "Skip domain notification steps" % mapped_nffg) return log.debug("Notify initiated domains: %s" % [d for d in self.domains.initiated]) # TODO - abstract/inter-domain tag rewrite # NFFGToolBox.rewrite_interdomain_tags(slices) mapping_result = True for domain, part in slices: log.debug( "Recreate missing TAG matching fields in domain part: %s..." % domain) # Temporarily rewrite/recreate TAGs here NFFGToolBox.recreate_match_TAGs(nffg=part, log=log) # Get Domain Manager domain_mgr = self.domains.get_component_by_domain( domain_name=domain) if domain_mgr is None: log.warning( "No DomainManager has been initialized for domain: %s! " "Skip install domain part..." % domain) continue log.log(VERBOSE, "Splitted domain: %s part:\n%s" % (domain, part.dump())) log.info("Delegate splitted part: %s to %s" % (part, domain_mgr)) # Rebind requirement link fragments as e2e reqs part = NFFGToolBox.rebind_e2e_req_links(nffg=part, log=log) # Check if need to reset domain before install if CONFIG.reset_domains_before_install(): log.debug("Reset %s domain before deploying mapped NFFG..." % domain_mgr.domain_name) domain_mgr.clear_domain() # Invoke DomainAdapter's install res = domain_mgr.install_nffg(part) # Update the DoV based on the mapping result covering some corner case if not res: log.error("Installation of %s in %s was unsuccessful!" % (part, domain)) # Note result according to others before mapping_result = mapping_result and res # If installation of the domain was performed without error if not res: log.warning("Skip DoV update with domain: %s! Cause: " "Domain installation was unsuccessful!" % domain) continue # If the domain manager does not poll the domain update here # else polling takes care of domain updating if isinstance(domain_mgr, AbstractRemoteDomainManager) and domain_mgr._poll: log.info("Skip explicit DoV update for domain: %s. " "Cause: polling enabled!" % domain) continue # If the internalDM is the only initiated mgr, we can override the # whole DoV if domain_mgr.IS_LOCAL_MANAGER: if mapped_nffg.is_SBB(): # If the request was a cleanup request, we can simply clean the DOV if mapped_nffg.is_bare(): log.debug( "Detected cleanup topology (no NF/Flowrule/SG_hop)! Clean DoV..." ) self.DoVManager.clean_domain(domain=domain) # If the reset contains some VNF, cannot clean or override else: log.warning( "Detected SingleBiSBiS topology! Local domain has been already " "cleared, skip DoV update...") # If the the topology was a GLOBAL view, just override the whole DoV elif not mapped_nffg.is_virtualized(): self.DoVManager.set_global_view(nffg=mapped_nffg) else: log.warning( "Detected virtualized Infrastructure node in mapped NFFG! Skip " "DoV update...") # In case of Local manager skip the rest of the update continue # Explicit domain update self.DoVManager.update_domain(domain=domain, nffg=part) log.debug("NF-FG installation is finished by %s" % self.__class__.__name__) # Post-mapping steps if mapping_result: log.info( "All installation process has been finished with success! ") # Notify remote visualizer about the installation result if it's needed notify_remote_visualizer( data=self.DoVManager.dov.get_resource_info(), id=LAYER_NAME) else: log.error("%s installation was not successful!" % mapped_nffg) return mapping_result
def StressTestCore(seed, loops, use_saps_once, vnf_sharing, multiple_scs, max_sc_count, vnf_sharing_same_sg, fullremap, batch_length, shareable_sg_count, sliding_share, poisson, topo_name, bw_factor, res_factor, lat_factor, bt_limit, bt_br_factor, outputfile, queue=None, shortest_paths_precalc=None, filehandler=None): """ If queue is given, the result will be put in that Queue object too. Meanwhile if shortest_paths_precalc is not given, it means the caller needs the shortest_paths, so we send it back. In this case the resulting test_lvl will be sent by the queue. NOTE: outputfile is only used inside the function if an exception is thrown and than it is logged there. """ total_vnf_count = 0 mapped_vnf_count = 0 network = None if topo_name == "picotopo": network = CarrierTopoBuilder.getPicoTopo() elif topo_name == "gwin": network = CarrierTopoBuilder.getSNDlib_dfn_gwin(save_to_file=True) max_test_lvl = 50000 test_lvl = 1 all_saps_ending = [s.id for s in network.saps] all_saps_beginning = [s.id for s in network.saps] running_nfs = OrderedDict() random.seed(0) random.jumpahead(seed) random.shuffle(all_saps_beginning) random.shuffle(all_saps_ending) shortest_paths = shortest_paths_precalc ppid_pid = "" # log.addHandler(logging.StreamHandler()) log.setLevel(logging.WARN) if filehandler is not None: log.addHandler(filehandler) if shortest_paths is not None and type(shortest_paths) != dict: excp = Exception( "StressTest received something else other than shortest_" "paths dictionary: %s" % type(shortest_paths)) if queue is not None: queue.put(excp) raise excp if queue is not None: ppid_pid = "%s.%s:" % (os.getppid(), os.getpid()) try: try: batch_count = 0 batched_request = NFFG(id="Benchmark-Req-" + str(test_lvl)) # built-in libs can change the state of random module during mapping. random_state = None while batched_request is not None: if test_lvl > max_test_lvl: break if (len(all_saps_ending) < batch_length or \ len(all_saps_beginning) < batch_length) and use_saps_once: log.warn( "Can't start batching because all SAPs should only be used" " once for SC origin and destination and there are not " "enough SAPs!") batched_request = None elif batch_count < batch_length or len( [nf for nf in request.nfs]) == 0: request, all_saps_beginning, all_saps_ending = \ generateRequestForCarrierTopo(test_lvl, all_saps_beginning, all_saps_ending, running_nfs, loops=loops, use_saps_once=use_saps_once, vnf_sharing_probabilty=vnf_sharing, vnf_sharing_same_sg=vnf_sharing_same_sg, multiSC=multiple_scs, max_sc_count=max_sc_count) if request is None: break else: batch_count += (random.expovariate(1.0) if poisson else 1) if poisson: log.debug( "Time passed since last batched mapping: %s" % batch_count) running_nfs[test_lvl] = [ nf for nf in request.nfs if nf.id.split("-")[1] == str(test_lvl) ] # using merge to create the union of the NFFG-s! batched_request = NFFGToolBox.merge_nffgs( batched_request, request) if len(running_nfs) > shareable_sg_count: # make the ordered dict function as FIFO running_nfs.popitem(last=False) test_lvl += 1 if not sliding_share and test_lvl % shareable_sg_count == 0: running_nfs = OrderedDict() log.debug("Batching Service Graph number %s..." % batch_count) else: batch_count = 0 total_vnf_count += len([nf for nf in batched_request.nfs]) random_state = random.getstate() network, shortest_paths = MappingAlgorithms.MAP( batched_request, network, full_remap=fullremap, enable_shortest_path_cache=True, bw_factor=bw_factor, res_factor=res_factor, lat_factor=lat_factor, shortest_paths=shortest_paths, return_dist=True, bt_limit=bt_limit, bt_branching_factor=bt_br_factor) log.debug(ppid_pid + "Mapping successful on test level %s with batch" " length %s!" % (test_lvl, batch_length)) random.setstate(random_state) mapped_vnf_count += len([nf for nf in batched_request.nfs]) batched_request = NFFG(id="Benchmark-Req-" + str(test_lvl)) except uet.MappingException as me: log.info(ppid_pid + "Mapping failed: %s" % me.msg) if not me.backtrack_possible: # NOTE: peak SC count is only corret to add to test_lvl if SC-s are # disjoint on VNFs. if poisson: log.warn( "Peak mapped VNF count is %s in the last run, test level: " "UNKNOWN because of Poisson" % me.peak_mapped_vnf_count) else: log.warn("Peak mapped VNF count is %s in the last run, test level: %s"% (me.peak_mapped_vnf_count, test_lvl - batch_length + \ (me.peak_sc_cnt if me.peak_sc_cnt is not None else 0))) mapped_vnf_count += me.peak_mapped_vnf_count log.warn( "All-time peak mapped VNF count: %s, All-time total VNF " "count %s, Acceptance ratio: %s" % (mapped_vnf_count, total_vnf_count, float(mapped_vnf_count) / total_vnf_count)) # break if request is None or batched_request is None: log.warn(ppid_pid + "Request generation reached its end!") # break except uet.UnifyException as ue: log.error(ppid_pid + ue.msg) log.error(ppid_pid + traceback.format_exc()) with open(outputfile, "a") as f: f.write("\n".join( ("UnifyException cought during StressTest: ", ue.msg, traceback.format_exc()))) if queue is not None: queue.put(str(ue.__class__)) return test_lvl - 1 except Exception as e: log.error(ppid_pid + traceback.format_exc()) with open(outputfile, "a") as f: f.write("\n".join(("Exception cought during StressTest: ", traceback.format_exc()))) if queue is not None: queue.put(str(e.__class__)) return test_lvl - 1 # put the result to the queue if queue is not None: log.info(ppid_pid + "Putting %s to communication queue" % (test_lvl - 1)) queue.put(test_lvl - 1) if shortest_paths_precalc is None: log.info(ppid_pid + "Returning shortest_paths!") return shortest_paths # if returned_test_lvl is 0, we failed at the very fist mapping! return test_lvl - 1