def modify_sliver(self, *, rid: ID, modify_sub_command: str, properties: dict) -> int: """ Modify sliver :param rid: :param modify_sub_command: :param properties: :return: """ try: controller = OrchestratorKernelSingleton.get( ).get_management_actor() reservation = controller.get_reservation(rid=rid) if reservation is None: raise OrchestratorException( "Unable to find reservation {}".format(rid)) config_props = reservation.get_config_properties() if config_props is None: raise OrchestratorException( "Unable to get configuration properties for reservation {}" .format(rid)) # Update the properties # TODO index = 0 controller.modify_reservation(rid=rid, modify_properties=properties) return index except Exception as e: raise OrchestratorException( "Unable to modify sliver reservation: {}".format(e))
def __validate_lease_end_time(self, lease_end_time: str) -> datetime: """ Validate Lease End Time :param lease_end_time: New End Time :return End Time :raises Exception if new end time is in past """ new_end_time = None if lease_end_time is None: new_end_time = datetime.utcnow() + timedelta(hours=Constants.DEFAULT_LEASE_IN_HOURS) return new_end_time try: new_end_time = datetime.strptime(lease_end_time, Constants.LEASE_TIME_FORMAT) except Exception as e: raise OrchestratorException(f"Lease End Time is not in format {Constants.LEASE_TIME_FORMAT}", http_error_code=BAD_REQUEST) now = datetime.utcnow() if new_end_time <= now: raise OrchestratorException(f"New term end time {new_end_time} is in the past! ", http_error_code=BAD_REQUEST) if (new_end_time - now) > Constants.DEFAULT_MAX_DURATION: self.logger.info(f"New term end time {new_end_time} exceeds system default " f"{Constants.DEFAULT_MAX_DURATION}, setting to system default: ") new_end_time = now + Constants.DEFAULT_MAX_DURATION return new_end_time
def check_modify_queue(self, *, ok_or_failed: ReservationIDWithModifyIndex): """ Check Modify Queue :param ok_or_failed: :return: """ try: self.lock.acquire() res_queue = self.modify_queue.get( ok_or_failed.get_reservation_id(), None) if res_queue is None: raise OrchestratorException( "no queue found for {}, skipping processing".format( ok_or_failed)) mop = res_queue.pop(0) if mop is None: raise OrchestratorException( "no modify operation found at top of the queue, proceeding" ) if mop.get() != ok_or_failed: raise OrchestratorException( "dequeued reservation {} which doesn't match expected {}". format(mop.get(), ok_or_failed)) if len(res_queue) > 0: mop = res_queue.pop(0) else: mop = None if mop is not None: mop_index = self.modify_sliver( rid=ok_or_failed.get_reservation_id(), modify_sub_command=mop.get_sub_command(), properties=mop.get_properties()) mop.override_index(mop_index) watch_list = [mop.get()] OrchestratorKernelSingleton.get().get_sut( ).add_modify_status_watch(watch=watch_list, act=None, callback=self) else: self.modify_queue.pop(ok_or_failed.get_reservation_id()) finally: self.lock.release()
def check_(self, *, controller: ABCMgmtControllerMixin, rid) -> Status: """ Check status :param controller: :param rid: :return: """ if not isinstance(rid, ID): return Status.NOTREADY try: reservations = controller.get_reservations(rid=rid) units = controller.get_reservation_units(rid=rid) if reservations is None or len(reservations) == 0: raise OrchestratorException( "Unable to obtain reservation information for {}".format( rid)) reservation = next(iter(reservations)) if reservation.get_state() == ReservationStates.Active: if units is None or len(units) == 0: return Status.NOTREADY elif reservation.get_state() == ReservationStates.Closed: return Status.OK elif reservation.get_state() == ReservationStates.Failed: return Status.NOTOK except Exception as e: self.logger.error("Exception occurred e: {}".format(e)) return Status.NOTREADY
def get_slice_graph(self, *, token: str, slice_id: str, graph_format_str: str) -> dict: """ Get User Slice :param token Fabric Identity Token :param slice_id Slice Id :param graph_format_str :raises Raises an exception in case of failure :returns Slice Graph on success """ try: controller = self.controller_state.get_management_actor() self.logger.debug(f"get_slice_graph invoked for Controller: {controller}") slice_guid = None if slice_id is not None: slice_guid = ID(uid=slice_id) slice_list = controller.get_slices(id_token=token, slice_id=slice_guid) if slice_list is None or len(slice_list) == 0: if controller.get_last_error() is not None: self.logger.error(controller.get_last_error()) raise OrchestratorException(f"User# has no Slices", http_error_code=NOT_FOUND) slice_obj = next(iter(slice_list)) if slice_obj.get_graph_id() is None: raise OrchestratorException(f"Slice# {slice_obj} does not have graph id") slice_model = FimHelper.get_graph(graph_id=slice_obj.get_graph_id()) graph_format = self.__translate_graph_format(graph_format=graph_format_str) if graph_format == GraphFormat.JSON_NODELINK: slice_model_str = slice_model.serialize_graph() slice_model = FimHelper.get_networkx_graph_from_string(graph_str=slice_model_str) if slice_model is None: raise OrchestratorException(f"Slice# {slice_obj} graph could not be loaded") slice_model_str = slice_model.serialize_graph(format=graph_format) return ResponseBuilder.get_slice_summary(slice_list=slice_list, slice_id=slice_id, slice_model=slice_model_str) except Exception as e: self.logger.error(traceback.format_exc()) self.logger.error(f"Exception occurred processing get_slice_graph e: {e}") raise e
def __validate_node_sliver(sliver: NodeSliver): """ Validate Network Node Sliver @param sliver Node Sliver @raises exception for invalid slivers """ if sliver.get_capacities() is None and sliver.get_capacity_hints() is None: raise OrchestratorException(message="Either Capacity or Capacity Hints must be specified!", http_error_code=BAD_REQUEST)
def discover_broker_query_model(self, *, controller: ABCMgmtControllerMixin, token: str = None, level: int = 10, delete_graph: bool = True, ignore_validation: bool = True, graph_format: GraphFormat = GraphFormat.GRAPHML) -> Tuple[str, ABCCBMPropertyGraph or None]: """ Discover all the available resources by querying Broker :param controller Management Controller Object :param token Fabric Token :param level: level of details :param delete_graph flag indicating if the loaded graph should be deleted or not :param ignore_validation flag indicating to ignore validating the graph (only needed for ADs) :param graph_format: Graph format :return tuple of dictionary containing the BQM and ABCCBMPropertyGraph (if delete_graph = False) """ broker = self.get_broker(controller=controller) if broker is None: raise OrchestratorException("Unable to determine broker proxy for this controller. " "Please check Orchestrator container configuration and logs.") model = controller.get_broker_query_model(broker=broker, id_token=token, level=level, graph_format=graph_format) if model is None: raise OrchestratorException(f"Could not discover types: {controller.get_last_error()}") graph_str = model.get_model() try: if graph_str is not None and graph_str != '': graph = None # Load graph only when GraphML if graph_format == GraphFormat.GRAPHML: graph = FimHelper.get_neo4j_cbm_graph_from_string_direct(graph_str=graph_str, ignore_validation=ignore_validation) if delete_graph: FimHelper.delete_graph(graph_id=graph.get_graph_id()) graph = None return graph_str, graph else: raise OrchestratorException(http_error_code=NOT_FOUND, message="Resource(s) not found!") except Exception as e: self.logger.error(traceback.format_exc()) raise e
def get_slivers(self, *, token: str, slice_id: str, sliver_id: str = None, include_notices: bool = True) -> dict: """ Get Slivers for a Slice :param token Fabric Identity Token :param slice_id Slice Id :param sliver_id Sliver Id :param include_notices include notices :raises Raises an exception in case of failure :returns List of reservations created for the Slice on success """ try: controller = self.controller_state.get_management_actor() self.logger.debug(f"get_slivers invoked for Controller: {controller}") slice_guid = None if slice_id is not None: slice_guid = ID(uid=slice_id) rid = None if sliver_id is not None: rid = ID(uid=sliver_id) reservations = controller.get_reservations(id_token=token, slice_id=slice_guid, rid=rid) if reservations is None: if controller.get_last_error() is not None: self.logger.error(controller.get_last_error()) if controller.get_last_error().status.code == ErrorCodes.ErrorNoSuchSlice: raise OrchestratorException(f"Slice# {slice_id} not found", http_error_code=NOT_FOUND) elif controller.get_last_error().status.code == ErrorCodes.ErrorNoSuchReservation: raise OrchestratorException(f"Reservation# {rid} not found", http_error_code=NOT_FOUND) raise OrchestratorException(f"Slice# {slice_id} has no reservations", http_error_code=NOT_FOUND) return ResponseBuilder.get_reservation_summary(res_list=reservations, include_notices=include_notices, include_sliver=True) except Exception as e: self.logger.error(traceback.format_exc()) self.logger.error(f"Exception occurred processing get_slivers e: {e}") raise e
def delete_slice(self, *, token: str, slice_id: str = None): """ Delete User Slice :param token Fabric Identity Token :param slice_id Slice Id :raises Raises an exception in case of failure """ try: controller = self.controller_state.get_management_actor() self.logger.debug(f"delete_slice invoked for Controller: {controller}") slice_guid = None if slice_id is not None: slice_guid = ID(uid=slice_id) slice_list = controller.get_slices(id_token=token, slice_id=slice_guid) if slice_list is None or len(slice_list) == 0: raise OrchestratorException(f"Slice# {slice_id} not found", http_error_code=NOT_FOUND) slice_object = next(iter(slice_list)) slice_state = SliceState(slice_object.get_state()) if slice_state == SliceState.Dead or slice_state == SliceState.Closing: raise OrchestratorException(f"Slice# {slice_id} already closed", http_error_code=BAD_REQUEST) if slice_state != SliceState.StableOK and slice_state != SliceState.StableError: self.logger.info(f"Unable to delete Slice# {slice_guid} that is not yet stable, try again later") raise OrchestratorException(f"Unable to delete Slice# {slice_guid} that is not yet stable, " f"try again later") controller.close_reservations(slice_id=slice_guid) except Exception as e: self.logger.error(traceback.format_exc()) self.logger.error(f"Exception occurred processing delete_slice e: {e}") raise e
def main(): try: from fabric_cf.actor.core.container.globals import Globals, GlobalsSingleton with GracefulInterruptHandler() as h: GlobalsSingleton.get().start(force_fresh=False) while not GlobalsSingleton.get().start_completed: time.sleep(0.001) from fabric_cf.orchestrator.core.orchestrator_kernel import OrchestratorKernelSingleton OrchestratorKernelSingleton.get() OrchestratorKernelSingleton.get().start_threads() runtime_config = GlobalsSingleton.get().get_config( ).get_runtime_config() # prometheus server prometheus_port = int( runtime_config.get( Constants.PROPERTY_CONF_PROMETHEUS_REST_PORT, None)) prometheus_client.start_http_server(prometheus_port) rest_port_str = runtime_config.get( Constants.PROPERTY_CONF_CONTROLLER_REST_PORT, None) if rest_port_str is None: raise OrchestratorException( "Invalid configuration rest port not specified") print("Starting REST") # start swagger app = connexion.App(__name__, specification_dir='swagger_server/swagger/') app.app.json_encoder = encoder.JSONEncoder app.add_api('swagger.yaml', arguments={'title': 'Fabric Orchestrator API'}, pythonic_params=True) # Start up the server to expose the metrics. waitress.serve(app, port=int(rest_port_str)) while True: time.sleep(0.0001) if h.interrupted: GlobalsSingleton.get().stop() OrchestratorKernelSingleton.get().stop_threads() except Exception: traceback.print_exc()
def demand_slice(self, *, controller_slice: OrchestratorSliceWrapper): """ Demand slice reservations. :param controller_slice: """ computed_reservations = controller_slice.get_computed_reservations() try: self.lock.acquire() for reservation in computed_reservations: if reservation.get_reservation_id( ) in controller_slice.demanded_reservations(): self.get_logger().debug( f"Reservation: {reservation.get_reservation_id()} already demanded" ) continue self.get_logger().debug( f"Issuing demand for reservation: {reservation.get_reservation_id()}" ) if reservation.get_state() != ReservationStates.Unknown.value: self.get_logger().debug( f"Reservation not in {reservation.get_state()} state, ignoring it" ) continue if not self.controller.demand_reservation( reservation=reservation): raise OrchestratorException( f"Could not demand resources: {self.controller.get_last_error()}" ) controller_slice.mark_demanded( rid=reservation.get_reservation_id()) self.get_logger().debug( f"Reservation #{reservation.get_reservation_id()} demanded successfully" ) except Exception as e: self.get_logger().error(traceback.format_exc()) self.get_logger().error( "Unable to get orchestrator or demand reservation: {}".format( e)) finally: self.lock.release()
def start(self): """ Start :return: """ try: self.thread_lock.acquire() if self.thread is not None: raise OrchestratorException( "This ReservationStatusUpdateThread has already been started" ) self.thread = threading.Thread(target=self.periodic) self.thread.setName(self.__class__.__name__) self.thread.setDaemon(True) self.thread.start() finally: self.thread_lock.release()
def create(self, *, slice_graph: ABCASMPropertyGraph) -> List[TicketReservationAvro]: """ Create a slice :param slice_graph: Slice Graph :return: List of computed reservations """ try: # Build Network Node reservations network_node_reservations, node_res_mapping = self.__build_network_node_reservations(slice_graph=slice_graph) # Build Network Service reservations network_service_reservations = self.__build_network_service_reservations(slice_graph=slice_graph, node_res_mapping=node_res_mapping) # Add Network Node reservations for r in network_node_reservations: self.controller.add_reservation(reservation=r) # Add Network Node reservations for r in network_service_reservations: self.controller.add_reservation(reservation=r) # Add to computed reservations for x in network_node_reservations: self.computed_reservations.append(x) self.rid_to_res[x.get_reservation_id()] = x for x in network_service_reservations: self.computed_reservations.append(x) self.rid_to_res[x.get_reservation_id()] = x return self.computed_reservations except OrchestratorException as e: self.logger.error("Exception occurred while generating reservations for slivers: {}".format(e)) raise e except Exception as e: self.logger.error("Exception occurred while generating reservations for slivers: {}".format(e)) raise OrchestratorException(message=f"Failure to build Slivers: {e}")
def __init__(self): if self.__instance is not None: raise OrchestratorException("Singleton can't be created twice !")
def create_slice(self, *, token: str, slice_name: str, slice_graph: str, ssh_key: str, lease_end_time: str) -> dict: """ Create a slice :param token Fabric Identity Token :param slice_name Slice Name :param slice_graph Slice Graph Model :param ssh_key: User ssh key :param lease_end_time: Lease End Time (UTC) :raises Raises an exception in case of failure :returns List of reservations created for the Slice on success """ if self.globals.is_maintenance_mode_on(): raise OrchestratorException(Constants.MAINTENANCE_MODE_ERROR) slice_id = None controller = None orchestrator_slice = None asm_graph = None try: end_time = self.__validate_lease_end_time(lease_end_time=lease_end_time) controller = self.controller_state.get_management_actor() self.logger.debug(f"create_slice invoked for Controller: {controller}") # Check if an Active slice exists already with the same name for the user existing_slices = controller.get_slices(id_token=token, slice_name=slice_name) if existing_slices is not None and len(existing_slices) != 0: for es in existing_slices: if es.get_state() != SliceState.Dead.value and es.get_state() != SliceState.Closing.value: raise OrchestratorException(f"Slice {slice_name} already exists") asm_graph = FimHelper.get_neo4j_asm_graph(slice_graph=slice_graph) broker = self.get_broker(controller=controller) if broker is None: raise OrchestratorException("Unable to determine broker proxy for this controller. " "Please check Orchestrator container configuration and logs.") slice_obj = SliceAvro() slice_obj.set_slice_name(slice_name) slice_obj.set_client_slice(True) slice_obj.set_description("Description") slice_obj.graph_id = asm_graph.get_graph_id() slice_obj.set_config_properties(value={Constants.USER_SSH_KEY: ssh_key}) slice_obj.set_lease_end(lease_end=end_time) self.logger.debug(f"Adding Slice {slice_name}") slice_id = controller.add_slice(slice_obj=slice_obj, id_token=token) if slice_id is None: self.logger.error(controller.get_last_error()) self.logger.error("Slice could not be added to Database") raise OrchestratorException("Slice could not be added to Database") self.logger.debug(f"Slice {slice_name}/{slice_id} added successfully") slice_obj.set_slice_id(slice_id=str(slice_id)) orchestrator_slice = OrchestratorSliceWrapper(controller=controller, broker=broker, slice_obj=slice_obj, logger=self.logger) orchestrator_slice.lock() # Create Slivers from Slice Graph; Compute Reservations from Slivers; # Add Reservations to relational database; computed_reservations = orchestrator_slice.create(slice_graph=asm_graph) # Process the Slice i.e. Demand the computed reservations i.e. Add them to the policy # Once added to the policy; Actor Tick Handler will do following asynchronously: # 1. Ticket message exchange with broker and # 2. Redeem message exchange with AM once ticket is granted by Broker self.controller_state.demand_slice(controller_slice=orchestrator_slice) # self.controller_state.get_sdt().process_slice(controller_slice=orchestrator_slice) return ResponseBuilder.get_reservation_summary(res_list=computed_reservations) except Exception as e: if slice_id is not None and controller is not None and asm_graph is not None: FimHelper.delete_graph(graph_id=asm_graph.graph_id) controller.remove_slice(slice_id=slice_id, id_token=token) self.logger.error(traceback.format_exc()) self.logger.error(f"Exception occurred processing create_slice e: {e}") raise e finally: if orchestrator_slice is not None: orchestrator_slice.unlock()
def check_(self, *, controller: ABCMgmtControllerMixin, rid) -> Status: """ Check Status :param controller: :param rid: :return: """ if not isinstance(rid, ReservationIDWithModifyIndex): return Status.NOTREADY try: reservations = controller.get_reservations( rid=rid.get_reservation_id()) if reservations is None or len(reservations) == 0: raise OrchestratorException( "Unable to obtain reservation information for {}".format( rid.get_reservation_id())) reservation = next(iter(reservations)) if reservation.get_state() != ReservationStates.Active or \ reservation.get_pending_state() != ReservationPendingStates.None_: self.logger.debug( "Returning NOTREADY for reservation {} in state ({}, {})". format(rid, reservation.get_state(), reservation.get_pending_state())) return Status.NOTREADY if reservation.get_state() == ReservationStates.Failed or \ reservation.get_pending_state() == ReservationStates.Closed: return Status.NOTOK units_list = controller.get_reservation_units( rid=rid.get_reservation_id()) if len(units_list) == 0: raise OrchestratorException( "No units associated with reservation {}".format( rid.get_reservation_id())) unit = units_list.__iter__().__next__() properties = unit.get_properties() code_property_name = Constants.UNIT_MODIFY_PROP_PREFIX + rid.get_modify_index() + \ Constants.UNIT_MODIFY_PROP_CODE_SUFFIX message_property_name = Constants.UNIT_MODIFY_PROP_PREFIX + rid.get_modify_index() + \ Constants.UNIT_MODIFY_PROP_MESSAGE_SUFFIX modify_failed = False modify_error_message = None modify_error_code = 0 for key, value in properties.items(): if not modify_failed and code_property_name == key: modify_error_code = int(value) if modify_error_code == 0: self.logger.info( "Reservation {} was modified successfully for modify index {}" .format(rid.get_reservation_id(), rid.get_modify_index())) return Status.OK else: modify_failed = True if message_property_name == key: modify_error_message = value if modify_failed: self.logger.info( "Reservation {} failed modify for modify index {} with code {} and message {}" .format(rid.get_reservation_id(), rid.get_modify_index(), modify_error_code, modify_error_message)) return Status.NOTOK except Exception as e: self.logger.error("Exception occurred e: {}".format(e)) return Status.NOTREADY
def __build_network_service_reservations(self, slice_graph: ABCASMPropertyGraph, node_res_mapping: Dict[str, str]) -> List[TicketReservationAvro]: """ Build Network Service Reservations @param slice_graph Slice graph @param node_res_mapping Mapping of Network Node sliver Id to reservation Id; Needed to add dependency information in network service slivers @return list of network service reservations """ reservations = [] for ns_id in slice_graph.get_all_network_service_nodes(): # Build Network Service Sliver sliver = slice_graph.build_deep_ns_sliver(node_id=ns_id) sliver_type = sliver.get_type() # Ignore Sliver types P4,OVS and MPLS if sliver_type in self.ignorable_ns: continue # Process only the currently supported Network Sliver types L2STS, L2PTP and L2Bridge elif sliver_type in self.supported_ns: self.logger.trace(f"Network Service Sliver: {sliver}") # Processing Interface Slivers if sliver.interface_info is not None: predecessor_reservations = [] for ifs in sliver.interface_info.interfaces.values(): # Get Mapping information for Interface Sliver from ASM # i.e. [Peer IFS, Peer NS Id, Component Name, Node Id] ifs_mapping = FimHelper.get_interface_sliver_mapping(ifs_node_id=ifs.node_id, slice_graph=slice_graph) if ifs_mapping is None: raise OrchestratorException(message=f"Peer connection point not found for ifs# {ifs}", http_error_code=BAD_REQUEST) # capacities (bw in Gbps, burst size is in Mbits) source: (b) # Set Capacities ifs.set_capacities(cap=ifs_mapping.get_peer_ifs().get_capacities()) # Set Labels ifs.set_labels(lab=ifs_mapping.get_peer_ifs().get_labels()) # Save the parent component name and the parent reservation id in the Node Map parent_res_id = node_res_mapping.get(ifs_mapping.get_node_id(), None) node_map = tuple([ifs_mapping.get_component_name(), parent_res_id]) ifs.set_node_map(node_map=node_map) self.logger.trace(f"Interface Sliver: {ifs}") if parent_res_id is not None and parent_res_id not in predecessor_reservations: predecessor_reservations.append(parent_res_id) # Generate reservation for the sliver reservation = self.reservation_converter.generate_reservation(sliver=sliver, slice_id=self.slice_obj.get_slice_id(), end_time=self.slice_obj.get_lease_end(), pred_list=predecessor_reservations) reservations.append(reservation) else: raise OrchestratorException(message="Not implemented", http_error_code=BAD_REQUEST) return reservations
def renew_slice(self, *, token: str, slice_id: str, new_lease_end_time: str): """ Renew a slice :param token Fabric Identity Token :param slice_id Slice Id :param new_lease_end_time: New Lease End Time in UTC in '%Y-%m-%d %H:%M:%S' format :raises Raises an exception in case of failure :return: """ if self.globals.is_maintenance_mode_on(): raise OrchestratorException(Constants.MAINTENANCE_MODE_ERROR) failed_to_extend_rid_list = [] try: controller = self.controller_state.get_management_actor() self.logger.debug(f"renew_slice invoked for Controller: {controller}") slice_guid = None if slice_id is not None: slice_guid = ID(uid=slice_id) slice_list = controller.get_slices(id_token=token, slice_id=slice_guid) if slice_list is None or len(slice_list) == 0: raise OrchestratorException(f"Slice# {slice_id} not found", http_error_code=NOT_FOUND) slice_object = next(iter(slice_list)) slice_state = SliceState(slice_object.get_state()) if slice_state == SliceState.Dead or slice_state == SliceState.Closing: raise OrchestratorException(f"Slice# {slice_id} already closed", http_error_code=BAD_REQUEST) if slice_state != SliceState.StableOK and slice_state != SliceState.StableError: self.logger.info(f"Unable to renew Slice# {slice_guid} that is not yet stable, try again later") raise OrchestratorException(f"Unable to renew Slice# {slice_guid} that is not yet stable, " f"try again later") new_end_time = self.__validate_lease_end_time(lease_end_time=new_lease_end_time) reservations = controller.get_reservations(id_token=token, slice_id=slice_id) if reservations is None or len(reservations) < 1: if controller.get_last_error() is not None: self.logger.error(controller.get_last_error()) raise OrchestratorException(f"Slice# {slice_id} has no reservations") self.logger.debug(f"There are {len(reservations)} reservations in the slice# {slice_id}") for r in reservations: res_state = ReservationStates(r.get_state()) if res_state == ReservationStates.Closed or res_state == ReservationStates.Failed or \ res_state == ReservationStates.CloseWait: continue current_end_time = ActorClock.from_milliseconds(milli_seconds=r.get_end()) if new_end_time < current_end_time: raise OrchestratorException(f"Attempted new term end time is shorter than current slice end time") self.logger.debug(f"Extending reservation with reservation# {r.get_reservation_id()}") result = controller.extend_reservation(reservation=ID(uid=r.get_reservation_id()), new_end_time=new_end_time) if not result: failed_to_extend_rid_list.append(r.get_reservation_id()) else: slice_object.set_lease_end(lease_end=new_end_time) if not controller.update_slice(slice_obj=slice_object): self.logger.error(f"Failed to update lease end time: {new_end_time} in Slice: {slice_object}") return ResponseBuilder.get_response_summary(rid_list=failed_to_extend_rid_list) except Exception as e: self.logger.error(traceback.format_exc()) self.logger.error(f"Exception occurred processing renew e: {e}") raise e