def target_on_stop_all_wd(self, params): # the platform node target of a clean up is always the master threading.Thread(target=self.wdm.delete_all_watchdogs).start() reason = "Stop all wd ack" result = RCMPMessage() result.create_ok_response(reason) return result
def target_on_rollback_update(self, params): # the platform node target of an update is always the master threading.Thread(target=self.execute_rollback_update, args=(params, )).start() reason = "Rollback update ack" result = RCMPMessage() result.create_ok_response(reason) return result
def target_on_pairing(self, params): # when a platform node target receives a pairing event means that it is paired # to the platform node of type R that has sent him the event # it has to notify the master about this situation so sends an update event self.throw_update_event(params) reason = "Pairing ack" result = RCMPMessage() result.create_ok_response(reason) return result
def target_on_update(self, params): # when the master receives the update means that the connection between the 2 peers # has accepted and we must finalize the flow saving this connection and starting # the service logic associated with the robot in this connection # the platform node target of an update is always the master threading.Thread(target=self.execute_update, args=(params, )).start() reason = "Update ack" result = RCMPMessage() result.create_ok_response(reason) return result
def notify(self, params): """Notify an event to the RCM platform node instance specified in the request. Return an RCMPMessage.""" result = RCMPMessage() self._logger.debug("event notify with params: %s" % params) # the events are asynchronous so we don't need a timeout to wait the response c = RCMPInterCommunicationClient(params[self.I_ADDRESS_KEY], t_out=2.0) try: c.connect() # propagate the request as it is c.send(RCMPMessage(params).get_txt()) # take the result of the operation # res = c.receive() res = c.receive_all_so_far() if res: result.set_content(res) else: reason = "No response received" result.create_error_response(reason) except Exception as e: reason = "Error notifying the event to the RCM platform node at %s: %s" % (params[self.I_ADDRESS_KEY], e) self._logger.error(reason) result.create_error_response(reason) finally: try: c.close() except Exception as e: # this exception doesn't change the result but we want to trace in the log self._logger.warning("Error closing the notifying client: %s" % e) return result
def delegate(self, params, t_out=None): """Delegate the command to the RCM platform node instance specified in the request. Return an RCMPMessage.""" result = RCMPMessage() # the timeout provide the capability to decrease or increase the time available to perform # the task and can be useful let it change in base of the required operation if t_out: c = RCMPInterCommunicationClient(params[self.I_ADDRESS_KEY], t_out=t_out) else: c = RCMPInterCommunicationClient(params[self.I_ADDRESS_KEY]) try: c.connect() # propagate the request as it is c.send(RCMPMessage(params).get_txt()) # take the result of the operation res = c.receive_all_so_far() if res: result.set_content(res) else: reason = "No response received" result.create_error_response(reason) except Exception as e: reason = "Error delegating the request to the RCM platform node at %s: %s" % ( params[self.I_ADDRESS_KEY], e) self._logger.error(reason) result.create_error_response(reason) finally: try: c.close() except Exception as e: # this exception doesn't change the result but we want to trace in the log self._logger.warning( "Error closing the delegating client: %s" % e) return result
def target_on_rollback_update_all(self, params): # the platform node target of a rollback update all is always the master # the all version means that we have to rollback all the update between # we are rolling back all the update of the master platform node, so we take the # name of the current platform node and use it to have the connections associated # with it params[self.PI_S_NAME_KEY] = self.pni[RCMPlatformNode.PNI_NAME] params[self.PI_REACHABLE_KEY] = self.pni[RCMPlatformNode.PNI_TYPE] threading.Thread(target=self.execute_rollback_update_all, args=(params, )).start() reason = "Rollback update all ack" result = RCMPMessage() result.create_ok_response(reason) return result
def delete_all_of_ss(self, ss_name, s_space): from rcmp_robotics_data import execute_int_robotics_data_query ss_info = execute_int_robotics_data_query(self.get_ss_info, ss_name, "Unable to get the service spaces info of %s" % ss_name) if ss_info: from rcmp_command import DELETE_L_SERVICE_SPACE, DELETE_SERVICE_SPACE, RCMPCommandHandler from rcmp_service_command import ServiceSpace cmd = {RCMPCommandHandler.COMMAND_KEY: DELETE_L_SERVICE_SPACE, PNodeInstance.I_ADDRESS_KEY: self.pni[RCMPlatformNode.PNI_ADDRESS], ServiceSpace.SS_ADDRESS_KEY: ss_info[0], ServiceSpace.SS_PORT_KEY: str(ss_info[1])} result = s_space.delete_local(cmd) else: result = RCMPMessage() result.create_error_response("Not enough info to delete services of '%s'" % ss_name) return result
def delegate(self, params, t_out=None): """Delegate the command to the RCM platform node instance specified in the request. Return an RCMPMessage.""" result = RCMPMessage() # the timeout provide the capability to decrease or increase the time available to perform # the task and can be useful let it change in base of the required operation if t_out: c = RCMPInterCommunicationClient(params[self.I_ADDRESS_KEY], t_out=t_out) else: c = RCMPInterCommunicationClient(params[self.I_ADDRESS_KEY]) try: c.connect() # propagate the request as it is c.send(RCMPMessage(params).get_txt()) # take the result of the operation res = c.receive_all_so_far() if res: result.set_content(res) else: reason = "No response received" result.create_error_response(reason) except Exception as e: reason = "Error delegating the request to the RCM platform node at %s: %s" % (params[self.I_ADDRESS_KEY], e) self._logger.error(reason) result.create_error_response(reason) finally: try: c.close() except Exception as e: # this exception doesn't change the result but we want to trace in the log self._logger.warning("Error closing the delegating client: %s" % e) return result
def execute_robotics_data_query(robotics_data_related_f, params=None, err_reason="Error"): """Execute a robotics data query. Manage the connection to RCMPRoboticsDataManager and call the function robotics_data_related_f to know what to do with that. Return an RCMPMessage.""" from rcmp_inter_communication import RCMPMessage rdm = RCMPRoboticsDataManager() try: rdm.connect() if params: result = robotics_data_related_f(rdm, params) else: result = robotics_data_related_f(rdm) except sqlite3.Error as se: reason = "%s: %s" % (err_reason, se) result = RCMPMessage() result.create_error_response(reason) finally: rdm.disconnect() return result
def target_on_paired(self, params): # when a platform node target receives a paired event means that the master received # the update event notifying that the two peers agreed in pairing each other; the # connections table in robotics data on the master is updated and the service logic # has been started # on local we have to launch the watchdog to monitor the other peer # connection self._logger.debug("Executing paired") self.wdm.add_watchdog(params) self.wdm.get_watchdog(params).start() reason = "Paired ack" result = RCMPMessage() result.create_ok_response(reason) if not self.pni[RCMPlatformNode.PNI_IS_MASTER]: # the paired event arrived so the RCMPlatformNode doesn't need to directly ping # the master to know if it is on the platform # TODO the write at this level means that maybe we have to synchronize pni self.pni[RCMPlatformNode.PNI_STATE] = RCMPlatformNode.PAIRED_STATE return result
def generate_internal_msg(self, request): msg_dict = {RCMPCommandHandler.COMMAND_KEY: READ_SERVICE_SPACE} if I_NAME_PARAM in request.args and request.args[ I_NAME_PARAM] and request.args[I_NAME_PARAM][0]: msg_dict[ServiceSpace.I_NAME_KEY] = request.args[I_NAME_PARAM][0] if SS_NAME_PARAM in request.args and request.args[ SS_NAME_PARAM] and request.args[SS_NAME_PARAM][0]: # this is a mandatory parameter msg_dict[ServiceSpace.SS_NAME_KEY] = request.args[SS_NAME_PARAM][0] else: raise SyntaxError("%s is missing in the http request" % SS_NAME_PARAM) return RCMPMessage(msg_dict)
def execute(self, operation=None, params=None): """Execute the required operation with the parameters passed. Return an RCMPMessage.""" result = RCMPMessage() if operation: if params and self.I_ADDRESS_KEY in params and params[self.I_ADDRESS_KEY] and \ params[self.I_ADDRESS_KEY] != self.pni[RCMPlatformNode.PNI_ADDRESS]: # perform the delegation to another platform node because the target # is not the current one result = self.notify(params) else: # perform the operation (the target is the current platform node) result = operation(params) return result
def generate_internal_msg(self, params): # in the case of post, this method receives params as parameter # instead of request: this is because in this case we don't pass # the http request usually (in the get case) do msg_dict = { RCMPCommandHandler.COMMAND_KEY: ROLLBACK_PROVISIONING_SLOGIC_INSTANCE } if SLG_NAME_PARAM in params and params[SLG_NAME_PARAM]: # this is a mandatory parameter msg_dict[SLogicInstance.SLG_NAME_KEY] = params[SLG_NAME_PARAM] else: raise SyntaxError("%s is missing in the http request" % SLG_NAME_PARAM) return RCMPMessage(msg_dict)
def handle(self, params): """Delegate a sub component to handle the event.""" result = RCMPMessage() event = params[self.EVENT_KEY] # events about instances p_node_instance = PNodeInstance(self.pni, self.wdm, self.msm) if event == PAIRING_EVENT: result = p_node_instance.pairing(params) elif event == UPDATE_EVENT: result = p_node_instance.update(params) elif event == PAIRED_EVENT: result = p_node_instance.paired(params) elif event == ROLLBACK_UPDATE_EVENT: result = p_node_instance.rollback_update(params) elif event == ROLLBACK_UPDATE_ALL_EVENT: result = p_node_instance.rollback_update_all(params) elif event == STOP_WD_EVENT: result = p_node_instance.stop_wd(params) elif event == STOP_ALL_WD_EVENT: result = p_node_instance.stop_all_wd(params) else: reason = "The event '%s' does not exist" % event result.create_error_response(reason) return result
def generate_internal_msg(self, request): if SLG_NAME_PARAM in request.args and request.args[SLG_NAME_PARAM]: if request.args[SLG_NAME_PARAM][0]: # we suppose that in case there is a parameter with service logic name # we want a READ_SLOGIC_INSTANCE msg_dict = { RCMPCommandHandler.COMMAND_KEY: READ_SLOGIC_INSTANCE, SLogicInstance.SLG_NAME_KEY: request.args[SLG_NAME_PARAM][0] } else: raise SyntaxError("%s in the http request but empty" % SLG_NAME_PARAM) else: # otherwise we suppose that we want the list of all available service logics # and so READ_SLOGIC_INSTANCE_LIST msg_dict = { RCMPCommandHandler.COMMAND_KEY: READ_SLOGIC_INSTANCE_LIST } return RCMPMessage(msg_dict)
def execute(self, m_operation=None, operation=None, params=None, t_out=None): """Execute the required operation with the parameters passed. Return an RCMPMessage.""" result = RCMPMessage() if m_operation: if self.pni[RCMPlatformNode.PNI_IS_MASTER]: # perform the operation that can be done only by the master result = m_operation(params) if operation: if params and self.I_ADDRESS_KEY in params and params[self.I_ADDRESS_KEY] and \ params[self.I_ADDRESS_KEY] != self.pni[RCMPlatformNode.PNI_ADDRESS]: # perform the delegation to another platform node because the target # is not the current one result = self.delegate(params, t_out) else: # perform the operation (the target is the current platform node) result = operation(params) return result
def render_GET(self, request): """Dispatch the get request to the internal communication server and return the result.""" ret = RCMPMessage() try: # created using default parameters means calling to a local server icc = RCMPInterCommunicationClient() msg = None try: icc.connect() msg = self.generate_internal_msg(request) icc.send(msg.get_txt()) res = icc.receive_all_so_far() if res: ret.set_content(res) else: reason = "No response received" request.setResponseCode(http.INTERNAL_SERVER_ERROR) ret.create_error_response(reason) except SyntaxError as se: reason = "Syntax error in the request: %s" % se self.logger.exception(reason) request.setResponseCode(http.BAD_REQUEST) ret.create_error_response(reason) except Exception as e: reason = "Unable to accomplish the request %s: %s" % (msg, e) self.logger.exception(reason) request.setResponseCode(http.INTERNAL_SERVER_ERROR) ret.create_error_response(reason) finally: icc.close() except Exception as e: reason = "Internal communication server error: %s" % e self.logger.exception(reason) request.setResponseCode(http.INTERNAL_SERVER_ERROR) ret.create_error_response(reason) request.setHeader("Content-Type", "application/json") request.setHeader("Access-Control-Allow-Origin", "*") request.setHeader("Access-Control-Allow-Headers", "Content-Type") request.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS") return ret.get_txt()
def render_POST(self, request): """Dispatch the post request to the internal communication server and return the result.""" ret = RCMPMessage() try: # created using default parameters means calling to a local server icc = RCMPInterCommunicationClient() msg = None try: icc.connect() ext_data = self.get_ext_data(request) msg = self.generate_internal_msg(ext_data) icc.send(msg.get_txt()) res = icc.receive_all_so_far() if res: ret.set_content(res) else: reason = "No response received" request.setResponseCode(http.INTERNAL_SERVER_ERROR) ret.create_error_response(reason) except SyntaxError as se: reason = "Syntax error in the request: %s" % se self.logger.exception(reason) request.setResponseCode(http.BAD_REQUEST) ret.create_error_response(reason) except Exception as e: reason = "Unable to accomplish the request %s: %s" % (msg, e) self.logger.exception(reason) request.setResponseCode(http.INTERNAL_SERVER_ERROR) ret.create_error_response(reason) finally: icc.close() except Exception as e: reason = "Internal communication server error: %s" % e self.logger.exception(reason) request.setResponseCode(http.INTERNAL_SERVER_ERROR) ret.create_error_response(reason) request.setHeader("Content-Type", "application/json") request.setHeader("Access-Control-Allow-Origin", "*") request.setHeader("Access-Control-Allow-Headers", "Content-Type") request.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS") return ret.get_txt()
def generate_internal_msg(self, request): # otherwise we suppose that we want the list of all available platform instances # and so READ_PNODE_INSTANCE_LIST msg_dict = {RCMPCommandHandler.COMMAND_KEY: READ_PNODE_INSTANCE_LIST} return RCMPMessage(msg_dict)
def handle(self, params): """Delegate a sub component to execute the command. There are 2 sets of commands: commands addressed to services and others to the platform node itself.""" from rcmp_service_command import ServiceSpace, ServiceLauncher, ServiceNode, SLogicInstance from rcmp_platform_command import PNodeInstance cmd = params[self.COMMAND_KEY] op = None result = RCMPMessage() # commands about services if cmd == START_SERVICE_NODE or cmd == KILL_SERVICE_NODE or \ cmd == READ_SERVICE_NODE or cmd == KILL_ALL_L_SERVICE_NODE: s_node = ServiceNode(self.pni, self.msm) if cmd == START_SERVICE_NODE: op = s_node.start elif cmd == KILL_SERVICE_NODE: op = s_node.stop elif cmd == READ_SERVICE_NODE: op = s_node.read_list elif cmd == KILL_ALL_L_SERVICE_NODE: op = s_node.stop_all_local elif cmd == CREATE_SERVICE_SPACE or cmd == DELETE_SERVICE_SPACE or \ cmd == READ_SERVICE_SPACE or cmd == READ_P_IP_SERVICE_SPACE or \ cmd == DELETE_ALL_L_SERVICE_SPACE: s_space = ServiceSpace(self.pni, self.msm) if cmd == CREATE_SERVICE_SPACE: op = s_space.create elif cmd == DELETE_SERVICE_SPACE: op = s_space.delete elif cmd == READ_SERVICE_SPACE: op = s_space.read elif cmd == READ_P_IP_SERVICE_SPACE: op = s_space.read_public_ip elif cmd == DELETE_L_SERVICE_SPACE: op = s_space.delete_local elif cmd == DELETE_ALL_L_SERVICE_SPACE: op = s_space.delete_all_local elif cmd == START_SERVICE_LAUNCHER or cmd == KILL_SERVICE_LAUNCHER or \ cmd == READ_SERVICE_LAUNCHER or cmd == KILL_ALL_L_SERVICE_LAUNCHER: s_launcher = ServiceLauncher(self.pni, self.msm) if cmd == START_SERVICE_LAUNCHER: op = s_launcher.start elif cmd == KILL_SERVICE_LAUNCHER: op = s_launcher.stop elif cmd == READ_SERVICE_LAUNCHER: op = s_launcher.read_list elif cmd == KILL_ALL_L_SERVICE_LAUNCHER: op = s_launcher.stop_all_local # commands about instances elif cmd == MAN_PROVISIONING_PNODE_INSTANCE or cmd == AUTO_PROVISIONING_PNODE_INSTANCE \ or cmd == ROLLBACK_PROVISIONING_PNODE_INSTANCE or cmd == READ_PNODE_INSTANCE_LIST \ or cmd == READ_PNODE_INSTANCE or cmd == PING_PNODE_INSTANCE \ or cmd == DISCOVERY_PNODE_INSTANCE: p_node_instance = PNodeInstance(self.pni, self.wdm) if cmd == MAN_PROVISIONING_PNODE_INSTANCE: op = p_node_instance.manual_provisioning elif cmd == AUTO_PROVISIONING_PNODE_INSTANCE: op = p_node_instance.automatic_provisioning elif cmd == ROLLBACK_PROVISIONING_PNODE_INSTANCE: op = p_node_instance.rollback_provisioning elif cmd == READ_PNODE_INSTANCE_LIST: op = p_node_instance.read_list elif cmd == READ_PNODE_INSTANCE: op = p_node_instance.read elif cmd == PING_PNODE_INSTANCE: op = p_node_instance.ping elif cmd == DISCOVERY_PNODE_INSTANCE: op = p_node_instance.discovery elif cmd == MAN_PROVISIONING_SLOGIC_INSTANCE or cmd == ROLLBACK_PROVISIONING_SLOGIC_INSTANCE \ or cmd == READ_SLOGIC_INSTANCE_LIST or cmd == READ_SLOGIC_INSTANCE: s_logic_instance = SLogicInstance(self.pni) if cmd == MAN_PROVISIONING_SLOGIC_INSTANCE: op = s_logic_instance.manual_provisioning elif cmd == ROLLBACK_PROVISIONING_SLOGIC_INSTANCE: op = s_logic_instance.rollback_provisioning elif cmd == READ_SLOGIC_INSTANCE_LIST: op = s_logic_instance.read_list elif cmd == READ_SLOGIC_INSTANCE: op = s_logic_instance.read else: reason = "The command '%s' does not exist" % cmd result.create_error_response(reason) if op is not None: result = op(params) return result
def run(self): self._logger.info("---- watchdog '%s' starting ----" % self.name) # rcm_d_c = None rcm_n2d_m = None # at this moment the robot is connected so we have to notify that to RCM driver if self.pni[RCMPlatformNode.PNI_RCM_DRIVER_PORT]: # this is done only in case we have rcm driver on this rcm node instance try: self._logger.info("watchdog '%s' running with rcm on port %d" % (self.name, self.pni[RCMPlatformNode.PNI_RCM_DRIVER_PORT])) rs_queue = Queue.Queue() es_queue = Queue.Queue() rcm_n2d_m = self.RCMNode2DriverManager(rs_queue, es_queue, self.pni[RCMPlatformNode.PNI_RCM_DRIVER_PORT]) rcm_n2d_m.connect() threading.Thread(target=rcm_n2d_m.sniff).start() result = rcm_n2d_m.notify_robot_status(self.pni[RCMPlatformNode.PNI_NAME] if self.pni[RCMPlatformNode.PNI_TYPE] == RCMPlatformNode.R_TYPE else self.paired_node_name, True) if result is None: self._logger.info("Entity creation status required too much time: don't wait anymore") except Exception as e: self._logger.error("watchdog error notifying robot connection to rcm driver: %s" % e) self._logger.info("'%s' connected to '%s': watchdog monitoring the connection" % (self.pni[RCMPlatformNode.PNI_NAME], self.paired_node_name)) # the first time we launch the watchdog we wait for 1 minute to give the master the time # to perform the service logic boot w_timeout = 30 while not self.stop_event.isSet(): stop_event_is_set = self.stop_event.wait(w_timeout) if not stop_event_is_set: # we are not stopping the watchdog so we have to do the operations result = RCMPMessage() try: try_execution(self.ping_pni, args=(self.paired_node_ip, ), to_increment=0) # the ping to the other peer went well so the connection is still active except Exception as e: # the ping to the other peer failed self._logger.debug("Watchdog ping to '%s' failed: %s" % (self.paired_node_name, e)) try: if rcm_n2d_m: # this is done only in case we have rcm_d_c because means that we are on the # platform node instance which runs the rcm driver result = rcm_n2d_m.notify_robot_status(self.pni[RCMPlatformNode.PNI_NAME] if self.pni[RCMPlatformNode.PNI_TYPE] == RCMPlatformNode.R_TYPE else self.paired_node_name, False) if result is None: self._logger.info("Entity removal status required too much time: don't wait anymore") except Exception as e: self._logger.error("watchdog error notifying robot disconnection to rcm driver: %s" % e) self._logger.info("'%s' disconnected from '%s'" % (self.pni[RCMPlatformNode.PNI_NAME], self.paired_node_name)) if self.pni[RCMPlatformNode.PNI_IS_MASTER]: # we are on the master so we cannot be offline from the platform: these means that # the other peer is offline self.remote_paired_node_offline() elif self.pni[RCMPlatformNode.PNI_MASTER_NODE_IP] == self.paired_node_ip: # the other peer is the master that cannot be offline from the platform, so we are # us that we are offline self.local_paired_node_offline() else: # the master platform node is not involved between paired so to verify who is # offline we try to ping the master try: try_execution(self.ping_pni, args=(self.pni[RCMPlatformNode.PNI_MASTER_NODE_IP], ), to_increment=0) # the ping to the master went well so the other peer is offline self.remote_paired_node_offline() except Exception as e: # the ping to the master failed so we are us offline result.create_error_response("Watchdog error: %s" % e) self.local_paired_node_offline() w_timeout = 10 self._logger.debug("---- watchdog '%s' ended ----" % self.name)
def run(self): self._logger.info("---- watchdog '%s' starting ----" % self.name) # rcm_d_c = None rcm_n2d_m = None # at this moment the robot is connected so we have to notify that to RCM driver if self.pni[RCMPlatformNode.PNI_RCM_DRIVER_PORT]: # this is done only in case we have rcm driver on this rcm node instance try: self._logger.info( "watchdog '%s' running with rcm on port %d" % (self.name, self.pni[RCMPlatformNode.PNI_RCM_DRIVER_PORT])) rs_queue = Queue.Queue() es_queue = Queue.Queue() rcm_n2d_m = self.RCMNode2DriverManager( rs_queue, es_queue, self.pni[RCMPlatformNode.PNI_RCM_DRIVER_PORT]) rcm_n2d_m.connect() threading.Thread(target=rcm_n2d_m.sniff).start() result = rcm_n2d_m.notify_robot_status( self.pni[RCMPlatformNode.PNI_NAME] if self.pni[RCMPlatformNode.PNI_TYPE] == RCMPlatformNode.R_TYPE else self.paired_node_name, True) if result is None: self._logger.info( "Entity creation status required too much time: don't wait anymore" ) except Exception as e: self._logger.error( "watchdog error notifying robot connection to rcm driver: %s" % e) self._logger.info( "'%s' connected to '%s': watchdog monitoring the connection" % (self.pni[RCMPlatformNode.PNI_NAME], self.paired_node_name)) # the first time we launch the watchdog we wait for 1 minute to give the master the time # to perform the service logic boot w_timeout = 30 while not self.stop_event.isSet(): stop_event_is_set = self.stop_event.wait(w_timeout) if not stop_event_is_set: # we are not stopping the watchdog so we have to do the operations result = RCMPMessage() try: try_execution(self.ping_pni, args=(self.paired_node_ip, ), to_increment=0) # the ping to the other peer went well so the connection is still active except Exception as e: # the ping to the other peer failed self._logger.debug("Watchdog ping to '%s' failed: %s" % (self.paired_node_name, e)) try: if rcm_n2d_m: # this is done only in case we have rcm_d_c because means that we are on the # platform node instance which runs the rcm driver result = rcm_n2d_m.notify_robot_status( self.pni[RCMPlatformNode.PNI_NAME] if self.pni[RCMPlatformNode.PNI_TYPE] == RCMPlatformNode.R_TYPE else self.paired_node_name, False) if result is None: self._logger.info( "Entity removal status required too much time: don't wait anymore" ) except Exception as e: self._logger.error( "watchdog error notifying robot disconnection to rcm driver: %s" % e) self._logger.info("'%s' disconnected from '%s'" % (self.pni[RCMPlatformNode.PNI_NAME], self.paired_node_name)) if self.pni[RCMPlatformNode.PNI_IS_MASTER]: # we are on the master so we cannot be offline from the platform: these means that # the other peer is offline self.remote_paired_node_offline() elif self.pni[RCMPlatformNode. PNI_MASTER_NODE_IP] == self.paired_node_ip: # the other peer is the master that cannot be offline from the platform, so we are # us that we are offline self.local_paired_node_offline() else: # the master platform node is not involved between paired so to verify who is # offline we try to ping the master try: try_execution( self.ping_pni, args=(self.pni[ RCMPlatformNode.PNI_MASTER_NODE_IP], ), to_increment=0) # the ping to the master went well so the other peer is offline self.remote_paired_node_offline() except Exception as e: # the ping to the master failed so we are us offline result.create_error_response("Watchdog error: %s" % e) self.local_paired_node_offline() w_timeout = 10 self._logger.debug("---- watchdog '%s' ended ----" % self.name)