Esempio n. 1
0
        def serve(self):

            casalog_call_origin = "MPICommandServer::serve"

            # First start command and ping status services
            casalog.post("Starting services...", "INFO", casalog_call_origin)
            self.start_services()

            # Notify to MPICommandClient that service is up and running
            self.__communicator.control_service_response_send(response=self.__monitor_server.get_status())

            # Keep serving until a stop signal service is received
            control_service_request = {}
            stop_service_requested = False
            while (not stop_service_requested) and (not self.__monitor_server.get_client_timeout()):

                # Check if there is an incoming control service msg
                msg_available = False
                try:
                    msg_available = self.__communicator.control_service_request_probe()
                except:
                    msg_available = False
                    formatted_traceback = traceback.format_exc()
                    casalog.post(
                        "Exception checking if control service msg is available: %s" % str(formatted_traceback),
                        "SEVERE",
                        casalog_call_origin,
                    )

                # Notify to MPICommandClient that control signal has been processed
                if msg_available:

                    # Receive control service msg
                    msg_received = False
                    control_service_request = {}
                    try:
                        control_service_request = self.__communicator.control_service_request_recv()
                        msg_received = True
                    except:
                        msg_received = False
                        formatted_traceback = traceback.format_exc()
                        casalog.post(
                            "Exception receiving control service msg: %s" % str(formatted_traceback),
                            "SEVERE",
                            casalog_call_origin,
                        )
                        continue

                    # Process control service msg
                    cmd = None
                    send_response = False
                    if msg_received:
                        try:
                            cmd = control_service_request["command"]
                            send_response = control_service_request["send_response"]
                            code = compile(cmd, casalog_call_origin, "exec")
                            exec(code)
                            casalog.post(
                                "Control signal %s successfully handled by server %s"
                                % (str(cmd), str(MPIEnvironment.mpi_processor_rank)),
                                "INFO",
                                casalog_call_origin,
                            )
                        except:
                            formatted_traceback = traceback.format_exc()
                            casalog.post(
                                "Exception handling control signal command %s in server %s: %s"
                                % (
                                    str(control_service_request),
                                    str(MPIEnvironment.mpi_processor_rank),
                                    str(formatted_traceback),
                                ),
                                "SEVERE",
                                casalog_call_origin,
                            )

                    # Notify to MPICommandClient that control signal has been processed
                    if send_response:
                        try:
                            self.__communicator.control_service_response_send(
                                response=self.__monitor_server.get_status()
                            )
                        except:
                            formatted_traceback = traceback.format_exc()
                            casalog.post(
                                "Exception sending response to control signal command %s in server %s: %s"
                                % (str(cmd), str(MPIEnvironment.mpi_processor_rank), str(formatted_traceback)),
                                "SEVERE",
                                casalog_call_origin,
                            )

                time.sleep(MPIEnvironment.mpi_stop_service_sleep_time)

            # Process stop service request
            if stop_service_requested:

                # Check if force mode is needed
                force_command_request_interruption = control_service_request["force_command_request_interruption"]
                finalize_mpi_environment = control_service_request["finalize_mpi_environment"]
                busy = self.__monitor_server.get_status("busy")
                if force_command_request_interruption and busy:
                    casalog.post(
                        "force-stop service signal received, stopping services, "
                        + "command request handler service will be interrupted...",
                        "INFO",
                        casalog_call_origin,
                    )
                else:
                    force_command_request_interruption = False
                    casalog.post("stop service signal received, stopping services...", "INFO", casalog_call_origin)
            else:
                force_command_request_interruption = True
                casalog.post(
                    "client timeout, forcing disconnection, " + "command request handler service will be interrupted..",
                    "INFO",
                    casalog_call_origin,
                )

            # Stop services
            self.stop_services(force_command_request_interruption)

            # Finalize MPI environment
            if finalize_mpi_environment:
                try:
                    casalog.post("Going to finalize MPI environment", "INFO", casalog_call_origin)
                    MPIEnvironment.finalize_mpi_environment()
                except:
                    formatted_traceback = traceback.format_exc()
                    casalog.post(
                        "Exception finalizing MPI environment %s" % str(formatted_traceback),
                        "SEVERE",
                        casalog_call_origin,
                    )

            # Exit
            casalog.post("Exiting", "INFO", casalog_call_origin)
Esempio n. 2
0
        def stop_services(self, force_command_request_interruption=False):

            # jagonzal: This method is called by the atexit module and if it fails it
            # causes ipython to crash, producing a report and waiting for user input
            # so we cannot risk under any circumstances such an event
            try:

                casalog_call_origin = "MPICommandClient::stop_services"

                if self.__life_cycle_state == 0:
                    casalog.post("Services not started", "WARN", casalog_call_origin)
                    return
                elif self.__life_cycle_state == 2:
                    casalog.post("MPICommandClient life cycle finalized", "WARN", casalog_call_origin)
                    return

                # Check if any server is in timeout condition before stopping the monitoring service
                server_rank_timeout = self.__monitor_client.get_server_timeout()
                finalize_mpi_environment = True
                if len(server_rank_timeout) > 0:
                    finalize_mpi_environment = False
                    force_command_request_interruption = True

                # Stop client monitoring services
                self.__monitor_client.stop_services()

                # Notify command requests which are going to be interrupted
                for command_request_id in self.__command_request_list:
                    if not self.__command_response_list.has_key(command_request_id):
                        server = self.__command_request_list[command_request_id]["server"]
                        status = self.__command_request_list[command_request_id]["status"]
                        casalog.post(
                            "Aborting command request with id# %s: %s"
                            % (str(command_request_id), str(self.__command_request_list[command_request_id])),
                            "SEVERE",
                            casalog_call_origin,
                        )

                # Stop client command request-response services
                self.__stop_command_request_queue_service()
                self.__stop_command_response_handler_service()

                # Shutdown plotms process
                self.__send_control_signal(
                    {"command": "pm.killApp()", "signal": "process_control"}, check_response=True
                )

                # Shutdown virtual frame buffer
                self.__send_control_signal(
                    {"command": "self.stop_virtual_frame_buffer()", "signal": "process_control"}, check_response=True
                )

                # Send stop signal to servers
                self.__send_control_signal(
                    {
                        "command": "stop_service_requested = True",
                        "signal": "stop",
                        "force_command_request_interruption": force_command_request_interruption,
                        "finalize_mpi_environment": finalize_mpi_environment,
                    },
                    check_response=False,
                )

                # Finalize MPI environment
                if finalize_mpi_environment:
                    try:
                        casalog.post("Going to finalize MPI environment", "INFO", casalog_call_origin)
                        MPIEnvironment.finalize_mpi_environment()
                    except:
                        formatted_traceback = traceback.format_exc()
                        casalog.post(
                            "Exception finalizing MPI environment %s" % str(formatted_traceback),
                            "SEVERE",
                            casalog_call_origin,
                        )
                else:
                    casalog.post(
                        "MPIServers with rank %s are in timeout condition, skipping MPI_Finalize()"
                        % str(server_rank_timeout),
                        "SEVERE",
                        casalog_call_origin,
                    )

                # UnMark MPI environment to be finalized by the MPICommunicator destructor
                # (Either because it is already finalized or due to a
                # server not responsive that prevents graceful finalization)
                self.__communicator.set_finalize_mpi_environment(False)

                # Set life cycle state
                self.__life_cycle_state = 2

                casalog.post("All services stopped", "INFO", casalog_call_origin)

            except:
                formatted_traceback = traceback.format_exc()
                print "Unhandled exception in MPICommandClient::stop_services %s" % (formatted_traceback)
Esempio n. 3
0
        def serve(self):
            
            casalog_call_origin = "MPICommandServer::serve"        
            
            # First start command and ping status services
            casalog.post("Starting services...","INFO",casalog_call_origin) 
            self.start_services()
            
            # Notify to MPICommandClient that service is up and running
            self.__communicator.control_service_response_send(response=self.__monitor_server.get_status())
            
            # Keep serving until a stop signal service is received
            control_service_request = {}
            stop_service_requested = False
            while ((not stop_service_requested) and (not self.__monitor_server.get_client_timeout())):
                
                # Check if there is an incoming control service msg
                msg_available = False
                try:
                    msg_available = self.__communicator.control_service_request_probe()
                except:
                    msg_available = False
                    formatted_traceback = traceback.format_exc()
                    casalog.post("Exception checking if control service msg is available: %s" 
                                 % str(formatted_traceback),"SEVERE",casalog_call_origin)                        
                
                # Notify to MPICommandClient that control signal has been processed
                if msg_available:
                    
                    # Receive control service msg
                    msg_received = False                    
                    control_service_request = {}
                    try:
                        control_service_request = self.__communicator.control_service_request_recv()
                        msg_received = True
                    except:
                        msg_received = False
                        formatted_traceback = traceback.format_exc()
                        casalog.post("Exception receiving control service msg: %s"
                                     % str(formatted_traceback),"SEVERE",casalog_call_origin)
                        continue
                    
                    # Process control service msg
                    cmd = None
                    send_response = False
                    if msg_received:
                        try:
                            cmd = control_service_request['command']
                            send_response = control_service_request['send_response']
                            code = compile(cmd, casalog_call_origin, 'exec')                                                   
                            exec(code)
                            casalog.post("Control signal %s successfully handled by server %s" 
                                         % (str(cmd),str(MPIEnvironment.mpi_processor_rank)),
                                         "INFO",casalog_call_origin)                            
                        except:
                            formatted_traceback = traceback.format_exc()
                            casalog.post("Exception handling control signal command %s in server %s: %s" 
                                         % (str(control_service_request),
                                            str(MPIEnvironment.mpi_processor_rank),
                                            str(formatted_traceback)),
                                         "SEVERE",casalog_call_origin)
                            
                    # Notify to MPICommandClient that control signal has been processed
                    if send_response:
                        try:
                            self.__communicator.control_service_response_send(response=self.__monitor_server.get_status())
                        except:
                            formatted_traceback = traceback.format_exc()
                            casalog.post("Exception sending response to control signal command %s in server %s: %s" 
                                         % (str(cmd),str(MPIEnvironment.mpi_processor_rank),str(formatted_traceback)),
                                         "SEVERE",casalog_call_origin)
                    
                time.sleep(MPIEnvironment.mpi_stop_service_sleep_time)
            
            # Process stop service request
            if stop_service_requested:
                
                # Check if force mode is needed
                force_command_request_interruption = control_service_request['force_command_request_interruption']
                finalize_mpi_environment = control_service_request['finalize_mpi_environment']
                busy = self.__monitor_server.get_status('busy')
                if force_command_request_interruption and busy:
                    casalog.post("force-stop service signal received, stopping services, " + 
                                 "command request handler service will be interrupted...","INFO",casalog_call_origin)
                else:
                    force_command_request_interruption = False
                    casalog.post("stop service signal received, stopping services...","INFO",casalog_call_origin)
            else:
                force_command_request_interruption = True
                casalog.post("client timeout, forcing disconnection, " +
                             "command request handler service will be interrupted.." ,"INFO",casalog_call_origin)
            
            # Stop services
            self.stop_services(force_command_request_interruption)
            
            # Finalize MPI environment   
            if finalize_mpi_environment:    
                try:
                    casalog.post("Going to finalize MPI environment","INFO",casalog_call_origin)
                    MPIEnvironment.finalize_mpi_environment()
                except:
                    formatted_traceback = traceback.format_exc()
                    casalog.post("Exception finalizing MPI environment %s" 
                                 % str(formatted_traceback),"SEVERE",casalog_call_origin)

            # Exit
            casalog.post("Exiting","INFO",casalog_call_origin)
Esempio n. 4
0
 def stop_services(self,force_command_request_interruption=False):
     
     # jagonzal: This method is called by the atexit module and if it fails it
     # causes ipython to crash, producing a report and waiting for user input
     # so we cannot risk under any circumstances such an event
     try:
         
         casalog_call_origin = "MPICommandClient::stop_services"
         
         if self.__life_cycle_state == 0:
             casalog.post("Services not started","WARN",casalog_call_origin)
             return       
         elif self.__life_cycle_state == 2:
             casalog.post("MPICommandClient life cycle finalized","WARN",casalog_call_origin)
             return      
         
         # Check if any server is in timeout condition before stopping the monitoring service
         server_rank_timeout = self.__monitor_client.get_server_timeout()
         finalize_mpi_environment = True
         if len(server_rank_timeout) > 0:
             finalize_mpi_environment = False
             force_command_request_interruption = True                               
         
         # Stop client monitoring services
         self.__monitor_client.stop_services()
         
         # Notify command requests which are going to be interrupted
         for command_request_id in self.__command_request_list:
                 if not self.__command_response_list.has_key(command_request_id):
                     server = self.__command_request_list[command_request_id]['server']
                     status = self.__command_request_list[command_request_id]['status']
                     casalog.post("Aborting command request with id# %s: %s" 
                                  % (str(command_request_id),str(self.__command_request_list[command_request_id])),
                                  "SEVERE",casalog_call_origin)
         
         # Stop client command request-response services
         self.__stop_command_request_queue_service()
         self.__stop_command_response_handler_service()          
         
         # Shutdown plotms process
         self.__send_control_signal({'command':'pm.killApp()',
                                     'signal':'process_control'},
                                    check_response=True)
         
         # Shutdown virtual frame buffer
         self.__send_control_signal({'command':'self.stop_virtual_frame_buffer()',
                                     'signal':'process_control'},
                                    check_response=True)      
             
         # Send stop signal to servers
         self.__send_control_signal({'command':'stop_service_requested = True',
                                     'signal':'stop',
                                     'force_command_request_interruption':force_command_request_interruption,
                                     'finalize_mpi_environment':finalize_mpi_environment},
                                    check_response=False)    
         
         # Finalize MPI environment            
         if finalize_mpi_environment:
             try:
                 casalog.post("Going to finalize MPI environment","INFO",casalog_call_origin)
                 MPIEnvironment.finalize_mpi_environment()
             except:
                 formatted_traceback = traceback.format_exc()
                 casalog.post("Exception finalizing MPI environment %s" 
                              % str(formatted_traceback),"SEVERE",casalog_call_origin)
         else:
             casalog.post("MPIServers with rank %s are in timeout condition, skipping MPI_Finalize()" 
                          % str(server_rank_timeout),"SEVERE",casalog_call_origin)
         
         # UnMark MPI environment to be finalized by the MPICommunicator destructor
         # (Either because it is already finalized or due to a 
         # server not responsive that prevents graceful finalization)  
         self.__communicator.set_finalize_mpi_environment(False)         
                           
         # Set life cycle state
         self.__life_cycle_state = 2            
         
         casalog.post("All services stopped","INFO",casalog_call_origin)
         
     except:
         formatted_traceback = traceback.format_exc()
         print "Unhandled exception in MPICommandClient::stop_services %s" %(formatted_traceback)