class __MPICommandServerImpl: """ Implementation of the MPICommandServer singleton interface """ def __init__(self, start_services=False): # Initialize command request handler service state self.__command_request_handler_service_on = False self.__command_request_handler_service_running = False self.__command_request_handler_service_thread = None # Instantiate MPICommunicator reference self.__communicator = MPICommunicator() # Instantiate MPIMonitorClient reference self.__monitor_server = MPIMonitorServer(False) # Initialize logfile descriptor self.__logfile_descriptor = open(casalog.logfile(), "a") # Initialize virtual frame buffer state self.__virtual_frame_buffer_port = None self.__virtual_frame_buffer_process = None # Automatically start services if start_services: self.start_services() ################################################################################################################ # Private methods ############################################################################################## ################################################################################################################ def __command_request_handler_service(self): casalog_call_origin = "MPICommandServer::command_request_handler_service" # Mark service as running self.__command_request_handler_service_running = True while self.__command_request_handler_service_on: # First check if there is a command request msg available msg_available = False try: msg_available = self.__communicator.command_request_probe() except Exception as instance: casalog.post( "Exception checking if command request msg is available: %s" % str(instance), "SEVERE", casalog_call_origin, ) msg_available = False # Then receive command request msg msg_received = False if msg_available: try: command_request = self.__communicator.command_request_recv() casalog.post( "Received command request msg: %s" % command_request["command"], MPIEnvironment.command_handling_log_level, casalog_call_origin, ) msg_received = True except: formatted_traceback = traceback.format_exc() casalog.post( "Exception receiving command request msg: %s" % str(formatted_traceback), "SEVERE", casalog_call_origin, ) msg_received = False # Finally process command request and send back response if msg_received: # Start timer command_start_time = time.time() # Update server state self.__monitor_server.set_status("busy", True) self.__monitor_server.set_status("command", command_request["command"]) self.__monitor_server.set_status("command_start_time", command_start_time) # Get command request id command_request_id = command_request["id"] # Prepare command response command_response = dict(command_request) # Set command start time command_response["command_start_time"] = command_start_time # Execute/Evaluate command request try: # Add dict-defined parameters to globals if isinstance(command_request["parameters"], dict): globals().update(command_request["parameters"]) # Execute command if command_request["mode"] == "eval": casalog.post( "Going to evaluate command request with id# %s as an expression via eval: %s" % (str(command_request_id), str(command_request["command"])), MPIEnvironment.command_handling_log_level, casalog_call_origin, ) command_response["ret"] = eval(command_request["command"]) elif command_request["mode"] == "exec": casalog.post( "Going to execute command request with id# %s as a statement via exec: %s" % (str(command_request_id), command_request["command"]), MPIEnvironment.command_handling_log_level, casalog_call_origin, ) code = compile(command_request["command"], casalog_call_origin, "exec") exec(code) command_response["ret"] = None elif command_request["mode"] == "push": casalog.post( "Command request with id# %s is a push operation" % str(command_request_id), MPIEnvironment.command_handling_log_level, casalog_call_origin, ) command_response["ret"] = None # Set command response parameters command_response["successful"] = True command_response["traceback"] = None except: formatted_traceback = traceback.format_exc() casalog.post( "Exception executing command request via %s: %s" % (command_request["mode"], str(formatted_traceback)), "SEVERE", casalog_call_origin, ) # Set command response parameters command_response["successful"] = False command_response["traceback"] = formatted_traceback command_response["ret"] = None # Variables are cleaned from the environment regardless of the result finally: # Clear parameter variables if isinstance(command_request["parameters"], dict) and command_request["mode"] != "push": for parameter in command_request["parameters"]: try: del globals()[parameter] except: formatted_traceback = traceback.format_exc() casalog.post( "Exception deleting parameter variable '%s' from global environment: %s" % (str(parameter), str(formatted_traceback)), "WARN", casalog_call_origin, ) # Set command stop time command_stop_time = time.time() command_response["command_stop_time"] = command_stop_time # Update server state self.__monitor_server.set_status("busy", False) self.__monitor_server.set_status("command", None) self.__monitor_server.set_status("command_start_time", None) # Send response back (successful or not) try: casalog.post( "Command request with id %s successfully processed in %s mode, sending back response ..." % (str(command_response["id"]), str(command_response["mode"])), MPIEnvironment.command_handling_log_level, casalog_call_origin, ) self.__communicator.command_response_send(response=command_response) except: formatted_traceback = traceback.format_exc() casalog.post( "Exception sending back command response: %s" % str(formatted_traceback), "SEVERE", casalog_call_origin, ) else: time.sleep(MPIEnvironment.mpi_command_request_handler_service_sleep_time) # Mark service as not running self.__command_request_handler_service_running = False def __start_command_request_handler_service(self): casalog_call_origin = "MPICommandServer::start_command_request_handler_service" if self.__command_request_handler_service_running: casalog.post("MPI command request handler service is already running", "WARN", casalog_call_origin) return True try: self.__command_request_handler_service_on = True self.__command_request_handler_service_thread = thread.start_new_thread( self.__command_request_handler_service, () ) except Exception as instance: self.__command_request_handler_service_on = False self.__command_request_handler_service_running = False casalog.post( "Exception starting MPI command request handler service: %s" % str(instance), "SEVERE", casalog_call_origin, ) return False while not self.__command_request_handler_service_running: time.sleep(MPIEnvironment.mpi_check_start_service_sleep_time) casalog.post("MPI command request handler service started", "INFO", casalog_call_origin) return True def __stop_command_request_handler_service(self): casalog_call_origin = "MPICommandServer::stop_command_request_handler_service" if not self.__command_request_handler_service_running: casalog.post("MPI command request handler service is not running", "WARN", casalog_call_origin) return self.__command_request_handler_service_on = False while self.__command_request_handler_service_running: time.sleep(MPIEnvironment.mpi_check_stop_service_sleep_time) casalog.post("MPI command request handler service stopped", "INFO", casalog_call_origin) ################################################################################################################ # Public methods ############################################################################################### ################################################################################################################ def start_virtual_frame_buffer(self): casalog_call_origin = "MPICommandServer::start_virtual_frame_buffer" displayport = os.getpid() while os.path.exists("/tmp/.X%d-lock" % displayport): displayport += 1 self.__virtual_frame_buffer_port = ":%d" % displayport self.__xauthfile = tempfile.NamedTemporaryFile() try: cookie = subprocess.check_output(["mcookie"], universal_newlines=True).strip() except: cookie = str(uuid.uuid4()).replace("-", "") # sometimes also works without auth, so accept failure subprocess.call( ["xauth", "-f", self.__xauthfile.name, "add", self.__virtual_frame_buffer_port, ".", cookie], stdout=self.__logfile_descriptor, stderr=self.__logfile_descriptor, ) try: self.__virtual_frame_buffer_process = subprocess.Popen( ["Xvfb", self.__virtual_frame_buffer_port, "-auth", self.__xauthfile.name], stdout=self.__logfile_descriptor, stderr=self.__logfile_descriptor, shell=False, ) os.environ["DISPLAY"] = self.__virtual_frame_buffer_port os.environ["XAUTHORITY"] = self.__xauthfile.name casalog.post( "Deployed virtual frame buffer at %s with pid %s" % (self.__virtual_frame_buffer_port, str(self.__virtual_frame_buffer_process.pid)), "INFO", casalog_call_origin, ) except: self.__virtual_frame_buffer_process = None formatted_traceback = traceback.format_exc() casalog.post( "Exception deploying virtual frame buffer at %s: %s" % (self.__virtual_frame_buffer_port, str(formatted_traceback)), "SEVERE", casalog_call_origin, ) def stop_virtual_frame_buffer(self): casalog_call_origin = "MPICommandServer::stop_virtual_frame_buffer" if self.__virtual_frame_buffer_process is not None: try: self.__virtual_frame_buffer_process.terminate() casalog.post( "Virtual frame buffer deployed at %s with pid %s successfully shutdown" % (self.__virtual_frame_buffer_port, str(self.__virtual_frame_buffer_process.pid)), "DEBUG", casalog_call_origin, ) self.__virtual_frame_buffer_process = None except: formatted_traceback = traceback.format_exc() casalog.post( "Exception shutting down virtual frame buffer deployed at %s with pid %s: %s" % ( self.__virtual_frame_buffer_port, str(self.__virtual_frame_buffer_process.pid), str(formatted_traceback), ), "SEVERE", casalog_call_origin, ) else: casalog.post("Virtual frame buffer not deployed", "WARN", casalog_call_origin) subprocess.call( ["xauth", "-f", self.__xauthfile.name, "remove", self.__virtual_frame_buffer_port], stdout=self.__logfile_descriptor, stderr=self.__logfile_descriptor, ) self.__xauthfile.close() def start_services(self): self.__monitor_server.start_services() self.__start_command_request_handler_service() self.start_virtual_frame_buffer() def stop_services(self, force_command_request_interruption=False): if self.__logfile_descriptor is not None: self.__logfile_descriptor.close() self.__logfile_descriptor = None self.__monitor_server.stop_services() if not force_command_request_interruption: self.__stop_command_request_handler_service() def serve(self): casalog_call_origin = "MPICommandServer::serve" # First start command and ping status services casalog.post("Starting services...", "INFO", casalog_call_origin) self.start_services() # Notify to MPICommandClient that service is up and running self.__communicator.control_service_response_send(response=self.__monitor_server.get_status()) # Keep serving until a stop signal service is received control_service_request = {} stop_service_requested = False while (not stop_service_requested) and (not self.__monitor_server.get_client_timeout()): # Check if there is an incoming control service msg msg_available = False try: msg_available = self.__communicator.control_service_request_probe() except: msg_available = False formatted_traceback = traceback.format_exc() casalog.post( "Exception checking if control service msg is available: %s" % str(formatted_traceback), "SEVERE", casalog_call_origin, ) # Notify to MPICommandClient that control signal has been processed if msg_available: # Receive control service msg msg_received = False control_service_request = {} try: control_service_request = self.__communicator.control_service_request_recv() msg_received = True except: msg_received = False formatted_traceback = traceback.format_exc() casalog.post( "Exception receiving control service msg: %s" % str(formatted_traceback), "SEVERE", casalog_call_origin, ) continue # Process control service msg cmd = None send_response = False if msg_received: try: cmd = control_service_request["command"] send_response = control_service_request["send_response"] code = compile(cmd, casalog_call_origin, "exec") exec(code) casalog.post( "Control signal %s successfully handled by server %s" % (str(cmd), str(MPIEnvironment.mpi_processor_rank)), "INFO", casalog_call_origin, ) except: formatted_traceback = traceback.format_exc() casalog.post( "Exception handling control signal command %s in server %s: %s" % ( str(control_service_request), str(MPIEnvironment.mpi_processor_rank), str(formatted_traceback), ), "SEVERE", casalog_call_origin, ) # Notify to MPICommandClient that control signal has been processed if send_response: try: self.__communicator.control_service_response_send( response=self.__monitor_server.get_status() ) except: formatted_traceback = traceback.format_exc() casalog.post( "Exception sending response to control signal command %s in server %s: %s" % (str(cmd), str(MPIEnvironment.mpi_processor_rank), str(formatted_traceback)), "SEVERE", casalog_call_origin, ) time.sleep(MPIEnvironment.mpi_stop_service_sleep_time) # Process stop service request if stop_service_requested: # Check if force mode is needed force_command_request_interruption = control_service_request["force_command_request_interruption"] finalize_mpi_environment = control_service_request["finalize_mpi_environment"] busy = self.__monitor_server.get_status("busy") if force_command_request_interruption and busy: casalog.post( "force-stop service signal received, stopping services, " + "command request handler service will be interrupted...", "INFO", casalog_call_origin, ) else: force_command_request_interruption = False casalog.post("stop service signal received, stopping services...", "INFO", casalog_call_origin) else: force_command_request_interruption = True casalog.post( "client timeout, forcing disconnection, " + "command request handler service will be interrupted..", "INFO", casalog_call_origin, ) # Stop services self.stop_services(force_command_request_interruption) # Finalize MPI environment if finalize_mpi_environment: try: casalog.post("Going to finalize MPI environment", "INFO", casalog_call_origin) MPIEnvironment.finalize_mpi_environment() except: formatted_traceback = traceback.format_exc() casalog.post( "Exception finalizing MPI environment %s" % str(formatted_traceback), "SEVERE", casalog_call_origin, ) # Exit casalog.post("Exiting", "INFO", casalog_call_origin)
class __MPICommandServerImpl: """ Implementation of the MPICommandServer singleton interface """ def __init__(self,start_services=False): # Initialize command request handler service state self.__command_request_handler_service_on = False self.__command_request_handler_service_running = False self.__command_request_handler_service_thread = None # Instantiate MPICommunicator reference self.__communicator = MPICommunicator() # Instantiate MPIMonitorClient reference self.__monitor_server = MPIMonitorServer(False) # Initialize logfile descriptor self.__logfile_descriptor = open(casalog.logfile(), 'a') # Initialize virtual frame buffer state self.__virtual_frame_buffer_port = None self.__virtual_frame_buffer_process = None # Automatically start services if start_services: self.start_services() ################################################################################################################ # Private methods ############################################################################################## ################################################################################################################ def __command_request_handler_service(self): casalog_call_origin = "MPICommandServer::command_request_handler_service" # Mark service as running self.__command_request_handler_service_running = True while (self.__command_request_handler_service_on): # First check if there is a command request msg available msg_available = False try: msg_available = self.__communicator.command_request_probe() except Exception as instance: casalog.post("Exception checking if command request msg is available: %s" % str(instance),"SEVERE",casalog_call_origin) msg_available = False # Then receive command request msg msg_received = False if (msg_available): try: command_request = self.__communicator.command_request_recv() casalog.post("Received command request msg: %s" % command_request['command'],MPIEnvironment.command_handling_log_level,casalog_call_origin) msg_received = True except: formatted_traceback = traceback.format_exc() casalog.post("Exception receiving command request msg: %s" % str(formatted_traceback),"SEVERE",casalog_call_origin) msg_received = False # Finally process command request and send back response if (msg_received): # Start timer command_start_time = time.time() # Update server state self.__monitor_server.set_status('busy',True) self.__monitor_server.set_status('command',command_request['command']) self.__monitor_server.set_status('command_start_time',command_start_time) # Get command request id command_request_id = command_request['id'] # Prepare command response command_response = dict(command_request) # Set command start time command_response['command_start_time'] = command_start_time # Execute/Evaluate command request try: # Add dict-defined parameters to globals if isinstance(command_request['parameters'],dict): globals().update(command_request['parameters']) # Execute command if command_request['mode']=='eval': casalog.post("Going to evaluate command request with id# %s as an expression via eval: %s" % (str(command_request_id),str(command_request['command'])), MPIEnvironment.command_handling_log_level,casalog_call_origin) command_response['ret'] = eval(command_request['command']) elif command_request['mode']=='exec': casalog.post("Going to execute command request with id# %s as a statement via exec: %s" % (str(command_request_id),command_request['command']), MPIEnvironment.command_handling_log_level,casalog_call_origin) code = compile(command_request['command'], casalog_call_origin, 'exec') exec(code) command_response['ret'] = None elif command_request['mode']=='push': casalog.post("Command request with id# %s is a push operation" % str(command_request_id), MPIEnvironment.command_handling_log_level,casalog_call_origin) command_response['ret'] = None # Set command response parameters command_response['successful'] = True command_response['traceback'] = None except: formatted_traceback = traceback.format_exc() casalog.post("Exception executing command request via %s: %s" % (command_request['mode'],str(formatted_traceback)),"SEVERE",casalog_call_origin) # Set command response parameters command_response['successful'] = False command_response['traceback']=formatted_traceback command_response['ret']=None # Variables are cleaned from the environment regardless of the result finally: # Clear parameter variables if isinstance(command_request['parameters'],dict) and command_request['mode']!='push': for parameter in command_request['parameters']: try: del globals()[parameter] except: formatted_traceback = traceback.format_exc() casalog.post("Exception deleting parameter variable '%s' from global environment: %s" % (str(parameter),str(formatted_traceback)),"WARN",casalog_call_origin) # Set command stop time command_stop_time = time.time() command_response['command_stop_time'] = command_stop_time # Update server state self.__monitor_server.set_status('busy',False) self.__monitor_server.set_status('command',None) self.__monitor_server.set_status('command_start_time',None) # Send response back (successful or not) try: casalog.post("Command request with id %s successfully processed in %s mode, sending back response ..." % (str(command_response['id']),str(command_response['mode'])), MPIEnvironment.command_handling_log_level,casalog_call_origin) self.__communicator.command_response_send(response=command_response) except: formatted_traceback = traceback.format_exc() casalog.post("Exception sending back command response: %s" % str(formatted_traceback),"SEVERE",casalog_call_origin) else: time.sleep(MPIEnvironment.mpi_command_request_handler_service_sleep_time) # Mark service as not running self.__command_request_handler_service_running = False def __start_command_request_handler_service(self): casalog_call_origin = "MPICommandServer::start_command_request_handler_service" if self.__command_request_handler_service_running: casalog.post("MPI command request handler service is already running","WARN",casalog_call_origin) return True try: self.__command_request_handler_service_on = True self.__command_request_handler_service_thread = thread.start_new_thread(self.__command_request_handler_service, ()) except Exception as instance: self.__command_request_handler_service_on = False self.__command_request_handler_service_running = False casalog.post("Exception starting MPI command request handler service: %s" % str(instance),"SEVERE",casalog_call_origin) return False while (not self.__command_request_handler_service_running): time.sleep(MPIEnvironment.mpi_check_start_service_sleep_time) casalog.post("MPI command request handler service started","INFO",casalog_call_origin) return True def __stop_command_request_handler_service(self): casalog_call_origin = "MPICommandServer::stop_command_request_handler_service" if not self.__command_request_handler_service_running: casalog.post("MPI command request handler service is not running","WARN",casalog_call_origin) return self.__command_request_handler_service_on = False while (self.__command_request_handler_service_running): time.sleep(MPIEnvironment.mpi_check_stop_service_sleep_time) casalog.post("MPI command request handler service stopped","INFO",casalog_call_origin) ################################################################################################################ # Public methods ############################################################################################### ################################################################################################################ def start_virtual_frame_buffer(self): casalog_call_origin = "MPICommandServer::start_virtual_frame_buffer" displayport = os.getpid() while os.path.exists('/tmp/.X%d-lock' % displayport): displayport += 1 self.__virtual_frame_buffer_port = ":%d" % displayport self.__xauthfile = tempfile.NamedTemporaryFile() try: cookie = subprocess.check_output(['mcookie'], universal_newlines=True).strip() except: cookie = str(uuid.uuid4()).replace('-', '') #sometimes also works without auth, so accept failure subprocess.call(['xauth', '-f', self.__xauthfile.name, 'add', self.__virtual_frame_buffer_port, '.', cookie], stdout=self.__logfile_descriptor, stderr=self.__logfile_descriptor) try: self.__virtual_frame_buffer_process = subprocess.Popen(['Xvfb',self.__virtual_frame_buffer_port, '-auth', self.__xauthfile.name], stdout=self.__logfile_descriptor, stderr=self.__logfile_descriptor, shell=False) os.environ['DISPLAY']=self.__virtual_frame_buffer_port os.environ['XAUTHORITY'] = self.__xauthfile.name casalog.post("Deployed virtual frame buffer at %s with pid %s" % (self.__virtual_frame_buffer_port, str(self.__virtual_frame_buffer_process.pid)), "INFO",casalog_call_origin) except: self.__virtual_frame_buffer_process = None formatted_traceback = traceback.format_exc() casalog.post("Exception deploying virtual frame buffer at %s: %s" % (self.__virtual_frame_buffer_port, str(formatted_traceback)), "SEVERE",casalog_call_origin) def stop_virtual_frame_buffer(self): casalog_call_origin = "MPICommandServer::stop_virtual_frame_buffer" if self.__virtual_frame_buffer_process is not None: try: self.__virtual_frame_buffer_process.terminate() casalog.post("Virtual frame buffer deployed at %s with pid %s successfully shutdown" % (self.__virtual_frame_buffer_port, str(self.__virtual_frame_buffer_process.pid)), "DEBUG",casalog_call_origin) self.__virtual_frame_buffer_process = None except: formatted_traceback = traceback.format_exc() casalog.post("Exception shutting down virtual frame buffer deployed at %s with pid %s: %s" % (self.__virtual_frame_buffer_port, str(self.__virtual_frame_buffer_process.pid), str(formatted_traceback)), "SEVERE",casalog_call_origin) else: casalog.post("Virtual frame buffer not deployed","WARN",casalog_call_origin) subprocess.call(['xauth', '-f', self.__xauthfile.name, 'remove', self.__virtual_frame_buffer_port], stdout=self.__logfile_descriptor, stderr=self.__logfile_descriptor) self.__xauthfile.close() def start_services(self): self.__monitor_server.start_services() self.__start_command_request_handler_service() self.start_virtual_frame_buffer() def stop_services(self,force_command_request_interruption=False): if self.__logfile_descriptor is not None: self.__logfile_descriptor.close() self.__logfile_descriptor = None self.__monitor_server.stop_services() if not force_command_request_interruption: self.__stop_command_request_handler_service() def serve(self): casalog_call_origin = "MPICommandServer::serve" # First start command and ping status services casalog.post("Starting services...","INFO",casalog_call_origin) self.start_services() # Notify to MPICommandClient that service is up and running self.__communicator.control_service_response_send(response=self.__monitor_server.get_status()) # Keep serving until a stop signal service is received control_service_request = {} stop_service_requested = False while ((not stop_service_requested) and (not self.__monitor_server.get_client_timeout())): # Check if there is an incoming control service msg msg_available = False try: msg_available = self.__communicator.control_service_request_probe() except: msg_available = False formatted_traceback = traceback.format_exc() casalog.post("Exception checking if control service msg is available: %s" % str(formatted_traceback),"SEVERE",casalog_call_origin) # Notify to MPICommandClient that control signal has been processed if msg_available: # Receive control service msg msg_received = False control_service_request = {} try: control_service_request = self.__communicator.control_service_request_recv() msg_received = True except: msg_received = False formatted_traceback = traceback.format_exc() casalog.post("Exception receiving control service msg: %s" % str(formatted_traceback),"SEVERE",casalog_call_origin) continue # Process control service msg cmd = None send_response = False if msg_received: try: cmd = control_service_request['command'] send_response = control_service_request['send_response'] code = compile(cmd, casalog_call_origin, 'exec') exec(code) casalog.post("Control signal %s successfully handled by server %s" % (str(cmd),str(MPIEnvironment.mpi_processor_rank)), "INFO",casalog_call_origin) except: formatted_traceback = traceback.format_exc() casalog.post("Exception handling control signal command %s in server %s: %s" % (str(control_service_request), str(MPIEnvironment.mpi_processor_rank), str(formatted_traceback)), "SEVERE",casalog_call_origin) # Notify to MPICommandClient that control signal has been processed if send_response: try: self.__communicator.control_service_response_send(response=self.__monitor_server.get_status()) except: formatted_traceback = traceback.format_exc() casalog.post("Exception sending response to control signal command %s in server %s: %s" % (str(cmd),str(MPIEnvironment.mpi_processor_rank),str(formatted_traceback)), "SEVERE",casalog_call_origin) time.sleep(MPIEnvironment.mpi_stop_service_sleep_time) # Process stop service request if stop_service_requested: # Check if force mode is needed force_command_request_interruption = control_service_request['force_command_request_interruption'] finalize_mpi_environment = control_service_request['finalize_mpi_environment'] busy = self.__monitor_server.get_status('busy') if force_command_request_interruption and busy: casalog.post("force-stop service signal received, stopping services, " + "command request handler service will be interrupted...","INFO",casalog_call_origin) else: force_command_request_interruption = False casalog.post("stop service signal received, stopping services...","INFO",casalog_call_origin) else: force_command_request_interruption = True casalog.post("client timeout, forcing disconnection, " + "command request handler service will be interrupted.." ,"INFO",casalog_call_origin) # Stop services self.stop_services(force_command_request_interruption) # Finalize MPI environment if finalize_mpi_environment: try: casalog.post("Going to finalize MPI environment","INFO",casalog_call_origin) MPIEnvironment.finalize_mpi_environment() except: formatted_traceback = traceback.format_exc() casalog.post("Exception finalizing MPI environment %s" % str(formatted_traceback),"SEVERE",casalog_call_origin) # Exit casalog.post("Exiting","INFO",casalog_call_origin)