def set_and_send_nb_job_to_scheduler(self): """Send the number of current compilation job running to the Scheduler""" RemoteJob.nb_job -= 1 ProtocolUtils.connect_send_close(self.__scheduler_address,\ self.__scheduler_port,\ [Protocol.JOB_DONE, "%s"%(RemoteJob.nb_job)])
def send_result_files_back_to_client(\ self,\ input_files,\ output_files): """Send the result (output files and output messages) to the client, and replace the temporary input file names in the various messages by the original ones.""" #clean temporary input files created for tmp_input_file in input_files.values(): os.unlink(tmp_input_file) # FIXME: don't forget to send stdout and stderr # then send output files back to the client for original_output_file_name in output_files.keys(): self.__client_socket.send(Protocol.FILE_COMMAND) ProtocolUtils.send_data(self.__client_socket, original_output_file_name) # send the file tmp_output_file = open(\ output_files[original_output_file_name], 'rb') if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : "\ + output_files[original_output_file_name]) ProtocolUtils.send_data(self.__client_socket, tmp_output_file.read()) tmp_output_file.close() os.unlink(output_files[original_output_file_name]) if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : "\ + output_files[original_output_file_name])
def clean_on_exit(self): ProtocolUtils.connect_send_close(self.get_config()["scheduler_address"],\ string.atoi(self.get_config()["scheduler_port"]),\ [Protocol.UNSUBSCRIBE_ME]) try: self.__listen_socket.shutdown(0) except socket.error: syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \ "Error when shutdown socket with Scheduler") self.__listen_socket.close() for open_socket in CompilerDaemon_mod.open_sockets: open_socket.shutdown(0) self.remove_lock_file()
def send_output_messages(self, msg_stdout, msg_stderr, input_files,\ output_files): # replace temporary filenames in messages for original_input_file in input_files.keys(): msg_stdout = msg_stdout.replace(\ input_files[original_input_file],\ original_input_file) msg_stderr = msg_stderr.replace(\ input_files[original_input_file],\ original_input_file) # send stdout messages ProtocolUtils.send_data(self.__client_socket, msg_stdout) # send stderr messages ProtocolUtils.send_data(self.__client_socket, msg_stderr)
def job_done(self): nb_job = ProtocolUtils.recv_data(self.__client_socket) try: nb_job = string.atoi(nb_job) except: syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \ "dms Scheduler - receive bad information about"\ + " number of job in work by CompilerDaemon") else: self.store.set_nb_job_and_inc_count(self.__client_address, nb_job) syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \ "dms Scheduler receve DONE msg from host "\ + self.__client_address\ + ", actualy %s"%(nb_job) + " job in work")
def run(self): request_type = ProtocolUtils.recv_data(self.__client_socket) if (request_type == Protocol.REQUEST_HOST): self.request_host() elif (request_type == Protocol.RECORD_ME): self.record_host() elif (request_type == Protocol.UNSUBSCRIBE_ME): self.unsubscribe() elif (request_type == Protocol.JOB_DONE): self.job_done() else: syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \ "dms Scheduler receve Unknown Message from "\ + self.__client_address\ + ":%s"%(self.__client_port)) print "Unknown Message" + request_type self.__client_socket.close()
def __execute_command_remotely(self, preprocessed_files): """Execute the command associated with this compiler instance on a remote host and get the results back.""" if __debug__: print >> sys.stderr, "Distant execution of the command : "\ + " ".join(self.__compiler_command.get_after_preprocessing_options()) # here, the scheduler give an hostname to which # we would send the compilation job #FIXME: catch exception here self.__scheduler_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.__scheduler_socket.connect((self.__config['scheduler_address'], int(self.__config['scheduler_port']))) ProtocolUtils.send_data(self.__scheduler_socket, Protocol.REQUEST_HOST) compiler_host = ProtocolUtils.recv_data(self.__scheduler_socket) self.__scheduler_socket.close() self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # here we connect to the compiler_host self.__socket.connect((compiler_host,\ Protocol.COMPILER_DAEMON_PORT)) # ask for a compilation job self.__socket.send(Protocol.COMPILE_COMMAND) # send the compilation command line ProtocolUtils.send_data(self.__socket, " ".join(self.__compiler_command.get_after_preprocessing_options())) # send each file along with their name if __debug__: print >> sys.stderr, "List of files to send : " + str(preprocessed_files) for (tmp_input_file, input_file_name) in preprocessed_files: if __debug__: print >> sys.stderr, "Sending of a preprocessed file: %s" % input_file_name self.__socket.send(Protocol.FILE_COMMAND) # send the file name ProtocolUtils.send_data(self.__socket, input_file_name) # send the file content # FIXME: currently we don't handle file size > 4 Gos tmp_input_file.seek(0) ProtocolUtils.send_data(self.__socket, tmp_input_file.read()) tmp_input_file.close() # unblock the compiler server self.__socket.send(Protocol.STOP_COMMAND) # get the output messages # first stdout, then stderr std_out_msg = ProtocolUtils.recv_data(self.__socket) if __debug__: print >> sys.stderr, "STDOUT message : " + std_out_msg std_err_msg = ProtocolUtils.recv_data(self.__socket) if __debug__: print >> sys.stderr, "STDERR message : " + std_err_msg # then get the output files content command = self.__socket.recv(Protocol.COMMAND_TYPE_SIZE) while command == Protocol.FILE_COMMAND: file_name = ProtocolUtils.recv_data(self.__socket) if __debug__: print >> sys.stderr, "file name : " + file_name #FIXME This is a bad patch to make DMS work well # but it have to be really fixed # Maybe the CompilerCommands class have to be reworked if "-o" in self.__compiler_command.get_command_args(): file = open(file_name, 'wb') else: file = open(os.path.basename(file_name), 'wb') file.write(ProtocolUtils.recv_data(self.__socket)) file.flush() file.close() if __debug__: print "File written." command = self.__socket.recv(Protocol.COMMAND_TYPE_SIZE) assert command == Protocol.EXIT_CODE_COMMAND print >> sys.stderr, "Command received : " + command # finally get the exit code exit_code = int(ProtocolUtils.recv_data(self.__socket)) if __debug__: print "Return code : " + str(exit_code) self.__socket.close() return exit_code
def send_exit_code(self, exit_code): """Send the exit code "exit_code" of the executed command to the client.""" self.__client_socket.send(Protocol.EXIT_CODE_COMMAND) ProtocolUtils.send_data(self.__client_socket, str(exit_code))
def compiler_job(self): """This method is called when a remote client ask a compiler daemon to compile some source code, and send it back the results of the compilation (object files generally). Return 0 if everything has been done without any error, and other values otherwise. """ if __debug__: print >> sys.stderr, "Execution d'un job de compilation" syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Launching compilation job.") # receive the length of the command line to be received compiler_command_line = ProtocolUtils.recv_data(self.__client_socket) if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Compilation command line received : "\ + compiler_command_line) compiler_command = CompilerCommandFactory.build_compiler_instance\ (compiler_command_line.split()) # get the content of the input files used in the command line we # have just received input_temp_files = {} output_temp_files = {} command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE) while command == Protocol.FILE_COMMAND: file_name = ProtocolUtils.recv_data(self.__client_socket) # FIXME: do we need to create the file inside the # critical section ? RemoteJob.tmp_file_creation_lock.acquire() if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ "Creating temporary file.") tmp_file_name = tempfile.mktemp(\ FileUtils.get_file_extension(file_name)) tmp_file = open(tmp_file_name, 'w') RemoteJob.tmp_file_creation_lock.release() if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ "Temporary file created.") input_temp_files[file_name] = tmp_file_name tmp_file.write(ProtocolUtils.recv_data(self.__client_socket)) tmp_file.flush() tmp_file.close() command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE) # replace original input files in the command line by the # temporary ones compiler_command.replace_input_files(input_temp_files) if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ "New compilation command line :"\ + " ".join(compiler_command.get_command_args())) # FIXME We should not use "-o" here, this is compiler dependant if "-o" in compiler_command.get_command_args(): if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ "-o option in command line.") try: index_output = compiler_command.get_command_args().index("-o") \ + 1 # FIXME: do we need to create the file inside the # critical section ? RemoteJob.tmp_file_creation_lock.acquire() tmp_output_file = tempfile.mktemp() tmp_output_file_hdl = open(tmp_output_file, 'w') tmp_output_file_hdl.close() RemoteJob.tmp_file_creation_lock.release() # associate the output tmp file with the original one output_file_name = compiler_command.get_command_args()\ [index_output] output_temp_files[output_file_name] = tmp_output_file # replace the output file name in the command line compiler_command.get_command_args(\ )[index_output] = tmp_output_file except IndexError: # if there is no file name after the -o option, # it means that the command line is wrong, but we # must execute it in order to send the error # msg back to the client pass else: # no output file specified with -o switch if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ "No -o option in command line.") for original_input_file_name in input_temp_files.keys(): stop_step = compiler_command.get_stop_step() orig_output_file_name = compiler_command.\ get_output_file_name_for_step(\ original_input_file_name,\ stop_step) output_temp_files[\ orig_output_file_name] = compiler_command.\ get_output_file_name_for_step(\ input_temp_files[original_input_file_name],\ stop_step) if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ "File to return to the client : "\ + output_temp_files[orig_output_file_name]) # execute the command in a subshell and get the stdout and stderr output if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ "Executing the following command line : " \ + compiler_command.get_local_compiler_path() + " " +\ " ".join(compiler_command.get_command_args()[1:])) #proc = popen2.Popen3(" ".join(compiler_command.get_command_args()), 1) # FIXME : VERY IMPORTANT # uncomment the next two line to replace the previous one proc = popen2.Popen3(compiler_command.get_local_compiler_path() + " "\ + " ".join(compiler_command.get_command_args()[1:]), 1) msg_stderr = proc.childerr.read() msg_stdout = proc.fromchild.read() proc.childerr.close() proc.fromchild.close() exit_code = proc.wait() self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\ output_temp_files) if os.WIFEXITED(exit_code): exit_code = os.WEXITSTATUS(exit_code) if os.WIFSIGNALED(exit_code): exit_code = os.WTERMSIG(exit_code) if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ "Exit code : " + str(exit_code)) if exit_code == 0: # send the result (output files and output messages) to the client self.send_result_files_back_to_client(input_temp_files,\ output_temp_files) if __debug__: syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ "Output files sent.") self.send_exit_code(exit_code) self.set_and_send_nb_job_to_scheduler()
def request_host(self): host = self.store.give_host() ProtocolUtils.send_data(self.__client_socket, host) syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \ "dms Scheduler give host %s to client "%(host)\ + self.__client_address)