Example #1
0
 def set_and_send_nb_job_to_scheduler(self):
     """Send the number of current compilation job running to
     the Scheduler"""
     RemoteJob.nb_job -= 1
     ProtocolUtils.connect_send_close(self.__scheduler_address,\
                                      self.__scheduler_port,\
                                      [Protocol.JOB_DONE, "%s"%(RemoteJob.nb_job)])
Example #2
0
    def send_result_files_back_to_client(\
        self,\
        input_files,\
        output_files):
        """Send the result (output files and output messages) to the client,
        and replace the temporary input file names in the various messages by
        the original ones."""

        #clean temporary input files created
        for tmp_input_file in input_files.values():
            os.unlink(tmp_input_file)
        # FIXME: don't forget to send stdout and stderr
        
        # then send output files back to the client
        for original_output_file_name in output_files.keys():
            self.__client_socket.send(Protocol.FILE_COMMAND)

            ProtocolUtils.send_data(self.__client_socket, original_output_file_name)

            # send the file
            tmp_output_file = open(\
                output_files[original_output_file_name], 'rb')
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : "\
                              + output_files[original_output_file_name])
            ProtocolUtils.send_data(self.__client_socket, tmp_output_file.read())                 
            tmp_output_file.close()
            os.unlink(output_files[original_output_file_name])
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : "\
                              + output_files[original_output_file_name])
Example #3
0
    def send_result_files_back_to_client(\
        self,\
        input_files,\
        output_files):
        """Send the result (output files and output messages) to the client,
        and replace the temporary input file names in the various messages by
        the original ones."""

        #clean temporary input files created
        for tmp_input_file in input_files.values():
            os.unlink(tmp_input_file)
        # FIXME: don't forget to send stdout and stderr

        # then send output files back to the client
        for original_output_file_name in output_files.keys():
            self.__client_socket.send(Protocol.FILE_COMMAND)

            ProtocolUtils.send_data(self.__client_socket,
                                    original_output_file_name)

            # send the file
            tmp_output_file = open(\
                output_files[original_output_file_name], 'rb')
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : "\
                              + output_files[original_output_file_name])
            ProtocolUtils.send_data(self.__client_socket,
                                    tmp_output_file.read())
            tmp_output_file.close()
            os.unlink(output_files[original_output_file_name])
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : "\
                              + output_files[original_output_file_name])
Example #4
0
 def set_and_send_nb_job_to_scheduler(self):
     """Send the number of current compilation job running to
     the Scheduler"""
     RemoteJob.nb_job -= 1
     ProtocolUtils.connect_send_close(self.__scheduler_address,\
                                      self.__scheduler_port,\
                                      [Protocol.JOB_DONE, "%s"%(RemoteJob.nb_job)])
Example #5
0
 def clean_on_exit(self):
     ProtocolUtils.connect_send_close(self.get_config()["scheduler_address"],\
                                      string.atoi(self.get_config()["scheduler_port"]),\
                                      [Protocol.UNSUBSCRIBE_ME])
     try:
         self.__listen_socket.shutdown(0)
     except socket.error:
         syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \
                       "Error when shutdown socket with Scheduler")
     self.__listen_socket.close()
     for open_socket in CompilerDaemon_mod.open_sockets:
         open_socket.shutdown(0)
     self.remove_lock_file()
Example #6
0
 def clean_on_exit(self):
     ProtocolUtils.connect_send_close(self.get_config()["scheduler_address"],\
                                      string.atoi(self.get_config()["scheduler_port"]),\
                                      [Protocol.UNSUBSCRIBE_ME])
     try:
         self.__listen_socket.shutdown(0)
     except socket.error:
         syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \
                       "Error when shutdown socket with Scheduler")
     self.__listen_socket.close()
     for open_socket in CompilerDaemon_mod.open_sockets:
         open_socket.shutdown(0)
     self.remove_lock_file()
Example #7
0
    def send_output_messages(self, msg_stdout, msg_stderr, input_files,\
                             output_files):
        # replace temporary filenames in messages
        for original_input_file in input_files.keys():
            msg_stdout = msg_stdout.replace(\
                input_files[original_input_file],\
                original_input_file)
            msg_stderr = msg_stderr.replace(\
                input_files[original_input_file],\
                original_input_file)
        # send stdout messages
        ProtocolUtils.send_data(self.__client_socket, msg_stdout)

        # send stderr messages
        ProtocolUtils.send_data(self.__client_socket, msg_stderr)
Example #8
0
    def send_output_messages(self, msg_stdout, msg_stderr, input_files,\
                             output_files):
        # replace temporary filenames in messages
        for original_input_file in input_files.keys():
            msg_stdout = msg_stdout.replace(\
                input_files[original_input_file],\
                original_input_file)
            msg_stderr = msg_stderr.replace(\
                input_files[original_input_file],\
                original_input_file)
        # send stdout messages
        ProtocolUtils.send_data(self.__client_socket, msg_stdout)

        # send stderr messages
        ProtocolUtils.send_data(self.__client_socket, msg_stderr)
Example #9
0
 def job_done(self):
     nb_job = ProtocolUtils.recv_data(self.__client_socket)
     try:
         nb_job = string.atoi(nb_job)
     except:
         syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \
                       "dms Scheduler - receive bad information about"\
                       + " number of job in work by CompilerDaemon")
     else:
         self.store.set_nb_job_and_inc_count(self.__client_address, nb_job)
         syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \
                       "dms Scheduler receve DONE msg from host "\
                       + self.__client_address\
                       + ", actualy %s"%(nb_job) + " job in work")
Example #10
0
 def run(self):
     request_type = ProtocolUtils.recv_data(self.__client_socket)
     if (request_type == Protocol.REQUEST_HOST):
         self.request_host()
     elif (request_type == Protocol.RECORD_ME):
         self.record_host()
     elif (request_type == Protocol.UNSUBSCRIBE_ME):
         self.unsubscribe()
     elif (request_type == Protocol.JOB_DONE):
         self.job_done()
     else:
         syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \
                       "dms Scheduler receve Unknown Message from "\
                       + self.__client_address\
                       + ":%s"%(self.__client_port))
         print "Unknown Message" + request_type
     self.__client_socket.close()
Example #11
0
    def __execute_command_remotely(self, preprocessed_files):
        """Execute the command associated with this compiler instance
        on a remote host and get the results back."""
        if __debug__:
            print >> sys.stderr, "Distant execution of the command : "\
                  + " ".join(self.__compiler_command.get_after_preprocessing_options())

        # here, the scheduler give an hostname to which 
        # we would send the compilation job

        #FIXME: catch exception here
        self.__scheduler_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        self.__scheduler_socket.connect((self.__config['scheduler_address'],
                                         int(self.__config['scheduler_port'])))

        ProtocolUtils.send_data(self.__scheduler_socket, Protocol.REQUEST_HOST)
        compiler_host = ProtocolUtils.recv_data(self.__scheduler_socket)
        self.__scheduler_socket.close()

        self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        
        # here we connect to the compiler_host
        self.__socket.connect((compiler_host,\
                               Protocol.COMPILER_DAEMON_PORT))
        # ask for a compilation job
        self.__socket.send(Protocol.COMPILE_COMMAND)

        # send the compilation command line

        ProtocolUtils.send_data(self.__socket,
                                " ".join(self.__compiler_command.get_after_preprocessing_options()))

        # send each file along with their name
        if __debug__:
            print >> sys.stderr, "List of files to send : " + str(preprocessed_files)
            
        for (tmp_input_file, input_file_name) in preprocessed_files:
            if __debug__:
                print >> sys.stderr, "Sending of a preprocessed file: %s" % input_file_name
            self.__socket.send(Protocol.FILE_COMMAND)

            # send the file name
            ProtocolUtils.send_data(self.__socket, input_file_name)

            # send the file content
            # FIXME: currently we don't handle file size > 4 Gos
            tmp_input_file.seek(0)
            ProtocolUtils.send_data(self.__socket, tmp_input_file.read())
            tmp_input_file.close()
            
        # unblock the compiler server
        self.__socket.send(Protocol.STOP_COMMAND)
        # get the output messages
        # first stdout, then stderr
        std_out_msg = ProtocolUtils.recv_data(self.__socket)
        if __debug__:
            print >> sys.stderr, "STDOUT message : " + std_out_msg

        std_err_msg = ProtocolUtils.recv_data(self.__socket)
        if __debug__:
            print >> sys.stderr, "STDERR message : " + std_err_msg

        # then get the output files content
        command = self.__socket.recv(Protocol.COMMAND_TYPE_SIZE)
                                     
        while command == Protocol.FILE_COMMAND:
            file_name = ProtocolUtils.recv_data(self.__socket)

            if __debug__:
                print >> sys.stderr, "file name : " + file_name
            #FIXME This is a bad patch to make DMS work well
            # but it have to be really fixed
            # Maybe the CompilerCommands class have to be reworked
            if "-o" in self.__compiler_command.get_command_args():
                file = open(file_name, 'wb')
            else:
                file = open(os.path.basename(file_name), 'wb')

            file.write(ProtocolUtils.recv_data(self.__socket))
            file.flush()
            file.close()

            if __debug__:
                print "File written."
            command = self.__socket.recv(Protocol.COMMAND_TYPE_SIZE)

        assert command == Protocol.EXIT_CODE_COMMAND
        print >> sys.stderr, "Command received : " + command
        # finally get the exit code
        exit_code = int(ProtocolUtils.recv_data(self.__socket))
        if __debug__:
            print "Return code : " + str(exit_code)
        self.__socket.close()
        return exit_code
Example #12
0
    def send_exit_code(self, exit_code):
        """Send the exit code "exit_code" of the executed command to the client."""

        self.__client_socket.send(Protocol.EXIT_CODE_COMMAND)
        ProtocolUtils.send_data(self.__client_socket, str(exit_code))
Example #13
0
    def compiler_job(self):
        """This method is called when a remote client
        ask a compiler daemon to compile some source code,
        and send it back the results of the compilation
        (object files generally).
        Return 0 if everything has been done without any error,
        and other values otherwise.
        """

        if __debug__:
            print >> sys.stderr, "Execution d'un job de compilation"
        syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
                      "Launching compilation job.")
        # receive the length of the command line to be received
        compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
                          "Compilation command line received : "\
                          + compiler_command_line)
        compiler_command = CompilerCommandFactory.build_compiler_instance\
                           (compiler_command_line.split())

        # get the content of the input files used in the command line we
        # have just received
        input_temp_files = {}
        output_temp_files = {}
        command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)

        while command == Protocol.FILE_COMMAND:

            file_name = ProtocolUtils.recv_data(self.__client_socket)

            # FIXME: do we need to create the file inside the
            # critical section ?
            RemoteJob.tmp_file_creation_lock.acquire()
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                              "Creating temporary file.")
            tmp_file_name = tempfile.mktemp(\
                FileUtils.get_file_extension(file_name))
            tmp_file = open(tmp_file_name, 'w')
            RemoteJob.tmp_file_creation_lock.release()
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                              "Temporary file created.")

            input_temp_files[file_name] = tmp_file_name
            tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))
            tmp_file.flush()
            tmp_file.close()
            command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)

        # replace original input files in the command line by the
        # temporary ones
        compiler_command.replace_input_files(input_temp_files)
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                          "New compilation command line :"\
                          + " ".join(compiler_command.get_command_args()))

        # FIXME We should not use "-o" here, this is compiler dependant
        if "-o" in compiler_command.get_command_args():
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                              "-o option in command line.")
            try:
                index_output = compiler_command.get_command_args().index("-o") \
                               + 1

                # FIXME: do we need to create the file inside the
                # critical section ?
                RemoteJob.tmp_file_creation_lock.acquire()
                tmp_output_file = tempfile.mktemp()
                tmp_output_file_hdl = open(tmp_output_file, 'w')
                tmp_output_file_hdl.close()
                RemoteJob.tmp_file_creation_lock.release()
                # associate the output tmp file with the original one
                output_file_name = compiler_command.get_command_args()\
                                  [index_output]
                output_temp_files[output_file_name] = tmp_output_file
                # replace the output file name in the command line
                compiler_command.get_command_args(\
                    )[index_output] = tmp_output_file
            except IndexError:
                # if there is no file name after the -o option,
                # it means that the command line is wrong, but we
                # must execute it in order to send the error
                # msg back to the client
                pass
        else:
            # no output file specified with -o switch
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                              "No -o option in command line.")
            for original_input_file_name in input_temp_files.keys():
                stop_step = compiler_command.get_stop_step()
                orig_output_file_name = compiler_command.\
                                        get_output_file_name_for_step(\
                    original_input_file_name,\
                    stop_step)
                output_temp_files[\
                    orig_output_file_name] = compiler_command.\
                    get_output_file_name_for_step(\
                    input_temp_files[original_input_file_name],\
                    stop_step)

                if __debug__:
                    syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                                  "File to return to the client : "\
                                  + output_temp_files[orig_output_file_name])

        # execute the command in a subshell and get the stdout and stderr output
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                          "Executing the following command line : " \
                          + compiler_command.get_local_compiler_path() + " " +\
                          " ".join(compiler_command.get_command_args()[1:]))

        #proc = popen2.Popen3(" ".join(compiler_command.get_command_args()), 1)

        # FIXME : VERY IMPORTANT
        # uncomment the next two line to replace the previous one
        proc = popen2.Popen3(compiler_command.get_local_compiler_path() + " "\
                             + " ".join(compiler_command.get_command_args()[1:]), 1)
        msg_stderr = proc.childerr.read()
        msg_stdout = proc.fromchild.read()
        proc.childerr.close()
        proc.fromchild.close()
        exit_code = proc.wait()

        self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\
                                  output_temp_files)

        if os.WIFEXITED(exit_code):
            exit_code = os.WEXITSTATUS(exit_code)
        if os.WIFSIGNALED(exit_code):
            exit_code = os.WTERMSIG(exit_code)
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                          "Exit code : " + str(exit_code))
        if exit_code == 0:
            # send the result (output files and output messages) to the client
            self.send_result_files_back_to_client(input_temp_files,\
                                                  output_temp_files)
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                              "Output files sent.")
        self.send_exit_code(exit_code)
        self.set_and_send_nb_job_to_scheduler()
Example #14
0
    def send_exit_code(self, exit_code):
        """Send the exit code "exit_code" of the executed command to the client."""

        self.__client_socket.send(Protocol.EXIT_CODE_COMMAND)
        ProtocolUtils.send_data(self.__client_socket, str(exit_code))
Example #15
0
    def compiler_job(self):
        """This method is called when a remote client
        ask a compiler daemon to compile some source code,
        and send it back the results of the compilation
        (object files generally).
        Return 0 if everything has been done without any error,
        and other values otherwise.
        """

        if __debug__:
            print >> sys.stderr, "Execution d'un job de compilation"
        syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
                      "Launching compilation job.")
        # receive the length of the command line to be received
        compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
                          "Compilation command line received : "\
                          + compiler_command_line)
        compiler_command = CompilerCommandFactory.build_compiler_instance\
                           (compiler_command_line.split())


        # get the content of the input files used in the command line we
        # have just received
        input_temp_files = {}
        output_temp_files = {}
        command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
                                            
        while command == Protocol.FILE_COMMAND:

            file_name = ProtocolUtils.recv_data(self.__client_socket)

            # FIXME: do we need to create the file inside the
            # critical section ?
            RemoteJob.tmp_file_creation_lock.acquire()
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                              "Creating temporary file.")
            tmp_file_name = tempfile.mktemp(\
                FileUtils.get_file_extension(file_name))
            tmp_file = open(tmp_file_name, 'w')
            RemoteJob.tmp_file_creation_lock.release()
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                              "Temporary file created.")

            input_temp_files[file_name] = tmp_file_name
            tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))
            tmp_file.flush()
            tmp_file.close()
            command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
                                                

        # replace original input files in the command line by the
        # temporary ones
        compiler_command.replace_input_files(input_temp_files)
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                          "New compilation command line :"\
                          + " ".join(compiler_command.get_command_args()))


        # FIXME We should not use "-o" here, this is compiler dependant
        if "-o" in compiler_command.get_command_args():
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                              "-o option in command line.")
            try:
                index_output = compiler_command.get_command_args().index("-o") \
                               + 1
                
                # FIXME: do we need to create the file inside the
                # critical section ?
                RemoteJob.tmp_file_creation_lock.acquire()
                tmp_output_file = tempfile.mktemp()
                tmp_output_file_hdl = open(tmp_output_file, 'w')
                tmp_output_file_hdl.close()
                RemoteJob.tmp_file_creation_lock.release()
                # associate the output tmp file with the original one
                output_file_name = compiler_command.get_command_args()\
                                  [index_output]
                output_temp_files[output_file_name] = tmp_output_file
                # replace the output file name in the command line
                compiler_command.get_command_args(\
                    )[index_output] = tmp_output_file
            except IndexError:
                # if there is no file name after the -o option,
                # it means that the command line is wrong, but we
                # must execute it in order to send the error
                # msg back to the client
                pass
        else:
            # no output file specified with -o switch
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                              "No -o option in command line.")
            for original_input_file_name in input_temp_files.keys():
                stop_step = compiler_command.get_stop_step()
                orig_output_file_name = compiler_command.\
                                        get_output_file_name_for_step(\
                    original_input_file_name,\
                    stop_step)
                output_temp_files[\
                    orig_output_file_name] = compiler_command.\
                    get_output_file_name_for_step(\
                    input_temp_files[original_input_file_name],\
                    stop_step)
                
                if __debug__:
                    syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                                  "File to return to the client : "\
                                  + output_temp_files[orig_output_file_name])
                
        # execute the command in a subshell and get the stdout and stderr output
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                          "Executing the following command line : " \
                          + compiler_command.get_local_compiler_path() + " " +\
                          " ".join(compiler_command.get_command_args()[1:]))

        #proc = popen2.Popen3(" ".join(compiler_command.get_command_args()), 1)

        # FIXME : VERY IMPORTANT
        # uncomment the next two line to replace the previous one
        proc = popen2.Popen3(compiler_command.get_local_compiler_path() + " "\
                             + " ".join(compiler_command.get_command_args()[1:]), 1)
        msg_stderr = proc.childerr.read()
        msg_stdout = proc.fromchild.read()
        proc.childerr.close()
        proc.fromchild.close()
        exit_code = proc.wait()

        self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\
                                  output_temp_files)

        if os.WIFEXITED(exit_code):
            exit_code = os.WEXITSTATUS(exit_code)
        if os.WIFSIGNALED(exit_code):
            exit_code = os.WTERMSIG(exit_code)
        if __debug__:
            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                          "Exit code : " + str(exit_code))
        if exit_code == 0:
            # send the result (output files and output messages) to the client
            self.send_result_files_back_to_client(input_temp_files,\
                                                  output_temp_files)
            if __debug__:
                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
                              "Output files sent.")
        self.send_exit_code(exit_code)
        self.set_and_send_nb_job_to_scheduler()
Example #16
0
 def request_host(self):
     host = self.store.give_host()
     ProtocolUtils.send_data(self.__client_socket, host)
     syslog.syslog(syslog.LOG_ERR | syslog.LOG_DAEMON, \
                   "dms Scheduler give host %s to client "%(host)\
                   + self.__client_address)