def get_start_step_of_file(input_file):
"""Return an integer which represent at which step
gcc will start the c
ompilation process for this file
1 is starting at the preprocessing level
2 is starting at the compilation level
3 is starting at the assembly level
"""
# Put here any known extensions for c++ source files
start_step_by_extension = {
".cc" : 1,
".cpp" : 1,
".cxx" : 1,
".S" : 1,
".i" : 2,
".s" : 3
}
file_extension = FileUtils.get_file_extension(input_file)
if file_extension in start_step_by_extension.keys():
return start_step_by_extension[file_extension]
# unknown extension for the gcc C compiler, so consider
# start step will be linking
return 4
def testGetFileExtension(self):
files = (("test.c", ".c"), ("test.o", ".o"), ("test.s", ".s"),
("test.S", ".S"), ("test.i", ".i"), ("test.ii", ".ii"),
("test", ""), ("test.c.o", ".o"), (".test.c", ".c"),
("../../../../.test.c",
".c"), ("./.test.c.c", ".c"), ("../../test.o.c.ii", ".ii"))
for file in files:
self.assertEqual(FileUtils.get_file_extension(file[0]), file[1])
def testGetFileExtension(self):
files = (
("test.c", ".c"),
("test.o", ".o"),
("test.s", ".s"),
("test.S", ".S"),
("test.i", ".i"),
("test.ii", ".ii"),
("test", ""),
("test.c.o", ".o"),
(".test.c", ".c"),
("../../../../.test.c", ".c"),
("./.test.c.c", ".c"),
("../../test.o.c.ii", ".ii")
)
for file in files:
self.assertEqual(FileUtils.get_file_extension(file[0]), file[1])
def get_start_step_of_file(input_file):
"""Return an integer which represent at which step
gcc will start the c
ompilation process for this file
1 is starting at the preprocessing level
2 is starting at the compilation level
3 is starting at the assembly level
"""
# Put here any known extensions for c++ source files
start_step_by_extension = {".cc": 1, ".cpp": 1, ".cxx": 1, ".S": 1, ".i": 2, ".s": 3}
file_extension = FileUtils.get_file_extension(input_file)
if file_extension in start_step_by_extension.keys():
return start_step_by_extension[file_extension]
# unknown extension for the gcc C compiler, so consider
# start step will be linking
return 4
def compiler_job(self):
"""This method is called when a remote client
ask a compiler daemon to compile some source code,
and send it back the results of the compilation
(object files generally).
Return 0 if everything has been done without any error,
and other values otherwise.
"""
if __debug__:
print >> sys.stderr, "Execution d'un job de compilation"
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
"Launching compilation job.")
# receive the length of the command line to be received
compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
"Compilation command line received : "\
+ compiler_command_line)
compiler_command = CompilerCommandFactory.build_compiler_instance\
(compiler_command_line.split())
# get the content of the input files used in the command line we
# have just received
input_temp_files = {}
output_temp_files = {}
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
while command == Protocol.FILE_COMMAND:
file_name = ProtocolUtils.recv_data(self.__client_socket)
# FIXME: do we need to create the file inside the
# critical section ?
RemoteJob.tmp_file_creation_lock.acquire()
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Creating temporary file.")
tmp_file_name = tempfile.mktemp(\
FileUtils.get_file_extension(file_name))
tmp_file = open(tmp_file_name, 'w')
RemoteJob.tmp_file_creation_lock.release()
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Temporary file created.")
input_temp_files[file_name] = tmp_file_name
tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))
tmp_file.flush()
tmp_file.close()
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
# replace original input files in the command line by the
# temporary ones
compiler_command.replace_input_files(input_temp_files)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"New compilation command line :"\
+ " ".join(compiler_command.get_command_args()))
# FIXME We should not use "-o" here, this is compiler dependant
if "-o" in compiler_command.get_command_args():
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"-o option in command line.")
try:
index_output = compiler_command.get_command_args().index("-o") \
+ 1
# FIXME: do we need to create the file inside the
# critical section ?
RemoteJob.tmp_file_creation_lock.acquire()
tmp_output_file = tempfile.mktemp()
tmp_output_file_hdl = open(tmp_output_file, 'w')
tmp_output_file_hdl.close()
RemoteJob.tmp_file_creation_lock.release()
# associate the output tmp file with the original one
output_file_name = compiler_command.get_command_args()\
[index_output]
output_temp_files[output_file_name] = tmp_output_file
# replace the output file name in the command line
compiler_command.get_command_args(\
)[index_output] = tmp_output_file
except IndexError:
# if there is no file name after the -o option,
# it means that the command line is wrong, but we
# must execute it in order to send the error
# msg back to the client
pass
else:
# no output file specified with -o switch
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"No -o option in command line.")
for original_input_file_name in input_temp_files.keys():
stop_step = compiler_command.get_stop_step()
orig_output_file_name = compiler_command.\
get_output_file_name_for_step(\
original_input_file_name,\
stop_step)
output_temp_files[\
orig_output_file_name] = compiler_command.\
get_output_file_name_for_step(\
input_temp_files[original_input_file_name],\
stop_step)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"File to return to the client : "\
+ output_temp_files[orig_output_file_name])
# execute the command in a subshell and get the stdout and stderr output
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"Executing the following command line : " \
+ compiler_command.get_local_compiler_path() + " " +\
" ".join(compiler_command.get_command_args()[1:]))
#proc = popen2.Popen3(" ".join(compiler_command.get_command_args()), 1)
# FIXME : VERY IMPORTANT
# uncomment the next two line to replace the previous one
proc = popen2.Popen3(compiler_command.get_local_compiler_path() + " "\
+ " ".join(compiler_command.get_command_args()[1:]), 1)
msg_stderr = proc.childerr.read()
msg_stdout = proc.fromchild.read()
proc.childerr.close()
proc.fromchild.close()
exit_code = proc.wait()
self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\
output_temp_files)
if os.WIFEXITED(exit_code):
exit_code = os.WEXITSTATUS(exit_code)
if os.WIFSIGNALED(exit_code):
exit_code = os.WTERMSIG(exit_code)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Exit code : " + str(exit_code))
if exit_code == 0:
# send the result (output files and output messages) to the client
self.send_result_files_back_to_client(input_temp_files,\
output_temp_files)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Output files sent.")
self.send_exit_code(exit_code)
self.set_and_send_nb_job_to_scheduler()
def compiler_job(self):
"""This method is called when a remote client
ask a compiler daemon to compile some source code,
and send it back the results of the compilation
(object files generally).
Return 0 if everything has been done without any error,
and other values otherwise.
"""
if __debug__:
print >> sys.stderr, "Execution d'un job de compilation"
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
"Launching compilation job.")
# receive the length of the command line to be received
compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
"Compilation command line received : "\
+ compiler_command_line)
compiler_command = CompilerCommandFactory.build_compiler_instance\
(compiler_command_line.split())
# get the content of the input files used in the command line we
# have just received
input_temp_files = {}
output_temp_files = {}
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
while command == Protocol.FILE_COMMAND:
file_name = ProtocolUtils.recv_data(self.__client_socket)
# FIXME: do we need to create the file inside the
# critical section ?
RemoteJob.tmp_file_creation_lock.acquire()
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Creating temporary file.")
tmp_file_name = tempfile.mktemp(\
FileUtils.get_file_extension(file_name))
tmp_file = open(tmp_file_name, 'w')
RemoteJob.tmp_file_creation_lock.release()
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Temporary file created.")
input_temp_files[file_name] = tmp_file_name
tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))
tmp_file.flush()
tmp_file.close()
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
# replace original input files in the command line by the
# temporary ones
compiler_command.replace_input_files(input_temp_files)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"New compilation command line :"\
+ " ".join(compiler_command.get_command_args()))
# FIXME We should not use "-o" here, this is compiler dependant
if "-o" in compiler_command.get_command_args():
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"-o option in command line.")
try:
index_output = compiler_command.get_command_args().index("-o") \
+ 1
# FIXME: do we need to create the file inside the
# critical section ?
RemoteJob.tmp_file_creation_lock.acquire()
tmp_output_file = tempfile.mktemp()
tmp_output_file_hdl = open(tmp_output_file, 'w')
tmp_output_file_hdl.close()
RemoteJob.tmp_file_creation_lock.release()
# associate the output tmp file with the original one
output_file_name = compiler_command.get_command_args()\
[index_output]
output_temp_files[output_file_name] = tmp_output_file
# replace the output file name in the command line
compiler_command.get_command_args(\
)[index_output] = tmp_output_file
except IndexError:
# if there is no file name after the -o option,
# it means that the command line is wrong, but we
# must execute it in order to send the error
# msg back to the client
pass
else:
# no output file specified with -o switch
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"No -o option in command line.")
for original_input_file_name in input_temp_files.keys():
stop_step = compiler_command.get_stop_step()
orig_output_file_name = compiler_command.\
get_output_file_name_for_step(\
original_input_file_name,\
stop_step)
output_temp_files[\
orig_output_file_name] = compiler_command.\
get_output_file_name_for_step(\
input_temp_files[original_input_file_name],\
stop_step)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"File to return to the client : "\
+ output_temp_files[orig_output_file_name])
# execute the command in a subshell and get the stdout and stderr output
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
"Executing the following command line : " \
+ compiler_command.get_local_compiler_path() + " " +\
" ".join(compiler_command.get_command_args()[1:]))
#proc = popen2.Popen3(" ".join(compiler_command.get_command_args()), 1)
# FIXME : VERY IMPORTANT
# uncomment the next two line to replace the previous one
proc = popen2.Popen3(compiler_command.get_local_compiler_path() + " "\
+ " ".join(compiler_command.get_command_args()[1:]), 1)
msg_stderr = proc.childerr.read()
msg_stdout = proc.fromchild.read()
proc.childerr.close()
proc.fromchild.close()
exit_code = proc.wait()
self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\
output_temp_files)
if os.WIFEXITED(exit_code):
exit_code = os.WEXITSTATUS(exit_code)
if os.WIFSIGNALED(exit_code):
exit_code = os.WTERMSIG(exit_code)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Exit code : " + str(exit_code))
if exit_code == 0:
# send the result (output files and output messages) to the client
self.send_result_files_back_to_client(input_temp_files,\
output_temp_files)
if __debug__:
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
"Output files sent.")
self.send_exit_code(exit_code)
self.set_and_send_nb_job_to_scheduler()
