def send_message(cls, connection, host, port, message_type, *args):
"""
Sends a message with the given type and optional args via the given connection.
If the connection is established, it is used. Otherwise a new
connection is created and will be returned.
:param connection: The connection to use for sending.
:type connection: socket.socket
:param host: The host to send the message to if the connection is not established
:type host: basestring
:param port: The port to use for establishing a new connection
:type port: int
:param message_type: The type of the message
:param args: Optional arguments
:return: The connection that has been used to send the message
:rtype: socket.socket
"""
message = str(message_type.value)
for arg in args:
message += ";%s" % arg
message += cls.JOB_END_TOKEN
return inform(message, conn=connection, ip_port=(host, port))
def send_message(cls, connection, host, port, message_type, *args):
"""
Sends a message with the given type and optional args via the given connection.
If the connection is established, it is used. Otherwise a new
connection is created and will be returned.
:param connection: The connection to use for sending.
:type connection: socket.socket
:param host: The host to send the message to if the connection is not established
:type host: basestring
:param port: The port to use for establishing a new connection
:type port: int
:param message_type: The type of the message
:param args: Optional arguments
:return: The connection that has been used to send the message
:rtype: socket.socket
"""
message = str(message_type.value)
for arg in args:
message += ";%s" % arg
message += cls.JOB_END_TOKEN
return inform(message, conn=connection, ip_port=(host, port))
def execute(self):
""" Execute all processes specified in the currently staged operation """
assert (self.state == "staged")
self._log("Operation - executing")
self.state = "executing"
# The handler that is used remotely for logging
handler_class = logging.handlers.SocketHandler
handler_args = {"host": self.host, "port": self.port}
# the communication properties to talk to LoadLevelerComHandler
backend_com = (self.SERVER_IP, self.SERVER_PORT)
print('--> Loadleveler Communication : \n\t\t host:%s, port:%s' %
(self.host, self.port))
# Prepare the directory where processes are stored before submitted
# to LoadLeveler
self.process_dir = os.sep.join(
[self.current_operation.result_directory, ".processes"])
if not os.path.exists(self.process_dir):
os.mkdir(self.process_dir)
# create and start server socket thread
self.listener = LoadLevelerComHandler(
self.sock,
self.result_handlers,
self.progress_bar,
self.LL_COMMAND_FILE_TEMPLATE,
operation_dir=self.current_operation.result_directory)
self.listener.start()
# create a client socket to talk to server socket thread
send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
send_socket.connect((self.SERVER_IP, self.SERVER_PORT))
# get first process from creation queue
process = self.current_operation.processes.get()
process_counter = 0
# Until not all Processes have been created, prepare all processes
# from the queue for remote execution and execute them
while process != False:
process.prepare(pySPACE.configuration, handler_class, handler_args,
backend_com)
# since preparing the process might be quite faster than executing
# it we need another queue where processes get out when they have
# finished execution
self.result_handlers.put(1)
# pickle the process object
proc_file_name = os.sep.join(
[self.process_dir,
"process_%d.pickle" % process_counter])
proc_file = open(proc_file_name, "w")
cPickle.dump(process, proc_file, cPickle.HIGHEST_PROTOCOL)
proc_file.close()
# fill out LoadLeveler template
llfile = self.LL_COMMAND_FILE_TEMPLATE % \
{"process_file_path": proc_file_name,
"server_port": self.SERVER_PORT,
"op_result_dir": self.current_operation.result_directory}
llfilepath = os.path.join(self.current_operation.result_directory,
"ll_call.cmd")
f = open(llfilepath, 'w')
f.write(llfile)
f.close()
# submit to LoadLeveler
while 1:
outlog, errlog = sub.Popen(["llsubmit", llfilepath],
stdout=sub.PIPE,
stderr=sub.PIPE).communicate()
if errlog == "":
break
else:
self._log("Warning: Job submission to LoadLeveler failed"\
" with %s. Job will be resubmitted." % errlog,
logging.WARNING)
time.sleep(1)
# parse job_id for monitoring
loadl_id = outlog.split("\"")[1].split(".")[-1]
# inform listener that we successfully submitted the job
send_socket = inform('submitted;%d;%s%s' % \
(process_counter, loadl_id, self.end_token),
send_socket, (self.SERVER_IP,self.SERVER_PORT))
# get next process and update process_counter
process = self.current_operation.processes.get()
process_counter += 1
# send message 'creation finished' to listener
send_socket = inform('creation finished' + self.end_token, send_socket,
(self.SERVER_IP, self.SERVER_PORT))
# give socket chance to process message
# time.sleep(0.001)
self.listener.creation_finished = True
send_socket.shutdown(socket.SHUT_RDWR)
send_socket.close()
def execute(self):
""" Execute all processes specified in the currently staged operation """
assert(self.state == "staged")
self._log("Operation - executing")
self.state = "executing"
# The handler that is used remotely for logging
handler_class = logging.handlers.SocketHandler
handler_args = {"host" : self.host, "port" : self.port}
# the communication properties to talk to LoadLevelerComHandler
backend_com = (self.SERVER_IP, self.SERVER_PORT)
print('--> Loadleveler Communication : \n\t\t host:%s, port:%s' % \
(self.SERVER_IP, self.SERVER_PORT))
# Prepare the directory where processes are stored before submitted
# to LoadLeveler
self.process_dir = os.sep.join([self.current_operation.result_directory,
".processes"])
if not os.path.exists(self.process_dir):
os.mkdir(self.process_dir)
# create and start server socket thread
self.listener = LoadLevelerComHandler(self.sock, self.result_handlers,
self.progress_bar,
self.LL_COMMAND_FILE_TEMPLATE,
operation_dir=self.current_operation.result_directory)
self.listener.start()
# create a client socket to talk to server socket thread
send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
send_socket.connect((self.SERVER_IP,self.SERVER_PORT))
# get first process from creation queue
process = self.current_operation.processes.get()
process_counter = 0
# Until not all Processes have been created, prepare all processes
# from the queue for remote execution and execute them
while process != False:
process.prepare(pySPACE.configuration, handler_class, handler_args,
backend_com)
# since preparing the process might be quite faster than executing
# it we need another queue where processes get out when they have
# finished execution
self.result_handlers.put(1)
# pickle the process object
proc_file_name = os.sep.join([self.process_dir,
"process_%d.pickle" % process_counter])
proc_file = open(proc_file_name, "w")
cPickle.dump(process, proc_file, cPickle.HIGHEST_PROTOCOL)
proc_file.close()
# fill out LoadLeveler template
llfile = self.LL_COMMAND_FILE_TEMPLATE % \
{"process_file_path": proc_file_name,
"server_port": self.SERVER_PORT,
"op_result_dir": self.current_operation.result_directory}
llfilepath = os.path.join(self.current_operation.result_directory,
"ll_call.cmd")
f=open(llfilepath,'w')
f.write(llfile)
f.close()
# submit to LoadLeveler
while 1:
outlog, errlog = sub.Popen(["llsubmit", llfilepath],
stdout=sub.PIPE, stderr=sub.PIPE).communicate()
if errlog == "":
break
else:
self._log("Warning: Job submission to LoadLeveler failed"\
" with %s. Job will be resubmitted." % errlog,
logging.WARNING)
time.sleep(1)
# parse job_id for monitoring
loadl_id = outlog.split("\"")[1].split(".")[-1]
# inform listener that we successfully submitted the job
send_socket = inform('submitted;%d;%s%s' % \
(process_counter, loadl_id, self.end_token),
send_socket, (self.SERVER_IP,self.SERVER_PORT))
# get next process and update process_counter
process = self.current_operation.processes.get()
process_counter+=1
# send message 'creation finished' to listener
send_socket = inform('creation finished'+self.end_token, send_socket,
(self.SERVER_IP,self.SERVER_PORT))
# give socket chance to process message
# time.sleep(0.001)
self.listener.creation_finished = True
send_socket.shutdown(socket.SHUT_RDWR)
send_socket.close()