def stage_in(self, operation):
""" Stage the current operation """
super(MulticoreBackend, self).stage_in(operation)
self.pool = multiprocessing.Pool(processes = self.pool_size)
# Set up progress bar
widgets = ['Operation progress: ', Percentage(), ' ', Bar(), ' ', ETA()]
self.progress_bar = ProgressBar(widgets = widgets,
maxval = self.current_operation.number_processes)
self.progress_bar.start()
self._log("Operation - staged")
self.state = "staged"
def stage_in(self, operation):
""" Stage the current operation """
super(LoadLevelerBackend, self).stage_in(operation)
# set up queue
self.result_handlers = multiprocessing.Queue(200)
# Set up progress bar
widgets = [
'Operation progress: ',
Percentage(), ' ',
Bar(), ' ',
ETA()
]
self.progress_bar = ProgressBar(
widgets=widgets, maxval=self.current_operation.number_processes)
self.progress_bar.start()
self._log("Operation - staged")
self.state = "staged"
def stage_in(self, operation):
"""
Stage the current operation
"""
super(MpiBackend, self).stage_in(operation)
# init of process lists, because backend is only initialized once
self.process_args_list = []
self.IndexCopyStart = 0
self.ProcessingSuccessful = True
self.TotalProcessesFinished = 0
self.CrashedProcesses = []
# Set up progress bar
widgets = [
'Operation progress: ',
Percentage(), ' ',
Bar(), ' ',
ETA()
]
self.progress_bar = ProgressBar(
widgets=widgets, maxval=self.current_operation.number_processes)
self.progress_bar.start()
# The handler that is used remotely for logging
handler_class = logging.handlers.SocketHandler
handler_args = {"host": self.host, "port": self.port}
# Set up stage in directory
stagein_dir = os.sep.join(
[self.current_operation.result_directory, ".stagein"])
# Check if hosts file is created in the right directoy
HostfileCreated = pySPACE.configuration.root_dir + "/" + 'hostsfile'
if (not os.path.isfile(HostfileCreated)):
print "***************************************************************************************************"
print "hostsfile not created !"
print "Please create the hosts file with a filename 'hostsfile' under ", pySPACE.configuration.root_dir
print "***************************************************************************************************"
raise UserWarning('Missing hostsfile.')
if not os.path.exists(stagein_dir):
os.mkdir(stagein_dir)
process = self.current_operation.processes.get()
print "Preparing processes. This might take a few minutes...."
# Until not all Processes have been created prepare all processes
# from the queue for remote execution and execute them
i = 0
while process != False:
process.prepare(pySPACE.configuration, handler_class, handler_args)
# since preparing the process might be quite faster than executing
# it we need another queue where processes get out when they have
# finished execution
#self.result_handlers.put(1)
# Execute all functions in the process pool but return immediately
#self.pool.apply_async(process, callback=self.dequeue_process)
proc_file_name = os.sep.join(
[stagein_dir, "process_%d.pickle" % i])
proc_file = open(proc_file_name, "w")
cPickle.dump(process, proc_file)
proc_file.close()
# Add task to job specification
self.process_args_list.append(proc_file_name)
# Get the next process
process = self.current_operation.processes.get()
i += 1
self._log("Operation - staged")
self.state = "staged"