def stage_in(self, operation): """ Stage the current operation """ super(MulticoreBackend, self).stage_in(operation) self.pool = multiprocessing.Pool(processes = self.pool_size) # Set up progress bar widgets = ['Operation progress: ', Percentage(), ' ', Bar(), ' ', ETA()] self.progress_bar = ProgressBar(widgets = widgets, maxval = self.current_operation.number_processes) self.progress_bar.start() self._log("Operation - staged") self.state = "staged"
def stage_in(self, operation): """ Stage the current operation """ super(LoadLevelerBackend, self).stage_in(operation) # set up queue self.result_handlers = multiprocessing.Queue(200) # Set up progress bar widgets = [ 'Operation progress: ', Percentage(), ' ', Bar(), ' ', ETA() ] self.progress_bar = ProgressBar( widgets=widgets, maxval=self.current_operation.number_processes) self.progress_bar.start() self._log("Operation - staged") self.state = "staged"
def stage_in(self, operation): """ Stage the current operation """ super(MpiBackend, self).stage_in(operation) # init of process lists, because backend is only initialized once self.process_args_list = [] self.IndexCopyStart = 0 self.ProcessingSuccessful = True self.TotalProcessesFinished = 0 self.CrashedProcesses = [] # Set up progress bar widgets = [ 'Operation progress: ', Percentage(), ' ', Bar(), ' ', ETA() ] self.progress_bar = ProgressBar( widgets=widgets, maxval=self.current_operation.number_processes) self.progress_bar.start() # The handler that is used remotely for logging handler_class = logging.handlers.SocketHandler handler_args = {"host": self.host, "port": self.port} # Set up stage in directory stagein_dir = os.sep.join( [self.current_operation.result_directory, ".stagein"]) # Check if hosts file is created in the right directoy HostfileCreated = pySPACE.configuration.root_dir + "/" + 'hostsfile' if (not os.path.isfile(HostfileCreated)): print "***************************************************************************************************" print "hostsfile not created !" print "Please create the hosts file with a filename 'hostsfile' under ", pySPACE.configuration.root_dir print "***************************************************************************************************" raise UserWarning('Missing hostsfile.') if not os.path.exists(stagein_dir): os.mkdir(stagein_dir) process = self.current_operation.processes.get() print "Preparing processes. This might take a few minutes...." # Until not all Processes have been created prepare all processes # from the queue for remote execution and execute them i = 0 while process != False: process.prepare(pySPACE.configuration, handler_class, handler_args) # since preparing the process might be quite faster than executing # it we need another queue where processes get out when they have # finished execution #self.result_handlers.put(1) # Execute all functions in the process pool but return immediately #self.pool.apply_async(process, callback=self.dequeue_process) proc_file_name = os.sep.join( [stagein_dir, "process_%d.pickle" % i]) proc_file = open(proc_file_name, "w") cPickle.dump(process, proc_file) proc_file.close() # Add task to job specification self.process_args_list.append(proc_file_name) # Get the next process process = self.current_operation.processes.get() i += 1 self._log("Operation - staged") self.state = "staged"