def parallel(self, *tasklist): """Run tasks in parallel""" pids = [] old_log_filename = self._logger.global_filename for i, task in enumerate(tasklist): assert isinstance(task, (tuple, list)) self._logger.global_filename = old_log_filename + (".%d" % i) def task_func(): # stub out _record_indent with a process-local one base_record_indent = self._record_indent proc_local = self._job_state.property_factory( '_state', '_record_indent.%d' % os.getpid(), base_record_indent, namespace='client') self.__class__._record_indent = proc_local task[0](*task[1:]) pids.append(parallel.fork_start(self.resultdir, task_func)) old_log_path = os.path.join(self.resultdir, old_log_filename) old_log = open(old_log_path, "a") exceptions = [] for i, pid in enumerate(pids): # wait for the task to finish try: parallel.fork_waitfor(self.resultdir, pid) except Exception, e: exceptions.append(e) # copy the logs from the subtask into the main log new_log_path = old_log_path + (".%d" % i) if os.path.exists(new_log_path): new_log = open(new_log_path) old_log.write(new_log.read()) new_log.close() old_log.flush() os.remove(new_log_path)
def __init__(self, address='', port=_DEFAULT_PORT, tmpdir=None): """ :param address: Address on which server must be started. :param port: Port of server. :param tmpdir: Dir where pid file is saved. """ if tmpdir: self.tmpdir = TempDir(tmpdir) else: self.tmpdir = autotemp.tempdir(unique_id='', prefix=("SyncListenServer_%d" % port)) self.sessions = {} self.exit_event = threading.Event() self.server_pid = parallel.fork_start( self.tmpdir.name, lambda: self._start_server(address, port))
def __init__(self, address='', port=_DEFAULT_PORT, tmpdir=None): """ :param address: Address on which server must be started. :param port: Port of server. :param tmpdir: Dir where pid file is saved. """ l = lambda: self._start_server(address, port) if tmpdir: self.tmpdir = TempDir(tmpdir) else: self.tmpdir = autotemp.tempdir(unique_id='', prefix="SyncListenServer_%d" % (port)) self.sessions = {} self.exit_event = threading.Event() self.server_pid = parallel.fork_start(self.tmpdir.name, l)
def _runtest(self, url, tag, timeout, args, dargs): try: pid = parallel.fork_start(self.resultdir, lambda: test.runtest(self, url, tag, args, dargs)) if timeout: logging.debug("Waiting for pid %d for %d seconds", pid, timeout) parallel.fork_waitfor_timed(self.resultdir, pid, timeout) else: parallel.fork_waitfor(self.resultdir, pid) except error.TestBaseException: # These are already classified with an error type (exit_status) raise except error.JobError: raise # Caught further up and turned into an ABORT. except Exception, e: # Converts all other exceptions thrown by the test regardless # of phase into a TestError(TestBaseException) subclass that # reports them with their full stack trace. raise error.UnhandledTestError(e)
def _runtest(self, url, tag, timeout, args, dargs): try: l = lambda : test.runtest(self, url, tag, args, dargs) pid = parallel.fork_start(self.resultdir, l) if timeout: logging.debug('Waiting for pid %d for %d seconds', pid, timeout) parallel.fork_waitfor_timed(self.resultdir, pid, timeout) else: parallel.fork_waitfor(self.resultdir, pid) except error.TestBaseException: # These are already classified with an error type (exit_status) raise except error.JobError: raise # Caught further up and turned into an ABORT. except Exception, e: # Converts all other exceptions thrown by the test regardless # of phase into a TestError(TestBaseException) subclass that # reports them with their full stack trace. raise error.UnhandledTestError(e)