def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result): currentaddr = Value("c", bytes_("")) currentstart = Value("d", time.time()) keyboardCaught = Event() p = Process( target=runner, args=( iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config), ), ) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught old = signal.signal(signal.SIGILL, signalhandler) p.start() signal.signal(signal.SIGILL, old) return p
def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result): from nose.plugins.multiprocess import Value, Event, Process from nose.plugins.multiprocess import signalhandler currentaddr = Value('c', bytes_('')) currentstart = Value('d', time.time()) keyboardCaught = Event() mp_context = {plugin.name: plugin.mp_context for plugin in _instantiate_plugins if 'mp_context' in dir(plugin)} p = Process(target=runner, args=( iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config), _instantiate_plugins, mp_context)) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught old = signal.signal(signal.SIGILL, signalhandler) p.start() signal.signal(signal.SIGILL, old) return p
def startProcess(self, iworker, testQueue, resultQueue, procesingQueue, shouldStop, result): currentaddr = Value('c', bytes_('')) currentstart = Value('d', time.time()) keyboardCaught = Event() p = Process(target=runner, args=(iworker, testQueue, resultQueue, procesingQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught old = signal.signal(signal.SIGILL, signalhandler) p.start() signal.signal(signal.SIGILL, old) return p
def __runner( ix, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, loaderClass, resultClass, config ): config = pickle.loads(config) dummy_parser = config.parserClass() if _instantiate_plugins is not None: for pluginclass in _instantiate_plugins: plugin = pluginclass() plugin.addOptions(dummy_parser, {}) config.plugins.addPlugin(plugin) config.plugins.configure(config.options, config) config.plugins.begin() log.debug("Worker %s executing, pid=%d", ix, os.getpid()) loader = loaderClass(config=config) loader.suiteClass.suiteClass = NoSharedFixtureContextSuite def get(): return testQueue.get(timeout=config.multiprocess_timeout) def makeResult(): stream = _WritelnDecorator(StringIO()) result = resultClass(stream, descriptions=1, verbosity=config.verbosity, config=config) plug_result = config.plugins.prepareTestResult(result) if plug_result: return plug_result return result def batch(result): failures = [(TestLet(c), err) for c, err in result.failures] errors = [(TestLet(c), err) for c, err in result.errors] errorClasses = {} for key, (storage, label, isfail) in list(result.errorClasses.items()): errorClasses[key] = ([(TestLet(c), err) for c, err in storage], label, isfail) return (result.stream.getvalue(), result.testsRun, failures, errors, errorClasses) for test_addr, arg in iter(get, "STOP"): if shouldStop.is_set(): log.exception("Worker %d STOPPED", ix) break result = makeResult() test = loader.loadTestsFromNames([test_addr]) test.testQueue = testQueue test.tasks = [] test.arg = arg log.debug("Worker %s Test is %s (%s)", ix, test_addr, test) try: if arg is not None: test_addr = test_addr + str(arg) currentaddr.value = bytes_(test_addr) currentstart.value = time.time() test(result) currentaddr.value = bytes_("") resultQueue.put((ix, test_addr, test.tasks, batch(result))) except KeyboardInterrupt as e: # TimedOutException: timeout = isinstance(e, TimedOutException) if timeout: keyboardCaught.set() if len(currentaddr.value): if timeout: msg = "Worker %s timed out, failing current test %s" else: msg = "Worker %s keyboard interrupt, failing current test %s" log.exception(msg, ix, test_addr) currentaddr.value = bytes_("") failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: if timeout: msg = "Worker %s test %s timed out" else: msg = "Worker %s test %s keyboard interrupt" log.debug(msg, ix, test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if not timeout: raise except SystemExit: currentaddr.value = bytes_("") log.exception("Worker %s system exit", ix) raise except: currentaddr.value = bytes_("") log.exception("Worker %s error running test or returning " "results", ix) failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if config.multiprocess_restartworker: break log.debug("Worker %s ending", ix)
def run(self, test): """ Execute the test (which may be a test suite). If the test is a suite, distribute it out among as many processes as have been configured, at as fine a level as is possible given the context fixtures defined in the suite or any sub-suites. """ log.debug("%s.run(%s) (%s)", self, test, os.getpid()) wrapper = self.config.plugins.prepareTest(test) if wrapper is not None: test = wrapper # plugins can decorate or capture the output stream wrapped = self.config.plugins.setOutputStream(self.stream) if wrapped is not None: self.stream = wrapped testQueue = Queue() resultQueue = Queue() tasks = [] completed = [] workers = [] to_teardown = [] shouldStop = Event() result = self._makeResult() start = time.time() self.collect(test, testQueue, tasks, to_teardown, result) log.debug("Starting %s workers", self.config.multiprocess_workers) for i in range(self.config.multiprocess_workers): p = self.startProcess(i, testQueue, resultQueue, shouldStop, result) workers.append(p) log.debug("Started worker process %s", i + 1) total_tasks = len(tasks) # need to keep track of the next time to check for timeouts in case # more than one process times out at the same time. nexttimeout = self.config.multiprocess_timeout thrownError = None try: while tasks: log.debug( "Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks, nexttimeout ) try: iworker, addr, newtask_addrs, batch_result = resultQueue.get(timeout=nexttimeout) log.debug("Results received for worker %d, %s, new tasks: %d", iworker, addr, len(newtask_addrs)) try: try: tasks.remove(addr) except ValueError: log.warn("worker %s failed to remove from tasks: %s", iworker, addr) total_tasks += len(newtask_addrs) tasks.extend(newtask_addrs) except KeyError: log.debug("Got result for unknown task? %s", addr) log.debug("current: %s", str(list(tasks)[0])) else: completed.append([addr, batch_result]) self.consolidate(result, batch_result) if self.config.stopOnError and not result.wasSuccessful(): # set the stop condition shouldStop.set() break if self.config.multiprocess_restartworker: log.debug("joining worker %s", iworker) # wait for working, but not that important if worker # cannot be joined in fact, for workers that add to # testQueue, they will not terminate until all their # items are read workers[iworker].join(timeout=1) if not shouldStop.is_set() and not testQueue.empty(): log.debug("starting new process on worker %s", iworker) workers[iworker] = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result) except Empty: log.debug( "Timed out with %s tasks pending " "(empty testQueue=%r): %s", len(tasks), testQueue.empty(), str(tasks), ) any_alive = False for iworker, w in enumerate(workers): if w.is_alive(): worker_addr = bytes_(w.currentaddr.value, "ascii") timeprocessing = time.time() - w.currentstart.value if len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout - 0.1: log.debug( "worker %d has finished its work item, " "but is not exiting? do we wait for it?", iworker, ) else: any_alive = True if len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout - 0.1: log.debug("timed out worker %s: %s", iworker, worker_addr) w.currentaddr.value = bytes_("") # If the process is in C++ code, sending a SIGILL # might not send a python KeybordInterrupt exception # therefore, send multiple signals until an # exception is caught. If this takes too long, then # terminate the process w.keyboardCaught.clear() startkilltime = time.time() while not w.keyboardCaught.is_set() and w.is_alive(): if time.time() - startkilltime > self.waitkilltime: # have to terminate... log.error("terminating worker %s", iworker) w.terminate() # there is a small probability that the # terminated process might send a result, # which has to be specially handled or # else processes might get orphaned. workers[iworker] = w = self.startProcess( iworker, testQueue, resultQueue, shouldStop, result ) break os.kill(w.pid, signal.SIGILL) time.sleep(0.1) if not any_alive and testQueue.empty(): log.debug("All workers dead") break nexttimeout = self.config.multiprocess_timeout for w in workers: if w.is_alive() and len(w.currentaddr.value) > 0: timeprocessing = time.time() - w.currentstart.value if timeprocessing <= self.config.multiprocess_timeout: nexttimeout = min(nexttimeout, self.config.multiprocess_timeout - timeprocessing) log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks)) except (KeyboardInterrupt, SystemExit) as e: log.info("parent received ctrl-c when waiting for test results") thrownError = e # resultQueue.get(False) result.addError(test, sys.exc_info()) try: for case in to_teardown: log.debug("Tearing down shared fixtures for %s", case) try: case.tearDown() except (KeyboardInterrupt, SystemExit): raise except: result.addError(case, sys.exc_info()) stop = time.time() # first write since can freeze on shutting down processes result.printErrors() result.printSummary(start, stop) self.config.plugins.finalize(result) if thrownError is None: log.debug("Tell all workers to stop") for w in workers: if w.is_alive(): testQueue.put("STOP", block=False) # wait for the workers to end for iworker, worker in enumerate(workers): if worker.is_alive(): log.debug("joining worker %s", iworker) worker.join() if worker.is_alive(): log.debug("failed to join worker %s", iworker) except (KeyboardInterrupt, SystemExit): log.info("parent received ctrl-c when shutting down: stop all processes") for worker in workers: if worker.is_alive(): worker.terminate() if thrownError: raise thrownError else: raise return result
def multiprocess_runner(ix, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, loaderClass, resultClass, config): """To replace the test runner of multiprocess. * Setup gae services at the beginning of every process * Clean datastore after each test """ from nose.pyversion import bytes_ try: from cStringIO import StringIO except ImportError: import StringIO from nose.plugins.multiprocess import _instantiate_plugins, \ NoSharedFixtureContextSuite, _WritelnDecorator, TestLet config = pickle.loads(config) dummy_parser = config.parserClass() if _instantiate_plugins is not None: for pluginclass in _instantiate_plugins: plugin = pluginclass() plugin.addOptions(dummy_parser, {}) config.plugins.addPlugin(plugin) config.plugins.configure(config.options, config) config.plugins.begin() log.debug("Worker %s executing, pid=%d", ix, os.getpid()) loader = loaderClass(config=config) loader.suiteClass.suiteClass = NoSharedFixtureContextSuite def get(): return testQueue.get(timeout=config.multiprocess_timeout) def makeResult(): stream = _WritelnDecorator(StringIO()) result = resultClass(stream, descriptions=1, verbosity=config.verbosity, config=config) plug_result = config.plugins.prepareTestResult(result) return plug_result if plug_result else result def batch(result): failures = [(TestLet(c), err) for c, err in result.failures] errors = [(TestLet(c), err) for c, err in result.errors] errorClasses = {} for key, (storage, label, isfail) in result.errorClasses.items(): errorClasses[key] = ([(TestLet(c), err) for c, err in storage], label, isfail) return (result.stream.getvalue(), result.testsRun, failures, errors, errorClasses) def setup_process_env(): """Runs just after the process starts to setup services.""" setup_gae_services() def after_each_test(): """Runs after each test to clean datastore.""" clean_datastore() # Setup gae services at the beginning of every process setup_process_env() for test_addr, arg in iter(get, 'STOP'): if shouldStop.is_set(): log.exception('Worker %d STOPPED', ix) break result = makeResult() test = loader.loadTestsFromNames([test_addr]) test.testQueue = testQueue test.tasks = [] test.arg = arg log.debug("Worker %s Test is %s (%s)", ix, test_addr, test) try: if arg is not None: test_addr = test_addr + str(arg) currentaddr.value = bytes_(test_addr) currentstart.value = time.time() test(result) currentaddr.value = bytes_('') resultQueue.put((ix, test_addr, test.tasks, batch(result))) # Clean datastore after each test after_each_test() except KeyboardInterrupt: keyboardCaught.set() if len(currentaddr.value) > 0: log.exception( 'Worker %s keyboard interrupt, failing ' 'current test %s', ix, test_addr) currentaddr.value = bytes_('') failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: log.debug('Worker %s test %s timed out', ix, test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) except SystemExit: currentaddr.value = bytes_('') log.exception('Worker %s system exit', ix) raise except: currentaddr.value = bytes_('') log.exception( "Worker %s error running test or returning " "results", ix) failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if config.multiprocess_restartworker: break log.debug("Worker %s ending", ix)
resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: if timeout: msg = 'Worker %s test %s timed out' else: msg = 'Worker %s test %s keyboard interrupt' log.debug(msg,ix,test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if not timeout: raise except SystemExit: currentaddr.value = bytes_('') log.exception('Worker %s system exit',ix) raise except: currentaddr.value = bytes_('') log.exception("Worker %s error running test or returning " "results",ix) failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if config.multiprocess_restartworker: break log.debug("Worker %s ending", ix) class NoSharedFixtureContextSuite(ContextSuite): """ Context suite that never fires shared fixtures. When a context sets _multiprocess_shared_, fixtures in that context are executed by the main process. Using this suite class prevents them
def multiprocess_runner(ix, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, loaderClass, resultClass, config): """To replace the test runner of multiprocess. * Setup gae services at the beginning of every process * Clean datastore after each test """ from nose import failure from nose.pyversion import bytes_ import time import pickle try: from cStringIO import StringIO except ImportError: import StringIO from nose.plugins.multiprocess import _instantiate_plugins, \ NoSharedFixtureContextSuite, _WritelnDecorator, TestLet config = pickle.loads(config) dummy_parser = config.parserClass() if _instantiate_plugins is not None: for pluginclass in _instantiate_plugins: plugin = pluginclass() plugin.addOptions(dummy_parser,{}) config.plugins.addPlugin(plugin) config.plugins.configure(config.options,config) config.plugins.begin() log.debug("Worker %s executing, pid=%d", ix,os.getpid()) loader = loaderClass(config=config) loader.suiteClass.suiteClass = NoSharedFixtureContextSuite def get(): return testQueue.get(timeout=config.multiprocess_timeout) def makeResult(): stream = _WritelnDecorator(StringIO()) result = resultClass(stream, descriptions=1, verbosity=config.verbosity, config=config) plug_result = config.plugins.prepareTestResult(result) if plug_result: return plug_result return result def batch(result): failures = [(TestLet(c), err) for c, err in result.failures] errors = [(TestLet(c), err) for c, err in result.errors] errorClasses = {} for key, (storage, label, isfail) in result.errorClasses.items(): errorClasses[key] = ([(TestLet(c), err) for c, err in storage], label, isfail) return ( result.stream.getvalue(), result.testsRun, failures, errors, errorClasses) def setup_process_env(): """Runs just after the process starts to setup services. """ setup_gae_services() def after_each_test(): """Runs after each test to clean datastore. """ clean_datastore() # Setup gae services at the beginning of every process setup_process_env() for test_addr, arg in iter(get, 'STOP'): if shouldStop.is_set(): log.exception('Worker %d STOPPED',ix) break result = makeResult() test = loader.loadTestsFromNames([test_addr]) test.testQueue = testQueue test.tasks = [] test.arg = arg log.debug("Worker %s Test is %s (%s)", ix, test_addr, test) try: if arg is not None: test_addr = test_addr + str(arg) currentaddr.value = bytes_(test_addr) currentstart.value = time.time() test(result) currentaddr.value = bytes_('') resultQueue.put((ix, test_addr, test.tasks, batch(result))) # Clean datastore after each test after_each_test() except KeyboardInterrupt: keyboardCaught.set() if len(currentaddr.value) > 0: log.exception('Worker %s keyboard interrupt, failing ' 'current test %s',ix,test_addr) currentaddr.value = bytes_('') failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: log.debug('Worker %s test %s timed out',ix,test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) except SystemExit: currentaddr.value = bytes_('') log.exception('Worker %s system exit',ix) raise except: currentaddr.value = bytes_('') log.exception("Worker %s error running test or returning " "results",ix) failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if config.multiprocess_restartworker: break log.debug("Worker %s ending", ix)
result.addError(case, sys.exc_info()) else: to_teardown.append(case) for _t in case: test_addr = self.addtask(testQueue, tasks, _t) log.debug("Queued shared-fixture test %s (%s) to %s", len(tasks), test_addr, testQueue) else: test_addr = self.addtask(testQueue, tasks, case) log.debug("Queued test %s (%s) to %s", len(tasks), test_addr, testQueue) log.debug("Starting %s workers", self.config.multiprocess_workers) for i in range(self.config.multiprocess_workers): currentaddr = Value('c', bytes_('')) currentstart = Value('d', 0.0) keyboardCaught = Event() p = Process(target=runner, args=(i, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught # p.setDaemon(True) p.start() workers.append(p) log.debug("Started worker process %s", i + 1)
def run(self, test): """ Execute the test (which may be a test suite). If the test is a suite, distribute it out among as many processes as have been configured, at as fine a level as is possible given the context fixtures defined in the suite or any sub-suites. """ log.debug("%s.run(%s) (%s)", self, test, os.getpid()) wrapper = self.config.plugins.prepareTest(test) if wrapper is not None: test = wrapper # plugins can decorate or capture the output stream wrapped = self.config.plugins.setOutputStream(self.stream) if wrapped is not None: self.stream = wrapped testQueue = Queue() resultQueue = Queue() tasks = [] completed = [] workers = [] to_teardown = [] shouldStop = Event() result = self._makeResult() start = time.time() # dispatch and collect results # put indexes only on queue because tests aren't picklable for case in self.nextBatch(test): log.debug("Next batch %s (%s)", case, type(case)) if (isinstance(case, nose.case.Test) and isinstance(case.test, failure.Failure)): log.debug("Case is a Failure") case(result) # run here to capture the failure continue # handle shared fixtures if isinstance(case, ContextSuite) and case.context is failure.Failure: log.debug("Case is a Failure") case(result) # run here to capture the failure continue elif isinstance(case, ContextSuite) and self.sharedFixtures(case): log.debug("%s has shared fixtures", case) try: case.setUp() except (KeyboardInterrupt, SystemExit): raise except: log.debug("%s setup failed", sys.exc_info()) result.addError(case, sys.exc_info()) else: to_teardown.append(case) for _t in case: test_addr = self.addtask(testQueue,tasks,_t) log.debug("Queued shared-fixture test %s (%s) to %s", len(tasks), test_addr, testQueue) else: test_addr = self.addtask(testQueue,tasks,case) log.debug("Queued test %s (%s) to %s", len(tasks), test_addr, testQueue) log.debug("Starting %s workers", self.config.multiprocess_workers) for i in range(self.config.multiprocess_workers): currentaddr = Value('c',bytes_('')) currentstart = Value('d',0.0) keyboardCaught = Event() p = Process(target=runner, args=(i, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught # p.setDaemon(True) p.start() workers.append(p) log.debug("Started worker process %s", i+1) total_tasks = len(tasks) # need to keep track of the next time to check for timeouts in case # more than one process times out at the same time. nexttimeout=self.config.multiprocess_timeout while tasks: log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks,nexttimeout) try: iworker, addr, newtask_addrs, batch_result = resultQueue.get( timeout=nexttimeout) log.debug('Results received for worker %d, %s, new tasks: %d', iworker,addr,len(newtask_addrs)) try: try: tasks.remove(addr) except ValueError: log.warn('worker %s failed to remove from tasks: %s', iworker,addr) total_tasks += len(newtask_addrs) for newaddr in newtask_addrs: tasks.append(newaddr) except KeyError: log.debug("Got result for unknown task? %s", addr) log.debug("current: %s",str(list(tasks)[0])) else: completed.append([addr,batch_result]) self.consolidate(result, batch_result) if (self.config.stopOnError and not result.wasSuccessful()): # set the stop condition shouldStop.set() break if self.config.multiprocess_restartworker: log.debug('joining worker %s',iworker) # wait for working, but not that important if worker # cannot be joined in fact, for workers that add to # testQueue, they will not terminate until all their # items are read workers[iworker].join(timeout=1) if not shouldStop.is_set() and not testQueue.empty(): log.debug('starting new process on worker %s',iworker) currentaddr = Value('c',bytes_('')) currentstart = Value('d',time.time()) keyboardCaught = Event() workers[iworker] = Process(target=runner, args=(iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) workers[iworker].currentaddr = currentaddr workers[iworker].currentstart = currentstart workers[iworker].keyboardCaught = keyboardCaught workers[iworker].start() except Empty: log.debug("Timed out with %s tasks pending " "(empty testQueue=%d): %s", len(tasks),testQueue.empty(),str(tasks)) any_alive = False for iworker, w in enumerate(workers): if w.is_alive(): worker_addr = bytes_(w.currentaddr.value,'ascii') timeprocessing = time.time() - w.currentstart.value if ( len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout-0.1): log.debug('worker %d has finished its work item, ' 'but is not exiting? do we wait for it?', iworker) else: any_alive = True if (len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout-0.1): log.debug('timed out worker %s: %s', iworker,worker_addr) w.currentaddr.value = bytes_('') # If the process is in C++ code, sending a SIGINT # might not send a python KeybordInterrupt exception # therefore, send multiple signals until an # exception is caught. If this takes too long, then # terminate the process w.keyboardCaught.clear() startkilltime = time.time() while not w.keyboardCaught.is_set() and w.is_alive(): if time.time()-startkilltime > self.waitkilltime: # have to terminate... log.error("terminating worker %s",iworker) w.terminate() currentaddr = Value('c',bytes_('')) currentstart = Value('d',time.time()) keyboardCaught = Event() workers[iworker] = Process(target=runner, args=(iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) workers[iworker].currentaddr = currentaddr workers[iworker].currentstart = currentstart workers[iworker].keyboardCaught = keyboardCaught workers[iworker].start() # there is a small probability that the # terminated process might send a result, # which has to be specially handled or # else processes might get orphaned. w = workers[iworker] break os.kill(w.pid, signal.SIGINT) time.sleep(0.1) if not any_alive and testQueue.empty(): log.debug("All workers dead") break nexttimeout=self.config.multiprocess_timeout for w in workers: if w.is_alive() and len(w.currentaddr.value) > 0: timeprocessing = time.time()-w.currentstart.value if timeprocessing <= self.config.multiprocess_timeout: nexttimeout = min(nexttimeout, self.config.multiprocess_timeout-timeprocessing) log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks)) for case in to_teardown: log.debug("Tearing down shared fixtures for %s", case) try: case.tearDown() except (KeyboardInterrupt, SystemExit): raise except: result.addError(case, sys.exc_info()) stop = time.time() # first write since can freeze on shutting down processes result.printErrors() result.printSummary(start, stop) self.config.plugins.finalize(result) log.debug("Tell all workers to stop") for w in workers: if w.is_alive(): testQueue.put('STOP', block=False) # wait for the workers to end try: for iworker,worker in enumerate(workers): if worker.is_alive(): log.debug('joining worker %s',iworker) worker.join()#10) if worker.is_alive(): log.debug('failed to join worker %s',iworker) except KeyboardInterrupt: log.info('parent received ctrl-c') for worker in workers: worker.terminate() worker.join() return result
def __runner(ix, testQueue, resultQueue, procesingQueue, currentaddr, currentstart, keyboardCaught, shouldStop, loaderClass, resultClass, config): config = pickle.loads(config) dummy_parser = config.parserClass() if _instantiate_plugins is not None: for pluginclass in _instantiate_plugins: plugin = pluginclass() plugin.addOptions(dummy_parser, {}) config.plugins.addPlugin(plugin) config.plugins.configure(config.options, config) config.plugins.begin() log.debug("Worker %s executing, pid=%d", ix, os.getpid()) loader = loaderClass(config=config) loader.suiteClass.suiteClass = NoSharedFixtureContextSuite def get(): return testQueue.get(timeout=config.multiprocess_timeout) def makeResult(): stream = _WritelnDecorator(StringIO()) result = resultClass(stream, descriptions=1, verbosity=config.verbosity, config=config) plug_result = config.plugins.prepareTestResult(result) if plug_result: return plug_result return result def batch(result): failures = [(TestLet(c), err) for c, err in result.failures] errors = [(TestLet(c), err) for c, err in result.errors] errorClasses = {} for key, (storage, label, isfail) in result.errorClasses.items(): errorClasses[key] = ([(TestLet(c), err) for c, err in storage], label, isfail) return (result.stream.getvalue(), result.testsRun, failures, errors, errorClasses) for test_addr, arg in iter(get, 'STOP'): if shouldStop.is_set(): log.exception('Worker %d STOPPED', ix) break result = makeResult() test = loader.loadTestsFromNames([test_addr]) test.testQueue = testQueue test.tasks = [] test.arg = arg log.debug("Worker %s Test is %s (%s)", ix, test_addr, test) try: if arg is not None: test_addr = test_addr + str(arg) procesingQueue.put(test_addr) currentaddr.value = bytes_(test_addr) currentstart.value = time.time() test(result) currentaddr.value = bytes_('') resultQueue.put((ix, test_addr, test.tasks, batch(result))) except KeyboardInterrupt, e: #TimedOutException: timeout = isinstance(e, TimedOutException) if timeout: keyboardCaught.set() if len(currentaddr.value): if timeout: msg = 'Worker %s timed out, failing current test %s' else: msg = 'Worker %s keyboard interrupt, failing current test %s' log.exception(msg, ix, test_addr) currentaddr.value = bytes_('') failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: if timeout: msg = 'Worker %s test %s timed out' else: msg = 'Worker %s test %s keyboard interrupt' log.debug(msg, ix, test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if not timeout: raise except SystemExit: currentaddr.value = bytes_('') log.exception('Worker %s system exit', ix) raise
def run(self, test): """ Execute the test (which may be a test suite). If the test is a suite, distribute it out among as many processes as have been configured, at as fine a level as is possible given the context fixtures defined in the suite or any sub-suites. """ log.debug("%s.run(%s) (%s)", self, test, os.getpid()) wrapper = self.config.plugins.prepareTest(test) if wrapper is not None: test = wrapper # plugins can decorate or capture the output stream wrapped = self.config.plugins.setOutputStream(self.stream) if wrapped is not None: self.stream = wrapped testQueue = Queue() resultQueue = Queue() procesingQueues = [] tasks = [] completed = [] workers = [] to_teardown = [] shouldStop = Event() result = self._makeResult() start = time.time() self.collect(test, testQueue, tasks, to_teardown, result) log.debug("Starting %s workers", self.config.multiprocess_workers) for i in range(self.config.multiprocess_workers): procesingQueue = Queue() procesingQueues.append(procesingQueue) p = self.startProcess(i, testQueue, resultQueue, procesingQueue, shouldStop, result) workers.append(p) log.debug("Started worker process %s", i + 1) total_tasks = len(tasks) # need to keep track of the next time to check for timeouts in case # more than one process times out at the same time. nexttimeout = self.config.multiprocess_timeout thrownError = None try: while tasks: log.debug( "Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks, nexttimeout) try: iworker, addr, newtask_addrs, batch_result = resultQueue.get( timeout=nexttimeout) log.debug( 'Results received for worker %d, %s, new tasks: %d', iworker, addr, len(newtask_addrs)) try: try: tasks.remove(addr) except ValueError: log.warn( 'worker %s failed to remove from tasks: %s', iworker, addr) total_tasks += len(newtask_addrs) tasks.extend(newtask_addrs) except KeyError: log.debug("Got result for unknown task? %s", addr) log.debug("current: %s", str(list(tasks)[0])) else: completed.append([addr, batch_result]) self.consolidate(result, batch_result) if (self.config.stopOnError and not result.wasSuccessful()): # set the stop condition shouldStop.set() break if self.config.multiprocess_restartworker: log.debug('joining worker %s', iworker) # wait for working, but not that important if worker # cannot be joined in fact, for workers that add to # testQueue, they will not terminate until all their # items are read workers[iworker].join(timeout=1) if not shouldStop.is_set() and not testQueue.empty(): log.debug('starting new process on worker %s', iworker) workers[iworker] = self.startProcess( iworker, testQueue, resultQueue, shouldStop, result) except Empty: log.debug( "Timed out with %s tasks pending " "(empty testQueue=%r): %s", len(tasks), testQueue.empty(), str(tasks)) any_alive = False for iworker, w in enumerate(workers): if w.is_alive(): worker_addr = bytes_(w.currentaddr.value, 'ascii') timeprocessing = time.time() - w.currentstart.value if (len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout - 0.1): log.debug( 'worker %d has finished its work item, ' 'but is not exiting? do we wait for it?', iworker) else: any_alive = True if (len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout - 0.1): log.debug('timed out worker %s: %s', iworker, worker_addr) w.currentaddr.value = bytes_('') # If the process is in C++ code, sending a SIGILL # might not send a python KeybordInterrupt exception # therefore, send multiple signals until an # exception is caught. If this takes too long, then # terminate the process w.keyboardCaught.clear() startkilltime = time.time() while not w.keyboardCaught.is_set( ) and w.is_alive(): if time.time( ) - startkilltime > self.waitkilltime: # have to terminate... log.error("terminating worker %s", iworker) w.terminate() # there is a small probability that the # terminated process might send a result, # which has to be specially handled or # else processes might get orphaned. workers[ iworker] = w = self.startProcess( iworker, testQueue, resultQueue, shouldStop, result) break os.kill(w.pid, signal.SIGILL) time.sleep(0.1) elif w.exitcode != signal.SIGILL: lastTask = None procesingQueue = procesingQueues[iworker] while not procesingQueue.empty(): lastTask = procesingQueue.get() if lastTask is not None: try: raise Exception( "The following task crashed a worker process: %s" % lastTask) except: result.addError(test, sys.exc_info()) if not any_alive and testQueue.empty(): log.debug("All workers dead") break nexttimeout = self.config.multiprocess_timeout for w in workers: if w.is_alive() and len(w.currentaddr.value) > 0: timeprocessing = time.time() - w.currentstart.value if timeprocessing <= self.config.multiprocess_timeout: nexttimeout = min( nexttimeout, self.config.multiprocess_timeout - timeprocessing) log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks)) except (KeyboardInterrupt, SystemExit), e: log.info('parent received ctrl-c when waiting for test results') thrownError = e #resultQueue.get(False) result.addError(test, sys.exc_info())
resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: if timeout: msg = 'Worker %s test %s timed out' else: msg = 'Worker %s test %s keyboard interrupt' log.debug(msg, ix, test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if not timeout: raise except SystemExit: currentaddr.value = bytes_('') log.exception('Worker %s system exit', ix) raise except: currentaddr.value = bytes_('') log.exception( "Worker %s error running test or returning " "results", ix) failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if config.multiprocess_restartworker: break log.debug("Worker %s ending", ix) class NoSharedFixtureContextSuite(ContextSuite): """ Context suite that never fires shared fixtures. When a context sets _multiprocess_shared_, fixtures in that context
resultQueue.put((ix, test_addr, test.tasks, batch(result))) else: if timeout: msg = "Worker %s test %s timed out" else: msg = "Worker %s test %s keyboard interrupt" log.debug(msg, ix, test_addr) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if not timeout: raise except SystemExit: currentaddr.value = bytes_("") log.exception("Worker %s system exit", ix) raise except: currentaddr.value = bytes_("") log.exception("Worker %s error running test or returning " "results", ix) failure.Failure(*sys.exc_info())(result) resultQueue.put((ix, test_addr, test.tasks, batch(result))) if config.multiprocess_restartworker: break log.debug("Worker %s ending", ix) class NoSharedFixtureContextSuite(ContextSuite): """ Context suite that never fires shared fixtures. When a context sets _multiprocess_shared_, fixtures in that context are executed by the main process. Using this suite class prevents them from executing in the runner process as well.