def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result): currentaddr = Value("c", bytes_("")) currentstart = Value("d", time.time()) keyboardCaught = Event() p = Process( target=runner, args=( iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config), ), ) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught old = signal.signal(signal.SIGILL, signalhandler) p.start() signal.signal(signal.SIGILL, old) return p
def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result): currentaddr = Value('c', bytes_('')) currentstart = Value('d', time.time()) keyboardCaught = Event() p = Process(target=runner, args=(iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught old = signal.signal(signal.SIGILL, signalhandler) p.start() signal.signal(signal.SIGILL, old) return p
log.debug("Queued test %s (%s) to %s", len(tasks), test_addr, testQueue) log.debug("Starting %s workers", self.config.multiprocess_workers) for i in range(self.config.multiprocess_workers): currentaddr = Value('c', bytes_('')) currentstart = Value('d', 0.0) keyboardCaught = Event() p = Process(target=runner, args=(i, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught # p.setDaemon(True) p.start() workers.append(p) log.debug("Started worker process %s", i + 1) total_tasks = len(tasks) # need to keep track of the next time to check for timeouts in case # more than one process times out at the same time. nexttimeout = self.config.multiprocess_timeout while tasks: log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks, nexttimeout) try: iworker, addr, newtask_addrs, batch_result = resultQueue.get( timeout=nexttimeout)
def run(self, test): """ Execute the test (which may be a test suite). If the test is a suite, distribute it out among as many processes as have been configured, at as fine a level as is possible given the context fixtures defined in the suite or any sub-suites. """ log.debug("%s.run(%s) (%s)", self, test, os.getpid()) wrapper = self.config.plugins.prepareTest(test) if wrapper is not None: test = wrapper # plugins can decorate or capture the output stream wrapped = self.config.plugins.setOutputStream(self.stream) if wrapped is not None: self.stream = wrapped testQueue = Queue() resultQueue = Queue() tasks = [] completed = [] workers = [] to_teardown = [] shouldStop = Event() result = self._makeResult() start = time.time() # dispatch and collect results # put indexes only on queue because tests aren't picklable for case in self.nextBatch(test): log.debug("Next batch %s (%s)", case, type(case)) if (isinstance(case, nose.case.Test) and isinstance(case.test, failure.Failure)): log.debug("Case is a Failure") case(result) # run here to capture the failure continue # handle shared fixtures if isinstance(case, ContextSuite) and case.context is failure.Failure: log.debug("Case is a Failure") case(result) # run here to capture the failure continue elif isinstance(case, ContextSuite) and self.sharedFixtures(case): log.debug("%s has shared fixtures", case) try: case.setUp() except (KeyboardInterrupt, SystemExit): raise except: log.debug("%s setup failed", sys.exc_info()) result.addError(case, sys.exc_info()) else: to_teardown.append(case) for _t in case: test_addr = self.addtask(testQueue,tasks,_t) log.debug("Queued shared-fixture test %s (%s) to %s", len(tasks), test_addr, testQueue) else: test_addr = self.addtask(testQueue,tasks,case) log.debug("Queued test %s (%s) to %s", len(tasks), test_addr, testQueue) log.debug("Starting %s workers", self.config.multiprocess_workers) for i in range(self.config.multiprocess_workers): currentaddr = Value('c',bytes_('')) currentstart = Value('d',0.0) keyboardCaught = Event() p = Process(target=runner, args=(i, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) p.currentaddr = currentaddr p.currentstart = currentstart p.keyboardCaught = keyboardCaught # p.setDaemon(True) p.start() workers.append(p) log.debug("Started worker process %s", i+1) total_tasks = len(tasks) # need to keep track of the next time to check for timeouts in case # more than one process times out at the same time. nexttimeout=self.config.multiprocess_timeout while tasks: log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks,nexttimeout) try: iworker, addr, newtask_addrs, batch_result = resultQueue.get( timeout=nexttimeout) log.debug('Results received for worker %d, %s, new tasks: %d', iworker,addr,len(newtask_addrs)) try: try: tasks.remove(addr) except ValueError: log.warn('worker %s failed to remove from tasks: %s', iworker,addr) total_tasks += len(newtask_addrs) for newaddr in newtask_addrs: tasks.append(newaddr) except KeyError: log.debug("Got result for unknown task? %s", addr) log.debug("current: %s",str(list(tasks)[0])) else: completed.append([addr,batch_result]) self.consolidate(result, batch_result) if (self.config.stopOnError and not result.wasSuccessful()): # set the stop condition shouldStop.set() break if self.config.multiprocess_restartworker: log.debug('joining worker %s',iworker) # wait for working, but not that important if worker # cannot be joined in fact, for workers that add to # testQueue, they will not terminate until all their # items are read workers[iworker].join(timeout=1) if not shouldStop.is_set() and not testQueue.empty(): log.debug('starting new process on worker %s',iworker) currentaddr = Value('c',bytes_('')) currentstart = Value('d',time.time()) keyboardCaught = Event() workers[iworker] = Process(target=runner, args=(iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) workers[iworker].currentaddr = currentaddr workers[iworker].currentstart = currentstart workers[iworker].keyboardCaught = keyboardCaught workers[iworker].start() except Empty: log.debug("Timed out with %s tasks pending " "(empty testQueue=%d): %s", len(tasks),testQueue.empty(),str(tasks)) any_alive = False for iworker, w in enumerate(workers): if w.is_alive(): worker_addr = bytes_(w.currentaddr.value,'ascii') timeprocessing = time.time() - w.currentstart.value if ( len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout-0.1): log.debug('worker %d has finished its work item, ' 'but is not exiting? do we wait for it?', iworker) else: any_alive = True if (len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout-0.1): log.debug('timed out worker %s: %s', iworker,worker_addr) w.currentaddr.value = bytes_('') # If the process is in C++ code, sending a SIGINT # might not send a python KeybordInterrupt exception # therefore, send multiple signals until an # exception is caught. If this takes too long, then # terminate the process w.keyboardCaught.clear() startkilltime = time.time() while not w.keyboardCaught.is_set() and w.is_alive(): if time.time()-startkilltime > self.waitkilltime: # have to terminate... log.error("terminating worker %s",iworker) w.terminate() currentaddr = Value('c',bytes_('')) currentstart = Value('d',time.time()) keyboardCaught = Event() workers[iworker] = Process(target=runner, args=(iworker, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, self.loaderClass, result.__class__, pickle.dumps(self.config))) workers[iworker].currentaddr = currentaddr workers[iworker].currentstart = currentstart workers[iworker].keyboardCaught = keyboardCaught workers[iworker].start() # there is a small probability that the # terminated process might send a result, # which has to be specially handled or # else processes might get orphaned. w = workers[iworker] break os.kill(w.pid, signal.SIGINT) time.sleep(0.1) if not any_alive and testQueue.empty(): log.debug("All workers dead") break nexttimeout=self.config.multiprocess_timeout for w in workers: if w.is_alive() and len(w.currentaddr.value) > 0: timeprocessing = time.time()-w.currentstart.value if timeprocessing <= self.config.multiprocess_timeout: nexttimeout = min(nexttimeout, self.config.multiprocess_timeout-timeprocessing) log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks)) for case in to_teardown: log.debug("Tearing down shared fixtures for %s", case) try: case.tearDown() except (KeyboardInterrupt, SystemExit): raise except: result.addError(case, sys.exc_info()) stop = time.time() # first write since can freeze on shutting down processes result.printErrors() result.printSummary(start, stop) self.config.plugins.finalize(result) log.debug("Tell all workers to stop") for w in workers: if w.is_alive(): testQueue.put('STOP', block=False) # wait for the workers to end try: for iworker,worker in enumerate(workers): if worker.is_alive(): log.debug('joining worker %s',iworker) worker.join()#10) if worker.is_alive(): log.debug('failed to join worker %s',iworker) except KeyboardInterrupt: log.info('parent received ctrl-c') for worker in workers: worker.terminate() worker.join() return result