def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result):
currentaddr = Value("c", bytes_(""))
currentstart = Value("d", time.time())
keyboardCaught = Event()
p = Process(
target=runner,
args=(
iworker,
testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config),
),
)
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
old = signal.signal(signal.SIGILL, signalhandler)
p.start()
signal.signal(signal.SIGILL, old)
return p
def startProcess(self, iworker, testQueue, resultQueue, shouldStop,
result):
currentaddr = Value('c', bytes_(''))
currentstart = Value('d', time.time())
keyboardCaught = Event()
p = Process(target=runner,
args=(iworker, testQueue, resultQueue, currentaddr,
currentstart, keyboardCaught, shouldStop,
self.loaderClass, result.__class__,
pickle.dumps(self.config)))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
old = signal.signal(signal.SIGILL, signalhandler)
p.start()
signal.signal(signal.SIGILL, old)
return p
log.debug("Queued test %s (%s) to %s", len(tasks), test_addr,
testQueue)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
currentaddr = Value('c', bytes_(''))
currentstart = Value('d', 0.0)
keyboardCaught = Event()
p = Process(target=runner,
args=(i, testQueue, resultQueue, currentaddr,
currentstart, keyboardCaught, shouldStop,
self.loaderClass, result.__class__,
pickle.dumps(self.config)))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
# p.setDaemon(True)
p.start()
workers.append(p)
log.debug("Started worker process %s", i + 1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout = self.config.multiprocess_timeout
while tasks:
log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs",
len(completed), total_tasks, nexttimeout)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(
timeout=nexttimeout)
def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
tasks = []
completed = []
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
# dispatch and collect results
# put indexes only on queue because tests aren't picklable
for case in self.nextBatch(test):
log.debug("Next batch %s (%s)", case, type(case))
if (isinstance(case, nose.case.Test) and
isinstance(case.test, failure.Failure)):
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
# handle shared fixtures
if isinstance(case, ContextSuite) and case.context is failure.Failure:
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
elif isinstance(case, ContextSuite) and self.sharedFixtures(case):
log.debug("%s has shared fixtures", case)
try:
case.setUp()
except (KeyboardInterrupt, SystemExit):
raise
except:
log.debug("%s setup failed", sys.exc_info())
result.addError(case, sys.exc_info())
else:
to_teardown.append(case)
for _t in case:
test_addr = self.addtask(testQueue,tasks,_t)
log.debug("Queued shared-fixture test %s (%s) to %s",
len(tasks), test_addr, testQueue)
else:
test_addr = self.addtask(testQueue,tasks,case)
log.debug("Queued test %s (%s) to %s",
len(tasks), test_addr, testQueue)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
currentaddr = Value('c',bytes_(''))
currentstart = Value('d',0.0)
keyboardCaught = Event()
p = Process(target=runner, args=(i, testQueue, resultQueue,
currentaddr, currentstart,
keyboardCaught, shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
# p.setDaemon(True)
p.start()
workers.append(p)
log.debug("Started worker process %s", i+1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout=self.config.multiprocess_timeout
while tasks:
log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs",
len(completed), total_tasks,nexttimeout)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(
timeout=nexttimeout)
log.debug('Results received for worker %d, %s, new tasks: %d',
iworker,addr,len(newtask_addrs))
try:
try:
tasks.remove(addr)
except ValueError:
log.warn('worker %s failed to remove from tasks: %s',
iworker,addr)
total_tasks += len(newtask_addrs)
for newaddr in newtask_addrs:
tasks.append(newaddr)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
log.debug("current: %s",str(list(tasks)[0]))
else:
completed.append([addr,batch_result])
self.consolidate(result, batch_result)
if (self.config.stopOnError
and not result.wasSuccessful()):
# set the stop condition
shouldStop.set()
break
if self.config.multiprocess_restartworker:
log.debug('joining worker %s',iworker)
# wait for working, but not that important if worker
# cannot be joined in fact, for workers that add to
# testQueue, they will not terminate until all their
# items are read
workers[iworker].join(timeout=1)
if not shouldStop.is_set() and not testQueue.empty():
log.debug('starting new process on worker %s',iworker)
currentaddr = Value('c',bytes_(''))
currentstart = Value('d',time.time())
keyboardCaught = Event()
workers[iworker] = Process(target=runner,
args=(iworker, testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
workers[iworker].currentaddr = currentaddr
workers[iworker].currentstart = currentstart
workers[iworker].keyboardCaught = keyboardCaught
workers[iworker].start()
except Empty:
log.debug("Timed out with %s tasks pending "
"(empty testQueue=%d): %s",
len(tasks),testQueue.empty(),str(tasks))
any_alive = False
for iworker, w in enumerate(workers):
if w.is_alive():
worker_addr = bytes_(w.currentaddr.value,'ascii')
timeprocessing = time.time() - w.currentstart.value
if ( len(worker_addr) == 0
and timeprocessing > self.config.multiprocess_timeout-0.1):
log.debug('worker %d has finished its work item, '
'but is not exiting? do we wait for it?',
iworker)
else:
any_alive = True
if (len(worker_addr) > 0
and timeprocessing > self.config.multiprocess_timeout-0.1):
log.debug('timed out worker %s: %s',
iworker,worker_addr)
w.currentaddr.value = bytes_('')
# If the process is in C++ code, sending a SIGINT
# might not send a python KeybordInterrupt exception
# therefore, send multiple signals until an
# exception is caught. If this takes too long, then
# terminate the process
w.keyboardCaught.clear()
startkilltime = time.time()
while not w.keyboardCaught.is_set() and w.is_alive():
if time.time()-startkilltime > self.waitkilltime:
# have to terminate...
log.error("terminating worker %s",iworker)
w.terminate()
currentaddr = Value('c',bytes_(''))
currentstart = Value('d',time.time())
keyboardCaught = Event()
workers[iworker] = Process(target=runner,
args=(iworker, testQueue, resultQueue,
currentaddr, currentstart,
keyboardCaught, shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
workers[iworker].currentaddr = currentaddr
workers[iworker].currentstart = currentstart
workers[iworker].keyboardCaught = keyboardCaught
workers[iworker].start()
# there is a small probability that the
# terminated process might send a result,
# which has to be specially handled or
# else processes might get orphaned.
w = workers[iworker]
break
os.kill(w.pid, signal.SIGINT)
time.sleep(0.1)
if not any_alive and testQueue.empty():
log.debug("All workers dead")
break
nexttimeout=self.config.multiprocess_timeout
for w in workers:
if w.is_alive() and len(w.currentaddr.value) > 0:
timeprocessing = time.time()-w.currentstart.value
if timeprocessing <= self.config.multiprocess_timeout:
nexttimeout = min(nexttimeout,
self.config.multiprocess_timeout-timeprocessing)
log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks))
for case in to_teardown:
log.debug("Tearing down shared fixtures for %s", case)
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
# first write since can freeze on shutting down processes
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
log.debug("Tell all workers to stop")
for w in workers:
if w.is_alive():
testQueue.put('STOP', block=False)
# wait for the workers to end
try:
for iworker,worker in enumerate(workers):
if worker.is_alive():
log.debug('joining worker %s',iworker)
worker.join()#10)
if worker.is_alive():
log.debug('failed to join worker %s',iworker)
except KeyboardInterrupt:
log.info('parent received ctrl-c')
for worker in workers:
worker.terminate()
worker.join()
return result
