def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result):
currentaddr = Value("c", bytes_(""))
currentstart = Value("d", time.time())
keyboardCaught = Event()
p = Process(
target=runner,
args=(
iworker,
testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config),
),
)
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
old = signal.signal(signal.SIGILL, signalhandler)
p.start()
signal.signal(signal.SIGILL, old)
return p
def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result):
from nose.plugins.multiprocess import Value, Event, Process
from nose.plugins.multiprocess import signalhandler
currentaddr = Value('c', bytes_(''))
currentstart = Value('d', time.time())
keyboardCaught = Event()
mp_context = {plugin.name: plugin.mp_context for plugin in _instantiate_plugins if 'mp_context' in dir(plugin)}
p = Process(target=runner,
args=(
iworker, testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config),
_instantiate_plugins,
mp_context))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
old = signal.signal(signal.SIGILL, signalhandler)
p.start()
signal.signal(signal.SIGILL, old)
return p
def startProcess(self, iworker, testQueue, resultQueue, procesingQueue,
shouldStop, result):
currentaddr = Value('c', bytes_(''))
currentstart = Value('d', time.time())
keyboardCaught = Event()
p = Process(target=runner,
args=(iworker, testQueue, resultQueue, procesingQueue,
currentaddr, currentstart, keyboardCaught,
shouldStop, self.loaderClass, result.__class__,
pickle.dumps(self.config)))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
old = signal.signal(signal.SIGILL, signalhandler)
p.start()
signal.signal(signal.SIGILL, old)
return p
def __runner(
ix, testQueue, resultQueue, currentaddr, currentstart, keyboardCaught, shouldStop, loaderClass, resultClass, config
):
config = pickle.loads(config)
dummy_parser = config.parserClass()
if _instantiate_plugins is not None:
for pluginclass in _instantiate_plugins:
plugin = pluginclass()
plugin.addOptions(dummy_parser, {})
config.plugins.addPlugin(plugin)
config.plugins.configure(config.options, config)
config.plugins.begin()
log.debug("Worker %s executing, pid=%d", ix, os.getpid())
loader = loaderClass(config=config)
loader.suiteClass.suiteClass = NoSharedFixtureContextSuite
def get():
return testQueue.get(timeout=config.multiprocess_timeout)
def makeResult():
stream = _WritelnDecorator(StringIO())
result = resultClass(stream, descriptions=1, verbosity=config.verbosity, config=config)
plug_result = config.plugins.prepareTestResult(result)
if plug_result:
return plug_result
return result
def batch(result):
failures = [(TestLet(c), err) for c, err in result.failures]
errors = [(TestLet(c), err) for c, err in result.errors]
errorClasses = {}
for key, (storage, label, isfail) in list(result.errorClasses.items()):
errorClasses[key] = ([(TestLet(c), err) for c, err in storage], label, isfail)
return (result.stream.getvalue(), result.testsRun, failures, errors, errorClasses)
for test_addr, arg in iter(get, "STOP"):
if shouldStop.is_set():
log.exception("Worker %d STOPPED", ix)
break
result = makeResult()
test = loader.loadTestsFromNames([test_addr])
test.testQueue = testQueue
test.tasks = []
test.arg = arg
log.debug("Worker %s Test is %s (%s)", ix, test_addr, test)
try:
if arg is not None:
test_addr = test_addr + str(arg)
currentaddr.value = bytes_(test_addr)
currentstart.value = time.time()
test(result)
currentaddr.value = bytes_("")
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
except KeyboardInterrupt as e: # TimedOutException:
timeout = isinstance(e, TimedOutException)
if timeout:
keyboardCaught.set()
if len(currentaddr.value):
if timeout:
msg = "Worker %s timed out, failing current test %s"
else:
msg = "Worker %s keyboard interrupt, failing current test %s"
log.exception(msg, ix, test_addr)
currentaddr.value = bytes_("")
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
if timeout:
msg = "Worker %s test %s timed out"
else:
msg = "Worker %s test %s keyboard interrupt"
log.debug(msg, ix, test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if not timeout:
raise
except SystemExit:
currentaddr.value = bytes_("")
log.exception("Worker %s system exit", ix)
raise
except:
currentaddr.value = bytes_("")
log.exception("Worker %s error running test or returning " "results", ix)
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if config.multiprocess_restartworker:
break
log.debug("Worker %s ending", ix)
def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
tasks = []
completed = []
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
self.collect(test, testQueue, tasks, to_teardown, result)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
p = self.startProcess(i, testQueue, resultQueue, shouldStop, result)
workers.append(p)
log.debug("Started worker process %s", i + 1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout = self.config.multiprocess_timeout
thrownError = None
try:
while tasks:
log.debug(
"Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks, nexttimeout
)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(timeout=nexttimeout)
log.debug("Results received for worker %d, %s, new tasks: %d", iworker, addr, len(newtask_addrs))
try:
try:
tasks.remove(addr)
except ValueError:
log.warn("worker %s failed to remove from tasks: %s", iworker, addr)
total_tasks += len(newtask_addrs)
tasks.extend(newtask_addrs)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
log.debug("current: %s", str(list(tasks)[0]))
else:
completed.append([addr, batch_result])
self.consolidate(result, batch_result)
if self.config.stopOnError and not result.wasSuccessful():
# set the stop condition
shouldStop.set()
break
if self.config.multiprocess_restartworker:
log.debug("joining worker %s", iworker)
# wait for working, but not that important if worker
# cannot be joined in fact, for workers that add to
# testQueue, they will not terminate until all their
# items are read
workers[iworker].join(timeout=1)
if not shouldStop.is_set() and not testQueue.empty():
log.debug("starting new process on worker %s", iworker)
workers[iworker] = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result)
except Empty:
log.debug(
"Timed out with %s tasks pending " "(empty testQueue=%r): %s",
len(tasks),
testQueue.empty(),
str(tasks),
)
any_alive = False
for iworker, w in enumerate(workers):
if w.is_alive():
worker_addr = bytes_(w.currentaddr.value, "ascii")
timeprocessing = time.time() - w.currentstart.value
if len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout - 0.1:
log.debug(
"worker %d has finished its work item, " "but is not exiting? do we wait for it?",
iworker,
)
else:
any_alive = True
if len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout - 0.1:
log.debug("timed out worker %s: %s", iworker, worker_addr)
w.currentaddr.value = bytes_("")
# If the process is in C++ code, sending a SIGILL
# might not send a python KeybordInterrupt exception
# therefore, send multiple signals until an
# exception is caught. If this takes too long, then
# terminate the process
w.keyboardCaught.clear()
startkilltime = time.time()
while not w.keyboardCaught.is_set() and w.is_alive():
if time.time() - startkilltime > self.waitkilltime:
# have to terminate...
log.error("terminating worker %s", iworker)
w.terminate()
# there is a small probability that the
# terminated process might send a result,
# which has to be specially handled or
# else processes might get orphaned.
workers[iworker] = w = self.startProcess(
iworker, testQueue, resultQueue, shouldStop, result
)
break
os.kill(w.pid, signal.SIGILL)
time.sleep(0.1)
if not any_alive and testQueue.empty():
log.debug("All workers dead")
break
nexttimeout = self.config.multiprocess_timeout
for w in workers:
if w.is_alive() and len(w.currentaddr.value) > 0:
timeprocessing = time.time() - w.currentstart.value
if timeprocessing <= self.config.multiprocess_timeout:
nexttimeout = min(nexttimeout, self.config.multiprocess_timeout - timeprocessing)
log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks))
except (KeyboardInterrupt, SystemExit) as e:
log.info("parent received ctrl-c when waiting for test results")
thrownError = e
# resultQueue.get(False)
result.addError(test, sys.exc_info())
try:
for case in to_teardown:
log.debug("Tearing down shared fixtures for %s", case)
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
# first write since can freeze on shutting down processes
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
if thrownError is None:
log.debug("Tell all workers to stop")
for w in workers:
if w.is_alive():
testQueue.put("STOP", block=False)
# wait for the workers to end
for iworker, worker in enumerate(workers):
if worker.is_alive():
log.debug("joining worker %s", iworker)
worker.join()
if worker.is_alive():
log.debug("failed to join worker %s", iworker)
except (KeyboardInterrupt, SystemExit):
log.info("parent received ctrl-c when shutting down: stop all processes")
for worker in workers:
if worker.is_alive():
worker.terminate()
if thrownError:
raise thrownError
else:
raise
return result
def multiprocess_runner(ix, testQueue, resultQueue, currentaddr, currentstart,
keyboardCaught, shouldStop, loaderClass, resultClass,
config):
"""To replace the test runner of multiprocess.
* Setup gae services at the beginning of every process
* Clean datastore after each test
"""
from nose.pyversion import bytes_
try:
from cStringIO import StringIO
except ImportError:
import StringIO
from nose.plugins.multiprocess import _instantiate_plugins, \
NoSharedFixtureContextSuite, _WritelnDecorator, TestLet
config = pickle.loads(config)
dummy_parser = config.parserClass()
if _instantiate_plugins is not None:
for pluginclass in _instantiate_plugins:
plugin = pluginclass()
plugin.addOptions(dummy_parser, {})
config.plugins.addPlugin(plugin)
config.plugins.configure(config.options, config)
config.plugins.begin()
log.debug("Worker %s executing, pid=%d", ix, os.getpid())
loader = loaderClass(config=config)
loader.suiteClass.suiteClass = NoSharedFixtureContextSuite
def get():
return testQueue.get(timeout=config.multiprocess_timeout)
def makeResult():
stream = _WritelnDecorator(StringIO())
result = resultClass(stream,
descriptions=1,
verbosity=config.verbosity,
config=config)
plug_result = config.plugins.prepareTestResult(result)
return plug_result if plug_result else result
def batch(result):
failures = [(TestLet(c), err) for c, err in result.failures]
errors = [(TestLet(c), err) for c, err in result.errors]
errorClasses = {}
for key, (storage, label, isfail) in result.errorClasses.items():
errorClasses[key] = ([(TestLet(c), err)
for c, err in storage], label, isfail)
return (result.stream.getvalue(), result.testsRun, failures, errors,
errorClasses)
def setup_process_env():
"""Runs just after the process starts to setup services."""
setup_gae_services()
def after_each_test():
"""Runs after each test to clean datastore."""
clean_datastore()
# Setup gae services at the beginning of every process
setup_process_env()
for test_addr, arg in iter(get, 'STOP'):
if shouldStop.is_set():
log.exception('Worker %d STOPPED', ix)
break
result = makeResult()
test = loader.loadTestsFromNames([test_addr])
test.testQueue = testQueue
test.tasks = []
test.arg = arg
log.debug("Worker %s Test is %s (%s)", ix, test_addr, test)
try:
if arg is not None:
test_addr = test_addr + str(arg)
currentaddr.value = bytes_(test_addr)
currentstart.value = time.time()
test(result)
currentaddr.value = bytes_('')
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
# Clean datastore after each test
after_each_test()
except KeyboardInterrupt:
keyboardCaught.set()
if len(currentaddr.value) > 0:
log.exception(
'Worker %s keyboard interrupt, failing '
'current test %s', ix, test_addr)
currentaddr.value = bytes_('')
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
log.debug('Worker %s test %s timed out', ix, test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
except SystemExit:
currentaddr.value = bytes_('')
log.exception('Worker %s system exit', ix)
raise
except:
currentaddr.value = bytes_('')
log.exception(
"Worker %s error running test or returning "
"results", ix)
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if config.multiprocess_restartworker:
break
log.debug("Worker %s ending", ix)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
if timeout:
msg = 'Worker %s test %s timed out'
else:
msg = 'Worker %s test %s keyboard interrupt'
log.debug(msg,ix,test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if not timeout:
raise
except SystemExit:
currentaddr.value = bytes_('')
log.exception('Worker %s system exit',ix)
raise
except:
currentaddr.value = bytes_('')
log.exception("Worker %s error running test or returning "
"results",ix)
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if config.multiprocess_restartworker:
break
log.debug("Worker %s ending", ix)
class NoSharedFixtureContextSuite(ContextSuite):
"""
Context suite that never fires shared fixtures.
When a context sets _multiprocess_shared_, fixtures in that context
are executed by the main process. Using this suite class prevents them
def multiprocess_runner(ix, testQueue, resultQueue, currentaddr, currentstart,
keyboardCaught, shouldStop, loaderClass, resultClass, config):
"""To replace the test runner of multiprocess.
* Setup gae services at the beginning of every process
* Clean datastore after each test
"""
from nose import failure
from nose.pyversion import bytes_
import time
import pickle
try:
from cStringIO import StringIO
except ImportError:
import StringIO
from nose.plugins.multiprocess import _instantiate_plugins, \
NoSharedFixtureContextSuite, _WritelnDecorator, TestLet
config = pickle.loads(config)
dummy_parser = config.parserClass()
if _instantiate_plugins is not None:
for pluginclass in _instantiate_plugins:
plugin = pluginclass()
plugin.addOptions(dummy_parser,{})
config.plugins.addPlugin(plugin)
config.plugins.configure(config.options,config)
config.plugins.begin()
log.debug("Worker %s executing, pid=%d", ix,os.getpid())
loader = loaderClass(config=config)
loader.suiteClass.suiteClass = NoSharedFixtureContextSuite
def get():
return testQueue.get(timeout=config.multiprocess_timeout)
def makeResult():
stream = _WritelnDecorator(StringIO())
result = resultClass(stream, descriptions=1,
verbosity=config.verbosity,
config=config)
plug_result = config.plugins.prepareTestResult(result)
if plug_result:
return plug_result
return result
def batch(result):
failures = [(TestLet(c), err) for c, err in result.failures]
errors = [(TestLet(c), err) for c, err in result.errors]
errorClasses = {}
for key, (storage, label, isfail) in result.errorClasses.items():
errorClasses[key] = ([(TestLet(c), err) for c, err in storage],
label, isfail)
return (
result.stream.getvalue(),
result.testsRun,
failures,
errors,
errorClasses)
def setup_process_env():
"""Runs just after the process starts to setup services.
"""
setup_gae_services()
def after_each_test():
"""Runs after each test to clean datastore.
"""
clean_datastore()
# Setup gae services at the beginning of every process
setup_process_env()
for test_addr, arg in iter(get, 'STOP'):
if shouldStop.is_set():
log.exception('Worker %d STOPPED',ix)
break
result = makeResult()
test = loader.loadTestsFromNames([test_addr])
test.testQueue = testQueue
test.tasks = []
test.arg = arg
log.debug("Worker %s Test is %s (%s)", ix, test_addr, test)
try:
if arg is not None:
test_addr = test_addr + str(arg)
currentaddr.value = bytes_(test_addr)
currentstart.value = time.time()
test(result)
currentaddr.value = bytes_('')
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
# Clean datastore after each test
after_each_test()
except KeyboardInterrupt:
keyboardCaught.set()
if len(currentaddr.value) > 0:
log.exception('Worker %s keyboard interrupt, failing '
'current test %s',ix,test_addr)
currentaddr.value = bytes_('')
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
log.debug('Worker %s test %s timed out',ix,test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
except SystemExit:
currentaddr.value = bytes_('')
log.exception('Worker %s system exit',ix)
raise
except:
currentaddr.value = bytes_('')
log.exception("Worker %s error running test or returning "
"results",ix)
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if config.multiprocess_restartworker:
break
log.debug("Worker %s ending", ix)
result.addError(case, sys.exc_info())
else:
to_teardown.append(case)
for _t in case:
test_addr = self.addtask(testQueue, tasks, _t)
log.debug("Queued shared-fixture test %s (%s) to %s",
len(tasks), test_addr, testQueue)
else:
test_addr = self.addtask(testQueue, tasks, case)
log.debug("Queued test %s (%s) to %s", len(tasks), test_addr,
testQueue)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
currentaddr = Value('c', bytes_(''))
currentstart = Value('d', 0.0)
keyboardCaught = Event()
p = Process(target=runner,
args=(i, testQueue, resultQueue, currentaddr,
currentstart, keyboardCaught, shouldStop,
self.loaderClass, result.__class__,
pickle.dumps(self.config)))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
# p.setDaemon(True)
p.start()
workers.append(p)
log.debug("Started worker process %s", i + 1)
def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
tasks = []
completed = []
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
# dispatch and collect results
# put indexes only on queue because tests aren't picklable
for case in self.nextBatch(test):
log.debug("Next batch %s (%s)", case, type(case))
if (isinstance(case, nose.case.Test) and
isinstance(case.test, failure.Failure)):
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
# handle shared fixtures
if isinstance(case, ContextSuite) and case.context is failure.Failure:
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
elif isinstance(case, ContextSuite) and self.sharedFixtures(case):
log.debug("%s has shared fixtures", case)
try:
case.setUp()
except (KeyboardInterrupt, SystemExit):
raise
except:
log.debug("%s setup failed", sys.exc_info())
result.addError(case, sys.exc_info())
else:
to_teardown.append(case)
for _t in case:
test_addr = self.addtask(testQueue,tasks,_t)
log.debug("Queued shared-fixture test %s (%s) to %s",
len(tasks), test_addr, testQueue)
else:
test_addr = self.addtask(testQueue,tasks,case)
log.debug("Queued test %s (%s) to %s",
len(tasks), test_addr, testQueue)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
currentaddr = Value('c',bytes_(''))
currentstart = Value('d',0.0)
keyboardCaught = Event()
p = Process(target=runner, args=(i, testQueue, resultQueue,
currentaddr, currentstart,
keyboardCaught, shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
# p.setDaemon(True)
p.start()
workers.append(p)
log.debug("Started worker process %s", i+1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout=self.config.multiprocess_timeout
while tasks:
log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs",
len(completed), total_tasks,nexttimeout)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(
timeout=nexttimeout)
log.debug('Results received for worker %d, %s, new tasks: %d',
iworker,addr,len(newtask_addrs))
try:
try:
tasks.remove(addr)
except ValueError:
log.warn('worker %s failed to remove from tasks: %s',
iworker,addr)
total_tasks += len(newtask_addrs)
for newaddr in newtask_addrs:
tasks.append(newaddr)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
log.debug("current: %s",str(list(tasks)[0]))
else:
completed.append([addr,batch_result])
self.consolidate(result, batch_result)
if (self.config.stopOnError
and not result.wasSuccessful()):
# set the stop condition
shouldStop.set()
break
if self.config.multiprocess_restartworker:
log.debug('joining worker %s',iworker)
# wait for working, but not that important if worker
# cannot be joined in fact, for workers that add to
# testQueue, they will not terminate until all their
# items are read
workers[iworker].join(timeout=1)
if not shouldStop.is_set() and not testQueue.empty():
log.debug('starting new process on worker %s',iworker)
currentaddr = Value('c',bytes_(''))
currentstart = Value('d',time.time())
keyboardCaught = Event()
workers[iworker] = Process(target=runner,
args=(iworker, testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
workers[iworker].currentaddr = currentaddr
workers[iworker].currentstart = currentstart
workers[iworker].keyboardCaught = keyboardCaught
workers[iworker].start()
except Empty:
log.debug("Timed out with %s tasks pending "
"(empty testQueue=%d): %s",
len(tasks),testQueue.empty(),str(tasks))
any_alive = False
for iworker, w in enumerate(workers):
if w.is_alive():
worker_addr = bytes_(w.currentaddr.value,'ascii')
timeprocessing = time.time() - w.currentstart.value
if ( len(worker_addr) == 0
and timeprocessing > self.config.multiprocess_timeout-0.1):
log.debug('worker %d has finished its work item, '
'but is not exiting? do we wait for it?',
iworker)
else:
any_alive = True
if (len(worker_addr) > 0
and timeprocessing > self.config.multiprocess_timeout-0.1):
log.debug('timed out worker %s: %s',
iworker,worker_addr)
w.currentaddr.value = bytes_('')
# If the process is in C++ code, sending a SIGINT
# might not send a python KeybordInterrupt exception
# therefore, send multiple signals until an
# exception is caught. If this takes too long, then
# terminate the process
w.keyboardCaught.clear()
startkilltime = time.time()
while not w.keyboardCaught.is_set() and w.is_alive():
if time.time()-startkilltime > self.waitkilltime:
# have to terminate...
log.error("terminating worker %s",iworker)
w.terminate()
currentaddr = Value('c',bytes_(''))
currentstart = Value('d',time.time())
keyboardCaught = Event()
workers[iworker] = Process(target=runner,
args=(iworker, testQueue, resultQueue,
currentaddr, currentstart,
keyboardCaught, shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
workers[iworker].currentaddr = currentaddr
workers[iworker].currentstart = currentstart
workers[iworker].keyboardCaught = keyboardCaught
workers[iworker].start()
# there is a small probability that the
# terminated process might send a result,
# which has to be specially handled or
# else processes might get orphaned.
w = workers[iworker]
break
os.kill(w.pid, signal.SIGINT)
time.sleep(0.1)
if not any_alive and testQueue.empty():
log.debug("All workers dead")
break
nexttimeout=self.config.multiprocess_timeout
for w in workers:
if w.is_alive() and len(w.currentaddr.value) > 0:
timeprocessing = time.time()-w.currentstart.value
if timeprocessing <= self.config.multiprocess_timeout:
nexttimeout = min(nexttimeout,
self.config.multiprocess_timeout-timeprocessing)
log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks))
for case in to_teardown:
log.debug("Tearing down shared fixtures for %s", case)
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
# first write since can freeze on shutting down processes
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
log.debug("Tell all workers to stop")
for w in workers:
if w.is_alive():
testQueue.put('STOP', block=False)
# wait for the workers to end
try:
for iworker,worker in enumerate(workers):
if worker.is_alive():
log.debug('joining worker %s',iworker)
worker.join()#10)
if worker.is_alive():
log.debug('failed to join worker %s',iworker)
except KeyboardInterrupt:
log.info('parent received ctrl-c')
for worker in workers:
worker.terminate()
worker.join()
return result
def __runner(ix, testQueue, resultQueue, procesingQueue, currentaddr,
currentstart, keyboardCaught, shouldStop, loaderClass,
resultClass, config):
config = pickle.loads(config)
dummy_parser = config.parserClass()
if _instantiate_plugins is not None:
for pluginclass in _instantiate_plugins:
plugin = pluginclass()
plugin.addOptions(dummy_parser, {})
config.plugins.addPlugin(plugin)
config.plugins.configure(config.options, config)
config.plugins.begin()
log.debug("Worker %s executing, pid=%d", ix, os.getpid())
loader = loaderClass(config=config)
loader.suiteClass.suiteClass = NoSharedFixtureContextSuite
def get():
return testQueue.get(timeout=config.multiprocess_timeout)
def makeResult():
stream = _WritelnDecorator(StringIO())
result = resultClass(stream,
descriptions=1,
verbosity=config.verbosity,
config=config)
plug_result = config.plugins.prepareTestResult(result)
if plug_result:
return plug_result
return result
def batch(result):
failures = [(TestLet(c), err) for c, err in result.failures]
errors = [(TestLet(c), err) for c, err in result.errors]
errorClasses = {}
for key, (storage, label, isfail) in result.errorClasses.items():
errorClasses[key] = ([(TestLet(c), err)
for c, err in storage], label, isfail)
return (result.stream.getvalue(), result.testsRun, failures, errors,
errorClasses)
for test_addr, arg in iter(get, 'STOP'):
if shouldStop.is_set():
log.exception('Worker %d STOPPED', ix)
break
result = makeResult()
test = loader.loadTestsFromNames([test_addr])
test.testQueue = testQueue
test.tasks = []
test.arg = arg
log.debug("Worker %s Test is %s (%s)", ix, test_addr, test)
try:
if arg is not None:
test_addr = test_addr + str(arg)
procesingQueue.put(test_addr)
currentaddr.value = bytes_(test_addr)
currentstart.value = time.time()
test(result)
currentaddr.value = bytes_('')
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
except KeyboardInterrupt, e: #TimedOutException:
timeout = isinstance(e, TimedOutException)
if timeout:
keyboardCaught.set()
if len(currentaddr.value):
if timeout:
msg = 'Worker %s timed out, failing current test %s'
else:
msg = 'Worker %s keyboard interrupt, failing current test %s'
log.exception(msg, ix, test_addr)
currentaddr.value = bytes_('')
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
if timeout:
msg = 'Worker %s test %s timed out'
else:
msg = 'Worker %s test %s keyboard interrupt'
log.debug(msg, ix, test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if not timeout:
raise
except SystemExit:
currentaddr.value = bytes_('')
log.exception('Worker %s system exit', ix)
raise
def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
procesingQueues = []
tasks = []
completed = []
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
self.collect(test, testQueue, tasks, to_teardown, result)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
procesingQueue = Queue()
procesingQueues.append(procesingQueue)
p = self.startProcess(i, testQueue, resultQueue, procesingQueue,
shouldStop, result)
workers.append(p)
log.debug("Started worker process %s", i + 1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout = self.config.multiprocess_timeout
thrownError = None
try:
while tasks:
log.debug(
"Waiting for results (%s/%s tasks), next timeout=%.3fs",
len(completed), total_tasks, nexttimeout)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(
timeout=nexttimeout)
log.debug(
'Results received for worker %d, %s, new tasks: %d',
iworker, addr, len(newtask_addrs))
try:
try:
tasks.remove(addr)
except ValueError:
log.warn(
'worker %s failed to remove from tasks: %s',
iworker, addr)
total_tasks += len(newtask_addrs)
tasks.extend(newtask_addrs)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
log.debug("current: %s", str(list(tasks)[0]))
else:
completed.append([addr, batch_result])
self.consolidate(result, batch_result)
if (self.config.stopOnError
and not result.wasSuccessful()):
# set the stop condition
shouldStop.set()
break
if self.config.multiprocess_restartworker:
log.debug('joining worker %s', iworker)
# wait for working, but not that important if worker
# cannot be joined in fact, for workers that add to
# testQueue, they will not terminate until all their
# items are read
workers[iworker].join(timeout=1)
if not shouldStop.is_set() and not testQueue.empty():
log.debug('starting new process on worker %s',
iworker)
workers[iworker] = self.startProcess(
iworker, testQueue, resultQueue, shouldStop,
result)
except Empty:
log.debug(
"Timed out with %s tasks pending "
"(empty testQueue=%r): %s", len(tasks),
testQueue.empty(), str(tasks))
any_alive = False
for iworker, w in enumerate(workers):
if w.is_alive():
worker_addr = bytes_(w.currentaddr.value, 'ascii')
timeprocessing = time.time() - w.currentstart.value
if (len(worker_addr) == 0 and timeprocessing >
self.config.multiprocess_timeout - 0.1):
log.debug(
'worker %d has finished its work item, '
'but is not exiting? do we wait for it?',
iworker)
else:
any_alive = True
if (len(worker_addr) > 0 and timeprocessing >
self.config.multiprocess_timeout - 0.1):
log.debug('timed out worker %s: %s', iworker,
worker_addr)
w.currentaddr.value = bytes_('')
# If the process is in C++ code, sending a SIGILL
# might not send a python KeybordInterrupt exception
# therefore, send multiple signals until an
# exception is caught. If this takes too long, then
# terminate the process
w.keyboardCaught.clear()
startkilltime = time.time()
while not w.keyboardCaught.is_set(
) and w.is_alive():
if time.time(
) - startkilltime > self.waitkilltime:
# have to terminate...
log.error("terminating worker %s",
iworker)
w.terminate()
# there is a small probability that the
# terminated process might send a result,
# which has to be specially handled or
# else processes might get orphaned.
workers[
iworker] = w = self.startProcess(
iworker, testQueue,
resultQueue, shouldStop,
result)
break
os.kill(w.pid, signal.SIGILL)
time.sleep(0.1)
elif w.exitcode != signal.SIGILL:
lastTask = None
procesingQueue = procesingQueues[iworker]
while not procesingQueue.empty():
lastTask = procesingQueue.get()
if lastTask is not None:
try:
raise Exception(
"The following task crashed a worker process: %s"
% lastTask)
except:
result.addError(test, sys.exc_info())
if not any_alive and testQueue.empty():
log.debug("All workers dead")
break
nexttimeout = self.config.multiprocess_timeout
for w in workers:
if w.is_alive() and len(w.currentaddr.value) > 0:
timeprocessing = time.time() - w.currentstart.value
if timeprocessing <= self.config.multiprocess_timeout:
nexttimeout = min(
nexttimeout, self.config.multiprocess_timeout -
timeprocessing)
log.debug("Completed %s tasks (%s remain)", len(completed),
len(tasks))
except (KeyboardInterrupt, SystemExit), e:
log.info('parent received ctrl-c when waiting for test results')
thrownError = e
#resultQueue.get(False)
result.addError(test, sys.exc_info())
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
if timeout:
msg = 'Worker %s test %s timed out'
else:
msg = 'Worker %s test %s keyboard interrupt'
log.debug(msg, ix, test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if not timeout:
raise
except SystemExit:
currentaddr.value = bytes_('')
log.exception('Worker %s system exit', ix)
raise
except:
currentaddr.value = bytes_('')
log.exception(
"Worker %s error running test or returning "
"results", ix)
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if config.multiprocess_restartworker:
break
log.debug("Worker %s ending", ix)
class NoSharedFixtureContextSuite(ContextSuite):
"""
Context suite that never fires shared fixtures.
When a context sets _multiprocess_shared_, fixtures in that context
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
else:
if timeout:
msg = "Worker %s test %s timed out"
else:
msg = "Worker %s test %s keyboard interrupt"
log.debug(msg, ix, test_addr)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if not timeout:
raise
except SystemExit:
currentaddr.value = bytes_("")
log.exception("Worker %s system exit", ix)
raise
except:
currentaddr.value = bytes_("")
log.exception("Worker %s error running test or returning " "results", ix)
failure.Failure(*sys.exc_info())(result)
resultQueue.put((ix, test_addr, test.tasks, batch(result)))
if config.multiprocess_restartworker:
break
log.debug("Worker %s ending", ix)
class NoSharedFixtureContextSuite(ContextSuite):
"""
Context suite that never fires shared fixtures.
When a context sets _multiprocess_shared_, fixtures in that context
are executed by the main process. Using this suite class prevents them
from executing in the runner process as well.