def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
tasks = []
completed = []
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
# dispatch and collect results
# put indexes only on queue because tests aren't picklable
for case in self.nextBatch(test):
log.debug("Next batch %s (%s)", case, type(case))
if isinstance(case, nose.case.Test) and isinstance(case.test, failure.Failure):
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
# handle shared fixtures
if isinstance(case, ContextSuite) and case.context is failure.Failure:
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
elif isinstance(case, ContextSuite) and self.sharedFixtures(case):
log.debug("%s has shared fixtures", case)
try:
case.setUp()
except (KeyboardInterrupt, SystemExit):
raise
except:
log.debug("%s setup failed", sys.exc_info())
result.addError(case, sys.exc_info())
else:
to_teardown.append(case)
for _t in case:
test_addr = self.addtask(testQueue, tasks, _t)
log.debug("Queued shared-fixture test %s (%s) to %s", len(tasks), test_addr, testQueue)
else:
test_addr = self.addtask(testQueue, tasks, case)
log.debug("Queued test %s (%s) to %s", len(tasks), test_addr, testQueue)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
currentaddr = Array("c", 1000)
currentaddr.value = bytes_("")
currentstart = Value("d")
keyboardCaught = Event()
p = Process(
target=runner,
args=(
i,
testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config),
),
)
p.currentaddr = currentaddr
p.currentstart = currentstart
p.keyboardCaught = keyboardCaught
# p.setDaemon(True)
p.start()
workers.append(p)
log.debug("Started worker process %s", i + 1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout = self.config.multiprocess_timeout
while tasks:
log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks, nexttimeout)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(timeout=nexttimeout)
log.debug("Results received for worker %d, %s, new tasks: %d", iworker, addr, len(newtask_addrs))
try:
try:
tasks.remove(addr)
except ValueError:
log.warn("worker %s failed to remove from tasks: %s", iworker, addr)
total_tasks += len(newtask_addrs)
for newaddr in newtask_addrs:
tasks.append(newaddr)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
log.debug("current: %s", str(list(tasks)[0]))
else:
completed.append([addr, batch_result])
self.consolidate(result, batch_result)
if self.config.stopOnError and not result.wasSuccessful():
# set the stop condition
shouldStop.set()
break
if self.config.multiprocess_restartworker:
log.debug("joining worker %s", iworker)
# wait for working, but not that important if worker
# cannot be joined in fact, for workers that add to
# testQueue, they will not terminate until all their
# items are read
workers[iworker].join(timeout=1)
if not shouldStop.is_set() and not testQueue.empty():
log.debug("starting new process on worker %s", iworker)
currentaddr = Array("c", 1000)
currentaddr.value = bytes_("")
currentstart = Value("d")
currentstart.value = time.time()
keyboardCaught = Event()
workers[iworker] = Process(
target=runner,
args=(
iworker,
testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config),
),
)
workers[iworker].currentaddr = currentaddr
workers[iworker].currentstart = currentstart
workers[iworker].keyboardCaught = keyboardCaught
workers[iworker].start()
except Empty:
log.debug(
"Timed out with %s tasks pending " "(empty testQueue=%d): %s",
len(tasks),
testQueue.empty(),
str(tasks),
)
any_alive = False
for iworker, w in enumerate(workers):
if w.is_alive():
worker_addr = bytes_(w.currentaddr.value, "ascii")
timeprocessing = time.time() - w.currentstart.value
if len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout - 0.1:
log.debug(
"worker %d has finished its work item, " "but is not exiting? do we wait for it?",
iworker,
)
if timeprocessing > self.config.multiprocess_timeout + 30:
log.error("worker %d force kill", iworker)
os.kill(w.pid, signal.SIGINT)
time.sleep(0.1)
else:
any_alive = True
if len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout - 0.1:
log.debug("timed out worker %s: %s", iworker, worker_addr)
w.currentaddr.value = bytes_("")
# If the process is in C++ code, sending a SIGINT
# might not send a python KeybordInterrupt exception
# therefore, send multiple signals until an
# exception is caught. If this takes too long, then
# terminate the process
w.keyboardCaught.clear()
startkilltime = time.time()
while not w.keyboardCaught.is_set() and w.is_alive():
if time.time() - startkilltime > self.waitkilltime:
# have to terminate...
log.error("terminating worker %s", iworker)
w.terminate()
currentaddr = Array("c", 1000)
currentaddr.value = bytes_("")
currentstart = Value("d")
currentstart.value = time.time()
keyboardCaught = Event()
workers[iworker] = Process(
target=runner,
args=(
iworker,
testQueue,
resultQueue,
currentaddr,
currentstart,
keyboardCaught,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config),
),
)
workers[iworker].currentaddr = currentaddr
workers[iworker].currentstart = currentstart
workers[iworker].keyboardCaught = keyboardCaught
workers[iworker].start()
# there is a small probability that the
# terminated process might send a result,
# which has to be specially handled or
# else processes might get orphaned.
w = workers[iworker]
break
os.kill(w.pid, signal.SIGINT)
time.sleep(0.1)
if not any_alive and testQueue.empty():
log.debug("All workers dead")
break
nexttimeout = self.config.multiprocess_timeout
for w in workers:
if w.is_alive() and len(w.currentaddr.value) > 0:
timeprocessing = time.time() - w.currentstart.value
if timeprocessing <= self.config.multiprocess_timeout:
nexttimeout = min(nexttimeout, self.config.multiprocess_timeout - timeprocessing)
log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks))
for case in to_teardown:
log.debug("Tearing down shared fixtures for %s", case)
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
# first write since can freeze on shutting down processes
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
log.debug("Tell all workers to stop")
for w in workers:
if w.is_alive():
testQueue.put("STOP", block=False)
# wait for the workers to end
try:
for iworker, worker in enumerate(workers):
if worker.is_alive():
log.debug("joining worker %s", iworker)
worker.join() # 10)
if worker.is_alive():
log.debug("failed to join worker %s", iworker)
except KeyboardInterrupt:
log.info("parent received ctrl-c")
for worker in workers:
worker.terminate()
worker.join()
return result
def run(self, test):
"""
This is the entry point for the run of the tests.
This runner starts the test queue server and spawns the test runner
clients.
(Part of TextTestRunner API)
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
tasks_queue = []
# contains a list of task addresses
remaining_tasks = []
# list of tuples like [task_address, batch_result]
# completed_tasks = []
to_teardown = []
# thrownError = None
# API
test = self.config.plugins.prepareTest(test) or test
self.stream = self.config.plugins.setOutputStream(self.stream) \
or self.stream
result = self._makeResult()
start = time.time()
# populates the queues
self.collect_tasks(test, tasks_queue, remaining_tasks, to_teardown,
result)
if self.config.options.gevented_timing_file:
try:
task_times = pickle.load(
open(self.config.options.gevented_timing_file, 'r'))
except:
log.debug('No task times to read for sorting.')
task_times = {}
if task_times:
log.debug('Unsorted tasks: {}'.format(tasks_queue))
tasks_queue.sort(
key=lambda t: task_times.get(get_task_key(t), 0),
reverse=True)
log.debug('Sorted tasks: {}'.format(tasks_queue))
else:
task_times = {}
queue_manager = TestsQueueManager(
tasks_queue,
loader_class=self.loaderClass,
result_class=result.__class__,
config=self.config,
task_times=task_times
)
server = WSGIServer(queue_manager)
server_port = server.start()
results_processor = queue_manager.start_test_results_processor(
remaining_tasks, result, self.stream, self.config.stopOnError)
number_of_workers = self.config.options.gevented_processes
run_clients(
number_of_workers, server_port
) # <-- blocks until all are done consuming the queue
queue_manager.run_clients_event.set()
results_processor.join()
# TODO: not call tests / set ups could have ran. see if we can prune
# the tearDown collection as result
for case in to_teardown:
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
if self.config.options.gevented_timing_file:
try:
pickle.dump(
queue_manager.task_times,
open(self.config.options.gevented_timing_file, 'w'))
except:
log.exception('Error saving task times to {}'.format(
self.config.options.gevented_timing_file))
# first write since can freeze on shutting down processes
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
# except (KeyboardInterrupt, SystemExit):
# if thrownError:
# raise thrownError
# else:
# raise
return result
def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in
the suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
tasks = []
completed = []
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
self.collect(test, testQueue, tasks, to_teardown, result)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
p = self.startProcess(i, testQueue, resultQueue, shouldStop, result)
workers.append(p)
log.debug("Started worker process %s", i + 1)
total_tasks = len(tasks)
# need to keep track of the next time to check for timeouts in case
# more than one process times out at the same time.
nexttimeout = self.config.multiprocess_timeout
thrownError = None
try:
while tasks:
log.debug(
"Waiting for results (%s/%s tasks), next timeout=%.3fs", len(completed), total_tasks, nexttimeout
)
try:
iworker, addr, newtask_addrs, batch_result = resultQueue.get(timeout=nexttimeout)
log.debug("Results received for worker %d, %s, new tasks: %d", iworker, addr, len(newtask_addrs))
try:
try:
tasks.remove(addr)
except ValueError:
log.warn("worker %s failed to remove from tasks: %s", iworker, addr)
total_tasks += len(newtask_addrs)
tasks.extend(newtask_addrs)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
log.debug("current: %s", str(list(tasks)[0]))
else:
completed.append([addr, batch_result])
self.consolidate(result, batch_result)
if self.config.stopOnError and not result.wasSuccessful():
# set the stop condition
shouldStop.set()
break
if self.config.multiprocess_restartworker:
log.debug("joining worker %s", iworker)
# wait for working, but not that important if worker
# cannot be joined in fact, for workers that add to
# testQueue, they will not terminate until all their
# items are read
workers[iworker].join(timeout=1)
if not shouldStop.is_set() and not testQueue.empty():
log.debug("starting new process on worker %s", iworker)
workers[iworker] = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result)
except Empty:
log.debug(
"Timed out with %s tasks pending " "(empty testQueue=%r): %s",
len(tasks),
testQueue.empty(),
str(tasks),
)
any_alive = False
for iworker, w in enumerate(workers):
if w.is_alive():
worker_addr = bytes_(w.currentaddr.value, "ascii")
timeprocessing = time.time() - w.currentstart.value
if len(worker_addr) == 0 and timeprocessing > self.config.multiprocess_timeout - 0.1:
log.debug(
"worker %d has finished its work item, " "but is not exiting? do we wait for it?",
iworker,
)
else:
any_alive = True
if len(worker_addr) > 0 and timeprocessing > self.config.multiprocess_timeout - 0.1:
log.debug("timed out worker %s: %s", iworker, worker_addr)
w.currentaddr.value = bytes_("")
# If the process is in C++ code, sending a SIGILL
# might not send a python KeybordInterrupt exception
# therefore, send multiple signals until an
# exception is caught. If this takes too long, then
# terminate the process
w.keyboardCaught.clear()
startkilltime = time.time()
while not w.keyboardCaught.is_set() and w.is_alive():
if time.time() - startkilltime > self.waitkilltime:
# have to terminate...
log.error("terminating worker %s", iworker)
w.terminate()
# there is a small probability that the
# terminated process might send a result,
# which has to be specially handled or
# else processes might get orphaned.
workers[iworker] = w = self.startProcess(
iworker, testQueue, resultQueue, shouldStop, result
)
break
os.kill(w.pid, signal.SIGILL)
time.sleep(0.1)
if not any_alive and testQueue.empty():
log.debug("All workers dead")
break
nexttimeout = self.config.multiprocess_timeout
for w in workers:
if w.is_alive() and len(w.currentaddr.value) > 0:
timeprocessing = time.time() - w.currentstart.value
if timeprocessing <= self.config.multiprocess_timeout:
nexttimeout = min(nexttimeout, self.config.multiprocess_timeout - timeprocessing)
log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks))
except (KeyboardInterrupt, SystemExit) as e:
log.info("parent received ctrl-c when waiting for test results")
thrownError = e
# resultQueue.get(False)
result.addError(test, sys.exc_info())
try:
for case in to_teardown:
log.debug("Tearing down shared fixtures for %s", case)
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
# first write since can freeze on shutting down processes
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
if thrownError is None:
log.debug("Tell all workers to stop")
for w in workers:
if w.is_alive():
testQueue.put("STOP", block=False)
# wait for the workers to end
for iworker, worker in enumerate(workers):
if worker.is_alive():
log.debug("joining worker %s", iworker)
worker.join()
if worker.is_alive():
log.debug("failed to join worker %s", iworker)
except (KeyboardInterrupt, SystemExit):
log.info("parent received ctrl-c when shutting down: stop all processes")
for worker in workers:
if worker.is_alive():
worker.terminate()
if thrownError:
raise thrownError
else:
raise
return result
def run(self, test):
"""
Execute the test (which may be a test suite). If the test is a suite,
distribute it out among as many processes as have been configured, at
as fine a level as is possible given the context fixtures defined in the
suite or any sub-suites.
"""
log.debug("%s.run(%s) (%s)", self, test, os.getpid())
wrapper = self.config.plugins.prepareTest(test)
if wrapper is not None:
test = wrapper
# plugins can decorate or capture the output stream
wrapped = self.config.plugins.setOutputStream(self.stream)
if wrapped is not None:
self.stream = wrapped
testQueue = Queue()
resultQueue = Queue()
tasks = {}
completed = {}
workers = []
to_teardown = []
shouldStop = Event()
result = self._makeResult()
start = time.time()
# dispatch and collect results
# put indexes only on queue because tests aren't picklable
for case in self.nextBatch(test):
log.debug("Next batch %s (%s)", case, type(case))
if (isinstance(case, nose.case.Test) and
isinstance(case.test, failure.Failure)):
log.debug("Case is a Failure")
case(result) # run here to capture the failure
continue
# handle shared fixtures
if isinstance(case, ContextSuite) and self.sharedFixtures(case):
log.debug("%s has shared fixtures", case)
try:
case.setUp()
except (KeyboardInterrupt, SystemExit):
raise
except:
log.debug("%s setup failed", sys.exc_info())
result.addError(case, sys.exc_info())
else:
to_teardown.append(case)
for _t in case:
test_addr = self.address(_t)
testQueue.put(test_addr, block=False)
tasks[test_addr] = None
log.debug("Queued shared-fixture test %s (%s) to %s",
len(tasks), test_addr, testQueue)
else:
test_addr = self.address(case)
testQueue.put(test_addr, block=False)
tasks[test_addr] = None
log.debug("Queued test %s (%s) to %s",
len(tasks), test_addr, testQueue)
log.debug("Starting %s workers", self.config.multiprocess_workers)
for i in range(self.config.multiprocess_workers):
p = Process(target=runner, args=(i,
testQueue,
resultQueue,
shouldStop,
self.loaderClass,
result.__class__,
pickle.dumps(self.config)))
# p.setDaemon(True)
p.start()
workers.append(p)
log.debug("Started worker process %s", i+1)
num_tasks = len(tasks)
while tasks:
log.debug("Waiting for results (%s/%s tasks)",
len(completed), num_tasks)
try:
addr, batch_result = resultQueue.get(
timeout=self.config.multiprocess_timeout)
log.debug('Results received for %s', addr)
try:
tasks.pop(addr)
except KeyError:
log.debug("Got result for unknown task? %s", addr)
else:
completed[addr] = batch_result
self.consolidate(result, batch_result)
if (self.config.stopOnError
and not result.wasSuccessful()):
# set the stop condition
shouldStop.set()
break
except Empty:
log.debug("Timed out with %s tasks pending", len(tasks))
any_alive = False
for w in workers:
if w.is_alive():
any_alive = True
break
if not any_alive:
log.debug("All workers dead")
break
log.debug("Completed %s/%s tasks (%s remain)",
len(completed), num_tasks, len(tasks))
for case in to_teardown:
log.debug("Tearing down shared fixtures for %s", case)
try:
case.tearDown()
except (KeyboardInterrupt, SystemExit):
raise
except:
result.addError(case, sys.exc_info())
stop = time.time()
result.printErrors()
result.printSummary(start, stop)
self.config.plugins.finalize(result)
# Tell all workers to stop
for w in workers:
if w.is_alive():
testQueue.put('STOP', block=False)
return result
