def threading_test_runner(num_threads, test_work_items):
"""Provides hand-wrapped pooling threading-based test runner adapter
with Ctrl-C support.
This concurrent test runner is based on the threading
library, and rolls its own worker pooling strategy so it
can handle Ctrl-C properly.
@param num_threads the number of worker processes to use.
@param test_work_items the iterable of test work item tuples
to run.
"""
# Initialize our global state.
initialize_global_vars_threading(num_threads, test_work_items)
# Create jobs.
job_queue = Queue.Queue()
for test_work_item in test_work_items:
job_queue.put(test_work_item)
result_queue = Queue.Queue()
# Create queues for started child pids. Terminating
# the threading threads does not terminate the
# child processes they spawn.
inferior_pid_events = Queue.Queue()
# Create workers. We don't use multiprocessing.pool.ThreadedPool
# due to challenges with handling ^C keyboard interrupts.
workers = []
for _ in range(num_threads):
worker = threading.Thread(
target=process_dir_worker_threading,
args=(job_queue,
result_queue,
inferior_pid_events))
worker.start()
workers.append(worker)
# Main loop: wait for all workers to finish and wait for
# the socket handlers to wrap up.
ctrl_c_loop(
# Main operation of loop
lambda: pump_workers_and_asyncore_map(
workers, RUNNER_PROCESS_ASYNC_MAP),
# Return True when we're done with the main loop.
lambda: workers_and_async_done(workers, RUNNER_PROCESS_ASYNC_MAP),
# Indicate what we do when we receive one or more Ctrl-Cs.
lambda ctrl_c_count: handle_ctrl_c(
ctrl_c_count, job_queue, workers, inferior_pid_events,
kill_all_worker_threads))
# Reap the test results.
test_results = []
while not result_queue.empty():
test_results.append(result_queue.get(block=False))
return test_results
def multiprocessing_test_runner(num_threads, test_work_items):
"""Provides hand-wrapped pooling test runner adapter with Ctrl-C support.
This concurrent test runner is based on the multiprocessing
library, and rolls its own worker pooling strategy so it
can handle Ctrl-C properly.
This test runner is known to have an issue running on
Windows platforms.
@param num_threads the number of worker processes to use.
@param test_work_items the iterable of test work item tuples
to run.
"""
# Initialize our global state.
initialize_global_vars_multiprocessing(num_threads, test_work_items)
# Create jobs.
job_queue = multiprocessing.Queue(len(test_work_items))
for test_work_item in test_work_items:
job_queue.put(test_work_item)
result_queue = multiprocessing.Queue(len(test_work_items))
# Create queues for started child pids. Terminating
# the multiprocess processes does not terminate the
# child processes they spawn. We can remove this tracking
# if/when we move to having the multiprocess process directly
# perform the test logic. The Queue size needs to be able to
# hold 2 * (num inferior dotest.py processes started) entries.
inferior_pid_events = multiprocessing.Queue(4096)
# Worker dictionary allows each worker to figure out its worker index.
manager = multiprocessing.Manager()
worker_index_map = manager.dict()
# Create workers. We don't use multiprocessing.Pool due to
# challenges with handling ^C keyboard interrupts.
workers = []
for _ in range(num_threads):
worker = multiprocessing.Process(
target=process_dir_worker_multiprocessing,
args=(output_lock,
test_counter,
total_tests,
test_name_len,
dotest_options,
job_queue,
result_queue,
inferior_pid_events,
worker_index_map))
worker.start()
workers.append(worker)
# Main loop: wait for all workers to finish and wait for
# the socket handlers to wrap up.
ctrl_c_loop(
# Main operation of loop
lambda: pump_workers_and_asyncore_map(
workers, RUNNER_PROCESS_ASYNC_MAP),
# Return True when we're done with the main loop.
lambda: workers_and_async_done(workers, RUNNER_PROCESS_ASYNC_MAP),
# Indicate what we do when we receive one or more Ctrl-Cs.
lambda ctrl_c_count: handle_ctrl_c(
ctrl_c_count, job_queue, workers, inferior_pid_events,
kill_all_worker_processes))
# Reap the test results.
test_results = []
while not result_queue.empty():
test_results.append(result_queue.get(block=False))
return test_results
