def _shutdown_unfinished_drivers_gracefully(not_done, test_driver_list):
"""Kills unfinished concurrent test drivers as gracefully as possible."""
# Prevent new tasks from running.
not_cancelled = []
for future in not_done:
# Note: Running tasks cannot be cancelled.
if not future.cancel():
not_cancelled.append(future)
# We try to terminate first, as ./launch_chrome will respond by shutting
# down the chrome process cleanly. The later kill() call will not do so,
# and could leave a running process behind. At this point, consider any
# tests that have not finished as incomplete.
for driver in test_driver_list:
driver.terminate()
# Give everyone ten seconds to terminate.
_, not_cancelled = concurrent.wait(not_cancelled, 10)
if not not_cancelled:
return
# There still remain some running tasks. Kill them.
for driver in test_driver_list:
driver.kill()
concurrent.wait(not_cancelled, 5)
def _shutdown_unfinished_drivers_gracefully(not_done, test_driver_list):
"""Kills unfinished concurrent test drivers as gracefully as possible."""
# Prevent new tasks from running.
not_cancelled = []
for future in not_done:
# Note: Running tasks cannot be cancelled.
if not future.cancel():
not_cancelled.append(future)
# We try to terminate first, as ./launch_chrome will respond by shutting
# down the chrome process cleanly. The later kill() call will not do so,
# and could leave a running process behind. At this point, consider any
# tests that have not finished as incomplete.
for driver in test_driver_list:
driver.terminate()
# Give everyone ten seconds to terminate.
_, not_cancelled = concurrent.wait(not_cancelled, 10)
if not not_cancelled:
return
# There still remain some running tasks. Kill them.
for driver in test_driver_list:
driver.kill()
concurrent.wait(not_cancelled, 5)
def test_first_exception(self):
future1 = concurrent.Future()
future2 = concurrent.Future()
# First of all, no futures are completed.
done, not_done = concurrent.wait([future1, future2], timeout=0.01,
return_when=concurrent.FIRST_EXCEPTION)
self.assertFalse(done) # Empty.
self.assertEqual({future1, future2}, not_done)
# Mark future1 as completed with an exception.
self.assertTrue(future1.set_running_or_notify_cancel())
future1.set_exception(AssertionError())
# Here, wait should return immediately even without timeout.
done, not_done = concurrent.wait([future1, future2],
return_when=concurrent.FIRST_EXCEPTION)
self.assertEqual({future1}, done)
self.assertEqual({future2}, not_done)
def test_first_completed(self):
future1 = concurrent.Future()
future2 = concurrent.Future()
# First of all, no futures are completed.
done, not_done = concurrent.wait([future1, future2], timeout=0.01,
return_when=concurrent.FIRST_COMPLETED)
self.assertFalse(done) # Empty.
self.assertEqual({future1, future2}, not_done)
# Mark future1 as completed.
self.assertTrue(future1.set_running_or_notify_cancel())
future1.set_result(WaitTest._DUMMY_RESULT)
# Here, wait should return immediately even without timeout.
done, not_done = concurrent.wait([future1, future2],
return_when=concurrent.FIRST_COMPLETED)
self.assertEqual({future1}, done)
self.assertEqual({future2}, not_done)
def test_all_completed_with_exception(self):
future1 = concurrent.Future()
self.assertTrue(future1.set_running_or_notify_cancel())
future1.set_exception(AssertionError())
# Finished with an exception is also considered as completed.
done, not_done = concurrent.wait([future1])
self.assertEqual({future1}, done)
self.assertFalse(not_done) # Empty.
def run_in_parallel(task_list, maximum_jobs):
"""Runs task_list in parallel on multiprocess.
Returns a list of NinjaGenerator created in subprocesses.
If |maximum_jobs| is set to 0, this function runs the ninja generation
synchronously in process.
"""
if maximum_jobs == 0:
executor = concurrent.SynchronousExecutor()
else:
executor = concurrent.ProcessPoolExecutor(max_workers=maximum_jobs)
result_list = []
with executor:
try:
# Submit initial tasks.
not_done = {
executor.submit(_run_task, generator_task)
for generator_task in task_list
}
while not_done:
# Wait any task is completed.
done, not_done = concurrent.wait(
not_done, return_when=concurrent.FIRST_COMPLETED)
for completed_future in done:
if completed_future.exception():
# An exception is raised in a task. Cancel remaining tasks and
# re-raise the exception.
for future in not_done:
future.cancel()
not_done = []
raise completed_future.exception()
# The task is completed successfully. Process the result.
result, request_task_list = completed_future.result()
if request_task_list:
# If sub tasks are requested, submit them.
assert not result
not_done.update(
executor.submit(_run_task, generator_task)
for generator_task in request_task_list)
continue
if result:
result_list.append(result)
except:
# An exception is raised. Terminate the running workers.
if isinstance(executor, concurrent.ProcessPoolExecutor):
executor.terminate()
raise
return result_list
def test_first_completed_with_exception(self):
future1 = concurrent.Future()
future2 = concurrent.Future()
# Mark future1 as completed with an exception.
self.assertTrue(future1.set_running_or_notify_cancel())
future1.set_exception(AssertionError())
# Here, wait should return immediately even without timeout.
done, not_done = concurrent.wait([future1, future2],
return_when=concurrent.FIRST_COMPLETED)
self.assertEqual({future1}, done)
self.assertEqual({future2}, not_done)
def run_in_parallel(task_list, maximum_jobs):
"""Runs task_list in parallel on multiprocess.
Returns a list of NinjaGenerator created in subprocesses.
If |maximum_jobs| is set to 0, this function runs the ninja generation
synchronously in process.
"""
if maximum_jobs == 0:
executor = concurrent.SynchronousExecutor()
else:
executor = concurrent.ProcessPoolExecutor(max_workers=maximum_jobs)
result_list = []
with executor:
try:
# Submit initial tasks.
not_done = {executor.submit(_run_task, generator_task)
for generator_task in task_list}
while not_done:
# Wait any task is completed.
done, not_done = concurrent.wait(
not_done, return_when=concurrent.FIRST_COMPLETED)
for completed_future in done:
if completed_future.exception():
# An exception is raised in a task. Cancel remaining tasks and
# re-raise the exception.
for future in not_done:
future.cancel()
not_done = []
raise completed_future.exception()
# The task is completed successfully. Process the result.
result, request_task_list = completed_future.result()
if request_task_list:
# If sub tasks are requested, submit them.
assert not result
not_done.update(
executor.submit(_run_task, generator_task)
for generator_task in request_task_list)
continue
if result:
result_list.append(result)
except:
# An exception is raised. Terminate the running workers.
if isinstance(executor, concurrent.ProcessPoolExecutor):
executor.terminate()
raise
return result_list
def test_all_completed(self):
future1 = concurrent.Future()
future2 = concurrent.Future()
# First of all, no futures are completed.
done, not_done = concurrent.wait([future1, future2], timeout=0.01)
self.assertFalse(done) # Empty.
self.assertEqual({future1, future2}, not_done)
# Mark future1 as completed.
self.assertTrue(future1.set_running_or_notify_cancel())
future1.set_result(WaitTest._DUMMY_RESULT)
done, not_done = concurrent.wait([future1, future2], timeout=0.01)
self.assertEqual({future1}, done)
self.assertEqual({future2}, not_done)
# Mark future2 as completed, too.
self.assertTrue(future2.set_running_or_notify_cancel())
future2.set_result(WaitTest._DUMMY_RESULT)
# Now, wait should return immediately, even timeout is not set.
done, not_done = concurrent.wait([future1, future2])
self.assertEqual({future1, future2}, done)
self.assertFalse(not_done) # Empty.
def test_first_exception_without_error(self):
future1 = concurrent.Future()
future2 = concurrent.Future()
# Mark both futures as completed without any exceptions.
self.assertTrue(future1.set_running_or_notify_cancel())
future1.set_result(WaitTest._DUMMY_RESULT)
self.assertTrue(future2.set_running_or_notify_cancel())
future2.set_result(WaitTest._DUMMY_RESULT)
# If all futures are completed, wait() should return immediately,
# even if return_when is set to FIRST_EXCEPTION.
done, not_done = concurrent.wait([future1, future2],
return_when=concurrent.FIRST_EXCEPTION)
self.assertEqual({future1, future2}, done)
self.assertFalse(not_done) # Empty.
def _run_suites(test_driver_list, args, prepare_only=False):
"""Runs the indicated suites."""
setup_output_directory(args.output_dir)
suite_results.initialize(test_driver_list, args, prepare_only)
if not test_driver_list:
return False
timeout = (args.total_timeout
if args.total_timeout and not prepare_only else None)
try:
with concurrent.ThreadPoolExecutor(args.jobs, daemon=True) as executor:
futures = [
executor.submit(_run_driver, driver, args, prepare_only)
for driver in test_driver_list
]
done, not_done = concurrent.wait(futures, timeout,
concurrent.FIRST_EXCEPTION)
try:
# Iterate over the results to propagate an exception if any of the tasks
# aborted by an error in the test drivers. Since such an error is due to
# broken script rather than normal failure in tests, we prefer just to
# die similarly as when Python errors occurred in the main thread.
for future in done:
future.result()
# No exception was raised but some timed-out tasks are remaining.
if not_done:
print '@@@STEP_TEXT@Integration test timed out@@@'
debug.write_frames(sys.stdout)
if args.warn_on_failure:
print '@@@STEP_WARNINGS@@@'
else:
print '@@@STEP_FAILURE@@@'
return False
# All tests passed (or failed) in time.
return True
finally:
if not_done:
_shutdown_unfinished_drivers_gracefully(
not_done, test_driver_list)
finally:
for driver in test_driver_list:
driver.finalize(args)
def _run_suites(test_driver_list, args, prepare_only=False):
"""Runs the indicated suites."""
setup_output_directory(args.output_dir)
suite_results.initialize(test_driver_list, args, prepare_only)
if not test_driver_list:
return False
timeout = (
args.total_timeout if args.total_timeout and not prepare_only else None)
try:
with concurrent.ThreadPoolExecutor(args.jobs, daemon=True) as executor:
futures = [executor.submit(_run_driver, driver, args, prepare_only)
for driver in test_driver_list]
done, not_done = concurrent.wait(futures, timeout,
concurrent.FIRST_EXCEPTION)
try:
# Iterate over the results to propagate an exception if any of the tasks
# aborted by an error in the test drivers. Since such an error is due to
# broken script rather than normal failure in tests, we prefer just to
# die similarly as when Python errors occurred in the main thread.
for future in done:
future.result()
# No exception was raised but some timed-out tasks are remaining.
if not_done:
print '@@@STEP_TEXT@Integration test timed out@@@'
debug.write_frames(sys.stdout)
if args.warn_on_failure:
print '@@@STEP_WARNINGS@@@'
else:
print '@@@STEP_FAILURE@@@'
return False
# All tests passed (or failed) in time.
return True
finally:
if not_done:
_shutdown_unfinished_drivers_gracefully(not_done, test_driver_list)
finally:
for driver in test_driver_list:
driver.finalize(args)
