def test_exception_no_fork(self):
with OutputCapture(level=logging.INFO) as captured:
with self.assertRaises(RuntimeError):
with TaskPool(workers=1, force_fork=False) as pool:
pool.do(exception, 'Testing exception')
pool.wait()
self.assertEqual(captured.webkitcorepy.log.getvalue(), '')
def test_single_no_fork(self):
with OutputCapture(level=logging.WARNING) as captured:
with TaskPool(workers=1, force_fork=False) as pool:
pool.do(action, 'a')
pool.do(log, logging.WARNING, '1')
pool.wait()
self.assertEqual(captured.stdout.getvalue(), 'action(a)\n')
self.assertEqual(captured.webkitcorepy.log.getvalue(), '1\n')
def test_teardown_arguments(self):
with OutputCapture() as captured:
with TaskPool(workers=4, teardown=teardown, teardownargs=['Teardown argument']) as pool:
for character in self.alphabet:
pool.do(action, character)
pool.wait()
self.assertEqual(
sorted(captured.webkitcorepy.log.getvalue().splitlines()),
['worker/{} Teardown argument'.format(x) for x in range(4)],
)
def test_setup(self):
with OutputCapture() as captured:
with TaskPool(workers=4, setup=setup) as pool:
for character in self.alphabet:
pool.do(action, character)
pool.wait()
self.assertEqual(
sorted(captured.webkitcorepy.log.getvalue().splitlines()),
['worker/{} Setting up'.format(x) for x in range(4)],
)
def test_exception(self):
with OutputCapture(level=logging.INFO) as captured:
with self.assertRaises(RuntimeError):
with TaskPool(workers=1, force_fork=True) as pool:
pool.do(exception, 'Testing exception')
pool.wait()
self.assertEqual(
captured.webkitcorepy.log.getvalue().splitlines(),
['worker/0 starting', 'worker/0 stopping'],
)
def test_callback(self):
sequence = []
with OutputCapture():
with TaskPool(workers=4) as pool:
for character in self.alphabet:
pool.do(action, character, callback=lambda value: sequence.append(value))
pool.wait()
self.assertEqual(
self.alphabet,
''.join(sorted(sequence)),
)
def test_multiple(self):
with OutputCapture(level=logging.INFO) as captured:
with TaskPool(workers=4) as pool:
for character in self.alphabet:
pool.do(action, character)
pool.wait()
lines = captured.stdout.getvalue().splitlines()
self.assertEquals(sorted(lines), ['action({})'.format(character) for character in self.alphabet])
self.assertEqual(
sorted(captured.webkitcorepy.log.getvalue().splitlines()),
sorted(['worker/{} starting'.format(number) for number in range(4)] + ['worker/{} stopping'.format(number) for number in range(4)]),
)
def run_test(self, test_input, shard_name):
self._batch_count += 1
stop_when_done = False
if 0 < self._batch_size <= self._batch_count:
self._batch_count = 0
stop_when_done = True
test_timeout_sec = self._timeout(test_input)
start = time.time()
TaskPool.Process.queue.send(
TaskPool.Task(
handle_started_test,
None,
TaskPool.Process.name,
test_input.test_name,
))
result = self._run_test_with_or_without_timeout(
test_input, test_timeout_sec, stop_when_done)
result.shard_name = shard_name
result.worker_name = TaskPool.Process.name
result.total_run_time = time.time() - start
result.test_number = self._num_tests
self._num_tests += 1
TaskPool.Process.queue.send(
TaskPool.Task(
handle_finished_test,
None,
TaskPool.Process.name,
result,
))
self._clean_up_after_test(test_input, result)
def run(self, tests, num_workers):
if not tests:
return
self.printer.write_update('Sharding tests ...')
shards = Runner._shard_tests(tests)
original_level = server_process_logger.level
server_process_logger.setLevel(logging.CRITICAL)
try:
if Runner.instance:
raise RuntimeError('Cannot nest API test runners')
Runner.instance = self
self._num_workers = min(num_workers, len(shards))
devices = None
if getattr(self.port, 'DEVICE_MANAGER', None):
devices = dict(
available_devices=self.port.DEVICE_MANAGER.
AVAILABLE_DEVICES,
initialized_devices=self.port.DEVICE_MANAGER.
INITIALIZED_DEVICES,
)
with TaskPool(
workers=self._num_workers,
setup=setup_shard,
setupkwargs=dict(port=self.port, devices=devices),
teardown=teardown_shard,
) as pool:
for name, tests in iteritems(shards):
pool.do(run_shard, name, *tests)
pool.wait()
finally:
server_process_logger.setLevel(original_level)
Runner.instance = None
def test_invalid_shutdown(self):
with OutputCapture():
with self.assertRaises(TaskPool.Exception):
with TaskPool(workers=1, teardown=teardown, grace_period=1, force_fork=True) as pool:
pool.do(wait, 2)
def run(self, name, *tests):
binary_name = name.split('.')[0]
remaining_tests = ['.'.join(test.split('.')[1:]) for test in tests]
# Try to run the shard in a single process.
while remaining_tests and not self._port.get_option('run_singly'):
starting_length = len(remaining_tests)
server_process = ServerProcess(
self._port,
binary_name,
Runner.command_for_port(self._port, [
self._port._build_path(binary_name),
'--gtest_filter={}'.format(':'.join(remaining_tests))
]),
env=self._port.environment_for_api_tests())
try:
deadline = time.time() + self._timeout
last_test = None
last_status = None
stdout_buffer = ''
server_process.start()
while remaining_tests:
stdout = string_utils.decode(
server_process.read_stdout_line(deadline),
target_type=str)
# If we've triggered a timeout, we don't know which test caused it. Break out and run singly.
if stdout is None and server_process.timed_out:
break
if stdout is None and server_process.has_crashed():
# It's possible we crashed before printing anything.
if last_status == Runner.STATUS_PASSED:
last_test = None
else:
last_status = Runner.STATUS_CRASHED
break
assert stdout is not None
stdout_split = stdout.rstrip().split(' ')
if len(stdout_split) != 2 or not (
stdout_split[0].startswith('**')
and stdout_split[0].endswith('**')):
stdout_buffer += stdout
continue
if last_test is not None:
remaining_tests.remove(last_test)
for line in stdout_buffer.splitlines(False):
_log.error(line)
TaskPool.Process.queue.send(
TaskPool.Task(
report_result,
None,
TaskPool.Process.name,
'{}.{}'.format(binary_name, last_test),
last_status,
stdout_buffer,
))
deadline = time.time() + self._timeout
stdout_buffer = ''
if '**PASS**' == stdout_split[0]:
last_status = Runner.STATUS_PASSED
else:
last_status = Runner.STATUS_FAILED
last_test = stdout_split[1]
# We assume that stderr is only relevant if there is a crash (meaning we triggered an assert)
if last_test:
remaining_tests.remove(last_test)
stdout_buffer += string_utils.decode(
server_process.pop_all_buffered_stdout(),
target_type=str)
stderr_buffer = string_utils.decode(
server_process.pop_all_buffered_stderr(),
target_type=str
) if last_status == Runner.STATUS_CRASHED else ''
for line in (stdout_buffer +
stderr_buffer).splitlines(keepends=False):
_log.error(line)
TaskPool.Process.queue.send(
TaskPool.Task(
report_result,
None,
TaskPool.Process.name,
'{}.{}'.format(binary_name, last_test),
last_status,
self._filter_noisy_output(stdout_buffer +
stderr_buffer),
))
if server_process.timed_out:
break
# If we weren't able to determine the results for any tests, we need to run what remains singly.
if starting_length == len(remaining_tests):
break
finally:
server_process.stop()
# Now, just try and run the rest of the tests singly.
for test in remaining_tests:
self._run_single_test(binary_name, test)
def _run_single_test(self, binary_name, test):
server_process = ServerProcess(
self._port,
binary_name,
Runner.command_for_port(self._port, [
self._port._build_path(binary_name),
'--gtest_filter={}'.format(test)
]),
env=self._port.environment_for_api_tests())
status = Runner.STATUS_RUNNING
if test.split('.')[1].startswith(
'DISABLED_') and not self._port.get_option('force'):
status = Runner.STATUS_DISABLED
stdout_buffer = ''
stderr_buffer = ''
try:
deadline = time.time() + self._timeout
if status != Runner.STATUS_DISABLED:
server_process.start()
while status == Runner.STATUS_RUNNING:
stdout_line, stderr_line = server_process.read_either_stdout_or_stderr_line(
deadline)
if not stderr_line and not stdout_line:
break
if stderr_line:
stderr_line = string_utils.decode(stderr_line,
target_type=str)
stderr_buffer += stderr_line
_log.error(stderr_line[:-1])
if stdout_line:
stdout_line = string_utils.decode(stdout_line,
target_type=str)
if '**PASS**' in stdout_line:
status = Runner.STATUS_PASSED
elif '**FAIL**' in stdout_line:
status = Runner.STATUS_FAILED
else:
stdout_buffer += stdout_line
_log.error(stdout_line[:-1])
if status == Runner.STATUS_DISABLED:
pass
elif server_process.timed_out:
status = Runner.STATUS_TIMEOUT
elif server_process.has_crashed():
status = Runner.STATUS_CRASHED
elif status == Runner.STATUS_RUNNING:
status = Runner.STATUS_FAILED
finally:
remaining_stderr = string_utils.decode(
server_process.pop_all_buffered_stderr(), target_type=str)
remaining_stdout = string_utils.decode(
server_process.pop_all_buffered_stdout(), target_type=str)
for line in (remaining_stdout +
remaining_stderr).splitlines(False):
_log.error(line)
output_buffer = stderr_buffer + stdout_buffer + remaining_stderr + remaining_stdout
server_process.stop()
TaskPool.Process.queue.send(
TaskPool.Task(
report_result,
None,
TaskPool.Process.name,
'{}.{}'.format(binary_name, test),
status,
self._filter_noisy_output(output_buffer),
))
def run_tests(self,
expectations,
test_inputs,
num_workers,
retrying,
device_type=None):
self._expectations = expectations
self._test_inputs = list(test_inputs)
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations, len(test_inputs))
self._current_run_results = run_results
self.printer.num_tests = len(test_inputs)
self.printer.num_started = 0
if not retrying:
self.printer.print_expected(
run_results,
self._expectations.model().get_tests_with_result_type)
self.printer.write_update('Sharding tests ...')
all_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes),
self._options.fully_parallel)
num_workers = min(num_workers, len(all_shards))
self.printer.print_workers_and_shards(num_workers, len(all_shards))
if self._options.dry_run:
return run_results
self.printer.write_update('Starting {} ...'.format(
pluralize(num_workers, "worker")))
devices = None
if getattr(self._port, 'DEVICE_MANAGER', None):
devices = dict(
available_devices=self._port.DEVICE_MANAGER.AVAILABLE_DEVICES,
initialized_devices=self._port.DEVICE_MANAGER.
INITIALIZED_DEVICES,
)
try:
LayoutTestRunner.instance = self
with TaskPool(
workers=num_workers,
setup=setup_shard,
setupkwargs=dict(
port=self._port,
devices=devices,
results_directory=self._results_directory,
retrying=self._retrying,
),
teardown=teardown_shard,
) as pool:
for shard in all_shards:
pool.do(
run_shard,
shard,
callback=lambda value: self.
_annotate_results_with_additional_failures(value),
)
pool.wait()
except TestRunInterruptedException as e:
_log.warning(e.reason)
run_results.interrupted = True
except KeyboardInterrupt:
self.printer.flush()
self.printer.writeln('Interrupted, exiting ...')
run_results.keyboard_interrupted = True
except Exception as e:
_log.debug('{}("{}") raised, exiting'.format(
e.__class__.__name__, str(e)))
raise
finally:
LayoutTestRunner.instance = None
return run_results