def _test_result_from_row(cls, row, table_title):
test_name = unicode(row.find("a").string)
failures = cls._failures_from_row(row, table_title)
# TestResult is a class designed to work with new-run-webkit-tests.
# old-run-webkit-tests does not save quite enough information in results.html for us to parse.
# FIXME: It's unclear if test_name should include LayoutTests or not.
return test_results.TestResult(test_name, failures)
def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
failures = []
if result_type == test_expectations.TIMEOUT:
failures = [test_failures.FailureTimeout()]
elif result_type == test_expectations.CRASH:
failures = [test_failures.FailureCrash()]
path = self._port._filesystem.join(self._port.layout_tests_dir(), test)
return test_results.TestResult(path,
failures=failures,
test_run_time=run_time)
def test_parse_layout_test_results(self):
failures = [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html'
expected_results = [test_results.TestResult(testname, failures)]
results = LayoutTestResults._parse_results_html(
self._example_results_html)
self.assertEqual(expected_results, results)
def _run_test_in_another_thread(self, test_input, thread_timeout_sec):
"""Run a test in a separate thread, enforcing a hard time limit.
Since we can only detect the termination of a thread, not any internal
state or progress, we can only run per-test timeouts when running test
files singly.
Args:
test_input: Object containing the test filename and timeout
thread_timeout_sec: time to wait before killing the driver process.
Returns:
A TestResult
"""
worker = self
driver = worker._port.create_driver(worker._worker_number)
driver.start()
class SingleTestThread(threading.Thread):
def run(self):
self.result = worker._run_single_test(driver, test_input)
thread = SingleTestThread()
thread.start()
thread.join(thread_timeout_sec)
result = getattr(thread, 'result', None)
if thread.isAlive():
# If join() returned with the thread still running, the
# DumpRenderTree is completely hung and there's nothing
# more we can do with it. We have to kill all the
# DumpRenderTrees to free it up. If we're running more than
# one DumpRenderTree thread, we'll end up killing the other
# DumpRenderTrees too, introducing spurious crashes. We accept
# that tradeoff in order to avoid losing the rest of this
# thread's results.
_log.error('Test thread hung: killing all DumpRenderTrees')
driver.stop()
if not result:
result = test_results.TestResult(test_input.filename,
failures=[],
test_run_time=0)
return result
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def prepare_lists_and_print_output(self):
"""Create appropriate subsets of test lists and returns a
ResultSummary object. Also prints expected test counts.
"""
# Remove skipped - both fixable and ignored - files from the
# top-level list of files to test.
num_all_test_files = len(self._test_files)
self._printer.print_expected("Found: %d tests" %
(len(self._test_files)))
if not num_all_test_files:
_log.critical('No tests to run.')
return None
skipped = set()
if num_all_test_files > 1 and not self._options.force:
skipped = self._expectations.get_tests_with_result_type(
test_expectations.SKIP)
self._test_files -= skipped
# Create a sorted list of test files so the subset chunk,
# if used, contains alphabetically consecutive tests.
self._test_files_list = list(self._test_files)
if self._options.randomize_order:
random.shuffle(self._test_files_list)
else:
self._test_files_list.sort()
# If the user specifies they just want to run a subset of the tests,
# just grab a subset of the non-skipped tests.
if self._options.run_chunk or self._options.run_part:
chunk_value = self._options.run_chunk or self._options.run_part
test_files = self._test_files_list
try:
(chunk_num, chunk_len) = chunk_value.split(":")
chunk_num = int(chunk_num)
assert(chunk_num >= 0)
test_size = int(chunk_len)
assert(test_size > 0)
except:
_log.critical("invalid chunk '%s'" % chunk_value)
return None
# Get the number of tests
num_tests = len(test_files)
# Get the start offset of the slice.
if self._options.run_chunk:
chunk_len = test_size
# In this case chunk_num can be really large. We need
# to make the slave fit in the current number of tests.
slice_start = (chunk_num * chunk_len) % num_tests
else:
# Validate the data.
assert(test_size <= num_tests)
assert(chunk_num <= test_size)
# To count the chunk_len, and make sure we don't skip
# some tests, we round to the next value that fits exactly
# all the parts.
rounded_tests = num_tests
if rounded_tests % test_size != 0:
rounded_tests = (num_tests + test_size -
(num_tests % test_size))
chunk_len = rounded_tests / test_size
slice_start = chunk_len * (chunk_num - 1)
# It does not mind if we go over test_size.
# Get the end offset of the slice.
slice_end = min(num_tests, slice_start + chunk_len)
files = test_files[slice_start:slice_end]
tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
(slice_end - slice_start), slice_start, slice_end, num_tests)
self._printer.print_expected(tests_run_msg)
# If we reached the end and we don't have enough tests, we run some
# from the beginning.
if slice_end - slice_start < chunk_len:
extra = chunk_len - (slice_end - slice_start)
extra_msg = (' last chunk is partial, appending [0:%d]' %
extra)
self._printer.print_expected(extra_msg)
tests_run_msg += "\n" + extra_msg
files.extend(test_files[0:extra])
tests_run_filename = self._fs.join(self._results_directory, "tests_run.txt")
self._fs.write_text_file(tests_run_filename, tests_run_msg)
len_skip_chunk = int(len(files) * len(skipped) /
float(len(self._test_files)))
skip_chunk_list = list(skipped)[0:len_skip_chunk]
skip_chunk = set(skip_chunk_list)
# Update expectations so that the stats are calculated correctly.
# We need to pass a list that includes the right # of skipped files
# to ParseExpectations so that ResultSummary() will get the correct
# stats. So, we add in the subset of skipped files, and then
# subtract them back out.
self._test_files_list = files + skip_chunk_list
self._test_files = set(self._test_files_list)
self.parse_expectations()
self._test_files = set(files)
self._test_files_list = files
else:
skip_chunk = skipped
result_summary = ResultSummary(self._expectations,
self._test_files | skip_chunk)
self._print_expected_results_of_type(result_summary,
test_expectations.PASS, "passes")
self._print_expected_results_of_type(result_summary,
test_expectations.FAIL, "failures")
self._print_expected_results_of_type(result_summary,
test_expectations.FLAKY, "flaky")
self._print_expected_results_of_type(result_summary,
test_expectations.SKIP, "skipped")
if self._options.force:
self._printer.print_expected('Running all tests, including '
'skips (--force)')
else:
# Note that we don't actually run the skipped tests (they were
# subtracted out of self._test_files, above), but we stub out the
# results here so the statistics can remain accurate.
for test in skip_chunk:
result = test_results.TestResult(test)
result.type = test_expectations.SKIP
result_summary.add(result, expected=True)
self._printer.print_expected('')
# Check to make sure we didn't filter out all of the tests.
if not len(self._test_files):
_log.info("All tests are being skipped")
return None
return result_summary