def _run_single_test(self, test, driver):
test_failed = False
driver_need_restart = False
output = driver.run_test(
DriverInput(test, self._options.time_out_ms, None, False))
if output.text == None:
test_failed = True
elif output.timeout:
self._printer.write('timeout: %s' %
test[self._webkit_base_dir_len + 1:])
test_failed = True
driver_need_restart = True
elif output.crash:
self._printer.write('crash: %s' %
test[self._webkit_base_dir_len + 1:])
driver_need_restart = True
test_failed = True
else:
got_a_result = False
for line in re.split('\n', output.text):
if self._result_regex.match(line):
self._buildbot_output.write("%s\n" % line)
got_a_result = True
elif not len(line) == 0:
test_failed = True
self._printer.write("%s" % line)
test_failed = test_failed or not got_a_result
if len(output.error):
self._printer.write('error:\n%s' % output.error)
test_failed = True
return test_failed, driver_need_restart
def _run_reftest(self):
test_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
total_test_time = test_output.test_time
expected_output = None
test_result = None
expected_text = self._port.expected_text(self._test_name)
expected_text_output = DriverOutput(text=expected_text, image=None, image_hash=None, audio=None)
# If the test crashed, or timed out, there's no point in running the reference at all.
# This can save a lot of execution time if we have a lot of crashes or timeouts.
if test_output.crash or test_output.timeout:
test_result = self._compare_output(expected_text_output, test_output)
if test_output.crash:
test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory,
self._test_name, test_output, expected_text_output, test_result.failures)
return test_result
# A reftest can have multiple match references and multiple mismatch references;
# the test fails if any mismatch matches and all of the matches don't match.
# To minimize the number of references we have to check, we run all of the mismatches first,
# then the matches, and short-circuit out as soon as we can.
# Note that sorting by the expectation sorts "!=" before "==" so this is easy to do.
putAllMismatchBeforeMatch = sorted
reference_test_names = []
for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files):
if self._port.lookup_virtual_test_base(self._test_name):
args = self._port.lookup_virtual_reference_args(self._test_name)
else:
args = self._port.lookup_physical_reference_args(self._test_name)
reference_test_name = self._port.relative_test_filename(reference_filename)
reference_test_names.append(reference_test_name)
driver_input = DriverInput(reference_test_name, self._timeout,
image_hash=test_output.image_hash, should_run_pixel_test=True, args=args)
expected_output = self._reference_driver.run_test(driver_input, self._stop_when_done)
total_test_time += expected_output.test_time
test_result = self._compare_output_with_reference(
expected_output, test_output, reference_filename, expectation == '!=')
if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures):
break
assert(expected_output)
if expected_text:
text_output = DriverOutput(text=test_output.text, image=None, image_hash=None, audio=None)
test_result.failures.extend(self._compare_output(expected_text_output, text_output).failures)
expected_output.text = expected_text_output.text
test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory,
self._test_name, test_output, expected_output, test_result.failures)
# FIXME: We don't really deal with a mix of reftest types properly. We pass in a set() to reftest_type
# and only really handle the first of the references in the result.
reftest_type = list(set([reference_file[0] for reference_file in self._reference_files]))
return TestResult(self._test_name, test_result.failures, total_test_time,
test_result.has_stderr, reftest_type=reftest_type, pid=test_result.pid,
references=reference_test_names)
def _run_single_test(self, test, driver, is_chromium_style):
test_failed = False
driver_need_restart = False
output = driver.run_test(
DriverInput(test, self._options.time_out_ms, None, False))
if output.text == None:
test_failed = True
elif output.timeout:
self._printer.write('timeout: %s' %
test[self._webkit_base_dir_len + 1:])
test_failed = True
driver_need_restart = True
elif output.crash:
self._printer.write('crash: %s' %
test[self._webkit_base_dir_len + 1:])
driver_need_restart = True
test_failed = True
else:
if is_chromium_style:
test_failed = self._process_chromium_style_test_result(
test, output)
else:
test_failed = self._process_parser_test_result(test, output)
if len(output.error):
self._printer.write('error:\n%s' % output.error)
test_failed = True
if test_failed:
self._printer.write('FAILED')
return test_failed, driver_need_restart
def _run_reftest(self):
test_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
total_test_time = 0
reference_output = None
test_result = None
# A reftest can have multiple match references and multiple mismatch references;
# the test fails if any mismatch matches and all of the matches don't match.
# To minimize the number of references we have to check, we run all of the mismatches first,
# then the matches, and short-circuit out as soon as we can.
# Note that sorting by the expectation sorts "!=" before "==" so this is easy to do.
putAllMismatchBeforeMatch = sorted
reference_test_names = []
for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files):
reference_test_name = self._port.relative_test_filename(reference_filename)
reference_test_names.append(reference_test_name)
reference_output = self._driver.run_test(DriverInput(reference_test_name, self._timeout, None, should_run_pixel_test=True), self._stop_when_done)
test_result = self._compare_output_with_reference(reference_output, test_output, reference_filename, expectation == '!=')
if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures):
break
total_test_time += test_result.test_run_time
assert(reference_output)
test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, test_output, reference_output, test_result.failures)
reftest_type = set([reference_file[0] for reference_file in self._reference_files])
return TestResult(self._test_name, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr, reftest_type=reftest_type, pid=test_result.pid, references=reference_test_names)
def _run_reftest(self):
driver_output1 = self._driver.run_test(self._driver_input())
reference_test_name = self._port.relative_test_filename(self._reference_filename)
driver_output2 = self._driver.run_test(DriverInput(reference_test_name, self._timeout, driver_output1.image_hash))
test_result = self._compare_output_with_reference(driver_output1, driver_output2)
test_result_writer.write_test_result(self._port, self._test_name, driver_output1, driver_output2, test_result.failures)
return test_result
def _driver_input(self):
# The image hash is used to avoid doing an image dump if the
# checksums match, so it should be set to a blank value if we
# are generating a new baseline. (Otherwise, an image from a
# previous run will be copied into the baseline."""
image_hash = None
if self._should_fetch_expected_checksum():
image_hash = self._port.expected_checksum(self._test_name)
return DriverInput(self._test_name, self._timeout, image_hash)
def run_single(self,
driver,
path_or_url,
time_out_ms,
should_run_pixel_test=False):
return driver.run_test(
DriverInput(path_or_url,
time_out_ms,
image_hash=None,
should_run_pixel_test=should_run_pixel_test))
def input_from_line(self, line):
vals = line.strip().split()
if len(vals) == 3:
uri, timeout, checksum = vals
else:
uri, timeout = vals
checksum = None
test_name = self._driver.uri_to_test(uri)
return DriverInput(test_name, timeout, checksum, self._options.pixel_tests)
def test_crashed_process_name(self):
self.driver._proc = Mock()
# Simulate a crash by having stdout close unexpectedly.
def mock_readline():
raise IOError
self.driver._proc.stdout.readline = mock_readline
self.driver.test_to_uri = lambda test: 'mocktesturi'
driver_output = self.driver.run_test(DriverInput(test_name='some/test.html', timeout=1, image_hash=None, is_reftest=False))
self.assertEqual(self.driver._port.driver_name(), driver_output.crashed_process_name)
def run_single(self,
driver,
test_path,
time_out_ms,
should_run_pixel_test=False):
return driver.run_test(DriverInput(
test_path,
time_out_ms,
image_hash=None,
should_run_pixel_test=should_run_pixel_test),
stop_when_done=False)
def input_from_line(self, line):
vals = line.strip().split("'")
if len(vals) == 1:
uri = vals[0]
checksum = None
else:
uri = vals[0]
checksum = vals[1]
if uri.startswith('http://') or uri.startswith('https://'):
test_name = self._driver.uri_to_test(uri)
else:
test_name = self._port.relative_test_filename(uri)
return DriverInput(test_name, 0, checksum, self._options.pixel_tests)
def test_crash_log(self):
self.driver._proc = Mock()
# Simulate a crash by having stdout close unexpectedly.
def mock_readline():
raise IOError
self.driver._proc.stdout.readline = mock_readline
self.driver._proc.pid = 1234
self.driver.test_to_uri = lambda test: 'mocktesturi'
self.driver._port.driver_name = lambda: 'mockdriver'
self.driver._port._get_crash_log = lambda name, pid, out, err, newer_than: 'mockcrashlog'
driver_output = self.driver.run_test(DriverInput(test_name='some/test.html', timeout=1, image_hash=None, should_run_pixel_test=False))
self.assertTrue(driver_output.crash)
self.assertEqual(driver_output.crashed_process_name, 'mockdriver')
self.assertEqual(driver_output.crashed_pid, 1234)
self.assertEqual(driver_output.crash_log, 'mockcrashlog')
def input_from_line(self, line):
vals = line.strip().split("'")
uri = vals[0]
checksum = None
should_run_pixel_tests = False
if len(vals) == 2 and vals[1] == '--pixel-test':
should_run_pixel_tests = True
elif len(vals) == 3 and vals[1] == '--pixel-test':
should_run_pixel_tests = True
checksum = vals[2]
elif len(vals) != 1:
raise NotImplementedError
if uri.startswith('http://') or uri.startswith('https://'):
test_name = self._driver.uri_to_test(uri)
else:
test_name = self._port.relative_test_filename(uri)
return DriverInput(test_name, 0, checksum, should_run_pixel_tests, args=[])
def run(self, driver, timeout_ms):
test_times = []
for i in range(0, 20):
output = driver.run_test(
DriverInput(self.path_or_url(), timeout_ms, None, False))
if self.run_failed(output):
return None
if i == 0:
continue
test_times.append(output.test_time * 1000)
test_times = sorted(test_times)
# Compute the mean and variance using a numerically stable algorithm.
squareSum = 0
mean = 0
valueSum = sum(test_times)
for i, time in enumerate(test_times):
delta = time - mean
sweep = i + 1.0
mean += delta / sweep
squareSum += delta * delta * (i / sweep)
middle = int(len(test_times) / 2)
results = {
'avg':
mean,
'min':
min(test_times),
'max':
max(test_times),
'median':
test_times[middle] if len(test_times) % 2 else
(test_times[middle - 1] + test_times[middle]) / 2,
'stdev':
math.sqrt(squareSum),
'unit':
'ms'
}
self.output_statistics(self.test_name(), results)
return {self.test_name(): results}
def _driver_input(self):
# The image hash is used to avoid doing an image dump if the
# checksums match, so it should be set to a blank value if we
# are generating a new baseline. (Otherwise, an image from a
# previous run will be copied into the baseline."""
image_hash = None
if self._should_fetch_expected_checksum():
image_hash = self._port.expected_checksum(self._test_name)
test_base = self._port.lookup_virtual_test_base(self._test_name)
if test_base:
# If the file actually exists under the virtual dir, we want to use it (largely for virtual references),
# but we want to use the extra command line args either way.
if self._filesystem.exists(self._port.abspath_for_test(self._test_name)):
test_name = self._test_name
else:
test_name = test_base
args = self._port.lookup_virtual_test_args(self._test_name)
else:
test_name = self._test_name
args = self._port.lookup_physical_test_args(self._test_name)
return DriverInput(test_name, self._timeout, image_hash, self._should_run_pixel_test, args)
def _run_reftest(self):
test_output = self._driver.run_test(self._driver_input())
total_test_time = 0
reference_output = None
test_result = None
# A reftest can have multiple match references and multiple mismatch references;
# the test fails if any mismatch matches and all of the matches don't match.
# To minimize the number of references we have to check, we run all of the mismatches first,
# then the matches, and short-circuit out as soon as we can.
# Note that sorting by the expectation sorts "!=" before "==" so this is easy to do.
putAllMismatchBeforeMatch = sorted
for expectation, reference_filename in putAllMismatchBeforeMatch(
self._reference_files):
reference_test_name = self._port.relative_test_filename(
reference_filename)
reference_output = self._driver.run_test(
DriverInput(reference_test_name,
self._timeout,
test_output.image_hash,
is_reftest=True))
test_result = self._compare_output_with_reference(
test_output, reference_output, reference_filename,
expectation == '!=')
if (expectation == '!=' and test_result.failures) or (
expectation == '==' and not test_result.failures):
break
total_test_time += test_result.test_run_time
assert (reference_output)
test_result_writer.write_test_result(self._port, self._test_name,
test_output, reference_output,
test_result.failures)
return TestResult(self._test_name, test_result.failures,
total_test_time + test_result.test_run_time,
test_result.has_stderr)
def run(self, driver, timeout_ms):
output = driver.run_test(
DriverInput(self.path_or_url(), timeout_ms, None, False))
if self.run_failed(output):
return None
return self.parse_output(output)
