def find_touched_tests(self,
new_or_modified_paths,
apply_skip_expectations=True):
potential_test_paths = []
for test_file in new_or_modified_paths:
if not test_file.startswith(self.LAYOUT_TESTS_DIRECTORY):
continue
test_file = self._strip_test_dir_prefix(test_file)
test_paths = self._port.potential_test_names_from_expected_file(
test_file)
if test_paths:
potential_test_paths.extend(test_paths)
else:
potential_test_paths.append(test_file)
if not potential_test_paths:
return None
tests = self._port.tests(list(set(potential_test_paths)))
if not apply_skip_expectations:
return tests
expectations = test_expectations.TestExpectations(
self._port, tests, force_expectations_pass=False)
expectations.parse_all_expectations()
tests_to_skip = self.skip_tests(potential_test_paths, tests,
expectations, None)
return [test for test in tests if test not in tests_to_skip]
def run_results(port, extra_skipped_tests=[]):
tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html',
'failures/expected/audio.html', 'failures/expected/text.html', 'passes/skipped/skip.html']
expectations = test_expectations.TestExpectations(port, tests)
if extra_skipped_tests:
expectations.add_extra_skipped_tests(extra_skipped_tests)
return test_run_results.TestRunResults(expectations, len(tests))
def lint(port, options):
host = port.host
if options.platform:
ports_to_lint = [port]
else:
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names()
]
files_linted = set()
lint_failed = False
for port_to_lint in ports_to_lint:
expectations_file = port_to_lint.path_to_test_expectations_file()
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(port_to_lint, is_lint_mode=True)
except test_expectations.ParseError, e:
lint_failed = True
_log.error('')
for warning in e.warnings:
_log.error(warning)
_log.error('')
files_linted.add(expectations_file)
def lint(host, options):
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
failures = []
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.expectations_dict()
for expectations_file in expectations_dict.keys():
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_dict={
expectations_file: expectations_dict[expectations_file]
},
is_lint_mode=True)
except test_expectations.ParseError as e:
_log.error('')
for warning in e.warnings:
_log.error(warning)
failures.append('%s: %s' % (expectations_file, warning))
_log.error('')
files_linted.add(expectations_file)
return failures
def lint(host, options):
# FIXME: Remove this when we remove the --chromium flag (crbug.com/245504).
if options.platform == 'chromium':
options.platform = None
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
lint_failed = False
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.expectations_dict()
for expectations_file in expectations_dict.keys():
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_dict={
expectations_file: expectations_dict[expectations_file]
},
is_lint_mode=True)
except test_expectations.ParseError as e:
lint_failed = True
_log.error('')
for warning in e.warnings:
_log.error(warning)
_log.error('')
files_linted.add(expectations_file)
return lint_failed
def test_summarized_results_flaky_pass_after_first_retry(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(
self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 1)
initial_results.add(get_result(test_name, test_expectations.CRASH),
False, False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1)
]
all_retry_results[0].add(
get_result(test_name, test_expectations.TIMEOUT), False, False)
all_retry_results[1].add(get_result(test_name, test_expectations.PASS),
True, False)
all_retry_results[2].add(get_result(test_name, test_expectations.PASS),
True, False)
summary = test_run_results.summarize_results(
self.port,
expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry=True)
self.assertTrue(
'is_unexpected' not in summary['tests']['passes']['text.html'])
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH TIMEOUT PASS PASS')
self.assertEquals(summary['num_flaky'], 1)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 0)
def _collect_tests(self,
paths, # type: List[str]
device_type_list, # type: List[Optional[DeviceType]]
):
aggregate_tests = set() # type: Set[Test]
aggregate_tests_to_run = set() # type: Set[Test]
tests_to_run_by_device = {} # type: Dict[Optional[DeviceType], List[Test]]
device_type_list = self._port.supported_device_types()
for device_type in device_type_list:
for_device_type = u'for {} '.format(device_type) if device_type else ''
self._printer.write_update(u'Collecting tests {}...'.format(for_device_type))
paths, tests = self._finder.find_tests(self._options, paths, device_type=device_type)
aggregate_tests.update(tests)
test_names = [test.test_path for test in tests]
self._printer.write_update(u'Parsing expectations {}...'.format(for_device_type))
self._expectations[device_type] = test_expectations.TestExpectations(self._port, test_names, force_expectations_pass=self._options.force, device_type=device_type)
self._expectations[device_type].parse_all_expectations()
tests_to_run = self._tests_to_run(tests, device_type=device_type)
tests_to_run_by_device[device_type] = [test for test in tests_to_run if test not in aggregate_tests_to_run]
aggregate_tests_to_run.update(tests_to_run_by_device[device_type])
aggregate_tests_to_skip = aggregate_tests - aggregate_tests_to_run
return tests_to_run_by_device, aggregate_tests_to_skip
def get_result_summary(self, test_names, expectations_str):
port.test_expectations = lambda: expectations_str
port.test_expectations_overrides = lambda: None
expectations = test_expectations.TestExpectations(self._port, test_names)
rs = result_summary.ResultSummary(expectations, test_names)
return test_names, rs, expectations
def run_results(port):
tests = [
'passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html', 'failures/expected/hang.html',
'failures/expected/audio.html'
]
expectations = test_expectations.TestExpectations(port, tests)
return test_run_results.TestRunResults(expectations, len(tests))
def get_result_summary(self, port, test_names, expectations_str):
expectations = test_expectations.TestExpectations(
port,
test_names,
expectations_str,
port.test_configuration(),
is_lint_mode=False)
return test_names, result_summary.ResultSummary(
expectations, test_names), expectations
def check_test_expectations(self, expectations_str, tests=None, overrides=None):
err = None
expectations = None
try:
expectations = test_expectations.TestExpectations(
port=self._port_obj, expectations=expectations_str, tests=tests,
test_config=self._port_obj.test_configuration(),
is_lint_mode=True, overrides=overrides)
except test_expectations.ParseError, error:
err = error
def test_look_for_new_crash_logs(self):
def get_manager():
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port, options=MockOptions(test_list=None, http=True, max_locked_shards=1), printer=Mock())
return manager
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
tests = ['failures/expected/crash.html']
expectations = test_expectations.TestExpectations(port, tests)
run_results = TestRunResults(expectations, len(tests))
manager = get_manager()
manager._look_for_new_crash_logs(run_results, time.time())
def lint(host, options, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(logging_stream)
logger.addHandler(handler)
try:
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
lint_failed = False
for port_to_lint in ports_to_lint:
if port_to_lint in ['gtk', 'wpe']:
port_to_lint._options.additional_expectations = [
'LayoutTests/platform/glib/TestExpectations'
]
expectations_dict = port_to_lint.expectations_dict()
# FIXME: This won't work if multiple ports share a TestExpectations file but support different modifiers in the file.
for expectations_file in expectations_dict.keys():
if expectations_file in files_linted:
continue
try:
expectations = test_expectations.TestExpectations(
port_to_lint,
expectations_to_lint={
expectations_file:
expectations_dict[expectations_file]
})
expectations.parse_all_expectations()
except test_expectations.ParseError as e:
lint_failed = True
_log.error('')
for warning in e.warnings:
_log.error(warning)
_log.error('')
files_linted.add(expectations_file)
if lint_failed:
_log.error('Lint failed.')
return -1
_log.info('Lint succeeded.')
return 0
finally:
logger.removeHandler(handler)
def test_look_for_new_crash_logs(self):
def get_manager_with_tests(test_names):
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
manager.collect_tests(test_names)
return manager
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
tests = ['failures/expected/crash.html']
expectations = test_expectations.TestExpectations(port, tests)
rs = result_summary.ResultSummary(expectations, tests)
manager = get_manager_with_tests(tests)
manager._look_for_new_crash_logs(rs, time.time())
def lint(host, options):
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.all_expectations_dict()
# There are some TestExpectations files that are not loaded by default
# in any Port, and are instead passed via --additional-expectations on
# some builders. We also want to inspect these files if they're present.
extra_files = (
'ASANExpectations',
'LeakExpectations',
'MSANExpectations',
)
for name in extra_files:
path = port_to_lint.layout_tests_dir() + '/' + name
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for expectations_file in expectations_dict:
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_dict={
expectations_file: expectations_dict[expectations_file]
},
is_lint_mode=True)
except test_expectations.ParseError as error:
_log.error('')
for warning in error.warnings:
_log.error(warning)
failures.append('%s: %s' % (expectations_file, warning))
_log.error('')
files_linted.add(expectations_file)
return failures
def lint(host, options, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(logging_stream)
logger.addHandler(handler)
try:
# FIXME: Remove this when we remove the --chromium flag (crbug.com/245504).
if options.platform == 'chromium':
options.platform = None
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
lint_failed = False
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.expectations_dict()
for expectations_file in expectations_dict.keys():
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_dict={
expectations_file:
expectations_dict[expectations_file]
},
is_lint_mode=True)
except test_expectations.ParseError as e:
lint_failed = True
_log.error('')
for warning in e.warnings:
_log.error(warning)
_log.error('')
files_linted.add(expectations_file)
if lint_failed:
_log.error('Lint failed.')
return -1
_log.info('Lint succeeded.')
return 0
finally:
logger.removeHandler(handler)
def check_test_expectations(self, expectations_str, tests=None, overrides=None):
err = None
expectations = None
# FIXME: We need to rework how we lint strings so that we can do it independently of what a
# port's existing expectations are. Linting should probably just call the parser directly.
# For now we override the port hooks. This will also need to be reworked when expectations
# can cascade arbitrarily, rather than just have expectations and overrides.
orig_expectations = self._port_obj.test_expectations
orig_overrides = self._port_obj.test_expectations_overrides
try:
self._port_obj.test_expectations = lambda: expectations_str
self._port_obj.test_expectations_overrides = lambda: overrides
expectations = test_expectations.TestExpectations(self._port_obj, tests, True)
except test_expectations.ParseError, error:
err = error
def print_expectations(self, args):
self._printer.write_update("Collecting tests ...")
try:
paths, test_names = self._collect_tests(args)
except IOError:
# This is raised if --test-list doesn't exist
return -1
self._printer.write_update("Parsing expectations ...")
self._expectations = test_expectations.TestExpectations(
self._port,
test_names,
force_expectations_pass=self._options.force)
self._expectations.parse_all_expectations()
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._printer.print_found(len(test_names), len(tests_to_run),
self._options.repeat_each,
self._options.iterations)
test_col_width = len(max(tests_to_run + list(tests_to_skip),
key=len)) + 1
default_device_tests = []
# Look for tests with custom device requirements.
custom_device_tests = defaultdict(list)
for test_file in tests_to_run:
custom_device = self._custom_device_for_test(test_file)
if custom_device:
custom_device_tests[custom_device].append(test_file)
else:
default_device_tests.append(test_file)
if custom_device_tests:
for device_class in custom_device_tests:
_log.debug('{} tests use device {}'.format(
len(custom_device_tests[device_class]), device_class))
self._print_expectations_for_subset(None, test_col_width, tests_to_run,
tests_to_skip)
for device_class in custom_device_tests:
device_tests = custom_device_tests[device_class]
self._print_expectations_for_subset(device_class, test_col_width,
device_tests)
return 0
def __init__(self,
running_port,
target_port,
platform,
options,
url_fetcher,
zip_factory,
scm,
logged_before=False):
"""
Args:
running_port: the Port the script is running on.
target_port: the Port the script uses to find port-specific
configuration information like the test_expectations.txt
file location and the list of test platforms.
platform: the test platform to rebaseline
options: the command-line options object.
url_fetcher: object that can fetch objects from URLs
zip_factory: optional object that can fetch zip files from URLs
scm: scm object for adding new baselines
logged_before: whether the previous running port logged anything.
"""
self._platform = platform
self._options = options
self._port = running_port
self._filesystem = running_port._filesystem
self._target_port = target_port
self._rebaseline_port = port.get(platform,
options,
filesystem=self._filesystem)
self._rebaselining_tests = set()
self._rebaselined_tests = []
self._logged_before = logged_before
self.did_log = False
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
expectations_str = self._rebaseline_port.test_expectations()
self._test_expectations = test_expectations.TestExpectations(
self._rebaseline_port, None, expectations_str,
self._rebaseline_port.test_configuration(), False)
self._url_fetcher = url_fetcher
self._zip_factory = zip_factory
self._scm = scm
def skipped_layout_tests(self, extra_test_files=None):
expectations_str = self.test_expectations()
overrides_str = self.test_expectations_overrides()
is_debug_mode = False
all_test_files = self.tests([])
if extra_test_files:
all_test_files.update(extra_test_files)
expectations = test_expectations.TestExpectations(
self,
all_test_files,
expectations_str,
self.test_configuration(),
is_lint_mode=False,
overrides=overrides_str)
return expectations.get_tests_with_result_type(test_expectations.SKIP)
def test_look_for_new_crash_logs(self):
def get_manager():
host = MockHost()
port = host.port_factory.get('test-mac-mac10.10')
manager = Manager(port,
options=optparse.Values({
'test_list': None,
'http': True,
'max_locked_shards': 1
}),
printer=FakePrinter())
return manager
host = MockHost()
port = host.port_factory.get('test-mac-mac10.10')
tests = ['failures/expected/crash.html']
expectations = test_expectations.TestExpectations(port, tests)
run_results = TestRunResults(expectations, len(tests))
manager = get_manager()
manager._look_for_new_crash_logs(run_results, time.time())
def test_timeout_then_unexpected_pass(self):
test_name = 'failures/expected/text.html'
expectations = test_expectations.TestExpectations(self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 1)
initial_results.add(get_result(test_name, test_expectations.TIMEOUT, run_time=1), False, False)
all_retry_results = [test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1)]
all_retry_results[0].add(get_result(test_name, test_expectations.LEAK, run_time=0.1), False, False)
all_retry_results[1].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
all_retry_results[2].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
summary = test_run_results.summarize_results(
self.port, expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry=True)
self.assertTrue('is_unexpected' in summary['tests']['failures']['expected']['text.html'])
self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL')
self.assertEquals(summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT LEAK PASS PASS')
self.assertEquals(summary['num_passes'], 1)
self.assertEquals(summary['num_regressions'], 0)
self.assertEquals(summary['num_flaky'], 0)
def lint(port, options, logging_stream):
host = port.host
logging.getLogger().setLevel(
logging.DEBUG if options.debug_rwt_logging else logging.INFO)
printer = printing.Printer(port,
options,
logging_stream,
logger=logging.getLogger())
if options.platform:
ports_to_lint = [port]
else:
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names()
]
files_linted = set()
lint_failed = False
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.expectations_dict()
# FIXME: This won't work if multiple ports share a TestExpectations file but support different modifiers in the file.
for expectations_file in expectations_dict.keys():
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_to_lint={
expectations_file: expectations_dict[expectations_file]
})
except test_expectations.ParseError, e:
lint_failed = True
_log.error('')
for warning in e.warnings:
_log.error(warning)
_log.error('')
files_linted.add(expectations_file)
def lint(host, options):
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.all_expectations_dict()
for path in port_to_lint.extra_expectations_files():
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for expectations_file in expectations_dict:
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_dict={
expectations_file: expectations_dict[expectations_file]
},
is_lint_mode=True)
except test_expectations.ParseError as error:
_log.error('')
for warning in error.warnings:
_log.error(warning)
failures.append('%s: %s' % (expectations_file, warning))
_log.error('')
files_linted.add(expectations_file)
return failures
def lint_test_expectations(files, configuration, cwd, increment_error_count=lambda: 0, line_numbers=None, host=Host()):
error_count = 0
files_linted = set()
ports_to_lint = [host.port_factory.get(name) for name in host.port_factory.all_port_names()]
for port in ports_to_lint:
for expectations_file in port.expectations_dict().keys():
style_error_handler = DefaultStyleErrorHandler(expectations_file, configuration, increment_error_count, line_numbers)
try:
if expectations_file in files_linted:
continue
expectations = test_expectations.TestExpectations(
port,
expectations_to_lint={expectations_file: port.expectations_dict()[expectations_file]})
expectations.parse_all_expectations()
except test_expectations.ParseError as e:
for warning in e.warnings:
if TestExpectationsChecker._should_log_linter_warning(warning, files, cwd, host):
style_error_handler(warning.line_number, 'test/expectations', 5, warning.error)
error_count += 1
files_linted.add(expectations_file)
return error_count
def print_expectations(self, args):
aggregate_test_names = set()
aggregate_tests_to_run = set()
aggregate_tests_to_skip = set()
tests_to_run_by_device = {}
device_type_list = self._port.DEFAULT_DEVICE_TYPES or [self._port.DEVICE_TYPE]
for device_type in device_type_list:
"""Run the tests and return a RunDetails object with the results."""
for_device_type = 'for {} '.format(device_type) if device_type else ''
self._printer.write_update('Collecting tests {}...'.format(for_device_type))
try:
paths, test_names = self._collect_tests(args, device_type=device_type)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(exit_code=-1)
self._printer.write_update('Parsing expectations {}...'.format(for_device_type))
self._expectations[device_type] = test_expectations.TestExpectations(self._port, test_names, force_expectations_pass=self._options.force, device_type=device_type)
self._expectations[device_type].parse_all_expectations()
aggregate_test_names.update(test_names)
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names, device_type=device_type)
aggregate_tests_to_skip.update(tests_to_skip)
tests_to_run_by_device[device_type] = [test for test in tests_to_run if test not in aggregate_tests_to_run]
aggregate_tests_to_run.update(tests_to_run)
aggregate_tests_to_skip = aggregate_tests_to_skip - aggregate_tests_to_run
self._printer.print_found(len(aggregate_test_names), len(aggregate_tests_to_run), self._options.repeat_each, self._options.iterations)
test_col_width = len(max(aggregate_tests_to_run.union(aggregate_tests_to_skip), key=len)) + 1
self._print_expectations_for_subset(device_type_list[0], test_col_width, tests_to_run_by_device[device_type_list[0]], aggregate_tests_to_skip)
for device_type in device_type_list[1:]:
self._print_expectations_for_subset(device_type, test_col_width, tests_to_run_by_device[device_type])
return 0
def test_timeout_then_unexpected_pass(self):
tests = ['failures/expected/image.html']
expectations = test_expectations.TestExpectations(self.port, tests)
initial_results = test_run_results.TestRunResults(
expectations, len(tests))
initial_results.add(
get_result('failures/expected/image.html',
test_expectations.TIMEOUT,
run_time=1), False, False)
retry_results = test_run_results.TestRunResults(
expectations, len(tests))
retry_results.add(
get_result('failures/expected/image.html',
test_expectations.PASS,
run_time=0.1), False, False)
summary = test_run_results.summarize_results(
self.port,
expectations,
initial_results,
retry_results,
enabled_pixel_tests_in_retry=True,
only_include_failing=True)
self.assertEquals(summary['num_regressions'], 0)
self.assertEquals(summary['num_passes'], 1)
def run(self, args):
"""Run the tests and return a RunDetails object with the results."""
self._printer.write_update("Collecting tests ...")
try:
paths, test_names = self._collect_tests(args)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(exit_code=-1)
self._printer.write_update("Parsing expectations ...")
self._expectations = test_expectations.TestExpectations(
self._port,
test_names,
force_expectations_pass=self._options.force)
self._expectations.parse_all_expectations()
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._printer.print_found(len(test_names), len(tests_to_run),
self._options.repeat_each,
self._options.iterations)
start_time = time.time()
# Check to make sure we're not skipping every test.
if not tests_to_run:
_log.critical('No tests to run.')
return test_run_results.RunDetails(exit_code=-1)
if not self._set_up_run(tests_to_run):
return test_run_results.RunDetails(exit_code=-1)
enabled_pixel_tests_in_retry = False
try:
initial_results = self._run_tests(
tests_to_run,
tests_to_skip,
self._options.repeat_each,
self._options.iterations,
int(self._options.child_processes),
retrying=False)
tests_to_retry = self._tests_to_retry(
initial_results,
include_crashes=self._port.should_retry_crashes())
# Don't retry failures when interrupted by user or failures limit exception.
retry_failures = self._options.retry_failures and not (
initial_results.interrupted
or initial_results.keyboard_interrupted)
if retry_failures and tests_to_retry:
enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed(
)
_log.info('')
_log.info("Retrying %s ..." %
pluralize(len(tests_to_retry), "unexpected failure"))
_log.info('')
retry_results = self._run_tests(tests_to_retry,
tests_to_skip=set(),
repeat_each=1,
iterations=1,
num_workers=1,
retrying=True)
if enabled_pixel_tests_in_retry:
self._options.pixel_tests = False
else:
retry_results = None
finally:
self._clean_up_run()
end_time = time.time()
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
_log.debug("looking for new crash logs")
self._look_for_new_crash_logs(initial_results, start_time)
if retry_results:
self._look_for_new_crash_logs(retry_results, start_time)
_log.debug("summarizing results")
summarized_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, retry_results,
enabled_pixel_tests_in_retry)
results_including_passes = None
if self._options.results_server_host:
results_including_passes = test_run_results.summarize_results(
self._port,
self._expectations,
initial_results,
retry_results,
enabled_pixel_tests_in_retry,
include_passes=True,
include_time_and_modifiers=True)
self._printer.print_results(end_time - start_time, initial_results,
summarized_results)
exit_code = -1
if not self._options.dry_run:
self._port.print_leaks_summary()
self._upload_json_files(summarized_results, initial_results,
results_including_passes, start_time,
end_time)
results_path = self._filesystem.join(self._results_directory,
"results.html")
self._copy_results_html_file(results_path)
if initial_results.keyboard_interrupted:
exit_code = INTERRUPTED_EXIT_STATUS
else:
if self._options.show_results and (
initial_results.unexpected_results_by_name or
(self._options.full_results_html
and initial_results.total_failures)):
self._port.show_results_html_file(results_path)
exit_code = self._port.exit_code_from_summarized_results(
summarized_results)
return test_run_results.RunDetails(exit_code, summarized_results,
initial_results, retry_results,
enabled_pixel_tests_in_retry)
def run(self, args):
"""Run the tests and return a RunDetails object with the results."""
start_time = time.time()
self._printer.write_update("Collecting tests ...")
running_all_tests = False
try:
paths, test_names, running_all_tests = self._collect_tests(args)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(
exit_code=test_run_results.NO_TESTS_EXIT_STATUS)
self._printer.write_update("Parsing expectations ...")
self._expectations = test_expectations.TestExpectations(
self._port, test_names)
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._printer.print_found(len(test_names), len(tests_to_run),
self._options.repeat_each,
self._options.iterations)
# Check to make sure we're not skipping every test.
if not tests_to_run:
_log.critical('No tests to run.')
return test_run_results.RunDetails(
exit_code=test_run_results.NO_TESTS_EXIT_STATUS)
exit_code = self._set_up_run(tests_to_run)
if exit_code:
return test_run_results.RunDetails(exit_code=exit_code)
# Don't retry failures if an explicit list of tests was passed in.
if self._options.retry_failures is None:
should_retry_failures = len(paths) < len(test_names)
else:
should_retry_failures = self._options.retry_failures
enabled_pixel_tests_in_retry = False
try:
self._start_servers(tests_to_run)
num_workers = self._port.num_workers(
int(self._options.child_processes))
initial_results = self._run_tests(tests_to_run, tests_to_skip,
self._options.repeat_each,
self._options.iterations,
num_workers)
# Don't retry failures when interrupted by user or failures limit exception.
should_retry_failures = should_retry_failures and not (
initial_results.interrupted
or initial_results.keyboard_interrupted)
tests_to_retry = self._tests_to_retry(initial_results)
all_retry_results = []
if should_retry_failures and tests_to_retry:
enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed(
)
for retry_attempt in xrange(1, self._options.num_retries + 1):
if not tests_to_retry:
break
_log.info('')
_log.info(
'Retrying %s, attempt %d of %d...',
grammar.pluralize('unexpected failure',
len(tests_to_retry)), retry_attempt,
self._options.num_retries)
retry_results = self._run_tests(
tests_to_retry,
tests_to_skip=set(),
repeat_each=1,
iterations=1,
num_workers=num_workers,
retry_attempt=retry_attempt)
all_retry_results.append(retry_results)
tests_to_retry = self._tests_to_retry(retry_results)
if enabled_pixel_tests_in_retry:
self._options.pixel_tests = False
finally:
self._stop_servers()
self._clean_up_run()
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
self._printer.write_update("looking for new crash logs")
self._look_for_new_crash_logs(initial_results, start_time)
for retry_attempt_results in all_retry_results:
self._look_for_new_crash_logs(retry_attempt_results, start_time)
_log.debug("summarizing results")
summarized_full_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry)
summarized_failing_results = test_run_results.summarize_results(
self._port,
self._expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry,
only_include_failing=True)
exit_code = summarized_failing_results['num_regressions']
if exit_code > test_run_results.MAX_FAILURES_EXIT_STATUS:
_log.warning('num regressions (%d) exceeds max exit status (%d)',
exit_code, test_run_results.MAX_FAILURES_EXIT_STATUS)
exit_code = test_run_results.MAX_FAILURES_EXIT_STATUS
if not self._options.dry_run:
self._write_json_files(summarized_full_results,
summarized_failing_results, initial_results,
running_all_tests)
if self._options.write_full_results_to:
self._filesystem.copyfile(
self._filesystem.join(self._results_directory,
"full_results.json"),
self._options.write_full_results_to)
self._upload_json_files()
results_path = self._filesystem.join(self._results_directory,
"results.html")
self._copy_results_html_file(results_path)
if initial_results.keyboard_interrupted:
exit_code = test_run_results.INTERRUPTED_EXIT_STATUS
else:
if initial_results.interrupted:
exit_code = test_run_results.EARLY_EXIT_STATUS
if self._options.show_results and (
exit_code or (self._options.full_results_html
and initial_results.total_failures)):
self._port.show_results_html_file(results_path)
self._printer.print_results(time.time() - start_time,
initial_results,
summarized_failing_results)
return test_run_results.RunDetails(exit_code, summarized_full_results,
summarized_failing_results,
initial_results, all_retry_results,
enabled_pixel_tests_in_retry)
def run(self, args):
"""Runs the tests and return a RunDetails object with the results."""
start_time = time.time()
self._printer.write_update('Collecting tests ...')
running_all_tests = False
if not args or any('external' in path for path in args):
self._printer.write_update(
'Generating MANIFEST.json for web-platform-tests ...')
WPTManifest.ensure_manifest(self._port.host)
self._printer.write_update('Completed generating manifest.')
self._printer.write_update('Collecting tests ...')
try:
paths, all_test_names, running_all_tests = self._collect_tests(
args)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(
exit_code=exit_codes.NO_TESTS_EXIT_STATUS)
# Create a sorted list of test files so the subset chunk,
# if used, contains alphabetically consecutive tests.
if self._options.order == 'natural':
all_test_names.sort(key=self._port.test_key)
elif self._options.order == 'random':
all_test_names.sort()
random.Random(self._options.seed).shuffle(all_test_names)
test_names, tests_in_other_chunks = self._finder.split_into_chunks(
all_test_names)
self._printer.write_update('Parsing expectations ...')
self._expectations = test_expectations.TestExpectations(
self._port, test_names)
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._expectations.remove_tests_from_expectations(
tests_in_other_chunks)
self._printer.print_found(len(all_test_names), len(test_names),
len(tests_to_run), self._options.repeat_each,
self._options.iterations)
# Check to make sure we're not skipping every test.
if not tests_to_run:
_log.critical('No tests to run.')
return test_run_results.RunDetails(
exit_code=exit_codes.NO_TESTS_EXIT_STATUS)
exit_code = self._set_up_run(tests_to_run)
if exit_code:
return test_run_results.RunDetails(exit_code=exit_code)
# Don't retry failures if an explicit list of tests was passed in.
if self._options.retry_failures is None:
should_retry_failures = len(paths) < len(test_names)
else:
should_retry_failures = self._options.retry_failures
try:
self._start_servers(tests_to_run)
if self._options.watch:
run_results = self._run_test_loop(tests_to_run, tests_to_skip)
else:
run_results = self._run_test_once(tests_to_run, tests_to_skip,
should_retry_failures)
initial_results, all_retry_results, enabled_pixel_tests_in_retry = run_results
finally:
self._stop_servers()
self._clean_up_run()
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
self._printer.write_update('Looking for new crash logs ...')
self._look_for_new_crash_logs(initial_results, start_time)
for retry_attempt_results in all_retry_results:
self._look_for_new_crash_logs(retry_attempt_results, start_time)
self._printer.write_update('Summarizing results ...')
summarized_full_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry)
summarized_failing_results = test_run_results.summarize_results(
self._port,
self._expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry,
only_include_failing=True)
exit_code = summarized_failing_results['num_regressions']
if exit_code > exit_codes.MAX_FAILURES_EXIT_STATUS:
_log.warning('num regressions (%d) exceeds max exit status (%d)',
exit_code, exit_codes.MAX_FAILURES_EXIT_STATUS)
exit_code = exit_codes.MAX_FAILURES_EXIT_STATUS
if not self._options.dry_run:
self._write_json_files(summarized_full_results,
summarized_failing_results, initial_results,
running_all_tests)
self._upload_json_files()
self._copy_results_html_file(self._results_directory,
'results.html')
self._copy_results_html_file(self._results_directory,
'legacy-results.html')
if initial_results.keyboard_interrupted:
exit_code = exit_codes.INTERRUPTED_EXIT_STATUS
else:
if initial_results.interrupted:
exit_code = exit_codes.EARLY_EXIT_STATUS
if self._options.show_results and (
exit_code or initial_results.total_failures):
self._port.show_results_html_file(
self._filesystem.join(self._results_directory,
'results.html'))
self._printer.print_results(time.time() - start_time,
initial_results)
return test_run_results.RunDetails(exit_code, summarized_full_results,
summarized_failing_results,
initial_results, all_retry_results,
enabled_pixel_tests_in_retry)
