def _setup_host(self):
"""Returns a mock host with fake values set up for testing."""
self.set_logging_level(logging.DEBUG)
host = MockHost()
host.port_factory = MockPortFactory(host)
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Android Weblayer - Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release',
'anDroid', 'android_Weblayer'],
'is_try_builder': True,
},
'MOCK Android Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release', 'anDroid',
'Android_Webview', 'Chrome_Android'],
'is_try_builder': True,
},
})
# Write dummy expectations
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(
path, self._raw_android_expectations)
return host
def _check_test_existence(host, port, path, expectations, wpt_tests):
failures = []
warnings = []
is_android_path = path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values()
for exp in expectations:
if not exp.test:
continue
# TODO(crbug.com/1102901): We currently can't clean up webgpu tests
# since they are not picked up by blinkpy
if exp.test.startswith('external/wpt/webgpu'):
continue
if exp.is_glob:
test_name = exp.test[:-1]
else:
test_name = exp.test
possible_error = "{}:{} Test does not exist: {}".format(
host.filesystem.basename(path), exp.lineno, exp.test)
if is_android_path and test_name not in wpt_tests:
# TODO(crbug.com/1110003): Change this lint back into a failure
# after figuring out how to clean up renamed or deleted generated
# tests from expectations files when wpt_update_expectations.py is
# run with the --clean-up-affected-tests-only command line argument.
warnings.append(possible_error)
_log.warning(possible_error)
elif not is_android_path and not port.test_exists(test_name):
failures.append(possible_error)
_log.error(possible_error)
return failures, warnings
def expectations_files(self):
"""Returns list of expectations files.
Each expectation file in the list will be cleaned of expectations
for tests that were removed and will also have test names renamed
for tests that were renamed. Also the files may have their expectations
updated using builder results.
"""
return (self.port.all_expectations_dict().keys() +
PRODUCTS_TO_EXPECTATION_FILE_PATHS.values())
def _check_expectations(host, port, path, test_expectations, options):
# Check for original expectation lines (from get_updated_lines) instead of
# expectations filtered for the current port (test_expectations).
expectations = test_expectations.get_updated_lines(path)
failures, warnings = _check_test_existence(
host, port, path, expectations)
failures.extend(_check_directory_glob(host, port, path, expectations))
failures.extend(_check_never_fix_tests(host, port, path, expectations))
if path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
failures.extend(_check_non_wpt_in_android_override(
host, port, path, expectations))
# TODO(crbug.com/1080691): Change this to failures once
# wpt_expectations_updater is fixed.
warnings.extend(
_check_redundant_virtual_expectations(host, port, path, expectations))
return failures, warnings
def _setup_host(self):
"""Returns a mock host with fake values set up for testing."""
self.set_logging_level(logging.DEBUG)
host = MockHost()
host.port_factory = MockPortFactory(host)
host.executive._output = ''
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Android Weblayer - Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release',
'anDroid', 'android_Weblayer'],
'is_try_builder': True,
},
'MOCK Android Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release', 'anDroid',
'Android_Webview', 'Chrome_Android'],
'is_try_builder': True,
},
})
host.filesystem.write_text_file(
host.port_factory.get().web_tests_dir() + '/external/' +
BASE_MANIFEST_NAME,
json.dumps({
'items': {
'testharness': {
'ghi.html': ['abcdef123', [None, {}]],
'van.html': ['abcdef123', [None, {}]],
},
},
}))
# Write dummy expectations
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(
path, self._raw_android_expectations)
host.filesystem.write_text_file(
ANDROID_DISABLED_TESTS, self._raw_android_never_fix_tests)
return host
def test_only_wpt_in_android_override_files(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
raw_expectations = ('# results: [ Failure ]\n'
'external/wpt/test.html [ Failure ]\n'
'non-wpt/test.html [ Failure ]\n')
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(path, raw_expectations)
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
port.test_exists = lambda _: True
port.tests = lambda _: {'external/wpt/test.html', 'non-wpt/test.html'}
failures, _ = lint_test_expectations.lint(host, options)
self.assertTrue(all('is for a non WPT test' in f for f in failures))
def _check_test_existence(host, port, path, expectations):
failures = []
warnings = []
if path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
return [], []
for exp in expectations:
if not exp.test:
continue
if exp.is_glob:
test_name = exp.test[:-1]
else:
test_name = exp.test
possible_error = "{}:{} Test does not exist: {}".format(
host.filesystem.basename(path), exp.lineno, exp.test)
if not port.test_exists(test_name):
failures.append(possible_error)
_log.error(possible_error)
return failures, warnings
def _check_test_existence(host, port, path, expectations, wpt_tests):
failures = []
is_android_path = path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values()
for exp in expectations:
if not exp.test:
continue
# TODO(crbug.com/1102901): We currently can't clean up webgpu tests
# since they are not picked up by blinkpy
if exp.test.startswith('external/wpt/webgpu'):
continue
if exp.is_glob:
test_name = exp.test[:-1]
else:
test_name = exp.test
if (is_android_path and test_name not in wpt_tests
or not is_android_path and not port.test_exists(test_name)):
error = "{}:{} Test does not exist: {}".format(
host.filesystem.basename(path), exp.lineno, exp.test)
_log.error(error)
failures.append(error)
return failures
def lint(host, options):
port = host.port_factory.get(options.platform)
# Add all extra expectation files to be linted.
options.additional_expectations.extend(
PRODUCTS_TO_EXPECTATION_FILE_PATHS.values() + [ANDROID_DISABLED_TESTS] + [
host.filesystem.join(port.web_tests_dir(), 'WPTOverrideExpectations'),
host.filesystem.join(port.web_tests_dir(), 'WebGPUExpectations'),
])
ports_to_lint = [
host.port_factory.get(name, options=options)
for name in host.port_factory.all_port_names(options.platform)
]
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
warnings = []
expectations_dict = {}
all_system_specifiers = set()
all_build_specifiers = set(ports_to_lint[0].ALL_BUILD_TYPES)
# TODO(crbug.com/986447) Remove the checks below after migrating the expectations
# parsing to Typ. All the checks below can be handled by Typ.
for port in ports_to_lint:
expectations_dict.update(port.all_expectations_dict())
config_macro_dict = port.configuration_specifier_macros()
if config_macro_dict:
all_system_specifiers.update(
{s.lower()
for s in config_macro_dict.keys()})
all_system_specifiers.update({
s.lower()
for s in reduce(lambda x, y: x + y, config_macro_dict.values())
})
for path in port.extra_expectations_files():
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for path, content in expectations_dict.items():
# Check the expectations file content
failures.extend(_check_expectations_file_content(content))
# Create a TestExpectations instance and see if an exception is raised
try:
test_expectations = TestExpectations(
ports_to_lint[0], expectations_dict={path: content})
# Check each expectation for issues
f, w = _check_expectations(host, ports_to_lint[0], path,
test_expectations, options)
failures += f
warnings += w
except ParseError as error:
_log.error(str(error))
failures.append(str(error))
_log.error('')
return failures, warnings
def mock_host(self):
host = MockHost()
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(path, '')
return host
def lint(host, options):
port = host.port_factory.get(options.platform)
wpt_tests = set(port.tests([host.filesystem.join('external', 'wpt')]))
# Add all extra expectation files to be linted.
options.additional_expectations.extend(
PRODUCTS_TO_EXPECTATION_FILE_PATHS.values() +
[ANDROID_DISABLED_TESTS] + [
host.filesystem.join(port.web_tests_dir(),
'WPTOverrideExpectations'),
host.filesystem.join(port.web_tests_dir(), 'WebGPUExpectations'),
])
ports_to_lint = [
host.port_factory.get(name, options=options)
for name in host.port_factory.all_port_names(options.platform)
]
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
warnings = []
expectations_dict = {}
all_system_specifiers = set()
all_build_specifiers = set(ports_to_lint[0].ALL_BUILD_TYPES)
# TODO(crbug.com/986447) Remove the checks below after migrating the expectations
# parsing to Typ. All the checks below can be handled by Typ.
for port in ports_to_lint:
expectations_dict.update(port.all_expectations_dict())
config_macro_dict = port.configuration_specifier_macros()
if config_macro_dict:
all_system_specifiers.update(
{s.lower()
for s in config_macro_dict.keys()})
all_system_specifiers.update({
s.lower()
for s in reduce(lambda x, y: x + y, config_macro_dict.values())
})
for path in port.extra_expectations_files():
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for path, content in expectations_dict.items():
# Check the expectations file content
failures.extend(_check_expectations_file_content(content))
# Create a TestExpectations instance and see if an exception is raised
try:
test_expectations = TestExpectations(
ports_to_lint[0], expectations_dict={path: content})
# Check each expectation for issues
f, w = _check_expectations(host, ports_to_lint[0], path,
test_expectations, options, wpt_tests)
failures += f
warnings += w
except ParseError as error:
_log.error(str(error))
failures.append(str(error))
_log.error('')
if any('Test does not exist' in f for f in failures):
# TODO(rmhasan): Instead of hardcoding '--android-product=ANDROID_WEBLAYER'
# add a general --android command line argument which will be used to
# put wpt_update_expectations.py into Android mode.
_log.info('')
_log.info(
('If there are expectations for deleted tests in '
'Android WPT override files then clean them by running '
'\'//third_party/blink/tools/wpt_update_expectations.py '
'--android-product=%s --clean-up-test-expectations-only\'') %
ANDROID_WEBLAYER)
_log.info('')
return failures, warnings
def expectations_files(self):
# We need to put all the Android expectation files in
# the _test_expectations member variable so that the
# files get cleaned in cleanup_test_expectations_files()
return (PRODUCTS_TO_EXPECTATION_FILE_PATHS.values() +
[ANDROID_DISABLED_TESTS])
