def test_depot_tools_base_not_found(self):
filesystem = MockFileSystem()
filesystem.path_to_module = lambda _: (
'/mock-checkout/third_party/blink/tools/blinkpy/common/'
'path_finder.py')
finder = PathFinder(filesystem)
self.assertIsNone(finder.depot_tools_base())
def ensure_manifest(host, path=None):
"""Updates the MANIFEST.json file, or generates if it does not exist.
Args:
path: The path to a WPT root (relative to web_tests, optional).
"""
if path is None:
path = host.filesystem.join('external', 'wpt')
finder = PathFinder(host.filesystem)
wpt_path = finder.path_from_web_tests(path)
manifest_path = host.filesystem.join(wpt_path, MANIFEST_NAME)
# TODO(crbug.com/853815): perhaps also cache the manifest for wpt_internal.
if 'external' in path:
base_manifest_path = finder.path_from_web_tests(
'external', BASE_MANIFEST_NAME)
if not host.filesystem.exists(base_manifest_path):
_log.error('Manifest base not found at "%s".',
base_manifest_path)
host.filesystem.write_text_file(base_manifest_path, '{}')
# Unconditionally replace MANIFEST.json with the base manifest even if
# the former exists, to avoid regenerating the manifest from scratch
# when the manifest version changes. Remove the destination first as
# copyfile will fail if the two files are hardlinked or symlinked.
if host.filesystem.exists(manifest_path):
host.filesystem.remove(manifest_path)
host.filesystem.copyfile(base_manifest_path, manifest_path)
WPTManifest.generate_manifest(host, wpt_path)
_log.debug('Manifest generation completed.')
def __init__(self, host, wpt_github=None, wpt_manifests=None):
self.host = host
self.wpt_github = wpt_github
self.port = host.port_factory.get()
self.executive = host.executive
self.fs = host.filesystem
self.finder = PathFinder(self.fs)
self.chromium_git = self.host.git(self.finder.chromium_base())
self.dest_path = self.finder.path_from_web_tests('external', 'wpt')
# A common.net.git_cl.GitCL instance.
self.git_cl = None
# Another Git instance with local WPT as CWD, which can only be
# instantiated after the working directory is created.
self.wpt_git = None
# The WPT revision we are importing and the one imported last time.
self.wpt_revision = None
self.last_wpt_revision = None
# A set of rebaselined tests and a dictionary of new test expectations
# mapping failing tests to platforms to
# wpt_expectations_updater.SimpleTestResult.
self.rebaselined_tests = set()
self.new_test_expectations = {}
self.verbose = False
args = ['--clean-up-affected-tests-only',
'--clean-up-test-expectations']
self._expectations_updater = WPTExpectationsUpdater(
self.host, args, wpt_manifests)
def __init__(self, host):
self.host = host
self.port = self.host.port_factory.get()
self.git_cl = GitCL(host)
self.finder = PathFinder(self.host.filesystem)
self.ports_with_no_results = set()
self.ports_with_all_pass = set()
def __init__(self, host, args=None, wpt_manifests=None):
self.host = host
self.port = self.host.port_factory.get()
self.finder = PathFinder(self.host.filesystem)
self.git_cl = GitCL(host)
self.git = self.host.git(self.finder.chromium_base())
self.configs_with_no_results = []
self.patchset = None
self.wpt_manifests = (
wpt_manifests or
[self.port.wpt_manifest(d) for d in self.port.WPT_DIRS])
# Get options from command line arguments.
parser = argparse.ArgumentParser(description=__doc__)
self.add_arguments(parser)
self.options = parser.parse_args(args or [])
if not (self.options.clean_up_test_expectations or
self.options.clean_up_test_expectations_only):
assert not self.options.clean_up_affected_tests_only, (
'Cannot use --clean-up-affected-tests-only without using '
'--clean-up-test-expectations or '
'--clean-up-test-expectations-only')
# Set up TestExpectations instance which contains all
# expectations files associated with the platform.
expectations_dict = {p: self.host.filesystem.read_text_file(p)
for p in self.expectations_files()}
self._test_expectations = TestExpectations(
self.port, expectations_dict=expectations_dict)
def _read_configuration_from_gn(fs, options):
"""Returns the configuration to used based on args.gn, if possible."""
build_directory = getattr(options, 'build_directory', 'out')
target = options.target
finder = PathFinder(fs)
path = fs.join(finder.chromium_base(), build_directory, target, 'args.gn')
if not fs.exists(path):
path = fs.join(finder.chromium_base(), build_directory, target,
'toolchain.ninja')
if not fs.exists(path):
# This does not appear to be a GN-based build directory, so we don't know
# how to interpret it.
return None
# toolchain.ninja exists, but args.gn does not; this can happen when
# `gn gen` is run with no --args.
return 'Debug'
args = fs.read_text_file(path)
for line in args.splitlines():
if re.match(r'^\s*is_debug\s*=\s*false(\s*$|\s*#.*$)', line):
return 'Release'
# If is_debug is set to anything other than false, or if it
# does not exist at all, we should use the default value (True).
return 'Debug'
def __init__(self, port_obj, output_dir):
super(PyWebSocket, self).__init__(port_obj, output_dir)
self._name = 'pywebsocket'
self._log_prefixes = (_WS_LOG_PREFIX,)
self._mappings = [{'port': _DEFAULT_WS_PORT}]
self._pid_file = self._filesystem.join(self._runtime_path, '%s.pid' % self._name)
self._port = _DEFAULT_WS_PORT
self._web_tests = self._port_obj.web_tests_dir()
self._web_socket_tests = self._filesystem.join(self._web_tests, 'http', 'tests', 'websocket')
time_str = time.strftime('%d%b%Y-%H%M%S')
log_file_name = _WS_LOG_PREFIX + time_str
self._error_log = self._filesystem.join(self._output_dir, log_file_name + '-err.txt')
pywebsocket_base = PathFinder(self._filesystem).path_from_chromium_base('third_party', 'pywebsocket', 'src')
pywebsocket_script = self._filesystem.join(pywebsocket_base, 'mod_pywebsocket', 'standalone.py')
self._start_cmd = [
sys.executable, '-u', pywebsocket_script,
'--server-host', 'localhost',
'--port', str(self._port),
'--document-root', self._web_socket_tests,
'--scan-dir', self._web_socket_tests,
'--cgi-paths', '/',
'--log-file', self._error_log,
'--websock-handlers-map-file', self._filesystem.join(self._web_socket_tests, 'handler_map.txt'),
]
# TODO(burnik): Check if this is really needed (and why). If not, just set PYTHONPATH.
self._env = self._port_obj.setup_environ_for_server()
self._env['PYTHONPATH'] = (pywebsocket_base + os.pathsep + self._env.get('PYTHONPATH', ''))
def __init__(self, host):
self._host = host
finder = PathFinder(host.filesystem)
luci_auth_bin = 'luci-auth.bat' if host.platform.is_win(
) else 'luci-auth'
self._luci_auth_path = host.filesystem.join(finder.depot_tools_base(),
luci_auth_bin)
def __init__(self, port, options, printer):
"""Initializes test runner data structures.
Args:
port: An object implementing platform-specific functionality.
options: An options argument which contains command line options.
printer: A Printer object to record updates to.
"""
self._port = port
self._filesystem = port.host.filesystem
self._options = options
self._printer = printer
self._expectations = None
self._http_server_started = False
self._wptserve_started = False
self._websockets_server_started = False
self._results_directory = self._port.results_directory()
self._artifacts_directory = self._port.artifacts_directory()
self._finder = WebTestFinder(self._port, self._options)
self._path_finder = PathFinder(port.host.filesystem)
self._sink = CreateTestResultSink(self._port)
self._runner = WebTestRunner(self._options, self._port, self._printer,
self._results_directory,
self._test_is_slow, self._sink)
def test_update(self):
host = MockHost()
filesystem = host.filesystem
finder = PathFinder(filesystem)
flag_expectations_file = finder.path_from_web_tests(
'FlagExpectations', 'foo')
filesystem.write_text_file(
flag_expectations_file,
'something/pass-unexpectedly-mac.html [ Fail ]')
self._setup_mock_results(host.buildbot)
cmd = ['update', '--flag=--foo']
TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
def results_url(build):
return '%s/%s/%s/%s/layout-test-results/results.html' % (
'https://test-results.appspot.com/data/layout_results',
build.builder_name, build.build_number,
'webkit_layout_tests%20%28with%20patch%29')
self.assertEqual(
host.stdout.getvalue(), '\n'.join([
'Fetching results...',
'-- Linux: %s' % results_url(self.linux_build),
'-- Mac: %s' % results_url(self.mac_build),
'-- Win: %s' % results_url(self.win_build), '',
'### 1 unexpected passes:', '',
'Bug(none) [ Mac ] something/pass-unexpectedly-mac.html [ Pass ]',
'', '### 2 unexpected failures:', '',
'Bug(none) something/fail-everywhere.html [ Failure ]',
'Bug(none) [ Linux Win ] something/fail-win-and-linux.html [ Failure ]',
''
]))
def _assert_optimization(self, results_by_directory, directory_to_new_results, baseline_dirname=''):
layout_tests_dir = PathFinder(self.fs).layout_tests_dir()
test_name = 'mock-test.html'
baseline_name = 'mock-test-expected.txt'
self.fs.write_text_file(
self.fs.join(layout_tests_dir, 'VirtualTestSuites'),
'[{"prefix": "gpu", "base": "fast/canvas", "args": ["--foo"]}]')
for dirname, contents in results_by_directory.items():
self.fs.write_binary_file(self.fs.join(layout_tests_dir, dirname, baseline_name), contents)
baseline_optimizer = BaselineOptimizer(self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names())
self.assertTrue(baseline_optimizer.optimize(
self.fs.join(baseline_dirname, test_name), 'txt'))
for dirname, contents in directory_to_new_results.items():
path = self.fs.join(layout_tests_dir, dirname, baseline_name)
if contents is None:
# Check files that are explicitly marked as absent.
self.assertFalse(self.fs.exists(path), '%s should not exist after optimization' % path)
else:
self.assertEqual(self.fs.read_binary_file(path), contents, 'Content of %s != "%s"' % (path, contents))
for dirname in results_by_directory:
path = self.fs.join(layout_tests_dir, dirname, baseline_name)
if dirname not in directory_to_new_results or directory_to_new_results[dirname] is None:
self.assertFalse(self.fs.exists(path), '%s should not exist after optimization' % path)
def __init__(self, host, source_repo_path):
"""Initializes variables to prepare for copying and converting files.
Args:
host: An instance of Host.
source_repo_path: Path to the local checkout of web-platform-tests.
"""
self.host = host
assert self.host.filesystem.exists(source_repo_path)
self.source_repo_path = source_repo_path
self.filesystem = self.host.filesystem
self.path_finder = PathFinder(self.filesystem)
self.layout_tests_dir = self.path_finder.layout_tests_dir()
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir, DEST_DIR_NAME,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (
self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
# This is just a FYI list of CSS properties that still need to be prefixed,
# which may be output after importing.
self._prefixed_properties = {}
def __init__(self, port_obj, output_dir):
super(WPTServe, self).__init__(port_obj, output_dir)
# These ports must match wpt_support/wpt.config.json
http_port, http_alt_port, https_port = (8001, 8081, 8444)
ws_port, wss_port = (9001, 9444)
self._name = 'wptserve'
self._log_prefixes = ('access_log', 'error_log')
self._mappings = [{
'port': http_port
}, {
'port': http_alt_port
}, {
'port': https_port,
'sslcert': True
}, {
'port': ws_port
}, {
'port': wss_port,
'sslcert': True
}]
# TODO(burnik): We can probably avoid PID files for WPT in the future.
fs = self._filesystem
self._pid_file = fs.join(self._runtime_path, '%s.pid' % self._name)
finder = PathFinder(fs)
path_to_pywebsocket = finder.path_from_chromium_base(
'third_party', 'pywebsocket', 'src')
path_to_wpt_support = finder.path_from_blink_tools(
'blinkpy', 'third_party', 'wpt')
path_to_wpt_root = fs.join(path_to_wpt_support, 'wpt')
path_to_wpt_tests = fs.abspath(
fs.join(self._port_obj.layout_tests_dir(), 'external', 'wpt'))
path_to_ws_handlers = fs.join(path_to_wpt_tests, 'websockets',
'handlers')
self._config_file = self._prepare_wptserve_config(path_to_wpt_support)
wpt_script = fs.join(path_to_wpt_root, 'wpt')
start_cmd = [
self._port_obj.host.executable, '-u', wpt_script, 'serve',
'--config', self._config_file, '--doc_root', path_to_wpt_tests
]
# TODO(burnik): Merge with default start_cmd once we roll in websockets.
if self._port_obj.host.filesystem.exists(path_to_ws_handlers):
start_cmd += ['--ws_doc_root', path_to_ws_handlers]
self._stdout = self._stderr = self._executive.DEVNULL
# TODO(burnik): We should stop setting the CWD once WPT can be run without it.
self._cwd = path_to_wpt_root
self._env = port_obj.host.environ.copy()
self._env.update({'PYTHONPATH': path_to_pywebsocket})
self._start_cmd = start_cmd
expiration_date = datetime.date(2025, 1, 4)
if datetime.date.today() > expiration_date - datetime.timedelta(30):
_log.error(
'Pre-generated keys and certificates are going to be expired at %s.'
' Please re-generate them by following steps in %s/README.chromium.',
expiration_date.strftime('%b %d %Y'), path_to_wpt_support)
def _add_base_manifest_to_mock_filesystem(self, filesystem):
path_finder = PathFinder(filesystem)
external_dir = path_finder.path_from_web_tests('external')
filesystem.maybe_make_directory(filesystem.join(external_dir, 'wpt'))
manifest_base_path = filesystem.join(external_dir, BASE_MANIFEST_NAME)
filesystem.files[manifest_base_path] = '{"manifest": "base"}'
def test_strip_web_tests_path(self):
finder = PathFinder(MockFileSystem())
path_with_web_tests = '/mock-checkout/' + RELATIVE_WEB_TESTS + 'external/wpt'
self.assertEqual(finder.strip_web_tests_path(path_with_web_tests),
'external/wpt')
path_without_web_tests = '/checkout/' + RELATIVE_WEB_TESTS + 'external/wpt'
self.assertEqual(finder.strip_web_tests_path(path_without_web_tests),
path_without_web_tests)
def test_layout_tests_dir_with_backslash_sep(self):
filesystem = MockFileSystem()
filesystem.sep = '\\'
filesystem.path_to_module = lambda _: (
'C:\\mock-checkout\\third_party\\blink\\tools\\blinkpy\\foo.py')
finder = PathFinder(filesystem)
self.assertEqual(finder.layout_tests_dir(),
'C:\\mock-checkout\\third_party\\blink\\web_tests')
def __init__(self, argv, host, git_cl):
self._args = parse_args(argv)
self._host = host
self._git_cl = git_cl
self._expectations = []
self._filesystem = self._host.filesystem
self._path_finder = PathFinder(self._filesystem)
self._git = self._host.git()
def get_port(self, target=None, configuration=None, files=None):
host = MockHost()
finder = PathFinder(host.filesystem)
files = files or {}
for path, contents in files.items():
host.filesystem.write_text_file(finder.path_from_chromium_base(path), contents)
options = optparse.Values({'target': target, 'configuration': configuration})
return factory.PortFactory(host).get(options=options)
def _add_base_manifest_to_mock_filesystem(self, filesystem):
path_finder = PathFinder(filesystem)
external_dir = path_finder.path_from_layout_tests('external')
filesystem.maybe_make_directory(filesystem.join(external_dir, 'wpt'))
manifest_base_path = filesystem.join(external_dir,
'WPT_BASE_MANIFEST.json')
filesystem.files[manifest_base_path] = '{}'
def __init__(self, host, chromium_git, local_wpt):
self.host = host
self.git = chromium_git
self.local_wpt = local_wpt
self.default_port = host.port_factory.get()
self.finder = PathFinder(host.filesystem)
self.owners_extractor = DirectoryOwnersExtractor(host.filesystem)
self.new_failures_by_directory = defaultdict(list)
def test_depot_tools_base_exists(self):
filesystem = MockFileSystem()
filesystem.path_to_module = lambda _: (
'/checkout/third_party/blink/tools/blinkpy/common/'
'path_finder.py')
filesystem.maybe_make_directory('/checkout/third_party/depot_tools')
finder = PathFinder(filesystem)
self.assertEqual(finder.depot_tools_base(),
'/checkout/third_party/depot_tools')
def _add_base_manifest_to_mock_filesystem(self, filesystem):
path_finder = PathFinder(filesystem)
external_dir = path_finder.path_from_layout_tests('external')
filesystem.maybe_make_directory(filesystem.join(external_dir, 'wpt'))
# This filename should match the constant BASE_MANIFEST_NAME.
manifest_base_path = filesystem.join(external_dir, 'WPT_BASE_MANIFEST_5.json')
filesystem.files[manifest_base_path] = '{"manifest": "base"}'
def test_strip_webdriver_tests_path(self):
finder = PathFinder(MockFileSystem())
path_with_webdriver_prefix = 'external/wpt/webdriver/' + 'foo/bar.py>>test'
self.assertEqual(
finder.strip_webdriver_tests_path(path_with_webdriver_prefix),
'foo/bar.py>>test')
path_without_webdriver_prefix = 'external/wpt' + 'bar/foo.py>>test'
self.assertEqual(
finder.strip_webdriver_tests_path(path_without_webdriver_prefix),
path_without_webdriver_prefix)
def __init__(self, argv, host, git_cl):
self._args = parse_args(argv)
self._host = host
self._git_cl = git_cl
self._expectations_model = TestExpectationsModel()
self._test_configuration_converter = TestConfigurationConverter(
set(BUILDER_CONFIGS.values()))
self._filesystem = self._host.filesystem
self._path_finder = PathFinder(self._filesystem)
self._git = self._host.git()
def generate_manifest(host, dest_path):
"""Generates MANIFEST.json on the specified directory."""
finder = PathFinder(host.filesystem)
wpt_exec_path = finder.path_from_blink_tools('blinkpy', 'third_party', 'wpt', 'wpt', 'wpt')
cmd = ['python', wpt_exec_path, 'manifest', '--no-download', '--tests-root', dest_path]
# ScriptError will be raised if the command fails.
host.executive.run_command(
cmd,
return_stderr=True # This will also include stderr in the exception message.
)
def _assert_reftest_optimization(self,
results_by_directory,
directory_to_new_results,
test_path='',
baseline_dirname=''):
web_tests_dir = PathFinder(self.fs).web_tests_dir()
self.fs.write_text_file(
self.fs.join(web_tests_dir, test_path, 'mock-test-expected.html'),
'ref')
self._assert_optimization(results_by_directory,
directory_to_new_results,
baseline_dirname,
suffix='png')
def __init__(self, host, args=None):
self.host = host
self.port = self.host.port_factory.get()
self.git_cl = GitCL(host)
self.finder = PathFinder(self.host.filesystem)
self.configs_with_no_results = []
self.configs_with_all_pass = []
self.patchset = None
# Get options from command line arguments.
parser = argparse.ArgumentParser(description=__doc__)
self.add_arguments(parser)
self.options = parser.parse_args(args or [])
def test_update_irrelevant_unexpected_pass(self):
host = MockHost()
filesystem = host.filesystem
finder = PathFinder(filesystem)
flag_expectations_file = finder.path_from_web_tests(
'FlagExpectations', 'foo')
self._setup_mock_results(host.buildbot)
cmd = ['update', '--flag=--foo']
# Unexpected passes that don't have flag-specific failure expectations
# should not be reported.
filesystem.write_text_file(flag_expectations_file, '')
TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
self.assertTrue('### 0 unexpected passes' in host.stdout.getvalue())
def __init__(self, host, chromium_git, local_wpt):
self.host = host
self.git = chromium_git
self.local_wpt = local_wpt
self._monorail_api = MonorailAPI
self.default_port = host.port_factory.get()
self.finder = PathFinder(host.filesystem)
self.owners_extractor = DirectoryOwnersExtractor(host)
self.new_failures_by_directory = defaultdict(list)
self.components_for_product = {ANDROID_WEBLAYER: ["Internals>WebLayer"]}
self.labels_for_product = {
ANDROID_WEBLAYER: ["Project-WebLayer-WebPlatformSupport", "WL-WPT-Compat"]
}
def _run_trigger_test(self, regenerate):
host = MockHost()
git = host.git()
git_cl = MockGitCL(host)
finder = PathFinder(host.filesystem)
flag_file = finder.path_from_layout_tests(
'additional-driver-flag.setting')
flag_expectations_file = finder.path_from_layout_tests(
'FlagExpectations', 'foo')
cmd = ['trigger', '--flag=--foo']
if regenerate:
cmd.append('--regenerate')
TryFlag(cmd, host, git_cl).run()
expected_added_paths = {flag_file}
expected_commits = [[
'Flag try job: force --foo for run_web_tests.py.'
]]
if regenerate:
expected_added_paths.add(flag_expectations_file)
expected_commits.append(
['Flag try job: clear expectations for --foo.'])
self.assertEqual(git.added_paths, expected_added_paths)
self.assertEqual(git.local_commits(), expected_commits)
self.assertEqual(
git_cl.calls,
[[
'git', 'cl', 'upload', '--bypass-hooks', '-f', '-m',
'Flag try job for --foo.'
],
[
'git', 'cl', 'try', '-B', 'luci.chromium.try', '-b',
'linux_chromium_rel_ng'
],
[
'git', 'cl', 'try', '-B', 'master.tryserver.chromium.mac',
'-b', 'mac_chromium_rel_ng'
],
[
'git', 'cl', 'try', '-B', 'master.tryserver.chromium.win',
'-b', 'win7_chromium_rel_ng'
]])