def __init__(self, repository_directory, host, options):
self._options = options
self._host = host
self._filesystem = host.filesystem
self._test_suites = []
self.repository_directory = repository_directory
self.test_repositories = self.load_test_repositories()
self.paths_to_skip = []
self.paths_to_import = []
for test_repository in self.test_repositories:
self.paths_to_skip.extend(
[self._filesystem.join(test_repository["name"], path) for path in test_repository["paths_to_skip"]]
)
self.paths_to_import.extend(
[self._filesystem.join(test_repository["name"], path) for path in test_repository["paths_to_import"]]
)
if not self._options.import_all:
webkit_finder = WebKitFinder(self._filesystem)
import_expectations_path = webkit_finder.path_from_webkit_base(
"LayoutTests", "imported", "w3c", "resources", "ImportExpectations"
)
self._init_paths_from_expectations(import_expectations_path)
def _read_configuration_from_gn(fs, options):
"""Return the configuration to used based on args.gn, if possible."""
# We should really default to 'out' everywhere at this point, but
# that's a separate cleanup CL.
build_directory = getattr(options, 'build_directory', None) or 'out'
target = options.target
finder = WebKitFinder(fs)
path = fs.join(finder.chromium_base(), build_directory, target, 'args.gn')
if not fs.exists(path):
path = fs.join(finder.chromium_base(), build_directory, target, 'toolchain.ninja')
if not fs.exists(path):
# This does not appear to be a GN-based build directory, so we don't know
# how to interpret it.
return None
# toolchain.ninja exists, but args.gn does not; this can happen when
# `gn gen` is run with no --args.
return 'Debug'
args = fs.read_text_file(path)
for l in args.splitlines():
m = re.match('^\s*is_debug\s*=\s*false(\s*$|\s*#.*$)', l)
if m:
return 'Release'
# if is_debug is set to anything other than false, or if it
# does not exist at all, we should use the default value (True).
return 'Debug'
def _test_prefix_list(self, issue_number, only_changed_tests):
"""Returns a collection of test, builder and file extensions to get new baselines for.
Args:
issue_number: The CL number of the change which needs new baselines.
only_changed_tests: Whether to only include baselines for tests that
are changed in this CL. If False, all new baselines for failing
tests will be downloaded, even for tests that were not modified.
Returns:
A dict containing information about which new baselines to download.
"""
builds_to_tests = self._builds_to_tests(issue_number)
if only_changed_tests:
files_in_cl = self.rietveld.changed_files(issue_number)
finder = WebKitFinder(self._tool.filesystem)
tests_in_cl = [finder.layout_test_name(f) for f in files_in_cl]
result = {}
for build, tests in builds_to_tests.iteritems():
for test in tests:
if only_changed_tests and test not in tests_in_cl:
continue
if test not in result:
result[test] = {}
result[test][build] = BASELINE_SUFFIX_LIST
return result
def __init__(self, host, source_directory, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
self.tests_w3c_relative_path = self.filesystem.join('imported', 'w3c')
self.layout_tests_path = webkit_finder.path_from_webkit_base('LayoutTests')
self.layout_tests_w3c_path = self.filesystem.join(self.layout_tests_path, self.tests_w3c_relative_path)
self.tests_download_path = webkit_finder.path_from_webkit_base('WebKitBuild', 'w3c-tests')
self._test_downloader = None
self._potential_test_resource_files = []
self.import_list = []
self._importing_downloaded_tests = source_directory is None
self._test_resource_files_json_path = self.filesystem.join(self.layout_tests_w3c_path, "resources", "resource-files.json")
self._test_resource_files = json.loads(self.filesystem.read_text_file(self._test_resource_files_json_path)) if self.filesystem.exists(self._test_resource_files_json_path) else None
self._tests_options_json_path = self.filesystem.join(self.layout_tests_path, 'tests-options.json')
self._tests_options = json.loads(self.filesystem.read_text_file(self._tests_options_json_path)) if self.filesystem.exists(self._tests_options_json_path) else None
self._slow_tests = []
if self.options.clean_destination_directory and self._test_resource_files:
self._test_resource_files["files"] = []
if self._tests_options:
self.remove_slow_from_w3c_tests_options()
def load_test_repositories():
filesystem = FileSystem()
webkit_finder = WebKitFinder(filesystem)
test_repositories_path = webkit_finder.path_from_webkit_base(
"LayoutTests", "imported", "w3c", "resources", "TestRepositories"
)
return json.loads(filesystem.read_text_file(test_repositories_path))
def get_port(self, target=None, configuration=None, files=None):
host = MockSystemHost()
wkf = WebKitFinder(host.filesystem)
files = files or {}
for path, contents in files.items():
host.filesystem.write_text_file(wkf.path_from_chromium_base(path), contents)
options = MockOptions(target=target, configuration=configuration)
return factory.PortFactory(host).get(options=options)
def get_port(self, target=None, configuration=None, files=None):
host = MockHost()
wkf = WebKitFinder(host.filesystem)
files = files or {}
for path, contents in files.items():
host.filesystem.write_text_file(wkf.path_from_chromium_base(path), contents)
options = optparse.Values({'target': target, 'configuration': configuration})
return factory.PortFactory(host).get(options=options)
def _run_pylint(self, path):
wkf = WebKitFinder(FileSystem())
executive = Executive()
return executive.run_command([sys.executable, wkf.path_from_depot_tools_base('pylint.py'),
'--output-format=parseable',
'--errors-only',
'--rcfile=' + wkf.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'pylintrc'),
path],
error_handler=executive.ignore_error)
def main():
filesystem = FileSystem()
wkf = WebKitFinder(filesystem)
tester = Tester(filesystem, wkf)
tester.add_tree(wkf.path_from_webkit_base('tools'), 'webkitpy')
tester.skip(('webkitpy.common.checkout.scm.scm_unittest',), 'are really, really, slow', 31818)
if sys.platform == 'win32':
tester.skip(('webkitpy.common.checkout', 'webkitpy.common.config', 'webkitpy.tool', 'webkitpy.w3c', 'webkitpy.layout_tests.layout_package.bot_test_expectations'), 'fail horribly on win32', 54526)
return not tester.run()
def _determine_driver_path_statically(cls, host, options):
config_object = config.Config(host.executive, host.filesystem)
build_directory = getattr(options, "build_directory", None)
finder = WebKitFinder(host.filesystem)
webkit_base = finder.webkit_base()
chromium_base = finder.chromium_base()
driver_name = cls.SKY_SHELL_NAME
if hasattr(options, "configuration") and options.configuration:
configuration = options.configuration
else:
configuration = config_object.default_configuration()
return cls._static_build_path(host.filesystem, build_directory, chromium_base, configuration, [driver_name])
def __init__(self, host, source_directory, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
self.import_list = []
def _run_pylint(self, path):
wkf = WebKitFinder(FileSystem())
executive = Executive()
env = os.environ.copy()
env['PYTHONPATH'] = ('%s%s%s' % (wkf.path_from_webkit_base('Tools', 'Scripts'),
os.pathsep,
wkf.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'thirdparty')))
return executive.run_command([sys.executable, wkf.path_from_depot_tools_base('pylint.py'),
'--output-format=parseable',
'--errors-only',
'--rcfile=' + wkf.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'pylintrc'),
path],
env=env,
error_handler=executive.ignore_error)
def __init__(self, host, source_directory, repo_dir, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.repo_dir = repo_dir
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
self.changeset = CHANGESET_NOT_AVAILABLE
self.import_list = []
def __init__(self, host):
self.host = host
self.executive = host.executive
self.fs = host.filesystem
self.finder = WebKitFinder(self.fs)
self.verbose = False
self.git_cl = None
def __init__(self, host, source_directory, repo_dir, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.repo_dir = self.filesystem.abspath(repo_dir)
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination, self.filesystem.basename(self.repo_dir))
self.changeset = CHANGESET_NOT_AVAILABLE
self.test_status = TEST_STATUS_UNKNOWN
self.import_list = []
def __init__(self, host):
self.host = host
self.executive = host.executive
self.fs = host.filesystem
self.finder = WebKitFinder(self.fs)
self.verbose = False
self.allow_local_blink_commits = False
self.keep_w3c_repos_around = False
def __init__(self, host, source_directory, repo_dir, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.repo_dir = repo_dir
self.destination_directory = webkit_finder.path_from_webkit_base(
"LayoutTests", options.destination)
self.changeset = CHANGESET_NOT_AVAILABLE
self.test_status = TEST_STATUS_UNKNOWN
self.import_list = []
def __init__(self,
port,
host,
script_name,
options,
downloader_class=TestDownloader,
create_webdriver_func=create_webdriver,
spawn_wpt_func=spawn_wpt):
self._port = port
self._host = host
self._finder = WebKitFinder(self._host.filesystem)
self._script_name = script_name
self._options = options
self._downloader_class = downloader_class
self._create_webdriver_func = create_webdriver_func
self._spawn_wpt_func = spawn_wpt_func
def __init__(self):
self._host = Host()
self._filesystem = self._host.filesystem
self._webkit_root = WebKitFinder(self._filesystem).webkit_base()
# These settings might vary between WebKit and Blink
self._css_property_file = self.path_from_webkit_root('Source', 'WebCore', 'css', 'CSSPropertyNames.in')
self._css_property_split_string = '='
self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
def __init__(self, host, source_repo_path, options):
self.host = host
self.source_repo_path = source_repo_path
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base(
'LayoutTests')
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir, options.destination,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (
self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
def __init__(self, port, driver, env, expectations):
self._port = port
self._driver = driver
self._env = env
self._expectations = expectations
self._results = []
self._tests_dir = WebKitFinder(
self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
self._server = WebDriverW3CWebServer(self._port)
def __init__(self, host, dir_to_import, top_of_repo, options):
self.host = host
self.dir_to_import = dir_to_import
self.top_of_repo = top_of_repo
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base(
'LayoutTests')
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(self.layout_tests_dir, options.destination,
self.filesystem.basename(self.top_of_repo)))
self.import_in_place = (
self.dir_to_import == self.destination_directory)
self.changeset = CHANGESET_NOT_AVAILABLE
self.import_list = []
def __init__(self, repository_directory, host, options):
self._options = options
self._host = host
self._filesystem = host.filesystem
self._test_suites = []
self.repository_directory = repository_directory
self.test_repositories = self.load_test_repositories(self._filesystem)
self.paths_to_skip = []
self.paths_to_import = []
for test_repository in self.test_repositories:
self.paths_to_skip.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_skip']])
self.paths_to_import.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_import']])
if not self._options.import_all:
webkit_finder = WebKitFinder(self._filesystem)
import_expectations_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'ImportExpectations')
self._init_paths_from_expectations(import_expectations_path)
def start(self):
assert not self._pid, '%s server is already running' % self._name
# Stop any stale servers left over from previous instances.
if self._port.host.filesystem.exists(self._pid_file):
try:
self._pid = int(
self._port.host.filesystem.read_text_file(self._pid_file))
self.stop()
except (ValueError, UnicodeDecodeError):
# These could be raised if the pid file is corrupt.
self._port.host.filesystem.remove(self._pid_file)
self._pid = None
_log.debug('Copying WebDriver WPT server config.json')
doc_root = os.path.join(
WebKitFinder(self._port.host.filesystem).path_from_webkit_base(
'WebDriverTests'), 'imported', 'w3c')
config_filename = os.path.join(doc_root, 'config.json')
config_json = self._port.host.filesystem.read_text_file(
config_filename).replace("%DOC_ROOT%", doc_root)
self._port.host.filesystem.write_text_file(
os.path.join(self._layout_doc_root, 'config.json'), config_json)
config = json.loads(config_json)
self._server_host = config['browser_host']
self._server_port = config['ports']['http'][0]
self._wsout = self._port.host.filesystem.open_text_file_for_writing(
self._output_log_path)
wpt_file = os.path.join(self._layout_doc_root, "wpt.py")
cmd = [
"python", wpt_file, "serve", "--config",
os.path.join(self._layout_doc_root, 'config.json')
]
self._process = self._port._executive.popen(
cmd,
cwd=self._layout_doc_root,
shell=False,
stdin=self._port._executive.PIPE,
stdout=self._wsout,
stderr=self._wsout)
self._pid = self._process.pid
self._port.host.filesystem.write_text_file(self._pid_file,
str(self._pid))
if not self._wait_for_server():
_log.error(
'WPT Server process exited prematurely with status code %s' %
self._process.returncode)
self.stop()
raise RuntimeError
_log.info('WebDriver WPT server listening at http://%s:%s/' %
(self._server_host, self._server_port))
def __init__(self, repository_directory, host, options):
self._options = options
self._host = host
self._filesystem = host.filesystem
self._test_suites = []
self.repository_directory = repository_directory
self.test_repositories = self.load_test_repositories(self._filesystem)
self.paths_to_skip = []
self.paths_to_import = []
for test_repository in self.test_repositories:
self.paths_to_skip.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_skip']])
self.paths_to_import.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_import']])
if not self._options.import_all:
webkit_finder = WebKitFinder(self._filesystem)
import_expectations_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'ImportExpectations')
self._init_paths_from_expectations(import_expectations_path)
def __init__(self, host, source_directory, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
self.tests_w3c_relative_path = self.filesystem.join("imported", "w3c")
self.layout_tests_w3c_path = webkit_finder.path_from_webkit_base("LayoutTests", self.tests_w3c_relative_path)
self.tests_download_path = webkit_finder.path_from_webkit_base("WebKitBuild", "w3c-tests")
self._test_downloader = None
self._potential_test_resource_files = []
self.import_list = []
self._importing_downloaded_tests = source_directory is None
def create_patch(self, git_commit=None, changed_files=None, git_index=None):
"""Returns a byte array (str()) representing the patch file.
Patch files are effectively binary since they may contain
files of multiple different encodings."""
if changed_files == []:
return ""
elif changed_files == None:
changed_files = []
script_path = WebKitFinder(self._filesystem).path_from_webkit_base("Tools", "Scripts", "svn-create-patch")
return self.run([script_path, "--no-style"] + changed_files,
cwd=self.checkout_root, return_stderr=False,
decode_output=False)
def _determine_driver_path_statically(cls, host, options):
config_object = config.Config(host.executive, host.filesystem)
build_directory = getattr(options, 'build_directory', None)
webkit_base = WebKitFinder(host.filesystem).webkit_base()
chromium_base = cls._chromium_base_dir(host.filesystem)
if hasattr(options, 'configuration') and options.configuration:
configuration = options.configuration
else:
configuration = config_object.default_configuration()
return cls._static_build_path(host.filesystem, build_directory,
chromium_base, webkit_base,
configuration, ['DumpRenderTree'])
def __init__(self, host, test_paths, options):
self.host = host
self.source_directory = options.source
self.options = options
self.test_paths = test_paths if test_paths else []
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
self.tests_w3c_relative_path = self.filesystem.join('imported', 'w3c')
self.layout_tests_path = webkit_finder.path_from_webkit_base('LayoutTests')
self.layout_tests_w3c_path = self.filesystem.join(self.layout_tests_path, self.tests_w3c_relative_path)
self.tests_download_path = webkit_finder.path_from_webkit_base('WebKitBuild', 'w3c-tests')
self._test_downloader = None
self._potential_test_resource_files = []
self.import_list = []
self._importing_downloaded_tests = self.source_directory is None
self._test_resource_files_json_path = self.filesystem.join(self.layout_tests_w3c_path, "resources", "resource-files.json")
self._test_resource_files = json.loads(self.filesystem.read_text_file(self._test_resource_files_json_path)) if self.filesystem.exists(self._test_resource_files_json_path) else None
self._tests_options_json_path = self.filesystem.join(self.layout_tests_path, 'tests-options.json')
self._tests_options = json.loads(self.filesystem.read_text_file(self._tests_options_json_path)) if self.filesystem.exists(self._tests_options_json_path) else None
self._slow_tests = []
if self.options.clean_destination_directory and self._test_resource_files:
self._test_resource_files["files"] = []
if self._tests_options:
self.remove_slow_from_w3c_tests_options()
def __init__(self):
self._host = Host()
self._filesystem = self._host.filesystem
self._webkit_root = WebKitFinder(self._filesystem).webkit_base()
# These settings might vary between WebKit and Blink
self._css_property_file = self.path_from_webkit_root('Source', 'WebCore', 'css', 'CSSPropertyNames.in')
self._css_property_split_string = '='
self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
prop_regex = '([\s{]|^)(' + "|".join(prop.replace('-webkit-', '') for prop in self.prefixed_properties) + ')(\s+:|:)'
self.prop_re = re.compile(prop_regex)
def setUp(self):
# FIXME: This should not need to touch the filesystem, however
# ChangeLog is difficult to mock at current.
self.filesystem = FileSystem()
self.temp_dir = str(self.filesystem.mkdtemp(suffix="changelogs"))
self.old_cwd = self.filesystem.getcwd()
self.filesystem.chdir(self.temp_dir)
self.webkit_base = WebKitFinder(self.filesystem).webkit_base()
# Trick commit-log-editor into thinking we're in a Subversion working copy so it won't
# complain about not being able to figure out what SCM is in use.
# FIXME: VCSTools.pm is no longer so easily fooled. It logs because "svn info" doesn't
# treat a bare .svn directory being part of an svn checkout.
self.filesystem.maybe_make_directory(".svn")
def do_POST(self):
json_raw_data = self.rfile.read(int(self.headers.getheader('content-length')))
json_data = json.loads(json_raw_data)
test_list = ''
for each in json_data['tests']:
test_list += each + ' '
filesystem = FileSystem()
webkit_finder = WebKitFinder(filesystem)
script_dir = webkit_finder.path_from_webkit_base('Tools', 'Scripts')
executable_path = script_dir + '/run-webkit-tests'
cmd = 'python ' + executable_path + ' --no-show-results '
cmd += test_list
process = subprocess.Popen(cmd, shell=True, cwd=script_dir, env=None, stdout=subprocess.PIPE, stderr=STDOUT)
self.send_response(200)
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Content-type', 'text/html')
self.end_headers()
while process.poll() is None:
html_output = '<br>' + str(process.stdout.readline())
self.wfile.write(html_output)
self.wfile.flush()
time.sleep(0.05)
process.wait()
def do_POST(self):
json_raw_data = self.rfile.read(int(self.headers.getheader('content-length')))
json_data = json.loads(json_raw_data)
test_list = ''
for each in json_data['tests']:
test_list += each + ' '
filesystem = FileSystem()
webkit_finder = WebKitFinder(filesystem)
script_dir = webkit_finder.path_from_webkit_base('Tools', 'Scripts')
executable_path = script_dir + "/run-webkit-tests"
cmd = "python " + executable_path + " --no-show-results "
cmd += test_list
process = subprocess.Popen(cmd, shell=True, cwd=script_dir, env=None, stdout=subprocess.PIPE, stderr=STDOUT)
self.send_response(200)
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header("Content-type", "text/html")
self.end_headers()
while process.poll() is None:
html_output = '<br>' + str(process.stdout.readline())
self.wfile.write(html_output)
self.wfile.flush()
time.sleep(0.05)
process.wait()
def __init__(self, host, source_repo_path, options):
self.host = host
self.source_repo_path = source_repo_path
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base(
'LayoutTests')
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir, options.destination,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (
self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
# This is just a FYI list of CSS properties that still need to be prefixed,
# which may be output after importing.
self._prefixed_properties = {}
def __init__(self,
new_path,
filename,
reference_support_info,
host=Host(),
convert_test_harness_links=True,
webkit_test_runner_options=''):
HTMLParser.__init__(self)
self._host = host
self._filesystem = self._host.filesystem
self._webkit_root = WebKitFinder(self._filesystem).webkit_base()
self.converted_data = []
self.converted_properties = []
self.converted_property_values = []
self.in_style_tag = False
self.style_data = []
self.filename = filename
self.reference_support_info = reference_support_info
self.webkit_test_runner_options = webkit_test_runner_options
self.has_started = False
resources_path = self.path_from_webkit_root('LayoutTests', 'resources')
resources_relpath = self._filesystem.relpath(resources_path, new_path)
self.new_test_harness_path = resources_relpath
self.convert_test_harness_links = convert_test_harness_links
# These settings might vary between WebKit and Blink
css_property_file = self.path_from_webkit_root('Source', 'WebCore',
'css',
'CSSProperties.json')
css_property_value_file = self.path_from_webkit_root(
'Source', 'WebCore', 'css', 'CSSValueKeywords.in')
self.test_harness_re = re.compile('/resources/testharness')
self.prefixed_properties = self.read_webkit_prefixed_css_property_list(
css_property_file)
prop_regex = '([\s{]|^)(' + "|".join(
prop.replace('-webkit-', '')
for prop in self.prefixed_properties) + ')(\s+:|:)'
self.prop_re = re.compile(prop_regex)
self.prefixed_property_values = self.legacy_read_webkit_prefixed_css_property_list(
css_property_value_file)
prop_value_regex = '(:\s*|^\s*)(' + "|".join(
value.replace('-webkit-', '')
for value in self.prefixed_property_values) + ')(\s*;|\s*}|\s*$)'
self.prop_value_re = re.compile(prop_value_regex)
def __init__(self, host, source_repo_path, dest_dir_name='external'):
"""Initializes variables to prepare for copying and converting files.
Args:
host: An instance of Host.
source_repo_path: Path to the local checkout of a
web-platform-tests or csswg-test repository.
dest_dir_name: The name of the directory under the layout tests
directory where imported tests should be copied to.
TODO(qyearsley): This can be made into a constant.
"""
self.host = host
assert self.host.filesystem.exists(source_repo_path)
self.source_repo_path = source_repo_path
self.dest_dir_name = dest_dir_name
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base(
'LayoutTests')
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir, dest_dir_name,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (
self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.is_wpt = self.filesystem.basename(self.source_repo_path) == 'wpt'
self.import_list = []
# This is just a FYI list of CSS properties that still need to be prefixed,
# which may be output after importing.
self._prefixed_properties = {}
def main():
filesystem = FileSystem()
wkf = WebKitFinder(filesystem)
tester = Tester(filesystem, wkf)
tester.add_tree(wkf.path_from_webkit_base('Tools', 'Scripts'), 'webkitpy')
tester.skip(('webkitpy.common.checkout.scm.scm_unittest',), 'are really, really, slow', 31818)
if sys.platform == 'win32':
tester.skip(('webkitpy.common.checkout', 'webkitpy.common.config', 'webkitpy.tool', 'webkitpy.w3c', 'webkitpy.layout_tests.layout_package.bot_test_expectations'), 'fail horribly on win32', 54526)
# This only needs to run on Unix, so don't worry about win32 for now.
appengine_sdk_path = '/usr/local/google_appengine'
if os.path.exists(appengine_sdk_path):
if not appengine_sdk_path in sys.path:
sys.path.append(appengine_sdk_path)
import dev_appserver
from google.appengine.dist import use_library
use_library('django', '1.2')
dev_appserver.fix_sys_path()
tester.add_tree(wkf.path_from_webkit_base('Tools', 'TestResultServer'))
else:
_log.info('Skipping TestResultServer tests; the Google AppEngine Python SDK is not installed.')
return not tester.run()
def main():
filesystem = FileSystem()
wkf = WebKitFinder(filesystem)
tester = Tester(filesystem, wkf)
tester.add_tree(wkf.path_from_webkit_base('Tools', 'Scripts'), 'webkitpy')
tester.skip(('webkitpy.common.checkout.scm.scm_unittest',), 'are really, really, slow', 31818)
if sys.platform == 'win32':
tester.skip(('webkitpy.common.checkout', 'webkitpy.common.config', 'webkitpy.tool', 'webkitpy.w3c', 'webkitpy.layout_tests.layout_package.bot_test_expectations'), 'fail horribly on win32', 54526)
# This only needs to run on Unix, so don't worry about win32 for now.
appengine_sdk_path = '/usr/local/google_appengine'
if os.path.exists(appengine_sdk_path):
if not appengine_sdk_path in sys.path:
sys.path.append(appengine_sdk_path)
import dev_appserver
from google.appengine.dist import use_library
use_library('django', '1.2')
dev_appserver.fix_sys_path()
tester.add_tree(wkf.path_from_webkit_base('Tools', 'TestResultServer'))
else:
_log.info('Skipping TestResultServer tests; the Google AppEngine Python SDK is not installed.')
return not tester.run()
def __init__(self, host, dir_to_import, top_of_repo, options):
self.host = host
self.dir_to_import = dir_to_import
self.top_of_repo = top_of_repo
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests')
self.destination_directory = self.filesystem.normpath(self.filesystem.join(self.layout_tests_dir, options.destination,
self.filesystem.basename(self.top_of_repo)))
self.import_in_place = (self.dir_to_import == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.top_of_repo)
self.import_list = []
def _assertOptimization(self,
results_by_directory,
expected_new_results_by_directory,
baseline_dirname='',
expected_files_to_delete=None,
host=None):
if not host:
host = MockHost()
fs = host.filesystem
webkit_base = WebKitFinder(fs).webkit_base()
baseline_name = 'mock-baseline-expected.txt'
fs.write_text_file(
fs.join(webkit_base, 'LayoutTests', 'VirtualTestSuites'),
'[{"prefix": "gpu", "base": "fast/canvas", "args": ["--foo"]}]')
for dirname, contents in results_by_directory.items():
path = fs.join(webkit_base, 'LayoutTests', dirname, baseline_name)
fs.write_binary_file(path, contents)
baseline_optimizer = BaselineOptimizer(
host,
host.port_factory.get(),
host.port_factory.all_port_names(),
skip_scm_commands=expected_files_to_delete is not None)
self.assertTrue(
baseline_optimizer.optimize(
fs.join(baseline_dirname, baseline_name)))
for dirname, contents in expected_new_results_by_directory.items():
path = fs.join(webkit_base, 'LayoutTests', dirname, baseline_name)
if contents is None:
self.assertTrue(not fs.exists(path)
or path in baseline_optimizer._files_to_delete)
else:
self.assertEqual(fs.read_binary_file(path), contents)
# Check that the files that were in the original set have been deleted where necessary.
for dirname in results_by_directory:
path = fs.join(webkit_base, 'LayoutTests', dirname, baseline_name)
if not dirname in expected_new_results_by_directory:
self.assertTrue(not fs.exists(path)
or path in baseline_optimizer._files_to_delete)
if expected_files_to_delete:
self.assertEqual(sorted(baseline_optimizer._files_to_delete),
sorted(expected_files_to_delete))
def __init__(self, new_path, filename, reference_support_info,
host=Host()):
HTMLParser.__init__(self)
self._host = host
self._filesystem = self._host.filesystem
self._webkit_root = WebKitFinder(self._filesystem).webkit_base()
self.converted_data = []
self.converted_properties = []
self.in_style_tag = False
self.style_data = []
self.filename = filename
self.reference_support_info = reference_support_info
resources_path = self.path_from_webkit_root('LayoutTests', 'resources')
resources_relpath = self._filesystem.relpath(resources_path, new_path)
self.resources_relpath = resources_relpath
# These settings might vary between WebKit and Blink
self._css_property_file = self.path_from_webkit_root(
'Source', 'core', 'css', 'CSSProperties.in')
# prefixed_properties is a list of properties where we will add "-webkit-" on the front of each name.
# prefixed_values is a map of properties where we we will prefix each value.
# Property name can be also prefixed if included in the prefixed_properties.
# renamed_properties and renamed_values is a map where simply adding a prefix isn't good enough.
# TODO(kojii): other than prefixed_properties need to hard-code here, clean this up when we unprefix.
self.prefixed_properties = self.read_webkit_prefixed_css_property_list(
)
self.prefixed_values = {
'unicode-bidi': ['isolate', 'isolate-override', 'plaintext']
}
self.renamed_properties = {
'text-combine-upright': '-webkit-text-combine'
}
self.renamed_values = {'text-combine-upright': {'all': 'horizontal'}}
prop_regex = '([\s{]|^)(' + "|".join(
itertools.chain(
self.renamed_properties.iterkeys(),
self.renamed_values.iterkeys(),
self.prefixed_values.iterkeys(),
(prop.replace('-webkit-', '')
for prop in self.prefixed_properties))) + ')(\s*:\s*[-\w]*)'
self.prop_re = re.compile(prop_regex)
def test_default_configuration__standalone(self):
# FIXME: This test runs a standalone python script to test
# reading the default configuration to work around any possible
# caching / reset bugs. See https://bugs.webkit.org/show_bug.cgi?id=49360
# for the motivation. We can remove this test when we remove the
# global configuration cache in config.py.
e = Executive()
fs = FileSystem()
c = config.Config(e, fs)
script = WebKitFinder(fs).path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'layout_tests', 'port', 'config_standalone.py')
# Note: don't use 'Release' here, since that's the normal default.
expected = 'Debug'
args = [sys.executable, script, '--mock', expected]
actual = e.run_command(args).rstrip()
self.assertEqual(actual, expected)
def _run_pylint(self, path):
wkf = WebKitFinder(FileSystem())
executive = Executive()
env = os.environ.copy()
env['PYTHONPATH'] = ('%s%s%s%s%s' % (wkf.path_from_webkit_base('Tools', 'Scripts'),
os.pathsep,
wkf.path_from_webkit_base('Source', 'build', 'scripts'),
os.pathsep,
wkf.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'thirdparty')))
return executive.run_command([sys.executable, wkf.path_from_depot_tools_base('pylint.py'),
'--output-format=parseable',
'--errors-only',
'--rcfile=' + wkf.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'pylintrc'),
path],
env=env,
error_handler=executive.ignore_error)
def __init__(self, host, dir_to_import, top_of_repo, options):
self.host = host
self.dir_to_import = dir_to_import
self.top_of_repo = top_of_repo
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests')
self.destination_directory = self.filesystem.normpath(self.filesystem.join(self.layout_tests_dir, options.destination,
self.filesystem.basename(self.top_of_repo)))
self.import_in_place = (self.dir_to_import == self.destination_directory)
self.changeset = CHANGESET_NOT_AVAILABLE
self.test_status = TEST_STATUS_UNKNOWN
self.import_list = []
def __init__(self, host, source_repo_path, options):
self.host = host
self.source_repo_path = source_repo_path
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base("LayoutTests")
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir, options.destination, self.filesystem.basename(self.source_repo_path)
)
)
self.import_in_place = self.source_repo_path == self.destination_directory
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
def _assert_optimization(self, results_by_directory, directory_to_new_results, baseline_dirname='', host=None):
host = host or MockHost()
fs = host.filesystem
webkit_base = WebKitFinder(fs).webkit_base()
baseline_name = 'mock-baseline-expected.txt'
fs.write_text_file(
fs.join(webkit_base, 'LayoutTests', 'VirtualTestSuites'),
'[{"prefix": "gpu", "base": "fast/canvas", "args": ["--foo"]}]')
for dirname, contents in results_by_directory.items():
fs.write_binary_file(fs.join(webkit_base, 'LayoutTests', dirname, baseline_name), contents)
baseline_optimizer = BaselineOptimizer(host, host.port_factory.get(), host.port_factory.all_port_names())
self.assertTrue(baseline_optimizer.optimize(fs.join(baseline_dirname, baseline_name)))
for dirname, contents in directory_to_new_results.items():
path = fs.join(webkit_base, 'LayoutTests', dirname, baseline_name)
if contents is not None:
self.assertEqual(fs.read_binary_file(path), contents)
def test_run(self,
verbose=0,
timing=False,
child_processes=1,
quiet=False):
options = MockOptions(verbose=verbose,
timing=timing,
child_processes=child_processes,
quiet=quiet,
pass_through=False)
stream = StringIO.StringIO()
loader = FakeLoader(('test1 (Foo)', '.', ''),
('test2 (Foo)', 'F', 'test2\nfailed'),
('test3 (Foo)', 'E', 'test3\nerred'))
runner = Runner(Printer(stream, options), loader,
WebKitFinder(FileSystem()))
runner.run(['Foo.test1', 'Foo.test2', 'Foo.test3'], 1)
self.assertEqual(runner.tests_run, 3)
self.assertEqual(len(runner.failures), 1)
self.assertEqual(len(runner.errors), 1)
def __init__(self, port):
self._port = port
_log.info('Using port %s' % self._port.name())
_log.info('Test configuration: %s' % self._port.test_configuration())
_log.info('Using display server %s' % (self._port._display_server))
self._display_driver = self._port.create_driver(worker_number=0, no_timeout=True)._make_driver(pixel_tests=False)
if not self._display_driver.check_driver(self._port):
raise RuntimeError("Failed to check driver %s" % self._display_driver.__class__.__name__)
driver = create_driver(self._port)
_log.info('Using driver at %s' % (driver.binary_path()))
_log.info('Browser: %s' % (driver.browser_name()))
_log.info('Parsing expectations')
self._tests_dir = WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
expectations_file = os.path.join(self._tests_dir, 'TestExpectations.json')
build_type = 'Debug' if self._port.get_option('debug') else 'Release'
self._expectations = TestExpectations(self._port.name(), expectations_file, build_type)
self._runners = [runner_cls(self._port, driver, self._display_driver, self._expectations) for runner_cls in self.RUNNER_CLASSES]
def __init__(self, host, source_repo_path, options):
self.host = host
self.source_repo_path = source_repo_path
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests')
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir,
options.destination,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
# This is just a FYI list of CSS properties that still need to be prefixed,
# which may be output after importing.
self._prefixed_properties = {}
def __init__(self,
host,
options,
gitClass=Git,
bugzillaClass=Bugzilla,
WPTGitHubClass=WPTGitHub,
WPTLinterClass=WPTLinter):
self._host = host
self._filesystem = host.filesystem
self._options = options
self._host.initialize_scm()
self._WPTGitHubClass = WPTGitHubClass
self._gitClass = gitClass
self._bugzilla = bugzillaClass()
self._bug_id = options.bug_id
if not self._bug_id:
if options.attachment_id:
self._bug_id = self._bugzilla.bug_id_for_attachment_id(
options.attachment_id)
elif options.git_commit:
self._bug_id = self._host.checkout().bug_id_for_this_commit(
options.git_commit)
if not self._options.repository_directory:
webkit_finder = WebKitFinder(self._filesystem)
self._options.repository_directory = WPTPaths.wpt_checkout_path(
webkit_finder)
self._linter = WPTLinterClass(self._options.repository_directory,
host.filesystem)
self._bugzilla_url = "https://bugs.webkit.org/show_bug.cgi?id=" + str(
self._bug_id)
self._commit_message = options.message
if not self._commit_message:
self._commit_message = 'WebKit export of ' + self._bugzilla_url if self._bug_id else 'Export made from a WebKit repository'
def __init__(self, new_path, filename, reference_support_info, host=Host()):
HTMLParser.__init__(self)
self._host = host
self._filesystem = self._host.filesystem
self._webkit_root = WebKitFinder(self._filesystem).webkit_base()
self.converted_data = []
self.converted_properties = []
self.in_style_tag = False
self.style_data = []
self.filename = filename
self.reference_support_info = reference_support_info
resources_path = self.path_from_webkit_root('LayoutTests', 'resources')
resources_relpath = self._filesystem.relpath(resources_path, new_path)
self.resources_relpath = resources_relpath
# These settings might vary between WebKit and Blink.
self._css_property_file = self.path_from_webkit_root('Source', 'core', 'css', 'CSSProperties.in')
self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
prop_regex = r'([\s{]|^)(' + '|'.join(
prop.replace('-webkit-', '') for prop in self.prefixed_properties) + r')(\s+:|:)'
self.prop_re = re.compile(prop_regex)
def __init__(self, host, source_directory, options):
self.host = host
self.source_directory = source_directory
self.options = options
self.filesystem = self.host.filesystem
webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = webkit_finder.webkit_base()
self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
self.layout_tests_w3c_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c')
self.tests_download_path = webkit_finder.path_from_webkit_base('WebKitBuild', 'w3c-tests')
self._test_downloader = None
self.import_list = []
self._importing_downloaded_tests = source_directory is None
def __init__(self,
host,
bugzilla_id,
is_bug_id=True,
bot_filter_name=None,
attachment_fetcher=bugzilla.Bugzilla(),
unzip=None):
self.host = host
self.filesystem = self.host.filesystem
self.bot_filter_name = bot_filter_name
self.unzip = unzip if unzip else lambda content: zipfile.ZipFile(
io.BytesIO(content))
self.layout_test_repository = WebKitFinder(
self.filesystem).path_from_webkit_base("LayoutTests")
if is_bug_id:
bug_info = attachment_fetcher.fetch_bug(bugzilla_id)
attachments = [
attachments
for attachments in bug_info.attachments(include_obsolete=False)
if attachments.is_patch()
]
if len(attachments) > 1:
raise RuntimeError(
"Found more than one non-obsolete patch in bug {}. Please specify which one to process."
.format(bugzilla_id))
if len(attachments) < 1:
raise RuntimeError(
"Couldn't find any non-obsolete patch in bug {}. Please specify which one to process."
.format(bugzilla_id))
self.patch = attachments[0]
else:
self.patch = attachment_fetcher.fetch_attachment(bugzilla_id)
if not self.patch.is_patch():
raise RuntimeError(
"Attachment {} its not a patch. Can't continue.".format(
bugzilla_id))
def __init__(self, host, build_dir):
super(DumpReaderMultipart, self).__init__(host, build_dir)
self._webkit_finder = WebKitFinder(host.filesystem)
self._breakpad_tools_available = None
self._generated_symbols = False
class DumpReaderMultipart(DumpReader):
"""Base class for Linux and Android breakpad dump reader."""
def __init__(self, host, build_dir):
super(DumpReaderMultipart, self).__init__(host, build_dir)
self._webkit_finder = WebKitFinder(host.filesystem)
self._breakpad_tools_available = None
self._generated_symbols = False
def check_is_functional(self):
return self._check_breakpad_tools_available()
def _get_pid_from_dump(self, dump_file):
dump = self._read_dump(dump_file)
if not dump:
return None
if 'pid' in dump:
return dump['pid'][0]
return None
def _get_stack_from_dump(self, dump_file):
dump = self._read_dump(dump_file)
if not dump:
return None
if not 'upload_file_minidump' in dump:
return None
self._generate_breakpad_symbols_if_necessary()
f, temp_name = self._host.filesystem.open_binary_tempfile('dmp')
f.write("\r\n".join(dump['upload_file_minidump']))
f.close()
cmd = [self._path_to_minidump_stackwalk(), temp_name, self._symbols_dir()]
try:
stack = self._host.executive.run_command(cmd, return_stderr=False)
except:
_log.warning('Failed to execute "%s"' % ' '.join(cmd))
stack = None
finally:
self._host.filesystem.remove(temp_name)
return stack
def _read_dump(self, dump_file):
with self._host.filesystem.open_binary_file_for_reading(dump_file) as f:
boundary = f.readline().strip()[2:]
f.seek(0)
try:
data = cgi.parse_multipart(f, {'boundary': boundary})
return data
except:
pass
return None
def _check_breakpad_tools_available(self):
if self._breakpad_tools_available != None:
return self._breakpad_tools_available
REQUIRED_BREAKPAD_TOOLS = [
'dump_syms',
'minidump_stackwalk',
]
result = True
for binary in REQUIRED_BREAKPAD_TOOLS:
full_path = self._host.filesystem.join(self._build_dir, binary)
if not self._host.filesystem.exists(full_path):
result = False
_log.error('Unable to find %s' % binary)
_log.error(' at %s' % full_path)
if not result:
_log.error(" Could not find breakpad tools, unexpected crashes won't be symbolized")
_log.error(' Did you build the target blink_tests?')
_log.error('')
self._breakpad_tools_available = result
return self._breakpad_tools_available
def _path_to_minidump_stackwalk(self):
return self._host.filesystem.join(self._build_dir, "minidump_stackwalk")
def _path_to_generate_breakpad_symbols(self):
return self._webkit_finder.path_from_chromium_base("components", "crash", "content", "tools", "generate_breakpad_symbols.py")
def _symbols_dir(self):
return self._host.filesystem.join(self._build_dir, 'content_shell.syms')
def _generate_breakpad_symbols_if_necessary(self):
if self._generated_symbols:
return
self._generated_symbols = True
_log.debug("Generating breakpad symbols")
queue = Queue.Queue()
thread = threading.Thread(target=_symbolize_keepalive, args=(queue,))
thread.start()
try:
for binary in self._binaries_to_symbolize():
_log.debug(' Symbolizing %s' % binary)
full_path = self._host.filesystem.join(self._build_dir, binary)
cmd = [
self._path_to_generate_breakpad_symbols(),
'--binary=%s' % full_path,
'--symbols-dir=%s' % self._symbols_dir(),
'--build-dir=%s' % self._build_dir,
]
try:
self._host.executive.run_command(cmd)
except:
_log.error('Failed to execute "%s"' % ' '.join(cmd))
finally:
queue.put(None)
thread.join()
_log.debug("Done generating breakpad symbols")
def _binaries_to_symbolize(self):
"""This routine must be implemented by subclasses.
Returns an array of binaries that need to be symbolized."""
raise NotImplementedError()
class TestImporter(object):
def __init__(self, host, dir_to_import, top_of_repo, options):
self.host = host
self.dir_to_import = dir_to_import
self.top_of_repo = top_of_repo
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests')
self.destination_directory = self.filesystem.normpath(self.filesystem.join(self.layout_tests_dir, options.destination,
self.filesystem.basename(self.top_of_repo)))
self.import_in_place = (self.dir_to_import == self.destination_directory)
self.changeset = CHANGESET_NOT_AVAILABLE
self.test_status = TEST_STATUS_UNKNOWN
self.import_list = []
def do_import(self):
_log.info("Importing %s into %s", self.dir_to_import, self.destination_directory)
self.find_importable_tests(self.dir_to_import)
self.load_changeset()
self.import_tests()
def load_changeset(self):
"""Returns the current changeset from mercurial or "Not Available"."""
try:
self.changeset = self.host.executive.run_command(['hg', 'tip']).split('changeset:')[1]
except (OSError, ScriptError):
self.changeset = CHANGESET_NOT_AVAILABLE
def find_importable_tests(self, directory):
# FIXME: use filesystem
paths_to_skip = self.find_paths_to_skip()
for root, dirs, files in os.walk(directory):
cur_dir = root.replace(self.layout_tests_dir + '/', '') + '/'
_log.info(' scanning ' + cur_dir + '...')
total_tests = 0
reftests = 0
jstests = 0
# "archive" and "data" dirs are internal csswg things that live in every approved directory.
# FIXME: skip 'incoming' tests for now, but we should rework the 'test_status' concept and
# support reading them as well.
DIRS_TO_SKIP = ('.git', '.hg', 'data', 'archive', 'incoming')
if dirs:
for d in DIRS_TO_SKIP:
if d in dirs:
dirs.remove(d)
for path in paths_to_skip:
path_base = path.replace(cur_dir, '')
path_full = self.filesystem.join(root, path_base)
if path_base in dirs:
dirs.remove(path_base)
if not self.options.dry_run and self.import_in_place:
_log.info(" pruning %s" % path_base)
self.filesystem.rmtree(path_full)
copy_list = []
for filename in files:
path_full = self.filesystem.join(root, filename)
path_base = path_full.replace(self.layout_tests_dir + '/', '')
if path_base in paths_to_skip:
if not self.options.dry_run and self.import_in_place:
_log.info(" pruning %s" % path_base)
self.filesystem.remove(path_full)
continue
# FIXME: This block should really be a separate function, but the early-continues make that difficult.
if filename.startswith('.') or filename.endswith('.pl'):
continue # For some reason the w3c repo contains random perl scripts we don't care about.
fullpath = os.path.join(root, filename)
mimetype = mimetypes.guess_type(fullpath)
if not 'html' in str(mimetype[0]) and not 'xml' in str(mimetype[0]):
copy_list.append({'src': fullpath, 'dest': filename})
continue
if root.endswith('resources'):
copy_list.append({'src': fullpath, 'dest': filename})
continue
test_parser = TestParser(vars(self.options), filename=fullpath)
test_info = test_parser.analyze_test()
if test_info is None:
continue
if 'reference' in test_info.keys():
reftests += 1
total_tests += 1
test_basename = os.path.basename(test_info['test'])
# Add the ref file, following WebKit style.
# FIXME: Ideally we'd support reading the metadata
# directly rather than relying on a naming convention.
# Using a naming convention creates duplicate copies of the
# reference files.
ref_file = os.path.splitext(test_basename)[0] + '-expected'
ref_file += os.path.splitext(test_basename)[1]
copy_list.append({'src': test_info['reference'], 'dest': ref_file})
copy_list.append({'src': test_info['test'], 'dest': filename})
# Update any support files that need to move as well to remain relative to the -expected file.
if 'refsupport' in test_info.keys():
for support_file in test_info['refsupport']:
source_file = os.path.join(os.path.dirname(test_info['reference']), support_file)
source_file = os.path.normpath(source_file)
# Keep the dest as it was
to_copy = {'src': source_file, 'dest': support_file}
# Only add it once
if not(to_copy in copy_list):
copy_list.append(to_copy)
elif 'jstest' in test_info.keys():
jstests += 1
total_tests += 1
copy_list.append({'src': fullpath, 'dest': filename})
else:
total_tests += 1
copy_list.append({'src': fullpath, 'dest': filename})
if not total_tests:
# We can skip the support directory if no tests were found.
if 'support' in dirs:
dirs.remove('support')
if copy_list:
# Only add this directory to the list if there's something to import
self.import_list.append({'dirname': root, 'copy_list': copy_list,
'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})
def find_paths_to_skip(self):
if self.options.ignore_expectations:
return set()
paths_to_skip = set()
port = self.host.port_factory.get()
w3c_import_expectations_path = self.webkit_finder.path_from_webkit_base('LayoutTests', 'W3CImportExpectations')
w3c_import_expectations = self.filesystem.read_text_file(w3c_import_expectations_path)
parser = TestExpectationParser(port, full_test_list=(), is_lint_mode=False)
expectation_lines = parser.parse(w3c_import_expectations_path, w3c_import_expectations)
for line in expectation_lines:
if 'SKIP' in line.expectations:
if line.specifiers:
_log.warning("W3CImportExpectations:%s should not have any specifiers" % line.line_numbers)
continue
paths_to_skip.add(line.name)
return paths_to_skip
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = os.path.relpath(orig_path, self.top_of_repo)
new_path = os.path.join(self.destination_directory, subpath)
if not(os.path.exists(new_path)):
os.makedirs(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = os.path.normpath(file_to_copy['src'])
if os.path.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory' % orig_filepath)
continue
if not(os.path.exists(orig_filepath)):
_log.warning('%s not found. Possible error in the test.', orig_filepath)
continue
new_filepath = os.path.join(new_path, file_to_copy['dest'])
if not(os.path.exists(os.path.dirname(new_filepath))):
if not self.import_in_place and not self.options.dry_run:
os.makedirs(os.path.dirname(new_filepath))
if not self.options.overwrite and os.path.exists(new_filepath):
_log.info(' skipping import of existing file ' + new_filepath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' importing %s', os.path.relpath(new_filepath, self.layout_tests_dir))
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0]) or 'css' in str(mimetype[0]):
converted_file = convert_for_webkit(new_path, filename=orig_filepath)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
outfile = open(new_filepath, 'wb')
outfile.write(converted_file[1])
outfile.close()
else:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ''))
if not self.import_in_place and not self.options.dry_run:
self.remove_deleted_files(new_path, copied_files)
self.write_import_log(new_path, copied_files, prefixed_properties)
_log.info('')
_log.info('Import complete')
_log.info('')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
_log.info('')
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
def setup_destination_directory(self):
""" Creates a destination directory that mirrors that of the source approved or submitted directory """
self.update_test_status()
start = self.dir_to_import.find(self.test_status)
new_subpath = self.dir_to_import[len(self.top_of_repo):]
destination_directory = os.path.join(self.destination_directory, new_subpath)
if not os.path.exists(destination_directory):
os.makedirs(destination_directory)
_log.info('Tests will be imported into: %s', destination_directory)
def update_test_status(self):
""" Sets the test status to either 'approved' or 'submitted' """
status = TEST_STATUS_UNKNOWN
directory_parts = self.dir_to_import.split(os.path.sep)
for test_status in VALID_TEST_STATUSES:
if test_status in directory_parts:
status = test_status
self.test_status = status
def remove_deleted_files(self, dir_to_import, new_file_list):
previous_file_list = []
import_log_file = os.path.join(dir_to_import, 'w3c-import.log')
if not os.path.exists(import_log_file):
return
import_log = open(import_log_file, 'r')
contents = import_log.readlines()
if 'List of files\n' in contents:
list_index = contents.index('List of files:\n') + 1
previous_file_list = [filename.strip() for filename in contents[list_index:]]
deleted_files = set(previous_file_list) - set(new_file_list)
for deleted_file in deleted_files:
_log.info('Deleting file removed from the W3C repo: %s', deleted_file)
deleted_file = os.path.join(self._webkit_root, deleted_file)
os.remove(deleted_file)
import_log.close()
def write_import_log(self, dir_to_import, file_list, prop_list):
now = datetime.datetime.now()
import_log = open(os.path.join(dir_to_import, 'w3c-import.log'), 'w')
import_log.write('The tests in this directory were imported from the W3C repository.\n')
import_log.write('Do NOT modify these tests directly in Webkit. Instead, push changes to the W3C CSS repo:\n\n')
import_log.write('http://hg.csswg.org/test\n\n')
import_log.write('Then run the Tools/Scripts/import-w3c-tests in Webkit to reimport\n\n')
import_log.write('Do NOT modify or remove this file\n\n')
import_log.write('------------------------------------------------------------------------\n')
import_log.write('Last Import: ' + now.strftime('%Y-%m-%d %H:%M') + '\n')
import_log.write('W3C Mercurial changeset: ' + self.changeset + '\n')
import_log.write('Test status at time of import: ' + self.test_status + '\n')
import_log.write('------------------------------------------------------------------------\n')
import_log.write('Properties requiring vendor prefixes:\n')
if prop_list:
for prop in prop_list:
import_log.write(prop + '\n')
else:
import_log.write('None\n')
import_log.write('------------------------------------------------------------------------\n')
import_log.write('List of files:\n')
for item in file_list:
import_log.write(item + '\n')
import_log.close()
class TestImporter(object):
def __init__(self, host, dir_to_import, top_of_repo, options):
self.host = host
self.dir_to_import = dir_to_import
self.top_of_repo = top_of_repo
self.options = options
self.filesystem = self.host.filesystem
self.webkit_finder = WebKitFinder(self.filesystem)
self._webkit_root = self.webkit_finder.webkit_base()
self.layout_tests_dir = self.webkit_finder.path_from_webkit_base("LayoutTests")
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(self.layout_tests_dir, options.destination, self.filesystem.basename(self.top_of_repo))
)
self.import_in_place = self.dir_to_import == self.destination_directory
self.dir_above_repo = self.filesystem.dirname(self.top_of_repo)
self.changeset = CHANGESET_NOT_AVAILABLE
self.import_list = []
def do_import(self):
_log.info("Importing %s into %s", self.dir_to_import, self.destination_directory)
self.find_importable_tests(self.dir_to_import)
self.load_changeset()
self.import_tests()
def load_changeset(self):
"""Returns the current changeset from mercurial or "Not Available"."""
try:
self.changeset = self.host.executive.run_command(["hg", "tip"]).split("changeset:")[1]
except (OSError, ScriptError):
self.changeset = CHANGESET_NOT_AVAILABLE
def find_importable_tests(self, directory):
# FIXME: use filesystem
paths_to_skip = self.find_paths_to_skip()
for root, dirs, files in os.walk(directory):
cur_dir = root.replace(self.dir_above_repo + "/", "") + "/"
_log.info(" scanning " + cur_dir + "...")
total_tests = 0
reftests = 0
jstests = 0
DIRS_TO_SKIP = (".git", ".hg")
if dirs:
for d in DIRS_TO_SKIP:
if d in dirs:
dirs.remove(d)
for path in paths_to_skip:
path_base = path.replace(self.options.destination + "/", "")
path_base = path_base.replace(cur_dir, "")
path_full = self.filesystem.join(root, path_base)
if path_base in dirs:
dirs.remove(path_base)
if not self.options.dry_run and self.import_in_place:
_log.info(" pruning %s" % path_base)
self.filesystem.rmtree(path_full)
else:
_log.info(" skipping %s" % path_base)
copy_list = []
for filename in files:
path_full = self.filesystem.join(root, filename)
path_base = path_full.replace(directory + "/", "")
path_base = self.destination_directory.replace(self.layout_tests_dir + "/", "") + "/" + path_base
if path_base in paths_to_skip:
if not self.options.dry_run and self.import_in_place:
_log.info(" pruning %s" % path_base)
self.filesystem.remove(path_full)
continue
else:
continue
# FIXME: This block should really be a separate function, but the early-continues make that difficult.
if filename.startswith(".") or filename.endswith(".pl"):
continue # For some reason the w3c repo contains random perl scripts we don't care about.
fullpath = os.path.join(root, filename)
mimetype = mimetypes.guess_type(fullpath)
if (
not "html" in str(mimetype[0])
and not "application/xhtml+xml" in str(mimetype[0])
and not "application/xml" in str(mimetype[0])
):
copy_list.append({"src": fullpath, "dest": filename})
continue
if root.endswith("resources"):
copy_list.append({"src": fullpath, "dest": filename})
continue
test_parser = TestParser(vars(self.options), filename=fullpath)
test_info = test_parser.analyze_test()
if test_info is None:
continue
if "reference" in test_info.keys():
reftests += 1
total_tests += 1
test_basename = os.path.basename(test_info["test"])
# Add the ref file, following WebKit style.
# FIXME: Ideally we'd support reading the metadata
# directly rather than relying on a naming convention.
# Using a naming convention creates duplicate copies of the
# reference files.
ref_file = os.path.splitext(test_basename)[0] + "-expected"
# Make sure to use the extension from the *reference*, not
# from the test, because at least flexbox tests use XHTML
# references but HTML tests.
ref_file += os.path.splitext(test_info["reference"])[1]
copy_list.append(
{
"src": test_info["reference"],
"dest": ref_file,
"reference_support_info": test_info["reference_support_info"],
}
)
copy_list.append({"src": test_info["test"], "dest": filename})
elif "jstest" in test_info.keys():
jstests += 1
total_tests += 1
copy_list.append({"src": fullpath, "dest": filename})
else:
total_tests += 1
copy_list.append({"src": fullpath, "dest": filename})
if copy_list:
# Only add this directory to the list if there's something to import
self.import_list.append(
{
"dirname": root,
"copy_list": copy_list,
"reftests": reftests,
"jstests": jstests,
"total_tests": total_tests,
}
)
def find_paths_to_skip(self):
if self.options.ignore_expectations:
return set()
paths_to_skip = set()
port = self.host.port_factory.get()
w3c_import_expectations_path = self.webkit_finder.path_from_webkit_base("LayoutTests", "W3CImportExpectations")
w3c_import_expectations = self.filesystem.read_text_file(w3c_import_expectations_path)
parser = TestExpectationParser(port, full_test_list=(), is_lint_mode=False)
expectation_lines = parser.parse(w3c_import_expectations_path, w3c_import_expectations)
for line in expectation_lines:
if "SKIP" in line.expectations:
if line.specifiers:
_log.warning("W3CImportExpectations:%s should not have any specifiers" % line.line_numbers)
continue
paths_to_skip.add(line.name)
return paths_to_skip
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy["total_tests"]
total_imported_reftests += dir_to_copy["reftests"]
total_imported_jstests += dir_to_copy["jstests"]
prefixed_properties = []
if not dir_to_copy["copy_list"]:
continue
orig_path = dir_to_copy["dirname"]
subpath = os.path.relpath(orig_path, self.top_of_repo)
new_path = os.path.join(self.destination_directory, subpath)
if not (os.path.exists(new_path)):
os.makedirs(new_path)
copied_files = []
for file_to_copy in dir_to_copy["copy_list"]:
# FIXME: Split this block into a separate function.
orig_filepath = os.path.normpath(file_to_copy["src"])
if os.path.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error("%s refers to a directory" % orig_filepath)
continue
if not (os.path.exists(orig_filepath)):
_log.warning("%s not found. Possible error in the test.", orig_filepath)
continue
new_filepath = os.path.join(new_path, file_to_copy["dest"])
if "reference_support_info" in file_to_copy.keys() and file_to_copy["reference_support_info"] != {}:
reference_support_info = file_to_copy["reference_support_info"]
else:
reference_support_info = None
if not (os.path.exists(os.path.dirname(new_filepath))):
if not self.import_in_place and not self.options.dry_run:
os.makedirs(os.path.dirname(new_filepath))
relpath = os.path.relpath(new_filepath, self.layout_tests_dir)
if not self.options.overwrite and os.path.exists(new_filepath):
_log.info(" skipping %s" % relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(" %s" % relpath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if "html" in str(mimetype[0]) or "xml" in str(mimetype[0]) or "css" in str(mimetype[0]):
converted_file = convert_for_webkit(
new_path, filename=orig_filepath, reference_support_info=reference_support_info
)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
outfile = open(new_filepath, "wb")
outfile.write(converted_file[1])
outfile.close()
else:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ""))
_log.info("")
_log.info("Import complete")
_log.info("")
_log.info("IMPORTED %d TOTAL TESTS", total_imported_tests)
_log.info("Imported %d reftests", total_imported_reftests)
_log.info("Imported %d JS tests", total_imported_jstests)
_log.info(
"Imported %d pixel/manual tests", total_imported_tests - total_imported_jstests - total_imported_reftests
)
_log.info("")
if total_prefixed_properties:
_log.info("Properties needing prefixes (by count):")
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(" %s: %s", prefixed_property, total_prefixed_properties[prefixed_property])
def setup_destination_directory(self):
""" Creates a destination directory that mirrors that of the source directory """
new_subpath = self.dir_to_import[len(self.top_of_repo) :]
destination_directory = os.path.join(self.destination_directory, new_subpath)
if not os.path.exists(destination_directory):
os.makedirs(destination_directory)
_log.info("Tests will be imported into: %s", destination_directory)
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import logging
import os
import sys
from webkitpy.common.system.filesystem import FileSystem
from webkitpy.common.webkit_finder import WebKitFinder
import webkitpy.thirdparty.autoinstalled.mozlog
import webkitpy.thirdparty.autoinstalled.mozprocess
from mozlog import structuredlog
w3c_tools_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests', 'imported', 'w3c', 'tools')
def _ensure_directory_in_path(directory):
if not directory in sys.path:
sys.path.insert(0, directory)
_ensure_directory_in_path(os.path.join(w3c_tools_dir, 'pytest'))
_ensure_directory_in_path(os.path.join(w3c_tools_dir, 'webdriver'))
_ensure_directory_in_path(os.path.join(w3c_tools_dir, 'wptrunner'))
_ensure_directory_in_path(os.path.join(w3c_tools_dir, "webdriver"))
from wptrunner.executors.base import WdspecExecutor, WebDriverProtocol
from wptrunner.webdriver_server import WebDriverServer
_log = logging.getLogger(__name__)
class DepsUpdater(object):
def __init__(self, host):
self.host = host
self.executive = host.executive
self.fs = host.filesystem
self.finder = WebKitFinder(self.fs)
self.verbose = False
self.allow_local_blink_commits = False
self.keep_w3c_repos_around = False
def main(self, argv=None):
self.parse_args(argv)
self.cd("")
if not self.checkout_is_okay():
return 1
self.print_("## noting the current Blink commitish")
blink_commitish = self.run(["git", "show-ref", "HEAD"])[1].split()[0]
wpt_import_text = self.update(
"web-platform-tests", "https://chromium.googlesource.com/external/w3c/web-platform-tests.git"
)
for resource in ["testharnessreport.js", "vendor-prefix.js"]:
source = self.path_from_webkit_base("LayoutTests", "resources", resource)
destination = self.path_from_webkit_base(
"LayoutTests", "imported", "web-platform-tests", "resources", resource
)
self.copyfile(source, destination)
self.run(["git", "add", destination])
css_import_text = self.update("csswg-test", "https://chromium.googlesource.com/external/w3c/csswg-test.git")
self.commit_changes_if_needed(blink_commitish, css_import_text, wpt_import_text)
return 0
def parse_args(self, argv):
parser = argparse.ArgumentParser()
parser.description = __doc__
parser.add_argument("-v", "--verbose", action="store_true", help="log what we are doing")
parser.add_argument(
"--allow-local-blink-commits",
action="store_true",
help="allow script to run even if we have local blink commits",
)
parser.add_argument(
"--keep-w3c-repos-around",
action="store_true",
help="leave the w3c repos around that were imported previously.",
)
args = parser.parse_args(argv)
self.allow_local_blink_commits = args.allow_local_blink_commits
self.keep_w3c_repos_around = args.keep_w3c_repos_around
self.verbose = args.verbose
def checkout_is_okay(self):
if self.run(["git", "diff", "--quiet", "HEAD"], exit_on_failure=False)[0]:
self.print_("## blink checkout is dirty, aborting")
return False
local_blink_commits = self.run(["git", "log", "--oneline", "origin/master..HEAD"])[1]
if local_blink_commits and not self.allow_local_blink_commits:
self.print_("## blink checkout has local commits, aborting")
return False
if self.fs.exists(self.path_from_webkit_base("web-platform-tests")):
self.print_("## web-platform-tests repo exists, aborting")
return False
if self.fs.exists(self.path_from_webkit_base("csswg-test")):
self.print_("## csswg-test repo exists, aborting")
return False
return True
def update(self, repo, url):
self.print_("## cloning %s" % repo)
self.cd("")
self.run(["git", "clone", url])
self.cd(re.compile(".*/([^/]+)\.git").match(url).group(1))
self.run(["git", "submodule", "update", "--init", "--recursive"])
self.print_("## noting the revision we are importing")
master_commitish = self.run(["git", "show-ref", "origin/master"])[1].split()[0]
self.print_("## cleaning out tests from LayoutTests/imported/%s" % repo)
dest_repo = self.path_from_webkit_base("LayoutTests", "imported", repo)
files_to_delete = self.fs.files_under(dest_repo, file_filter=self.is_not_baseline)
for subpath in files_to_delete:
self.remove("LayoutTests", "imported", subpath)
self.print_("## importing the tests")
src_repo = self.path_from_webkit_base(repo)
import_path = self.path_from_webkit_base("Tools", "Scripts", "import-w3c-tests")
self.run([self.host.executable, import_path, "-d", "imported", src_repo])
self.cd("")
self.run(["git", "add", "--all", "LayoutTests/imported/%s" % repo])
self.print_("## deleting manual tests")
files_to_delete = self.fs.files_under(dest_repo, file_filter=self.is_manual_test)
for subpath in files_to_delete:
self.remove("LayoutTests", "imported", subpath)
self.print_("## deleting any orphaned baselines")
previous_baselines = self.fs.files_under(dest_repo, file_filter=self.is_baseline)
for subpath in previous_baselines:
full_path = self.fs.join(dest_repo, subpath)
if self.fs.glob(full_path.replace("-expected.txt", "*")) == [full_path]:
self.fs.remove(full_path)
if not self.keep_w3c_repos_around:
self.print_("## deleting %s repo" % repo)
self.cd("")
self.rmtree(repo)
return "imported %s@%s" % (repo, master_commitish)
def commit_changes_if_needed(self, blink_commitish, css_import_text, wpt_import_text):
if self.run(["git", "diff", "--quiet", "HEAD"], exit_on_failure=False)[0]:
self.print_("## commiting changes")
commit_msg = (
"update-w3c-deps import using blink %s:\n"
"\n"
"%s\n"
"%s\n" % (blink_commitish, css_import_text, wpt_import_text)
)
path_to_commit_msg = self.path_from_webkit_base("commit_msg")
if self.verbose:
self.print_("cat > %s <<EOF" % path_to_commit_msg)
self.print_(commit_msg)
self.print_("EOF")
self.fs.write_text_file(path_to_commit_msg, commit_msg)
self.run(["git", "commit", "-a", "-F", path_to_commit_msg])
self.remove(path_to_commit_msg)
self.print_("## Done: changes imported and committed")
else:
self.print_("## Done: no changes to import")
def is_manual_test(self, fs, dirname, basename):
return basename.endswith("-manual.html") or basename.endswith("-manual.htm")
def is_baseline(self, fs, dirname, basename):
return basename.endswith("-expected.txt")
def is_not_baseline(self, fs, dirname, basename):
return not self.is_baseline(fs, dirname, basename)
def run(self, cmd, exit_on_failure=True):
if self.verbose:
self.print_(" ".join(cmd))
proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self.executive.PIPE)
out, err = proc.communicate()
if proc.returncode or self.verbose:
self.print_("# ret> %d" % proc.returncode)
if out:
for line in out.splitlines():
self.print_("# out> %s" % line)
if err:
for line in err.splitlines():
self.print_("# err> %s" % line)
if exit_on_failure and proc.returncode:
self.host.exit(proc.returncode)
return proc.returncode, out
def cd(self, *comps):
dest = self.path_from_webkit_base(*comps)
if self.verbose:
self.print_("cd %s" % dest)
self.fs.chdir(dest)
def copyfile(self, source, destination):
if self.verbose:
self.print_("cp %s %s" % (source, destination))
self.fs.copyfile(source, destination)
def remove(self, *comps):
dest = self.path_from_webkit_base(*comps)
if self.verbose:
self.print_("rm %s" % dest)
self.fs.remove(dest)
def rmtree(self, *comps):
dest = self.path_from_webkit_base(*comps)
if self.verbose:
self.print_("rm -fr %s" % dest)
self.fs.rmtree(dest)
def path_from_webkit_base(self, *comps):
return self.finder.path_from_webkit_base(*comps)
def print_(self, msg):
self.host.print_(msg)
