def _prepare_config(self, options, args, tool):
results_directory = args[0]
host = Host()
host.initialize_scm()
print 'Parsing full_results.json...'
results_json_path = host.filesystem.join(results_directory, 'full_results.json')
results_json = json_results_generator.load_json(host.filesystem, results_json_path)
port = tool.port_factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = host.filesystem.listdir(host.filesystem.join(layout_tests_directory, 'platform'))
self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, host)
print 'Gathering current baselines...'
self._gather_baselines(results_json)
return {
'test_config': self._test_config,
"results_json": results_json,
"platforms_json": {
'platforms': platforms,
'defaultPlatform': port.name(),
},
}
def _prepare_config(self, options, args, tool):
results_directory = args[0]
host = Host()
host.initialize_scm()
print 'Parsing full_results.json...'
results_json_path = host.filesystem.join(results_directory, 'full_results.json')
results_json = json_results_generator.load_json(host.filesystem, results_json_path)
port = tool.port_factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = host.filesystem.listdir(host.filesystem.join(layout_tests_directory, 'platform'))
self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, host)
print 'Gathering current baselines...'
self._gather_baselines(results_json)
return {
'test_config': self._test_config,
"results_json": results_json,
"platforms_json": {
'platforms': platforms,
'defaultPlatform': port.name(),
},
}
def main(self):
args = sys.argv[1:]
host = Host()
host.initialize_scm()
stderr = self._engage_awesome_stderr_hacks()
# Checking for the verbose flag before calling check_webkit_style_parser()
# lets us enable verbose logging earlier.
is_verbose = "-v" in args or "--verbose" in args
checker.configure_logging(stream=stderr, is_verbose=is_verbose)
_log.debug("Verbose logging enabled.")
parser = checker.check_webkit_style_parser()
(paths, options) = parser.parse(args)
configuration = checker.check_webkit_style_configuration(options)
paths = change_directory(host.filesystem,
checkout_root=host.scm().checkout_root,
paths=paths)
style_processor = StyleProcessor(configuration)
file_reader = TextFileReader(host.filesystem, style_processor)
if paths and not options.diff_files:
file_reader.process_paths(paths)
file_reader.do_association_check(host.scm().checkout_root)
else:
changed_files = paths if options.diff_files else None
patch = host.scm().create_patch(options.git_commit,
changed_files=changed_files,
git_index=options.git_index)
patch_checker = PatchReader(file_reader)
patch_checker.check(patch)
error_count = style_processor.error_count
file_count = file_reader.file_count
delete_only_file_count = file_reader.delete_only_file_count
_log.info("Total errors found: %d in %d files" %
(error_count, file_count))
# We fail when style errors are found or there are no checked files.
return error_count > 0 or (file_count == 0
and delete_only_file_count == 0)
def get_test_baselines(test_file, test_config):
# FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic.
class AllPlatformsPort(Port):
def __init__(self, host):
super(AllPlatformsPort, self).__init__(host, 'mac')
self._platforms_by_directory = dict([
(self._webkit_baseline_path(p), p)
for p in test_config.platforms
])
def baseline_search_path(self):
return self._platforms_by_directory.keys()
def platform_from_directory(self, directory):
return self._platforms_by_directory[directory]
test_path = test_config.filesystem.join(test_config.layout_tests_directory,
test_file)
# FIXME: This should get the Host from the test_config to be mockable!
host = Host()
host.initialize_scm()
host.filesystem = test_config.filesystem
all_platforms_port = AllPlatformsPort(host)
all_test_baselines = {}
for baseline_extension in ('.txt', '.checksum', '.png'):
test_baselines = test_config.test_port.expected_baselines(
test_file, baseline_extension)
baselines = all_platforms_port.expected_baselines(test_file,
baseline_extension,
all_baselines=True)
for platform_directory, expected_filename in baselines:
if not platform_directory:
continue
if platform_directory == test_config.layout_tests_directory:
platform = 'base'
else:
platform = all_platforms_port.platform_from_directory(
platform_directory)
platform_baselines = all_test_baselines.setdefault(platform, {})
was_used_for_test = (platform_directory,
expected_filename) in test_baselines
platform_baselines[baseline_extension] = was_used_for_test
return all_test_baselines
def main(self):
args = sys.argv[1:]
host = Host()
host.initialize_scm()
stderr = self._engage_awesome_stderr_hacks()
# Checking for the verbose flag before calling check_webkit_style_parser()
# lets us enable verbose logging earlier.
is_verbose = "-v" in args or "--verbose" in args
checker.configure_logging(stream=stderr, is_verbose=is_verbose)
_log.debug("Verbose logging enabled.")
parser = checker.check_webkit_style_parser()
(paths, options) = parser.parse(args)
configuration = checker.check_webkit_style_configuration(options)
paths = change_directory(host.filesystem, checkout_root=host.scm().checkout_root, paths=paths)
style_processor = StyleProcessor(configuration)
file_reader = TextFileReader(host.filesystem, style_processor)
if paths and not options.diff_files:
file_reader.process_paths(paths)
else:
changed_files = paths if options.diff_files else None
patch = host.scm().create_patch(options.git_commit, changed_files=changed_files)
patch_checker = PatchReader(file_reader)
patch_checker.check(patch)
error_count = style_processor.error_count
file_count = file_reader.file_count
delete_only_file_count = file_reader.delete_only_file_count
_log.info("Total errors found: %d in %d files" % (error_count, file_count))
# We fail when style errors are found or there are no checked files.
return error_count > 0
def get_test_baselines(test_file, test_config):
# FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic.
class AllPlatformsPort(Port):
def __init__(self, host):
super(AllPlatformsPort, self).__init__(host, 'mac')
self._platforms_by_directory = dict([(self._webkit_baseline_path(p), p) for p in test_config.platforms])
def baseline_search_path(self):
return self._platforms_by_directory.keys()
def platform_from_directory(self, directory):
return self._platforms_by_directory[directory]
test_path = test_config.filesystem.join(test_config.layout_tests_directory, test_file)
# FIXME: This should get the Host from the test_config to be mockable!
host = Host()
host.initialize_scm()
host.filesystem = test_config.filesystem
all_platforms_port = AllPlatformsPort(host)
all_test_baselines = {}
for baseline_extension in ('.txt', '.checksum', '.png'):
test_baselines = test_config.test_port.expected_baselines(test_file, baseline_extension)
baselines = all_platforms_port.expected_baselines(test_file, baseline_extension, all_baselines=True)
for platform_directory, expected_filename in baselines:
if not platform_directory:
continue
if platform_directory == test_config.layout_tests_directory:
platform = 'base'
else:
platform = all_platforms_port.platform_from_directory(platform_directory)
platform_baselines = all_test_baselines.setdefault(platform, {})
was_used_for_test = (platform_directory, expected_filename) in test_baselines
platform_baselines[baseline_extension] = was_used_for_test
return all_test_baselines
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
# The GTK+ and EFL ports only supports WebKit2, so they always use WKTR.
if self._port.name().startswith("gtk") or self._port.name().startswith("efl"):
self._options.webkit_test_runner = True
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._timestamp = time.time()
self._utc_timestamp = datetime.datetime.utcnow()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--no-build", dest="build", action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True,
help="Do no generate results JSON and results page."),
optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str",
help="Path to generate a JSON file at; may contain previous results if it already exists."),
optparse.make_option("--reset-results", action="store_true",
help="Clears the content in the generated JSON file before adding the results."),
optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str",
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option("--description",
help="Add a description to the output JSON file if one is generated"),
optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results",
help="Don't launch a browser with results after the tests are done"),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--dump-render-tree", "-1", action="store_false", default=True, dest="webkit_test_runner",
help="Use DumpRenderTree rather than WebKitTestRunner."),
optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True,
help="Run all tests, including the ones in the Skipped list."),
optparse.make_option("--profile", action="store_true",
help="Output per-test profile information."),
optparse.make_option("--profiler", action="store",
help="Output per-test profile information, using the specified profiler."),
optparse.make_option("--additional-drt-flag", action="append",
default=[], help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
optparse.make_option("--driver-name", type="string",
help="Alternative DumpRenderTree binary to use"),
optparse.make_option("--repeat", default=1, type="int",
help="Specify number of times to run test set (default: 1)."),
optparse.make_option("--test-runner-count", default=-1, type="int",
help="Specify number of times to invoke test runner for each performance test."),
optparse.make_option("--wrapper",
help="wrapper command to insert before invocations of "
"DumpRenderTree or WebKitTestRunner; option is split on whitespace before "
"running. (Example: --wrapper='valgrind --smc-check=all')"),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
test_extensions = ['.html', '.svg']
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
if filesystem.exists(filesystem.join(self._base_path, arg)):
paths.append(arg)
else:
relpath = filesystem.relpath(arg, self._base_path)
if filesystem.exists(filesystem.join(self._base_path, relpath)):
paths.append(filesystem.normpath(relpath))
else:
_log.warn('Path was not found:' + arg)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
test_runner_count = DEFAULT_TEST_RUNNER_COUNT
if self._options.test_runner_count > 0:
test_runner_count = self._options.test_runner_count
elif self._options.profile:
test_runner_count = 1
for path in test_files:
relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/')
if self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path, test_runner_count=test_runner_count)
tests.append(test)
return tests
def run(self):
if "Debug" == self._port.get_option("configuration"):
_log.warning("""****************************************************
* WARNING: run-perf-tests is running in DEBUG mode *
****************************************************""")
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
# Check that the system dependencies (themes, fonts, ...) are correct.
if not self._port.check_sys_deps(needs_http=False):
_log.error("Failed to check system dependencies.")
self._port.stop_helper()
return self.EXIT_CODE_BAD_PREPARATION
run_count = 0
repeat = self._options.repeat
while (run_count < repeat):
run_count += 1
tests = self._collect_tests()
runs = ' (Run %d of %d)' % (run_count, repeat) if repeat > 1 else ''
_log.info("Running %d tests%s" % (len(tests), runs))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()))
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_results()
if exit_code:
return exit_code
if self._options.generate_results and not self._options.profile:
test_results_server = self._options.test_results_server
if test_results_server and not self._upload_json(test_results_server, self._output_json_path()):
return self.EXIT_CODE_FAILED_UPLOADING
if self._options.show_results:
self._port.show_results_html_file(self._results_page_path())
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _results_page_path(self):
return self._host.filesystem.splitext(self._output_json_path())[0] + '.html'
def _generate_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number)
if options.slave_config_json_path:
output = self._merge_slave_config_json(options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
filesystem = self._host.filesystem
json_output = json.dumps(output)
filesystem.write_text_file(output_json_path, json_output)
template_path = filesystem.join(self._port.perf_tests_dir(), 'resources/results-template.html')
template = filesystem.read_text_file(template_path)
absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir())
results_page = template.replace('%AbsolutePathToWebKitTrunk%', absolute_path_to_trunk)
results_page = results_page.replace('%PeformanceTestsResultsJSON%', json_output)
filesystem.write_text_file(self._results_page_path(), results_page)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
revisions = {}
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm()
revision = scm.svn_revision(path)
revisions[name] = {'revision': revision, 'timestamp': scm.timestamp_of_revision(path, revision)}
meta_info = {
'description': description,
'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp),
'platform': platform,
'revisions': revisions,
'builderName': builder_name,
'buildNumber': int(build_number) if build_number else None}
contents = {'tests': {}}
for key, value in meta_info.items():
if value:
contents[key] = value
for metric in self._results:
tests = contents['tests']
path = metric.path()
for i in range(0, len(path)):
is_last_token = i + 1 == len(path)
url = view_source_url('PerformanceTests/' + '/'.join(path[0:i + 1]))
test_name = path[i]
# FIXME: This is a temporary workaround for the fact perf dashboard doesn't support renaming tests.
if test_name == 'Speedometer':
test_name = 'DoYouEvenBench'
tests.setdefault(test_name, {'url': url})
current_test = tests[test_name]
if is_last_token:
current_test['url'] = view_source_url('PerformanceTests/' + metric.test_file_name())
current_test.setdefault('metrics', {})
assert metric.name() not in current_test['metrics']
test_results = {'current': metric.grouped_iteration_values()}
if metric.aggregator():
test_results['aggregators'] = [metric.aggregator()]
current_test['metrics'][metric.name()] = test_results
else:
current_test.setdefault('tests', {})
tests = current_test['tests']
return contents
@staticmethod
def _datetime_in_ES5_compatible_iso_format(datetime):
return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')
def _merge_slave_config_json(self, slave_config_json_path, contents):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" % slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
for key in slave_config:
contents['builder' + key.capitalize()] = slave_config[key]
return contents
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_default_branch = "webkit-trunk"
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = "PerformanceTestsResults.json"
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option(
"--debug",
action="store_const",
const="Debug",
dest="configuration",
help="Set the configuration to Debug",
),
optparse.make_option(
"--release",
action="store_const",
const="Release",
dest="configuration",
help="Set the configuration to Release",
),
optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option(
"--chromium",
action="store_const",
const="chromium",
dest="platform",
help="Alias for --platform=chromium",
),
optparse.make_option(
"--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2."),
),
optparse.make_option("--build-number", help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default).",
),
optparse.make_option(
"--no-build",
dest="build",
action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date.",
),
optparse.make_option(
"--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)",
),
optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"),
optparse.make_option(
"--pause-before-testing",
dest="pause_before_testing",
action="store_true",
default=False,
help="Pause before running the tests to let user attach a performance monitor.",
),
optparse.make_option(
"--no-results",
action="store_false",
dest="generate_results",
default=True,
help="Do no generate results JSON and results page.",
),
optparse.make_option(
"--output-json-path",
action="callback",
callback=_expand_path,
type="str",
help="Path to generate a JSON file at; may contain previous results if it already exists.",
),
optparse.make_option(
"--reset-results",
action="store_true",
help="Clears the content in the generated JSON file before adding the results.",
),
optparse.make_option(
"--slave-config-json-path",
action="callback",
callback=_expand_path,
type="str",
help="Only used on bots. Path to a slave configuration file.",
),
optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help="Don't launch a browser with results after the tests are done",
),
optparse.make_option(
"--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present.",
),
optparse.make_option(
"--webkit-test-runner",
"-2",
action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree.",
),
optparse.make_option(
"--replay", dest="replay", action="store_true", default=False, help="Run replay tests."
),
optparse.make_option(
"--force",
dest="skipped",
action="store_true",
default=False,
help="Run all tests, including the ones in the Skipped list.",
),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
test_extensions = [".html", ".svg"]
if self._options.replay:
test_extensions.append(".replay")
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set([".svn", "resources"])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(path).replace("\\", "/")
if self._port.skips_perf_test(relative_path) and not self._options.skipped:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path)
tests.append(test)
return tests
def run(self):
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
if self._options.generate_results:
exit_code = self._generate_and_show_results()
if exit_code:
return exit_code
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _generate_and_show_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(
self._timestamp, options.description, options.platform, options.builder_name, options.build_number
)
if options.slave_config_json_path:
output = self._merge_slave_config_json(options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
results_page_path = self._host.filesystem.splitext(output_json_path)[0] + ".html"
self._generate_output_files(output_json_path, results_page_path, output)
if options.test_results_server:
if not self._upload_json(options.test_results_server, output_json_path):
return self.EXIT_CODE_FAILED_UPLOADING
if options.show_results:
self._port.show_results_html_file(results_page_path)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
contents = {"results": self._results}
if description:
contents["description"] = description
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm()
contents[name + "-revision"] = scm.svn_revision(path)
# FIXME: Add --branch or auto-detect the branch we're in
for key, value in {
"timestamp": int(timestamp),
"branch": self._default_branch,
"platform": platform,
"builder-name": builder_name,
"build-number": int(build_number) if build_number else None,
}.items():
if value:
contents[key] = value
return contents
def _merge_slave_config_json(self, slave_config_json_path, output):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" % slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
return dict(slave_config.items() + output.items())
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
# Timeouts are controlled by the Python Driver, so DRT/WTR runs with no-timeout.
self._options.additional_drt_flag.append('--no-timeout')
# The GTK+ and EFL ports only supports WebKit2, so they always use WKTR.
if self._port.name().startswith("gtk") or self._port.name().startswith(
"efl"):
self._options.webkit_test_runner = True
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._timestamp = time.time()
self._utc_timestamp = datetime.datetime.utcnow()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option('--debug',
action='store_const',
const='Debug',
dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release',
action='store_const',
const='Release',
dest="configuration",
help='Set the configuration to Release'),
optparse.make_option(
"--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option(
"--builder-name",
help=
("The name of the builder shown on the waterfall running this script e.g. google-mac-2."
)),
optparse.make_option(
"--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help=
"Check to ensure the DumpRenderTree build is up-to-date (default)."
),
optparse.make_option(
"--no-build",
dest="build",
action="store_false",
help=
"Don't check to see if the DumpRenderTree build is up-to-date."
),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option("--time-out-ms",
default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--no-timeout",
action="store_true",
default=False,
help="Disable test timeouts"),
optparse.make_option(
"--no-results",
action="store_false",
dest="generate_results",
default=True,
help="Do no generate results JSON and results page."),
optparse.make_option(
"--output-json-path",
action='callback',
callback=_expand_path,
type="str",
help=
"Path to generate a JSON file at; may contain previous results if it already exists."
),
optparse.make_option(
"--reset-results",
action="store_true",
help=
"Clears the content in the generated JSON file before adding the results."
),
optparse.make_option(
"--slave-config-json-path",
action='callback',
callback=_expand_path,
type="str",
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option(
"--description",
help=
"Add a description to the output JSON file if one is generated"
),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help=
"Don't launch a browser with results after the tests are done"
),
optparse.make_option(
"--test-results-server",
help=
"Upload the generated JSON file to the specified server when --output-json-path is present."
),
optparse.make_option(
"--dump-render-tree",
"-1",
action="store_false",
default=True,
dest="webkit_test_runner",
help="Use DumpRenderTree rather than WebKitTestRunner."),
optparse.make_option(
"--force",
dest="use_skipped_list",
action="store_false",
default=True,
help="Run all tests, including the ones in the Skipped list."),
optparse.make_option("--profile",
action="store_true",
help="Output per-test profile information."),
optparse.make_option(
"--profiler",
action="store",
help=
"Output per-test profile information, using the specified profiler."
),
optparse.make_option(
"--additional-drt-flag",
action="append",
default=[],
help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
optparse.make_option(
"--driver-name",
type="string",
help="Alternative DumpRenderTree binary to use"),
optparse.make_option(
"--repeat",
default=1,
type="int",
help="Specify number of times to run test set (default: 1)."),
optparse.make_option(
"--test-runner-count",
default=-1,
type="int",
help=
"Specify number of times to invoke test runner for each performance test."
),
optparse.make_option(
"--wrapper",
help="wrapper command to insert before invocations of "
"DumpRenderTree or WebKitTestRunner; option is split on whitespace before "
"running. (Example: --wrapper='valgrind --smc-check=all')"),
optparse.make_option(
'--display-server',
choices=['xvfb', 'xorg', 'weston', 'wayland'],
default='xvfb',
help=
'"xvfb": Use a virtualized X11 server. "xorg": Use the current X11 session. '
'"weston": Use a virtualized Weston server. "wayland": Use the current wayland session.'
),
]
return optparse.OptionParser(
option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
test_extensions = ['.html', '.svg']
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
if filesystem.exists(filesystem.join(self._base_path, arg)):
paths.append(arg)
else:
relpath = filesystem.relpath(arg, self._base_path)
if filesystem.exists(filesystem.join(self._base_path,
relpath)):
paths.append(filesystem.normpath(relpath))
else:
_log.warn('Path was not found:' + arg)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths,
skipped_directories, _is_test_file)
tests = []
test_runner_count = DEFAULT_TEST_RUNNER_COUNT
if self._options.test_runner_count > 0:
test_runner_count = self._options.test_runner_count
elif self._options.profile:
test_runner_count = 1
for path in test_files:
relative_path = filesystem.relpath(path, self._base_path).replace(
'\\', '/')
if self._options.use_skipped_list and self._port.skips_perf_test(
relative_path) and filesystem.normpath(
relative_path) not in paths:
continue
test = PerfTestFactory.create_perf_test(
self._port,
relative_path,
path,
test_runner_count=test_runner_count)
tests.append(test)
return tests
def run(self):
if "Debug" == self._port.get_option("configuration"):
_log.warning(
"""****************************************************
* WARNING: run-perf-tests is running in DEBUG mode *
****************************************************""")
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" %
self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
# Check that the system dependencies (themes, fonts, ...) are correct.
if not self._port.check_sys_deps(needs_http=False):
_log.error("Failed to check system dependencies.")
self._port.stop_helper()
return self.EXIT_CODE_BAD_PREPARATION
run_count = 0
repeat = self._options.repeat
while (run_count < repeat):
run_count += 1
tests = self._collect_tests()
runs = ' (Run %d of %d)' % (run_count,
repeat) if repeat > 1 else ''
_log.info("Running %d tests%s" % (len(tests), runs))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(
sorted(list(tests), key=lambda test: test.test_name()))
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_results()
if exit_code:
return exit_code
if self._options.generate_results and not self._options.profile:
test_results_server = self._options.test_results_server
if test_results_server and not self._upload_json(
test_results_server, self._output_json_path()):
return self.EXIT_CODE_FAILED_UPLOADING
if self._options.show_results:
self._port.show_results_html_file(self._results_page_path())
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(),
self._DEFAULT_JSON_FILENAME)
def _results_page_path(self):
return self._host.filesystem.splitext(
self._output_json_path())[0] + '.html'
def _generate_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp,
options.description,
options.platform,
options.builder_name,
options.build_number)
if options.slave_config_json_path:
output = self._merge_slave_config_json(
options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
filesystem = self._host.filesystem
json_output = json.dumps(output)
filesystem.write_text_file(output_json_path, json_output)
template_path = filesystem.join(self._port.perf_tests_dir(),
'resources/results-template.html')
template = filesystem.read_text_file(template_path)
absolute_path_to_trunk = filesystem.dirname(
self._port.perf_tests_dir())
results_page = template.replace('%AbsolutePathToWebKitTrunk%',
absolute_path_to_trunk)
results_page = results_page.replace('%PeformanceTestsResultsJSON%',
json_output)
filesystem.write_text_file(self._results_page_path(), results_page)
def _generate_results_dict(self, timestamp, description, platform,
builder_name, build_number):
revisions = {}
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive
).detect_scm_system(path) or self._host.scm()
revision = scm.native_revision(path)
revisions[name] = {
'revision': revision,
'timestamp': scm.timestamp_of_native_revision(path, revision)
}
meta_info = {
'description':
description,
'buildTime':
self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp),
'platform':
platform,
'revisions':
revisions,
'builderName':
builder_name,
'buildNumber':
int(build_number) if build_number else None
}
contents = {'tests': {}}
for key, value in meta_info.items():
if value:
contents[key] = value
for metric in self._results:
tests = contents['tests']
path = metric.path()
for i in range(0, len(path)):
is_last_token = i + 1 == len(path)
url = view_source_url('PerformanceTests/' +
'/'.join(path[0:i + 1]))
test_name = path[i]
# FIXME: This is a temporary workaround for the fact perf dashboard doesn't support renaming tests.
if test_name == 'Speedometer':
test_name = 'DoYouEvenBench'
tests.setdefault(test_name, {'url': url})
current_test = tests[test_name]
if is_last_token:
current_test['url'] = view_source_url(
'PerformanceTests/' + metric.test_file_name())
current_test.setdefault('metrics', {})
assert metric.name() not in current_test['metrics']
test_results = {
'current': metric.grouped_iteration_values()
}
if metric.aggregator():
test_results['aggregators'] = [metric.aggregator()]
current_test['metrics'][metric.name()] = test_results
else:
current_test.setdefault('tests', {})
tests = current_test['tests']
return contents
@staticmethod
def _datetime_in_ES5_compatible_iso_format(datetime):
return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')
def _merge_slave_config_json(self, slave_config_json_path, contents):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" %
slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(
slave_config_json_path)
slave_config = json.load(slave_config_json)
for key in slave_config:
contents['builder' + key.capitalize()] = slave_config[key]
return contents
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" %
(slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
self._needs_http = None
self._has_http_lock = False
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
optparse.make_option("--chromium-android",
action="store_const", const='chromium-android', dest='platform', help='Alias for --platform=chromium-android'),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--no-build", dest="build", action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--pause-before-testing", dest="pause_before_testing", action="store_true", default=False,
help="Pause before running the tests to let user attach a performance monitor."),
optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True,
help="Do no generate results JSON and results page."),
optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str",
help="Path to generate a JSON file at; may contain previous results if it already exists."),
optparse.make_option("--reset-results", action="store_true",
help="Clears the content in the generated JSON file before adding the results."),
optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str",
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option("--description",
help="Add a description to the output JSON file if one is generated"),
optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results",
help="Don't launch a browser with results after the tests are done"),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--webkit-test-runner", "-2", action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
optparse.make_option("--replay", dest="replay", action="store_true", default=False,
help="Run replay tests."),
optparse.make_option("--force", dest="skipped", action="store_true", default=False,
help="Run all tests, including the ones in the Skipped list."),
optparse.make_option("--profile", action="store_true",
help="Output per-test profile information."),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(path).replace('\\', '/')
if self._port.skips_perf_test(relative_path) and not self._options.skipped:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path)
tests.append(test)
return tests
def _start_servers(self):
if self._needs_http:
self._port.acquire_http_lock()
self._port.start_http_server(number_of_servers=2)
self._has_http_lock = True
def _stop_servers(self):
if self._has_http_lock:
self._port.stop_http_server()
self._port.release_http_lock()
def run(self):
self._needs_http = self._port.requires_http_server()
if not self._port.check_build(needs_http=self._needs_http):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
try:
self._start_servers()
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
finally:
self._stop_servers()
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_and_show_results()
if exit_code:
return exit_code
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _generate_and_show_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number)
if options.slave_config_json_path:
output = self._merge_slave_config_json(options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
results_page_path = self._host.filesystem.splitext(output_json_path)[0] + '.html'
self._generate_output_files(output_json_path, results_page_path, output)
if options.test_results_server:
if not self._upload_json(options.test_results_server, output_json_path):
return self.EXIT_CODE_FAILED_UPLOADING
if options.show_results:
self._port.show_results_html_file(results_page_path)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
contents = {'results': self._results}
if description:
contents['description'] = description
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm()
contents[name + '-revision'] = scm.svn_revision(path)
# FIXME: Add --branch or auto-detect the branch we're in
for key, value in {'timestamp': int(timestamp), 'branch': self._default_branch, 'platform': platform,
'builder-name': builder_name, 'build-number': int(build_number) if build_number else None}.items():
if value:
contents[key] = value
return contents
def _merge_slave_config_json(self, slave_config_json_path, output):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" % slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
return dict(slave_config.items() + output.items())
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error))
return None
class W3CTestConverter(object):
def __init__(self):
self._host = Host()
self._filesystem = self._host.filesystem
self._host.initialize_scm()
self._webkit_root = self._host.scm().checkout_root
# These settings might vary between WebKit and Blink
self._css_property_file = self.path_from_webkit_root(
'Source', 'core', 'css', 'CSSPropertyNames.in')
self._css_property_split_string = 'alias_for='
self.prefixed_properties = self.read_webkit_prefixed_css_property_list(
)
def path_from_webkit_root(self, *comps):
return self._filesystem.abspath(
self._filesystem.join(self._webkit_root, *comps))
def read_webkit_prefixed_css_property_list(self):
prefixed_properties = []
contents = self._filesystem.read_text_file(self._css_property_file)
for line in contents.splitlines():
# Find lines starting with the -webkit- prefix.
match = re.match('-webkit-[\w|-]*', line)
if match:
# Ignore lines where both the prefixed and non-prefixed property
# are supported - denoted by -webkit-some-property = some-property.
fields = line.split(self._css_property_split_string)
if len(fields) == 2 and fields[1].strip() in fields[0].strip():
continue
prefixed_properties.append(match.group(0))
return prefixed_properties
def convert_for_webkit(self, new_path, filename):
""" Converts a file's |contents| so it will function correctly in its |new_path| in Webkit.
Returns the list of modified properties and the modified text if the file was modifed, None otherwise."""
contents = self._filesystem.read_binary_file(filename)
if filename.endswith('.css'):
return self.convert_css(contents, filename)
return self.convert_html(new_path, contents, filename)
def convert_css(self, contents, filename):
return self.add_webkit_prefix_to_unprefixed_properties(
contents, filename)
def convert_html(self, new_path, contents, filename):
doc = BeautifulSoup(contents)
did_modify_paths = self.convert_testharness_paths(
doc, new_path, filename)
converted_properties_and_content = self.convert_prefixed_properties(
doc, filename)
return converted_properties_and_content if (
did_modify_paths or converted_properties_and_content[0]) else None
def convert_testharness_paths(self, doc, new_path, filename):
""" Update links to testharness.js in the BeautifulSoup |doc| to point to the copy in |new_path|.
Returns whether the document was modified."""
# Look for the W3C-style path to any testharness files - scripts (.js) or links (.css)
pattern = re.compile('/resources/testharness')
script_tags = doc.findAll(src=pattern)
link_tags = doc.findAll(href=pattern)
testharness_tags = script_tags + link_tags
if not testharness_tags:
return False
resources_path = self.path_from_webkit_root('LayoutTests', 'resources')
resources_relpath = self._filesystem.relpath(resources_path, new_path)
for tag in testharness_tags:
# FIXME: We need to handle img, audio, video tags also.
attr = 'src'
if tag.name != 'script':
attr = 'href'
if not attr in tag.attrMap:
# FIXME: Figure out what to do w/ invalid tags. For now, we return False
# and leave the document unmodified, which means that it'll probably fail to run.
_log.error("Missing an attr in %s" % filename)
return False
old_path = tag[attr]
new_tag = Tag(doc, tag.name, tag.attrs)
new_tag[attr] = re.sub(pattern, resources_relpath + '/testharness',
old_path)
self.replace_tag(tag, new_tag)
return True
def convert_prefixed_properties(self, doc, filename):
""" Searches a BeautifulSoup |doc| for any CSS properties requiring the -webkit- prefix and converts them.
Returns the list of converted properties and the modified document as a string """
converted_properties = []
# Look for inline and document styles.
inline_styles = doc.findAll(style=re.compile('.*'))
style_tags = doc.findAll('style')
all_styles = inline_styles + style_tags
for tag in all_styles:
# Get the text whether in a style tag or style attribute.
style_text = ''
if tag.name == 'style':
if not tag.contents:
continue
style_text = tag.contents[0]
else:
style_text = tag['style']
updated_style_text = self.add_webkit_prefix_to_unprefixed_properties(
style_text, filename)
# Rewrite tag only if changes were made.
if updated_style_text[0]:
converted_properties.extend(updated_style_text[0])
new_tag = Tag(doc, tag.name, tag.attrs)
new_tag.insert(0, updated_style_text[1])
self.replace_tag(tag, new_tag)
return (converted_properties, doc.prettify())
def add_webkit_prefix_to_unprefixed_properties(self, text, filename):
""" Searches |text| for instances of properties requiring the -webkit- prefix and adds the prefix to them.
Returns the list of converted properties and the modified text."""
converted_properties = []
for prefixed_property in self.prefixed_properties:
# FIXME: add in both the prefixed and unprefixed versions, rather than just replacing them?
# That might allow the imported test to work in other browsers more easily.
unprefixed_property = prefixed_property.replace('-webkit-', '')
# Look for the various ways it might be in the CSS
# Match the the property preceded by either whitespace or left curly brace
# or at the beginning of the string (for inline style attribute)
pattern = '([\s{]|^)' + unprefixed_property + '(\s+:|:)'
if re.search(pattern, text):
_log.info('converting %s -> %s' %
(unprefixed_property, prefixed_property))
converted_properties.append(prefixed_property)
text = re.sub(pattern, prefixed_property + ':', text)
# FIXME: Handle the JS versions of these properties and GetComputedStyle, too.
return (converted_properties, text)
def replace_tag(self, old_tag, new_tag):
index = old_tag.parent.contents.index(old_tag)
old_tag.parent.insert(index, new_tag)
old_tag.extract()
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
self._utc_timestamp = datetime.datetime.utcnow()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
optparse.make_option("--chromium-android",
action="store_const", const='chromium-android', dest='platform', help='Alias for --platform=chromium-android'),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--no-build", dest="build", action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True,
help="Do no generate results JSON and results page."),
optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str",
help="Path to generate a JSON file at; may contain previous results if it already exists."),
optparse.make_option("--reset-results", action="store_true",
help="Clears the content in the generated JSON file before adding the results."),
optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str",
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option("--description",
help="Add a description to the output JSON file if one is generated"),
optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results",
help="Don't launch a browser with results after the tests are done"),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--webkit-test-runner", "-2", action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
optparse.make_option("--replay", dest="replay", action="store_true", default=False,
help="Run replay tests."),
optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True,
help="Run all tests, including the ones in the Skipped list."),
optparse.make_option("--profile", action="store_true",
help="Output per-test profile information."),
optparse.make_option("--profiler", action="store",
help="Output per-test profile information, using the specified profiler."),
optparse.make_option("--additional-drt-flag", action="append",
default=[], help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
if filesystem.exists(filesystem.join(self._base_path, arg)):
paths.append(arg)
else:
relpath = filesystem.relpath(arg, self._base_path)
if filesystem.exists(filesystem.join(self._base_path, relpath)):
paths.append(filesystem.normpath(relpath))
else:
_log.warn('Path was not found:' + arg)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/')
if self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path)
tests.append(test)
return tests
def _start_http_servers(self):
self._port.acquire_http_lock()
self._port.start_http_server(number_of_servers=2)
def _stop_http_servers(self):
self._port.stop_http_server()
self._port.release_http_lock()
def run(self):
needs_http = self._port.requires_http_server()
if not self._port.check_build(needs_http=needs_http):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
try:
if needs_http:
self._start_http_servers()
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
finally:
if needs_http:
self._stop_http_servers()
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_and_show_results()
if exit_code:
return exit_code
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _generate_and_show_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number)
if options.slave_config_json_path:
output = self._merge_slave_config_json(options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
results_page_path = self._host.filesystem.splitext(output_json_path)[0] + '.html'
self._generate_output_files(output_json_path, results_page_path, output)
if options.test_results_server:
if options.test_results_server == 'webkit-perf.appspot.com':
options.test_results_server = 'perf.webkit.org'
if not self._upload_json(options.test_results_server, output_json_path):
return self.EXIT_CODE_FAILED_UPLOADING
if options.show_results:
self._port.show_results_html_file(results_page_path)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
contents = {'tests': {}}
if description:
contents['description'] = description
revisions = {}
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm()
revision = scm.svn_revision(path)
revisions[name] = {'revision': str(revision), 'timestamp': scm.timestamp_of_latest_commit(path, revision)}
meta_info = {
'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp),
'platform': platform,
'revisions': revisions,
'builderName': builder_name,
'buildNumber': int(build_number) if build_number else None}
for key, value in meta_info.items():
if value:
contents[key] = value
# FIXME: Make this function shorter once we've transitioned to use perf.webkit.org.
for metric_full_name, result in self._results.iteritems():
if not isinstance(result, dict): # We can't reports results without indivisual measurements.
continue
assert metric_full_name.count(':') <= 1
test_full_name, _, metric = metric_full_name.partition(':')
if not metric:
metric = {'fps': 'FrameRate', 'runs/s': 'Runs', 'ms': 'Time'}[result['unit']]
tests = contents['tests']
path = test_full_name.split('/')
for i in range(0, len(path)):
# FIXME: We shouldn't assume HTML extension.
is_last_token = i + 1 == len(path)
url = 'http://trac.webkit.org/browser/trunk/PerformanceTests/' + '/'.join(path[0:i + 1])
if is_last_token:
url += '.html'
tests.setdefault(path[i], {'url': url})
current_test = tests[path[i]]
if is_last_token:
current_test.setdefault('metrics', {})
assert metric not in current_test['metrics']
current_test['metrics'][metric] = {'current': result['values']}
else:
current_test.setdefault('tests', {})
tests = current_test['tests']
return contents
@staticmethod
def _datetime_in_ES5_compatible_iso_format(datetime):
return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')
def _merge_slave_config_json(self, slave_config_json_path, contents):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" % slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
for key in slave_config:
contents['builder' + key.capitalize()] = slave_config[key]
return contents
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._timestamp = time.time()
self._utc_timestamp = datetime.datetime.utcnow()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option('-t', '--target', dest='configuration',
help='Specify the target build subdirectory under src/out/'),
optparse.make_option("--platform",
help="Specify port/platform being tested (e.g. mac)"),
optparse.make_option("--chromium",
action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
optparse.make_option("--android",
action="store_const", const='android', dest='platform', help='Alias for --platform=android'),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up to date (default)."),
optparse.make_option("--no-build", dest="build", action="store_false",
help="Don't check to see if the DumpRenderTree build is up to date."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True,
help="Do no generate results JSON and results page."),
optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str",
help="Path to generate a JSON file at; may contain previous results if it already exists."),
optparse.make_option("--reset-results", action="store_true",
help="Clears the content in the generated JSON file before adding the results."),
optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str",
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option("--description",
help="Add a description to the output JSON file if one is generated"),
optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results",
help="Don't launch a browser with results after the tests are done"),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True,
help="Run all tests, including the ones in the Skipped list."),
optparse.make_option("--profile", action="store_true",
help="Output per-test profile information."),
optparse.make_option("--profiler", action="store",
help="Output per-test profile information, using the specified profiler."),
optparse.make_option("--additional-driver-flag", action="append",
default=[], help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
optparse.make_option("--driver-name", type="string",
help="Alternative DumpRenderTree binary to use"),
optparse.make_option("--content-shell", action="store_true",
help="Use Content Shell instead of DumpRenderTree"),
optparse.make_option("--repeat", default=1, type="int",
help="Specify number of times to run test set (default: 1)."),
optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUNNER_COUNT, type="int",
help="Specify number of times to invoke test runner for each performance test."),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
test_extensions = ['.html', '.svg']
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
if filesystem.exists(filesystem.join(self._base_path, arg)):
paths.append(arg)
else:
relpath = filesystem.relpath(arg, self._base_path)
if filesystem.exists(filesystem.join(self._base_path, relpath)):
paths.append(filesystem.normpath(relpath))
else:
_log.warning('Path was not found:' + arg)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/')
if self._options.use_skipped_list and self._port.skips_perf_test(
relative_path) and filesystem.normpath(relative_path) not in paths:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path,
test_runner_count=self._options.test_runner_count)
tests.append(test)
return tests
def _start_http_servers(self):
self._port.acquire_http_lock()
self._port.start_http_server(number_of_servers=2)
def _stop_http_servers(self):
self._port.stop_http_server()
self._port.release_http_lock()
def run(self):
needs_http = self._port.requires_http_server()
class FakePrinter(object):
def write_update(self, msg):
print msg
def write_throttled_update(self, msg):
pass
if self._port.check_build(needs_http=needs_http, printer=FakePrinter()):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
run_count = 0
repeat = self._options.repeat
while run_count < repeat:
run_count += 1
tests = self._collect_tests()
runs = ' (Run %d of %d)' % (run_count, repeat) if repeat > 1 else ''
_log.info("Running %d tests%s", len(tests), runs)
try:
if needs_http:
self._start_http_servers()
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()))
finally:
if needs_http:
self._stop_http_servers()
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_results()
if exit_code:
return exit_code
if self._options.generate_results and not self._options.profile:
test_results_server = self._options.test_results_server
if test_results_server and not self._upload_json(test_results_server, self._output_json_path()):
return self.EXIT_CODE_FAILED_UPLOADING
if self._options.show_results:
self._port.show_results_html_file(self._results_page_path())
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _results_page_path(self):
return self._host.filesystem.splitext(self._output_json_path())[0] + '.html'
def _generate_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp, options.description,
options.platform, options.builder_name, options.build_number)
if options.slave_config_json_path:
output = self._merge_slave_config_json(options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
filesystem = self._host.filesystem
json_output = json.dumps(output)
filesystem.write_text_file(output_json_path, json_output)
template_path = filesystem.join(self._port.perf_tests_dir(), 'resources/results-template.html')
template = filesystem.read_text_file(template_path)
absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir())
results_page = template.replace('%AbsolutePathToWebKitTrunk%', absolute_path_to_trunk)
results_page = results_page.replace('%PeformanceTestsResultsJSON%', json_output)
filesystem.write_text_file(self._results_page_path(), results_page)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
revisions = {}
path = self._port.repository_path()
scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm()
revision = str(scm.commit_position(path))
revisions['chromium'] = {'revision': revision, 'timestamp': scm.timestamp_of_revision(path, revision)}
meta_info = {
'description': description,
'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp),
'platform': platform,
'revisions': revisions,
'builderName': builder_name,
'buildNumber': int(build_number) if build_number else None}
contents = {'tests': {}}
for key, value in meta_info.items():
if value:
contents[key] = value
for test, metrics in self._results:
for metric_name, iteration_values in metrics.iteritems():
if not isinstance(iteration_values, list): # We can't reports results without individual measurements.
continue
tests = contents['tests']
path = test.test_name_without_file_extension().split('/')
for i in range(0, len(path)):
is_last_token = i + 1 == len(path)
url = self.view_source_url(
'PerformanceTests/' + (test.test_name() if is_last_token else '/'.join(path[0:i + 1])))
tests.setdefault(path[i], {'url': url})
current_test = tests[path[i]]
if is_last_token:
current_test.setdefault('metrics', {})
assert metric_name not in current_test['metrics']
current_test['metrics'][metric_name] = {'current': iteration_values}
else:
current_test.setdefault('tests', {})
tests = current_test['tests']
return contents
@staticmethod
def view_source_url(path_from_blink):
return 'https://chromium.googlesource.com/chromium/src/+/master/third_party/WebKit/%s' % path_from_blink
@staticmethod
def _datetime_in_ES5_compatible_iso_format(datetime):
return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')
def _merge_slave_config_json(self, slave_config_json_path, contents):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s", slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
for key in slave_config:
contents['builder' + key.capitalize()] = slave_config[key]
return contents
except Exception as error:
_log.error("Failed to merge slave configuration JSON file %s: %s", slave_config_json_path, error)
return None
def _merge_outputs_if_needed(self, output_json_path, output):
if self._options.reset_results or not self._host.filesystem.isfile(output_json_path):
return [output]
try:
existing_outputs = json.loads(self._host.filesystem.read_text_file(output_json_path))
return existing_outputs + [output]
except Exception as error:
_log.error("Failed to merge output JSON file %s: %s", output_json_path, error)
return None
def _upload_json(self, test_results_server, json_path, host_path="/api/report", file_uploader=FileUploader):
url = "https://%s%s" % (test_results_server, host_path)
uploader = file_uploader(url, 120)
try:
response = uploader.upload_single_text_file(self._host.filesystem, 'application/json', json_path)
except Exception as error:
_log.error("Failed to upload JSON file to %s in 120s: %s", url, error)
return False
response_body = [line.strip('\n') for line in response]
if response_body != ['OK']:
try:
parsed_response = json.loads('\n'.join(response_body))
except:
_log.error("Uploaded JSON to %s but got a bad response:", url)
for line in response_body:
_log.error(line)
return False
if parsed_response.get('status') != 'OK':
_log.error("Uploaded JSON to %s but got an error:", url)
_log.error(json.dumps(parsed_response, indent=4))
return False
_log.info("JSON file uploaded to %s.", url)
return True
def _run_tests_set(self, tests):
failures = 0
self._results = []
for i, test in enumerate(tests):
_log.info('Running %s (%d of %d)', test.test_name(), i + 1, len(tests))
start_time = time.time()
metrics = test.run(self._options.time_out_ms)
if metrics:
self._results.append((test, metrics))
else:
failures += 1
_log.error('FAILED')
_log.info('Finished: %f s', time.time() - start_time)
_log.info('')
return failures
class W3CTestConverter(object):
def __init__(self):
self._host = Host()
self._filesystem = self._host.filesystem
self._host.initialize_scm()
self._webkit_root = self._host.scm().checkout_root
# These settings might vary between WebKit and Blink
self._css_property_file = self.path_from_webkit_root('Source', 'core', 'css', 'CSSPropertyNames.in')
self._css_property_split_string = 'alias_for='
self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
def path_from_webkit_root(self, *comps):
return self._filesystem.abspath(self._filesystem.join(self._webkit_root, *comps))
def read_webkit_prefixed_css_property_list(self):
prefixed_properties = []
contents = self._filesystem.read_text_file(self._css_property_file)
for line in contents.splitlines():
# Find lines starting with the -webkit- prefix.
match = re.match('-webkit-[\w|-]*', line)
if match:
# Ignore lines where both the prefixed and non-prefixed property
# are supported - denoted by -webkit-some-property = some-property.
fields = line.split(self._css_property_split_string)
if len(fields) == 2 and fields[1].strip() in fields[0].strip():
continue
prefixed_properties.append(match.group(0))
return prefixed_properties
def convert_for_webkit(self, new_path, filename):
""" Converts a file's |contents| so it will function correctly in its |new_path| in Webkit.
Returns the list of modified properties and the modified text if the file was modifed, None otherwise."""
contents = self._filesystem.read_binary_file(filename)
if filename.endswith('.css'):
return self.convert_css(contents, filename)
return self.convert_html(new_path, contents, filename)
def convert_css(self, contents, filename):
return self.add_webkit_prefix_to_unprefixed_properties(contents, filename)
def convert_html(self, new_path, contents, filename):
doc = BeautifulSoup(contents)
did_modify_paths = self.convert_testharness_paths(doc, new_path, filename)
converted_properties_and_content = self.convert_prefixed_properties(doc, filename)
return converted_properties_and_content if (did_modify_paths or converted_properties_and_content[0]) else None
def convert_testharness_paths(self, doc, new_path, filename):
""" Update links to testharness.js in the BeautifulSoup |doc| to point to the copy in |new_path|.
Returns whether the document was modified."""
# Look for the W3C-style path to any testharness files - scripts (.js) or links (.css)
pattern = re.compile('/resources/testharness')
script_tags = doc.findAll(src=pattern)
link_tags = doc.findAll(href=pattern)
testharness_tags = script_tags + link_tags
if not testharness_tags:
return False
resources_path = self.path_from_webkit_root('LayoutTests', 'resources')
resources_relpath = self._filesystem.relpath(resources_path, new_path)
for tag in testharness_tags:
# FIXME: We need to handle img, audio, video tags also.
attr = 'src'
if tag.name != 'script':
attr = 'href'
if not attr in tag.attrMap:
# FIXME: Figure out what to do w/ invalid tags. For now, we return False
# and leave the document unmodified, which means that it'll probably fail to run.
_log.error("Missing an attr in %s" % filename)
return False
old_path = tag[attr]
new_tag = Tag(doc, tag.name, tag.attrs)
new_tag[attr] = re.sub(pattern, resources_relpath + '/testharness', old_path)
self.replace_tag(tag, new_tag)
return True
def convert_prefixed_properties(self, doc, filename):
""" Searches a BeautifulSoup |doc| for any CSS properties requiring the -webkit- prefix and converts them.
Returns the list of converted properties and the modified document as a string """
converted_properties = []
# Look for inline and document styles.
inline_styles = doc.findAll(style=re.compile('.*'))
style_tags = doc.findAll('style')
all_styles = inline_styles + style_tags
for tag in all_styles:
# Get the text whether in a style tag or style attribute.
style_text = ''
if tag.name == 'style':
if not tag.contents:
continue
style_text = tag.contents[0]
else:
style_text = tag['style']
updated_style_text = self.add_webkit_prefix_to_unprefixed_properties(style_text, filename)
# Rewrite tag only if changes were made.
if updated_style_text[0]:
converted_properties.extend(updated_style_text[0])
new_tag = Tag(doc, tag.name, tag.attrs)
new_tag.insert(0, updated_style_text[1])
self.replace_tag(tag, new_tag)
return (converted_properties, doc.prettify())
def add_webkit_prefix_to_unprefixed_properties(self, text, filename):
""" Searches |text| for instances of properties requiring the -webkit- prefix and adds the prefix to them.
Returns the list of converted properties and the modified text."""
converted_properties = []
for prefixed_property in self.prefixed_properties:
# FIXME: add in both the prefixed and unprefixed versions, rather than just replacing them?
# That might allow the imported test to work in other browsers more easily.
unprefixed_property = prefixed_property.replace('-webkit-', '')
# Look for the various ways it might be in the CSS
# Match the the property preceded by either whitespace or left curly brace
# or at the beginning of the string (for inline style attribute)
pattern = '([\s{]|^)' + unprefixed_property + '(\s+:|:)'
if re.search(pattern, text):
_log.info('converting %s -> %s' % (unprefixed_property, prefixed_property))
converted_properties.append(prefixed_property)
text = re.sub(pattern, prefixed_property + ':', text)
# FIXME: Handle the JS versions of these properties and GetComputedStyle, too.
return (converted_properties, text)
def replace_tag(self, old_tag, new_tag):
index = old_tag.parent.contents.index(old_tag)
old_tag.parent.insert(index, new_tag)
old_tag.extract()
def parse_args(args):
option_group_definitions = []
option_group_definitions.append(("Platform options", platform_options()))
option_group_definitions.append(
("Configuration options", configuration_options()))
option_group_definitions.append(
("Printing Options", printing.print_options()))
option_group_definitions.append(("Feature Switches", [
optparse.make_option(
"--complex-text",
action="store_true",
default=False,
help=
"Use the complex text code path for all text (OS X and Windows only)"
),
optparse.make_option("--accelerated-drawing",
action="store_true",
default=False,
help="Use accelerated drawing (OS X only)"),
optparse.make_option(
"--remote-layer-tree",
action="store_true",
default=False,
help="Use the remote layer tree drawing model (OS X WebKit2 only)"
),
optparse.make_option(
"--internal-feature",
type="string",
action="append",
default=[],
help=
"Enable (disable) an internal feature (--internal-feature FeatureName[=true|false])"
),
optparse.make_option(
"--experimental-feature",
type="string",
action="append",
default=[],
help=
"Enable (disable) an experimental feature (--experimental-feature FeatureName[=true|false])"
),
]))
option_group_definitions.append((
"WebKit Options",
[
optparse.make_option(
"--gc-between-tests",
action="store_true",
default=False,
help="Force garbage collection between each test"),
optparse.make_option(
"-l",
"--leaks",
action="store_true",
default=False,
help="Enable leaks checking (OS X and Gtk+ only)"),
optparse.make_option("-g",
"--guard-malloc",
action="store_true",
default=False,
help="Enable Guard Malloc (OS X only)"),
optparse.make_option(
"--threaded",
action="store_true",
default=False,
help="Run a concurrent JavaScript thread with each test"),
optparse.make_option(
"--dump-render-tree",
"-1",
action="store_false",
default=True,
dest="webkit_test_runner",
help="Use DumpRenderTree rather than WebKitTestRunner."),
# FIXME: We should merge this w/ --build-directory and only have one flag.
optparse.make_option(
"--root",
action="store",
help=
"Path to a directory containing the executables needed to run tests."
),
]))
option_group_definitions.append((
"Results Options",
[
optparse.make_option("-p",
"--pixel",
"--pixel-tests",
action="store_true",
dest="pixel_tests",
help="Enable pixel-to-pixel PNG comparisons"),
optparse.make_option(
"--no-pixel",
"--no-pixel-tests",
action="store_false",
dest="pixel_tests",
help="Disable pixel-to-pixel PNG comparisons"),
optparse.make_option(
"--no-sample-on-timeout",
action="store_false",
default=True,
dest="sample_on_timeout",
help="Don't run sample on timeout (OS X only)"),
optparse.make_option("--no-ref-tests",
action="store_true",
dest="no_ref_tests",
help="Skip all ref tests"),
optparse.make_option(
"--ignore-render-tree-dump-results",
action="store_true",
dest="ignore_render_tree_dump_results",
help=
"Don't compare or save results for render tree dump tests (they still run and crashes are reported)"
),
optparse.make_option(
"--tolerance",
help="Ignore image differences less than this percentage (some "
"ports may ignore this option)",
type="float"),
optparse.make_option("--results-directory",
help="Location of test results"),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option(
"--add-platform-exceptions",
action="store_true",
default=False,
help=
"Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"
),
optparse.make_option(
"--new-baseline",
action="store_true",
default=False,
help="Save generated results as new baselines "
"into the *most-specific-platform* directory, overwriting whatever's "
"already there. Equivalent to --reset-results --add-platform-exceptions"
),
optparse.make_option(
"--reset-results",
action="store_true",
default=False,
help="Reset expectations to the "
"generated results in their existing location."),
optparse.make_option(
"--no-new-test-results",
action="store_false",
dest="new_test_results",
default=True,
help="Don't create new baselines when no expected results exist"
),
optparse.make_option(
"--treat-ref-tests-as-pixel-tests",
action="store_true",
default=False,
help=
"Run ref tests, but treat them as if they were traditional pixel tests"
),
#FIXME: we should support a comma separated list with --pixel-test-directory as well.
optparse.make_option(
"--pixel-test-directory",
action="append",
default=[],
dest="pixel_test_directories",
help=
"A directory where it is allowed to execute tests as pixel tests. "
"Specify multiple times to add multiple directories. "
"This option implies --pixel-tests. If specified, only those tests "
"will be executed as pixel tests that are located in one of the "
"directories enumerated with the option. Some ports may ignore this "
"option while others can have a default value that can be overridden here."
),
optparse.make_option(
"--skip-failing-tests",
action="store_true",
default=False,
help="Skip tests that are marked as failing or flaky. "
"Note: When using this option, you might miss new crashes "
"in these tests."),
optparse.make_option(
"--additional-drt-flag",
action="append",
default=[],
help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
optparse.make_option(
"--driver-name",
type="string",
help="Alternative DumpRenderTree binary to use"),
optparse.make_option(
"--additional-platform-directory",
action="append",
default=[],
help="Additional directory where to look for test "
"baselines (will take precendence over platform baselines). "
"Specify multiple times to add multiple search path entries."),
optparse.make_option(
"--additional-expectations",
action="append",
default=[],
help=
"Path to a test_expectations file that will override previous expectations. "
"Specify multiple times for multiple sets of overrides."),
optparse.make_option(
"--compare-port",
action="store",
default=None,
help="Use the specified port's baselines first"),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help="Don't launch a browser with results after the tests "
"are done"),
optparse.make_option(
"--full-results-html",
action="store_true",
default=False,
help=
"Show all failures in results.html, rather than only regressions"
),
optparse.make_option(
"--clobber-old-results",
action="store_true",
default=False,
help="Clobbers test results from previous runs."),
optparse.make_option(
"--http",
action="store_true",
dest="http",
default=True,
help="Run HTTP and WebSocket tests (default)"),
optparse.make_option("--no-http",
action="store_false",
dest="http",
help="Don't run HTTP and WebSocket tests"),
optparse.make_option("--no-http-servers",
action="store_false",
dest="start_http_servers_if_needed",
default=True,
help="Don't start HTTP servers"),
optparse.make_option(
"--ignore-metrics",
action="store_true",
dest="ignore_metrics",
default=False,
help="Ignore rendering metrics related information from test "
"output, only compare the structure of the rendertree."),
optparse.make_option(
"--nocheck-sys-deps",
action="store_true",
default=False,
help="Don't check the system dependencies (themes)"),
optparse.make_option("--java",
action="store_true",
default=False,
help="Build java support files"),
optparse.make_option(
"--layout-tests-directory",
action="store",
default=None,
help="Override the default layout test directory.",
dest="layout_tests_dir")
]))
option_group_definitions.append(("Testing Options", [
optparse.make_option("--build", dest="build",
action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date "
"(default)."),
optparse.make_option("--no-build", dest="build",
action="store_false", help="Don't check to see if the "
"DumpRenderTree build is up-to-date."),
optparse.make_option("-n", "--dry-run", action="store_true",
default=False,
help="Do everything but actually run the tests or upload results."),
optparse.make_option("--wrapper",
help="wrapper command to insert before invocations of "
"DumpRenderTree or WebKitTestRunner; option is split on whitespace before "
"running. (Example: --wrapper='valgrind --smc-check=all')"),
optparse.make_option("-i", "--ignore-tests", action="append", default=[],
help="directories or test to ignore (may specify multiple times)"),
optparse.make_option("--test-list", action="append",
help="read list of tests to run from file", metavar="FILE"),
optparse.make_option("--skipped", action="store", default="default",
help=("control how tests marked SKIP are run. "
"'default' == Skip tests unless explicitly listed on the command line, "
"'ignore' == Run them anyway, "
"'only' == only run the SKIP tests, "
"'always' == always skip, even if listed on the command line.")),
optparse.make_option("--force", action="store_true", default=False,
help="Run all tests with PASS as expected result, even those marked SKIP in the test list or " + \
"those which are device-specific (implies --skipped=ignore)"),
optparse.make_option("--time-out-ms", "--timeout",
help="Set the timeout for each test in milliseconds"),
optparse.make_option("--order", action="store", default="natural",
help=("determine the order in which the test cases will be run. "
"'none' == use the order in which the tests were listed either in arguments or test list, "
"'natural' == use the natural order (default), "
"'random' == randomize the test order.")),
optparse.make_option("--run-chunk",
help=("Run a specified chunk (n:l), the nth of len l, "
"of the layout tests")),
optparse.make_option("--run-part", help=("Run a specified part (n:m), "
"the nth of m parts, of the layout tests")),
optparse.make_option("--batch-size",
help=("Run a the tests in batches (n), after every n tests, "
"DumpRenderTree is relaunched."), type="int", default=None),
optparse.make_option("--run-singly", action="store_true",
default=False, help="run a separate DumpRenderTree for each test (implies --verbose)"),
optparse.make_option("--child-processes",
help="Number of DumpRenderTrees to run in parallel."),
# FIXME: Display default number of child processes that will run.
optparse.make_option("-f", "--fully-parallel", action="store_true",
help="run all tests in parallel"),
optparse.make_option("--exit-after-n-failures", type="int", default=None,
help="Exit after the first N failures instead of running all "
"tests"),
optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
default=None, help="Exit after the first N crashes instead of "
"running all tests"),
optparse.make_option("--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)"),
optparse.make_option("--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"),
optparse.make_option("--retry-failures", action="store_true",
default=True,
help="Re-try any tests that produce unexpected results (default)"),
optparse.make_option("--no-retry-failures", action="store_false",
dest="retry_failures",
help="Don't re-try any tests that produce unexpected results."),
optparse.make_option("--max-locked-shards", type="int", default=0,
help="Set the maximum number of locked shards"),
optparse.make_option("--additional-env-var", type="string", action="append", default=[],
help="Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"),
optparse.make_option("--profile", action="store_true",
help="Output per-test profile information."),
optparse.make_option("--profiler", action="store",
help="Output per-test profile information, using the specified profiler."),
optparse.make_option("--no-timeout", action="store_true", default=False, help="Disable test timeouts"),
optparse.make_option('--display-server', choices=['xvfb', 'xorg', 'weston', 'wayland'], default='xvfb',
help='"xvfb": Use a virtualized X11 server. "xorg": Use the current X11 session. '
'"weston": Use a virtualized Weston server. "wayland": Use the current wayland session.'),
optparse.make_option("--world-leaks", action="store_true", default=False, help="Check for world leaks (currently, only documents). Differs from --leaks in that this uses internal instrumentation, rather than external tools."),
optparse.make_option("--accessibility-isolated-tree", action="store_true", default=False, help="Runs tests in accessibility isolated tree mode."),
]))
option_group_definitions.append(("iOS Options", [
optparse.make_option(
'--no-install',
action='store_const',
const=False,
default=True,
dest='install',
help='Skip install step for device and simulator testing'),
optparse.make_option(
'--version',
help=
'Specify the version of iOS to be used. By default, this will adopt the runtime for iOS Simulator.'
),
optparse.make_option(
'--device-type',
help=
'iOS Simulator device type identifier (default: i386 -> iPhone 5, x86_64 -> iPhone SE)'
),
optparse.make_option(
'--dedicated-simulators',
action="store_true",
default=False,
help=
"If set, dedicated iOS simulators will always be created. If not set, the script will attempt to use any currently running simulator."
),
optparse.make_option(
'--show-touches',
action="store_true",
default=False,
help=
"If set, a small dot will be shown where the generated touches are. Helpful for debugging touch tests."
),
]))
option_group_definitions.append(("Miscellaneous Options", [
optparse.make_option(
"--lint-test-files",
action="store_true",
default=False,
help=
("Makes sure the test files parse for all configurations. Does not run any tests."
)),
optparse.make_option(
"--print-expectations",
action="store_true",
default=False,
help=
("Print the expected outcome for the given test, or all tests listed in TestExpectations. Does not run any tests."
)),
optparse.make_option(
"--webgl-test-suite",
action="store_true",
default=False,
help=
("Run exhaustive webgl list, including test ordinarily skipped for performance reasons. Equivalent to '--additional-expectations=LayoutTests/webgl/TestExpectations webgl'"
)),
optparse.make_option(
"--use-gpu-process",
action="store_true",
default=False,
help=
("Enable all GPU process related features, also set additional expectations and the result report flavor."
)),
optparse.make_option(
"--prefer-integrated-gpu",
action="store_true",
default=False,
help=
("Prefer using the lower-power integrated GPU on a dual-GPU system. Note that other running applications and the tests themselves can override this request."
)),
]))
option_group_definitions.append(("Web Platform Test Server Options", [
optparse.make_option(
"--wptserver-doc-root",
type="string",
help=
("Set web platform server document root, relative to LayoutTests directory"
)),
]))
# FIXME: Remove this group once the old results dashboards are deprecated.
option_group_definitions.append(("Legacy Result Options", [
optparse.make_option("--master-name",
help="The name of the buildbot master."),
optparse.make_option(
"--build-name",
default="DUMMY_BUILD_NAME",
help=(
"The name of the builder used in its path, e.g. webkit-rel.")),
optparse.make_option(
"--build-slave",
default="DUMMY_BUILD_SLAVE",
help=("The name of the worker used. e.g. apple-macpro-6.")),
optparse.make_option(
"--test-results-server",
action="append",
default=[],
help=
("If specified, upload results json files to this appengine server."
)),
optparse.make_option(
"--results-server-host",
action="append",
default=[],
help=(
"If specified, upload results JSON file to this results server."
)),
optparse.make_option(
"--additional-repository-name",
help=("The name of an additional subversion or git checkout")),
optparse.make_option(
"--additional-repository-path",
help=
("The path to an additional subversion or git checkout (requires --additional-repository-name)"
)),
optparse.make_option(
"--allowed-host",
type="string",
action="append",
default=[],
help=
("If specified, tests are allowed to make requests to the specified hostname."
))
]))
option_group_definitions.append(('Upload Options', upload_options()))
option_parser = optparse.OptionParser(usage="%prog [options] [<path>...]")
for group_name, group_options in option_group_definitions:
option_group = optparse.OptionGroup(option_parser, group_name)
option_group.add_options(group_options)
option_parser.add_option_group(option_group)
options, args = option_parser.parse_args(args)
if options.webgl_test_suite:
if not args:
args.append('webgl')
host = Host()
host.initialize_scm()
options.additional_expectations.insert(
0,
host.filesystem.join(host.scm().checkout_root,
'LayoutTests/webgl/TestExpectations'))
if options.use_gpu_process:
host = Host()
host.initialize_scm()
options.additional_expectations.insert(
0,
host.filesystem.join(host.scm().checkout_root,
'LayoutTests/gpu-process/TestExpectations'))
if not options.internal_feature:
options.internal_feature = []
options.internal_feature.append('UseGPUProcessForMediaEnabled')
options.internal_feature.append('CaptureAudioInGPUProcessEnabled')
options.internal_feature.append('CaptureVideoInGPUProcessEnabled')
options.internal_feature.append(
'UseGPUProcessForCanvasRenderingEnabled')
options.internal_feature.append('UseGPUProcessForDOMRenderingEnabled')
options.internal_feature.append('UseGPUProcessForWebGLEnabled')
if not options.experimental_feature:
options.experimental_feature = []
options.experimental_feature.append(
'WebRTCPlatformCodecsInGPUProcessEnabled')
if options.result_report_flavor:
raise RuntimeError(
'--use-gpu-process implicitly sets the result flavor, this should not be overridden'
)
options.result_report_flavor = 'gpuprocess'
return options, args
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._timestamp = time.time()
self._utc_timestamp = datetime.datetime.utcnow()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option('--debug',
action='store_const',
const='Debug',
dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release',
action='store_const',
const='Release',
dest="configuration",
help='Set the configuration to Release'),
optparse.make_option(
"--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const",
const='chromium',
dest='platform',
help='Alias for --platform=chromium'),
optparse.make_option("--chromium-android",
action="store_const",
const='chromium-android',
dest='platform',
help='Alias for --platform=chromium-android'),
optparse.make_option(
"--builder-name",
help=
("The name of the builder shown on the waterfall running this script e.g. google-mac-2."
)),
optparse.make_option(
"--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help=
"Check to ensure the DumpRenderTree build is up-to-date (default)."
),
optparse.make_option(
"--no-build",
dest="build",
action="store_false",
help=
"Don't check to see if the DumpRenderTree build is up-to-date."
),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option("--time-out-ms",
default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option(
"--no-results",
action="store_false",
dest="generate_results",
default=True,
help="Do no generate results JSON and results page."),
optparse.make_option(
"--output-json-path",
action='callback',
callback=_expand_path,
type="str",
help=
"Path to generate a JSON file at; may contain previous results if it already exists."
),
optparse.make_option(
"--reset-results",
action="store_true",
help=
"Clears the content in the generated JSON file before adding the results."
),
optparse.make_option(
"--slave-config-json-path",
action='callback',
callback=_expand_path,
type="str",
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option(
"--description",
help=
"Add a description to the output JSON file if one is generated"
),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help=
"Don't launch a browser with results after the tests are done"
),
optparse.make_option(
"--test-results-server",
help=
"Upload the generated JSON file to the specified server when --output-json-path is present."
),
optparse.make_option("--replay",
dest="replay",
action="store_true",
default=False,
help="Run replay tests."),
optparse.make_option(
"--force",
dest="use_skipped_list",
action="store_false",
default=True,
help="Run all tests, including the ones in the Skipped list."),
optparse.make_option("--profile",
action="store_true",
help="Output per-test profile information."),
optparse.make_option(
"--profiler",
action="store",
help=
"Output per-test profile information, using the specified profiler."
),
optparse.make_option(
"--additional-drt-flag",
action="append",
default=[],
help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
optparse.make_option(
"--driver-name",
type="string",
help="Alternative DumpRenderTree binary to use"),
optparse.make_option(
"--content-shell",
action="store_true",
help="Use Content Shell instead of DumpRenderTree"),
optparse.make_option(
"--repeat",
default=1,
type="int",
help="Specify number of times to run test set (default: 1)."),
optparse.make_option(
"--test-runner-count",
default=DEFAULT_TEST_RUNNER_COUNT,
type="int",
help=
"Specify number of times to invoke test runner for each performance test."
),
]
return optparse.OptionParser(
option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
if filesystem.exists(filesystem.join(self._base_path, arg)):
paths.append(arg)
else:
relpath = filesystem.relpath(arg, self._base_path)
if filesystem.exists(filesystem.join(self._base_path,
relpath)):
paths.append(filesystem.normpath(relpath))
else:
_log.warn('Path was not found:' + arg)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths,
skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = filesystem.relpath(path, self._base_path).replace(
'\\', '/')
if self._options.use_skipped_list and self._port.skips_perf_test(
relative_path) and filesystem.normpath(
relative_path) not in paths:
continue
test = PerfTestFactory.create_perf_test(
self._port,
relative_path,
path,
test_runner_count=self._options.test_runner_count)
tests.append(test)
return tests
def _start_http_servers(self):
self._port.acquire_http_lock()
self._port.start_http_server(number_of_servers=2)
def _stop_http_servers(self):
self._port.stop_http_server()
self._port.release_http_lock()
def run(self):
needs_http = self._port.requires_http_server()
if not self._port.check_build(needs_http=needs_http):
_log.error("Build not up to date for %s" %
self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
run_count = 0
repeat = self._options.repeat
while (run_count < repeat):
run_count += 1
tests = self._collect_tests()
runs = ' (Run %d of %d)' % (run_count,
repeat) if repeat > 1 else ''
_log.info("Running %d tests%s" % (len(tests), runs))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
try:
if needs_http:
self._start_http_servers()
unexpected = self._run_tests_set(
sorted(list(tests), key=lambda test: test.test_name()))
finally:
if needs_http:
self._stop_http_servers()
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_results()
if exit_code:
return exit_code
if self._options.generate_results and not self._options.profile:
test_results_server = self._options.test_results_server
if test_results_server and not self._upload_json(
test_results_server, self._output_json_path()):
return self.EXIT_CODE_FAILED_UPLOADING
if self._options.show_results:
self._port.show_results_html_file(self._results_page_path())
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(),
self._DEFAULT_JSON_FILENAME)
def _results_page_path(self):
return self._host.filesystem.splitext(
self._output_json_path())[0] + '.html'
def _generate_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp,
options.description,
options.platform,
options.builder_name,
options.build_number)
if options.slave_config_json_path:
output = self._merge_slave_config_json(
options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
filesystem = self._host.filesystem
json_output = json.dumps(output)
filesystem.write_text_file(output_json_path, json_output)
template_path = filesystem.join(self._port.perf_tests_dir(),
'resources/results-template.html')
template = filesystem.read_text_file(template_path)
absolute_path_to_trunk = filesystem.dirname(
self._port.perf_tests_dir())
results_page = template.replace('%AbsolutePathToWebKitTrunk%',
absolute_path_to_trunk)
results_page = results_page.replace('%PeformanceTestsResultsJSON%',
json_output)
filesystem.write_text_file(self._results_page_path(), results_page)
def _generate_results_dict(self, timestamp, description, platform,
builder_name, build_number):
revisions = {}
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive
).detect_scm_system(path) or self._host.scm()
revision = scm.svn_revision(path)
revisions[name] = {
'revision': revision,
'timestamp': scm.timestamp_of_revision(path, revision)
}
meta_info = {
'description':
description,
'buildTime':
self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp),
'platform':
platform,
'revisions':
revisions,
'builderName':
builder_name,
'buildNumber':
int(build_number) if build_number else None
}
contents = {'tests': {}}
for key, value in meta_info.items():
if value:
contents[key] = value
for test, metrics in self._results:
for metric_name, iteration_values in metrics.iteritems():
if not isinstance(
iteration_values, list
): # We can't reports results without individual measurements.
continue
tests = contents['tests']
path = test.test_name_without_file_extension().split('/')
for i in range(0, len(path)):
is_last_token = i + 1 == len(path)
url = view_source_url('PerformanceTests/' +
(test.test_name() if is_last_token
else '/'.join(path[0:i + 1])))
tests.setdefault(path[i], {'url': url})
current_test = tests[path[i]]
if is_last_token:
current_test.setdefault('metrics', {})
assert metric_name not in current_test['metrics']
current_test['metrics'][metric_name] = {
'current': iteration_values
}
else:
current_test.setdefault('tests', {})
tests = current_test['tests']
return contents
@staticmethod
def _datetime_in_ES5_compatible_iso_format(datetime):
return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')
def _merge_slave_config_json(self, slave_config_json_path, contents):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" %
slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(
slave_config_json_path)
slave_config = json.load(slave_config_json)
for key in slave_config:
contents['builder' + key.capitalize()] = slave_config[key]
return contents
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" %
(slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_default_branch = "webkit-trunk"
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = "PerformanceTestsResults.json"
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host.initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._timestamp = time.time()
self._utc_timestamp = datetime.datetime.utcnow()
@staticmethod
def _parse_args(args=None):
def _expand_path(option, opt_str, value, parser):
path = os.path.expandvars(os.path.expanduser(value))
setattr(parser.values, option.dest, path)
perf_option_list = [
optparse.make_option(
"--debug",
action="store_const",
const="Debug",
dest="configuration",
help="Set the configuration to Debug",
),
optparse.make_option(
"--release",
action="store_const",
const="Release",
dest="configuration",
help="Set the configuration to Release",
),
optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option(
"--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2."),
),
optparse.make_option("--build-number", help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default).",
),
optparse.make_option(
"--no-build",
dest="build",
action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date.",
),
optparse.make_option(
"--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)",
),
optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"),
optparse.make_option(
"--no-results",
action="store_false",
dest="generate_results",
default=True,
help="Do no generate results JSON and results page.",
),
optparse.make_option(
"--output-json-path",
action="callback",
callback=_expand_path,
type="str",
help="Path to generate a JSON file at; may contain previous results if it already exists.",
),
optparse.make_option(
"--reset-results",
action="store_true",
help="Clears the content in the generated JSON file before adding the results.",
),
optparse.make_option(
"--slave-config-json-path",
action="callback",
callback=_expand_path,
type="str",
help="Only used on bots. Path to a slave configuration file.",
),
optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help="Don't launch a browser with results after the tests are done",
),
optparse.make_option(
"--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present.",
),
optparse.make_option(
"--webkit-test-runner",
"-2",
action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree.",
),
optparse.make_option(
"--replay", dest="replay", action="store_true", default=False, help="Run replay tests."
),
optparse.make_option(
"--force",
dest="use_skipped_list",
action="store_false",
default=True,
help="Run all tests, including the ones in the Skipped list.",
),
optparse.make_option("--profile", action="store_true", help="Output per-test profile information."),
optparse.make_option(
"--profiler", action="store", help="Output per-test profile information, using the specified profiler."
),
optparse.make_option(
"--additional-drt-flag",
action="append",
default=[],
help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags.",
),
optparse.make_option("--driver-name", type="string", help="Alternative DumpRenderTree binary to use"),
optparse.make_option(
"--repeat", default=1, type="int", help="Specify number of times to run test set (default: 1)."
),
optparse.make_option(
"--test-runner-count",
default=DEFAULT_TEST_RUNNER_COUNT,
type="int",
help="Specify number of times to invoke test runner for each performance test.",
),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
test_extensions = [".html", ".svg"]
if self._options.replay:
test_extensions.append(".replay")
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
if filesystem.exists(filesystem.join(self._base_path, arg)):
paths.append(arg)
else:
relpath = filesystem.relpath(arg, self._base_path)
if filesystem.exists(filesystem.join(self._base_path, relpath)):
paths.append(filesystem.normpath(relpath))
else:
_log.warn("Path was not found:" + arg)
skipped_directories = set([".svn", "resources"])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = filesystem.relpath(path, self._base_path).replace("\\", "/")
if (
self._options.use_skipped_list
and self._port.skips_perf_test(relative_path)
and filesystem.normpath(relative_path) not in paths
):
continue
test = PerfTestFactory.create_perf_test(
self._port, relative_path, path, test_runner_count=self._options.test_runner_count
)
tests.append(test)
return tests
def _start_http_servers(self):
self._port.acquire_http_lock()
self._port.start_http_server(number_of_servers=2)
def _stop_http_servers(self):
self._port.stop_http_server()
self._port.release_http_lock()
def run(self):
needs_http = self._port.requires_http_server()
if not self._port.check_build(needs_http=needs_http):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
run_count = 0
repeat = self._options.repeat
while run_count < repeat:
run_count += 1
tests = self._collect_tests()
runs = " (Run %d of %d)" % (run_count, repeat) if repeat > 1 else ""
_log.info("Running %d tests%s" % (len(tests), runs))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
try:
if needs_http:
self._start_http_servers()
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()))
finally:
if needs_http:
self._stop_http_servers()
if self._options.generate_results and not self._options.profile:
exit_code = self._generate_results()
if exit_code:
return exit_code
if self._options.generate_results and not self._options.profile:
test_results_server = self._options.test_results_server
if test_results_server and not self._upload_json(test_results_server, self._output_json_path()):
return self.EXIT_CODE_FAILED_UPLOADING
if self._options.show_results:
self._port.show_results_html_file(self._results_page_path())
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _results_page_path(self):
return self._host.filesystem.splitext(self._output_json_path())[0] + ".html"
def _generate_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(
self._timestamp, options.description, options.platform, options.builder_name, options.build_number
)
if options.slave_config_json_path:
output = self._merge_slave_config_json(options.slave_config_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
output = self._merge_outputs_if_needed(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
filesystem = self._host.filesystem
json_output = json.dumps(output)
filesystem.write_text_file(output_json_path, json_output)
template_path = filesystem.join(self._port.perf_tests_dir(), "resources/results-template.html")
template = filesystem.read_text_file(template_path)
absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir())
results_page = template.replace("%AbsolutePathToWebKitTrunk%", absolute_path_to_trunk)
results_page = results_page.replace("%PeformanceTestsResultsJSON%", json_output)
filesystem.write_text_file(self._results_page_path(), results_page)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
revisions = {}
for (name, path) in self._port.repository_paths():
scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm()
revision = scm.svn_revision(path)
revisions[name] = {"revision": revision, "timestamp": scm.timestamp_of_revision(path, revision)}
meta_info = {
"description": description,
"buildTime": self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp),
"platform": platform,
"revisions": revisions,
"builderName": builder_name,
"buildNumber": int(build_number) if build_number else None,
}
contents = {"tests": {}}
for key, value in meta_info.items():
if value:
contents[key] = value
for test, metrics in self._results:
for metric_name, iteration_values in metrics.iteritems():
if not isinstance(iteration_values, list): # We can't reports results without individual measurements.
continue
tests = contents["tests"]
path = test.test_name_without_file_extension().split("/")
for i in range(0, len(path)):
is_last_token = i + 1 == len(path)
url = view_source_url(
"PerformanceTests/" + (test.test_name() if is_last_token else "/".join(path[0 : i + 1]))
)
tests.setdefault(path[i], {"url": url})
current_test = tests[path[i]]
if is_last_token:
current_test.setdefault("metrics", {})
assert metric_name not in current_test["metrics"]
current_test["metrics"][metric_name] = {"current": iteration_values}
else:
current_test.setdefault("tests", {})
tests = current_test["tests"]
return contents
@staticmethod
def _datetime_in_ES5_compatible_iso_format(datetime):
return datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")
def _merge_slave_config_json(self, slave_config_json_path, contents):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" % slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
for key in slave_config:
contents["builder" + key.capitalize()] = slave_config[key]
return contents
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error))
return None
