def get_test_baselines(test_file, test_config):
# FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic.
class AllPlatformsPort(WebKitPort):
def __init__(self, host):
WebKitPort.__init__(self, host, 'mac')
self._platforms_by_directory = dict([
(self._webkit_baseline_path(p), p)
for p in test_config.platforms
])
def baseline_search_path(self):
return self._platforms_by_directory.keys()
def platform_from_directory(self, directory):
return self._platforms_by_directory[directory]
test_path = test_config.filesystem.join(test_config.layout_tests_directory,
test_file)
# FIXME: This should get the Host from the test_config to be mockable!
host = Host()
host._initialize_scm()
host.filesystem = test_config.filesystem
all_platforms_port = AllPlatformsPort(host)
all_test_baselines = {}
for baseline_extension in ('.txt', '.checksum', '.png'):
test_baselines = test_config.test_port.expected_baselines(
test_file, baseline_extension)
baselines = all_platforms_port.expected_baselines(test_file,
baseline_extension,
all_baselines=True)
for platform_directory, expected_filename in baselines:
if not platform_directory:
continue
if platform_directory == test_config.layout_tests_directory:
platform = 'base'
else:
platform = all_platforms_port.platform_from_directory(
platform_directory)
platform_baselines = all_test_baselines.setdefault(platform, {})
was_used_for_test = (platform_directory,
expected_filename) in test_baselines
platform_baselines[baseline_extension] = was_used_for_test
return all_test_baselines
def main(self):
args = sys.argv[1:]
host = Host()
host._initialize_scm()
stderr = self._engage_awesome_stderr_hacks()
# Checking for the verbose flag before calling check_webkit_style_parser()
# lets us enable verbose logging earlier.
is_verbose = "-v" in args or "--verbose" in args
checker.configure_logging(stream=stderr, is_verbose=is_verbose)
_log.debug("Verbose logging enabled.")
parser = checker.check_webkit_style_parser()
(paths, options) = parser.parse(args)
configuration = checker.check_webkit_style_configuration(options)
paths = change_directory(host.filesystem,
checkout_root=host.scm().checkout_root,
paths=paths)
style_processor = StyleProcessor(configuration)
file_reader = TextFileReader(host.filesystem, style_processor)
if paths and not options.diff_files:
file_reader.process_paths(paths)
else:
changed_files = paths if options.diff_files else None
patch = host.scm().create_patch(options.git_commit,
changed_files=changed_files)
patch_checker = PatchReader(file_reader)
patch_checker.check(patch)
error_count = style_processor.error_count
file_count = file_reader.file_count
delete_only_file_count = file_reader.delete_only_file_count
_log.info("Total errors found: %d in %d files" %
(error_count, file_count))
# We fail when style errors are found or there are no checked files.
return error_count > 0 or (file_count == 0
and delete_only_file_count == 0)
def run(self):
options = self._options
# FIXME: This should get the Host from the owner of this object
# so this function can be properly mocked!
host = Host()
host._initialize_scm()
port_obj = host.port_factory.get(self._platform_name, options)
# The unix multiprocessing implementation clones the
# log handler configuration into the child processes,
# but the win implementation doesn't.
configure_logging = (sys.platform == 'win32')
# FIXME: this won't work if the calling process is logging
# somewhere other than sys.stderr and sys.stdout, but I'm not sure
# if this will be an issue in practice.
printer = printing.Printer(port_obj, options, sys.stderr, sys.stdout, configure_logging)
self._client.run(port_obj)
printer.cleanup()
def main(args):
"""Bootstrap function that sets up the object references we need and calls real_main()."""
options, target_options = parse_options(args)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
if options.verbose:
log_level = logging.DEBUG
log_handler = DebugLogHandler()
else:
log_level = logging.INFO
log_handler = NormalLogHandler()
logger = logging.getLogger()
logger.setLevel(log_level)
logger.addHandler(log_handler)
host = Host()
host._initialize_scm()
target_port_obj = host.port_factory.get(None, target_options)
host_port_obj = get_host_port_object(host.port_factory, options)
if not host_port_obj or not target_port_obj:
return 1
url_fetcher = urlfetcher.UrlFetcher(host.filesystem)
# We use the default zip factory method.
zip_factory = None
# FIXME: SCM module doesn't handle paths that aren't relative to the checkout_root consistently.
host_port_obj._filesystem.chdir(host.scm().checkout_root)
ret_code = real_main(host, options, target_options, host_port_obj,
target_port_obj, url_fetcher, zip_factory)
if not ret_code and log_handler.num_failures:
ret_code = 1
print ''
if ret_code:
print 'Rebaselining failed.'
else:
print 'Rebaselining succeeded.'
return ret_code
def main(args):
"""Bootstrap function that sets up the object references we need and calls real_main()."""
options, target_options = parse_options(args)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
if options.verbose:
log_level = logging.DEBUG
log_handler = DebugLogHandler()
else:
log_level = logging.INFO
log_handler = NormalLogHandler()
logger = logging.getLogger()
logger.setLevel(log_level)
logger.addHandler(log_handler)
host = Host()
host._initialize_scm()
target_port_obj = host.port_factory.get(None, target_options)
host_port_obj = get_host_port_object(host.port_factory, options)
if not host_port_obj or not target_port_obj:
return 1
url_fetcher = urlfetcher.UrlFetcher(host_port_obj._filesystem)
# We use the default zip factory method.
zip_factory = None
# FIXME: SCM module doesn't handle paths that aren't relative to the checkout_root consistently.
host_port_obj._filesystem.chdir(host.scm().checkout_root)
ret_code = real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher, zip_factory, host.scm())
if not ret_code and log_handler.num_failures:
ret_code = 1
print ""
if ret_code:
print "Rebaselining failed."
else:
print "Rebaselining succeeded."
return ret_code
def main(self):
args = sys.argv[1:]
host = Host()
host._initialize_scm()
stderr = self._engage_awesome_stderr_hacks()
# Checking for the verbose flag before calling check_webkit_style_parser()
# lets us enable verbose logging earlier.
is_verbose = "-v" in args or "--verbose" in args
checker.configure_logging(stream=stderr, is_verbose=is_verbose)
_log.debug("Verbose logging enabled.")
parser = checker.check_webkit_style_parser()
(paths, options) = parser.parse(args)
configuration = checker.check_webkit_style_configuration(options)
paths = change_directory(host.filesystem, checkout_root=host.scm().checkout_root, paths=paths)
style_processor = StyleProcessor(configuration)
file_reader = TextFileReader(host.filesystem, style_processor)
if paths and not options.diff_files:
file_reader.process_paths(paths)
else:
changed_files = paths if options.diff_files else None
patch = host.scm().create_patch(options.git_commit, changed_files=changed_files)
patch_checker = PatchReader(file_reader)
patch_checker.check(patch)
error_count = style_processor.error_count
file_count = file_reader.file_count
delete_only_file_count = file_reader.delete_only_file_count
_log.info("Total errors found: %d in %d files" % (error_count, file_count))
# We fail when style errors are found or there are no checked files.
return error_count > 0 or (file_count == 0 and delete_only_file_count == 0)
def get_test_baselines(test_file, test_config):
# FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic.
class AllPlatformsPort(WebKitPort):
def __init__(self, host):
WebKitPort.__init__(self, host)
self._platforms_by_directory = dict([(self._webkit_baseline_path(p), p) for p in test_config.platforms])
def baseline_search_path(self):
return self._platforms_by_directory.keys()
def platform_from_directory(self, directory):
return self._platforms_by_directory[directory]
test_path = test_config.filesystem.join(test_config.layout_tests_directory, test_file)
# FIXME: This should get the Host from the test_config to be mockable!
host = Host()
host._initialize_scm()
host.filesystem = test_config.filesystem
all_platforms_port = AllPlatformsPort(host)
all_test_baselines = {}
for baseline_extension in ('.txt', '.checksum', '.png'):
test_baselines = test_config.test_port.expected_baselines(test_file, baseline_extension)
baselines = all_platforms_port.expected_baselines(test_file, baseline_extension, all_baselines=True)
for platform_directory, expected_filename in baselines:
if not platform_directory:
continue
if platform_directory == test_config.layout_tests_directory:
platform = 'base'
else:
platform = all_platforms_port.platform_from_directory(platform_directory)
platform_baselines = all_test_baselines.setdefault(platform, {})
was_used_for_test = (platform_directory, expected_filename) in test_baselines
platform_baselines[baseline_extension] = was_used_for_test
return all_test_baselines
test_name = self._port.uri_to_test_name(test_input.uri)
actual_text = port.expected_text(test_name)
actual_image = ''
actual_checksum = ''
if self._options.pixel_tests and test_input.checksum:
actual_checksum = port.expected_checksum(test_name)
if actual_checksum != test_input.checksum:
actual_image = port.expected_image(test_name)
self._stdout.write("#URL:%s\n" % test_input.uri)
if self._options.pixel_tests and test_input.checksum:
self._stdout.write("#MD5:%s\n" % actual_checksum)
self._host.filesystem.write_binary_file(self._options.pixel_path,
actual_image)
self._stdout.write(actual_text)
# FIXME: (See above FIXME as well). Chromium DRT appears to always
# ensure the text output has a trailing newline. Mac DRT does not.
if not actual_text.endswith('\n'):
self._stdout.write('\n')
self._stdout.write('#EOF\n')
self._stdout.flush()
if __name__ == '__main__':
# FIXME: Why is this using a real Host object instead of MockHost?
host = Host()
host._initialize_scm()
sys.exit(main(sys.argv[1:], host, sys.stdin, sys.stdout, sys.stderr))
class PerfTestsRunner(object):
_perf_tests_base_dir = 'PerformanceTests'
_result_regex = re.compile('^RESULT .*$')
def __init__(self,
perf_tests_dir,
regular_output=sys.stderr,
buildbot_output=sys.stdout,
args=None):
self._perf_tests_dir = perf_tests_dir
self._buildbot_output = buildbot_output
self._options, self._args = self._parse_args(args)
self._host = Host()
self._host._initialize_scm()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
self._printer = printing.Printer(self._port,
self._options,
regular_output,
buildbot_output,
configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
def _parse_args(self, args=None):
print_options = printing.print_options()
perf_option_list = [
optparse.make_option('--debug',
action='store_const',
const='Debug',
dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release',
action='store_const',
const='Release',
dest="configuration",
help='Set the configuration to Release'),
optparse.make_option(
"--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option("--time-out-ms",
default=30000,
help="Set the timeout for each test"),
]
option_list = (perf_option_list + print_options)
return optparse.OptionParser(option_list=option_list).parse_args(args)
def _collect_tests(self, webkit_base, filesystem=None):
"""Return the list of tests found."""
def _is_test_file(filesystem, dirname, filename):
return filename.endswith('.html')
filesystem = filesystem or self._host.filesystem
base_dir = filesystem.join(webkit_base, self._perf_tests_base_dir,
self._perf_tests_dir)
return find_files.find(filesystem,
base_dir,
paths=self._args,
file_filter=_is_test_file)
def run(self):
if self._options.help_printing:
self._printer.help_printing()
self._printer.cleanup()
return 0
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" %
self._port._path_to_driver())
return -1
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected = -1
try:
tests = self._collect_tests(self._port.webkit_base())
unexpected = self._run_tests_set(tests, self._port)
finally:
self._printer.cleanup()
return unexpected
def _run_tests_set(self, tests, port):
result_count = len(tests)
expected = 0
unexpected = 0
self._printer.print_one_line_summary(result_count, 0, 0)
driver_need_restart = False
driver = None
for test in tests:
if driver_need_restart:
_log.debug("%s killing driver" % test)
driver.stop()
driver = None
if not driver:
driver = port.create_driver(worker_number=1)
test_failed, driver_need_restart = self._run_single_test(
test, driver)
if test_failed:
unexpected = unexpected + 1
else:
expected = expected + 1
self._printer.print_one_line_summary(result_count, expected,
unexpected)
if driver:
driver.stop()
return unexpected
def _run_single_test(self, test, driver):
test_failed = False
driver_need_restart = False
output = driver.run_test(
DriverInput(test, self._options.time_out_ms, None, False))
if output.text == None:
test_failed = True
elif output.timeout:
self._printer.write('timeout: %s' %
test[self._webkit_base_dir_len + 1:])
test_failed = True
driver_need_restart = True
elif output.crash:
self._printer.write('crash: %s' %
test[self._webkit_base_dir_len + 1:])
driver_need_restart = True
test_failed = True
else:
got_a_result = False
for line in re.split('\n', output.text):
if self._result_regex.match(line):
self._buildbot_output.write("%s\n" % line)
got_a_result = True
elif not len(line) == 0:
test_failed = True
self._printer.write("%s" % line)
test_failed = test_failed or not got_a_result
if len(output.error):
self._printer.write('error:\n%s' % output.error)
test_failed = True
return test_failed, driver_need_restart
def main():
options, args = parse_args()
host = Host()
host._initialize_scm()
port = host.port_factory.get(options.platform, options)
return run(port, options, args)
class PerfTestsRunner(object):
_perf_tests_base_dir = 'PerformanceTests'
_test_directories_for_chromium_style_tests = ['inspector']
_default_branch = 'webkit-trunk'
_EXIT_CODE_BAD_BUILD = -1
_EXIT_CODE_BAD_JSON = -2
_EXIT_CODE_FAILED_UPLOADING = -3
def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None, port=None):
self._buildbot_output = buildbot_output
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host._initialize_scm()
self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
print_options = printing.print_options()
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--output-json-path",
help="Filename of the JSON file that summaries the results"),
optparse.make_option("--source-json-path",
help="Path to a JSON file to be merged into the JSON file when --output-json-path is present"),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present"),
]
option_list = (perf_option_list + print_options)
return optparse.OptionParser(option_list=option_list).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
def _is_test_file(filesystem, dirname, filename):
return filename.endswith('.html')
skipped_directories = set(['.svn', 'resources'])
tests = find_files.find(self._host.filesystem, self._base_path, self._args, skipped_directories, _is_test_file)
return [test for test in tests if not self._port.skips_perf_test(self._port.relative_perf_test_filename(test))]
def run(self):
if self._options.help_printing:
self._printer.help_printing()
self._printer.cleanup()
return 0
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self._EXIT_CODE_BAD_BUILD
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected = -1
try:
tests = self._collect_tests()
unexpected = self._run_tests_set(sorted(list(tests)), self._port)
finally:
self._printer.cleanup()
options = self._options
if self._options.output_json_path:
# FIXME: Add --branch or auto-detect the branch we're in
test_results_server = options.test_results_server
branch = self._default_branch if test_results_server else None
build_number = int(options.build_number) if options.build_number else None
if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path,
branch, options.platform, options.builder_name, build_number) and not unexpected:
return self._EXIT_CODE_BAD_JSON
if test_results_server and not self._upload_json(test_results_server, options.output_json_path):
return self._EXIT_CODE_FAILED_UPLOADING
return unexpected
def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number):
contents = {'timestamp': int(timestamp), 'results': self._results}
for (name, path) in self._port.repository_paths():
contents[name + '-revision'] = self._host.scm().svn_revision(path)
for key, value in {'branch': branch, 'platform': platform, 'builder-name': builder_name, 'build-number': build_number}.items():
if value:
contents[key] = value
filesystem = self._host.filesystem
succeeded = False
if source_json_path:
try:
source_json_file = filesystem.open_text_file_for_reading(source_json_path)
source_json = json.load(source_json_file)
contents = dict(source_json.items() + contents.items())
succeeded = True
except IOError, error:
_log.error("Failed to read %s: %s" % (source_json_path, error))
except ValueError, error:
_log.error("Failed to parse %s: %s" % (source_json_path, error))
except TypeError, error:
_log.error("Failed to merge JSON files: %s" % error)
class PerfTestsRunner(object):
_perf_tests_base_dir = 'PerformanceTests'
_result_regex = re.compile('^RESULT .*$')
def __init__(self, perf_tests_dir, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None):
self._perf_tests_dir = perf_tests_dir
self._buildbot_output = buildbot_output
self._options, self._args = self._parse_args(args)
self._host = Host()
self._host._initialize_scm()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
def _parse_args(self, args=None):
print_options = printing.print_options()
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=30000,
help="Set the timeout for each test"),
]
option_list = (perf_option_list + print_options)
return optparse.OptionParser(option_list=option_list).parse_args(args)
def _collect_tests(self, webkit_base, filesystem=None):
"""Return the list of tests found."""
def _is_test_file(filesystem, dirname, filename):
return filename.endswith('.html')
filesystem = filesystem or self._host.filesystem
base_dir = filesystem.join(webkit_base, self._perf_tests_base_dir, self._perf_tests_dir)
return find_files.find(filesystem, base_dir, paths=self._args, file_filter=_is_test_file)
def run(self):
if self._options.help_printing:
self._printer.help_printing()
self._printer.cleanup()
return 0
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return -1
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected = -1
try:
tests = self._collect_tests(self._port.webkit_base())
unexpected = self._run_tests_set(tests, self._port)
finally:
self._printer.cleanup()
return unexpected
def _run_tests_set(self, tests, port):
result_count = len(tests)
expected = 0
unexpected = 0
self._printer.print_one_line_summary(result_count, 0, 0)
driver_need_restart = False
driver = None
for test in tests:
if driver_need_restart:
_log.debug("%s killing driver" % test)
driver.stop()
driver = None
if not driver:
driver = port.create_driver(worker_number=1)
test_failed, driver_need_restart = self._run_single_test(test, driver)
if test_failed:
unexpected = unexpected + 1
else:
expected = expected + 1
self._printer.print_one_line_summary(result_count, expected, unexpected)
if driver:
driver.stop()
return unexpected
def _run_single_test(self, test, driver):
test_failed = False
driver_need_restart = False
output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
if output.text == None:
test_failed = True
elif output.timeout:
self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
test_failed = True
driver_need_restart = True
elif output.crash:
self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
driver_need_restart = True
test_failed = True
else:
got_a_result = False
for line in re.split('\n', output.text):
if self._result_regex.match(line):
self._buildbot_output.write("%s\n" % line)
got_a_result = True
elif not len(line) == 0:
test_failed = True
self._printer.write("%s" % line)
test_failed = test_failed or not got_a_result
if len(output.error):
self._printer.write('error:\n%s' % output.error)
test_failed = True
return test_failed, driver_need_restart
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
self._host._initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
perf_option_list = [
optparse.make_option('--debug',
action='store_const',
const='Debug',
dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release',
action='store_const',
const='Release',
dest="configuration",
help='Set the configuration to Release'),
optparse.make_option(
"--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const",
const='chromium',
dest='platform',
help='Alias for --platform=chromium'),
optparse.make_option(
"--builder-name",
help=
("The name of the builder shown on the waterfall running this script e.g. google-mac-2."
)),
optparse.make_option(
"--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help=
"Check to ensure the DumpRenderTree build is up-to-date (default)."
),
optparse.make_option(
"--no-build",
dest="build",
action="store_false",
help=
"Don't check to see if the DumpRenderTree build is up-to-date."
),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option("--time-out-ms",
default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option(
"--pause-before-testing",
dest="pause_before_testing",
action="store_true",
default=False,
help=
"Pause before running the tests to let user attach a performance monitor."
),
optparse.make_option(
"--no-results",
action="store_false",
dest="generate_results",
default=True,
help="Do no generate results JSON and results page."),
optparse.make_option(
"--output-json-path",
help=
"Path to generate a JSON file at; may contain previous results if it already exists."
),
optparse.make_option(
"--source-json-path", # FIXME: Rename it to signify the fact it's a slave configuration.
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option(
"--description",
help=
"Add a description to the output JSON file if one is generated"
),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help=
"Don't launch a browser with results after the tests are done"
),
optparse.make_option(
"--test-results-server",
help=
"Upload the generated JSON file to the specified server when --output-json-path is present."
),
optparse.make_option(
"--webkit-test-runner",
"-2",
action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
optparse.make_option("--replay",
dest="replay",
action="store_true",
default=False,
help="Run replay tests."),
optparse.make_option(
"--force",
dest="skipped",
action="store_true",
default=False,
help="Run all tests, including the ones in the Skipped list."),
]
return optparse.OptionParser(
option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths,
skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(
path).replace('\\', '/')
if self._port.skips_perf_test(
relative_path) and not self._options.skipped:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path,
path)
tests.append(test)
return tests
def run(self):
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" %
self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(
sorted(list(tests), key=lambda test: test.test_name()), self._port)
if self._options.generate_results:
exit_code = self._generate_and_show_results()
if exit_code:
return exit_code
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(),
self._DEFAULT_JSON_FILENAME)
def _generate_and_show_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp,
options.description,
options.platform,
options.builder_name,
options.build_number)
if options.source_json_path:
output = self._merge_slave_config_json(options.source_json_path,
output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
test_results_server = options.test_results_server
results_page_path = None
if not test_results_server:
output = self._merge_outputs(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
results_page_path = self._host.filesystem.splitext(
output_json_path)[0] + '.html'
self._generate_output_files(output_json_path, results_page_path,
output)
if test_results_server:
if not self._upload_json(test_results_server, output_json_path):
return self.EXIT_CODE_FAILED_UPLOADING
elif options.show_results:
self._port.show_results_html_file(results_page_path)
def _generate_results_dict(self, timestamp, description, platform,
builder_name, build_number):
contents = {'results': self._results}
if description:
contents['description'] = description
for (name, path) in self._port.repository_paths():
contents[name + '-revision'] = self._host.scm().svn_revision(path)
# FIXME: Add --branch or auto-detect the branch we're in
for key, value in {
'timestamp': int(timestamp),
'branch': self._default_branch,
'platform': platform,
'builder-name': builder_name,
'build-number': int(build_number) if build_number else None
}.items():
if value:
contents[key] = value
return contents
def _merge_slave_config_json(self, slave_config_json_path, output):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" %
slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(
slave_config_json_path)
slave_config = json.load(slave_config_json)
return dict(slave_config.items() + output.items())
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" %
(slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
_EXIT_CODE_BAD_BUILD = -1
_EXIT_CODE_BAD_JSON = -2
_EXIT_CODE_FAILED_UPLOADING = -3
_EXIT_CODE_BAD_PREPARATION = -4
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host._initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--no-build", dest="build", action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--pause-before-testing", dest="pause_before_testing", action="store_true", default=False,
help="Pause before running the tests to let user attach a performance monitor."),
optparse.make_option("--output-json-path",
help="Filename of the JSON file that summaries the results."),
optparse.make_option("--source-json-path",
help="Path to a JSON file to be merged into the JSON file when --output-json-path is present."),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--webkit-test-runner", "-2", action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
optparse.make_option("--replay", dest="replay", action="store_true", default=False,
help="Run replay tests."),
optparse.make_option("--force", dest="skipped", action="store_true", default=False,
help="Run all tests, including the ones in the Skipped list."),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(path).replace('\\', '/')
if self._port.skips_perf_test(relative_path) and not self._options.skipped:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path)
tests.append(test)
return tests
def run(self):
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self._EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self._EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
options = self._options
if self._options.output_json_path:
# FIXME: Add --branch or auto-detect the branch we're in
test_results_server = options.test_results_server
branch = self._default_branch if test_results_server else None
build_number = int(options.build_number) if options.build_number else None
if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path,
branch, options.platform, options.builder_name, build_number) and not unexpected:
return self._EXIT_CODE_BAD_JSON
if test_results_server and not self._upload_json(test_results_server, options.output_json_path):
return self._EXIT_CODE_FAILED_UPLOADING
return unexpected
def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number):
contents = {'timestamp': int(timestamp), 'results': self._results}
for (name, path) in self._port.repository_paths():
contents[name + '-revision'] = self._host.scm().svn_revision(path)
for key, value in {'branch': branch, 'platform': platform, 'builder-name': builder_name, 'build-number': build_number}.items():
if value:
contents[key] = value
filesystem = self._host.filesystem
succeeded = False
if source_json_path:
try:
source_json_file = filesystem.open_text_file_for_reading(source_json_path)
source_json = json.load(source_json_file)
contents = dict(source_json.items() + contents.items())
succeeded = True
except IOError, error:
_log.error("Failed to read %s: %s" % (source_json_path, error))
except ValueError, error:
_log.error("Failed to parse %s: %s" % (source_json_path, error))
except TypeError, error:
_log.error("Failed to merge JSON files: %s" % error)
class PerfTestsRunner(object):
_perf_tests_base_dir = 'PerformanceTests'
_test_directories_for_chromium_style_tests = ['inspector']
_default_branch = 'webkit-trunk'
_EXIT_CODE_BAD_BUILD = -1
_EXIT_CODE_BAD_JSON = -2
_EXIT_CODE_FAILED_UPLOADING = -3
def __init__(self,
regular_output=sys.stderr,
buildbot_output=sys.stdout,
args=None,
port=None):
self._buildbot_output = buildbot_output
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
self._host._initialize_scm()
self._printer = printing.Printer(self._port,
self._options,
regular_output,
buildbot_output,
configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
print_options = printing.print_options()
perf_option_list = [
optparse.make_option('--debug',
action='store_const',
const='Debug',
dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release',
action='store_const',
const='Release',
dest="configuration",
help='Set the configuration to Release'),
optparse.make_option(
"--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option(
"--builder-name",
help=
("The name of the builder shown on the waterfall running this script e.g. google-mac-2."
)),
optparse.make_option(
"--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help=
"Check to ensure the DumpRenderTree build is up-to-date (default)."
),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option("--time-out-ms",
default=240 * 1000,
help="Set the timeout for each test"),
optparse.make_option(
"--output-json-path",
help="Filename of the JSON file that summaries the results"),
optparse.make_option(
"--source-json-path",
help=
"Path to a JSON file to be merged into the JSON file when --output-json-path is present"
),
optparse.make_option(
"--test-results-server",
help=
"Upload the generated JSON file to the specified server when --output-json-path is present"
),
]
option_list = (perf_option_list + print_options)
return optparse.OptionParser(option_list=option_list).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
def _is_test_file(filesystem, dirname, filename):
return filename.endswith('.html')
skipped_directories = set(['.svn', 'resources'])
tests = find_files.find(self._host.filesystem, self._base_path,
self._args, skipped_directories, _is_test_file)
return [
test for test in tests if not self._port.skips_perf_test(
self._port.relative_perf_test_filename(test))
]
def run(self):
if self._options.help_printing:
self._printer.help_printing()
self._printer.cleanup()
return 0
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" %
self._port._path_to_driver())
return self._EXIT_CODE_BAD_BUILD
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected = -1
try:
tests = self._collect_tests()
unexpected = self._run_tests_set(sorted(list(tests)), self._port)
finally:
self._printer.cleanup()
options = self._options
if self._options.output_json_path:
# FIXME: Add --branch or auto-detect the branch we're in
test_results_server = options.test_results_server
branch = self._default_branch if test_results_server else None
build_number = int(
options.build_number) if options.build_number else None
if not self._generate_json(
self._timestamp, options.output_json_path,
options.source_json_path, branch, options.platform,
options.builder_name, build_number) and not unexpected:
return self._EXIT_CODE_BAD_JSON
if test_results_server and not self._upload_json(
test_results_server, options.output_json_path):
return self._EXIT_CODE_FAILED_UPLOADING
return unexpected
def _generate_json(self, timestamp, output_json_path, source_json_path,
branch, platform, builder_name, build_number):
revision = self._host.scm().head_svn_revision()
contents = {
'timestamp': int(timestamp),
'revision': revision,
'results': self._results
}
for key, value in {
'branch': branch,
'platform': platform,
'builder-name': builder_name,
'build-number': build_number
}.items():
if value:
contents[key] = value
filesystem = self._host.filesystem
if source_json_path:
try:
source_json_file = filesystem.open_text_file_for_reading(
source_json_path)
source_json = json.load(source_json_file)
contents = dict(source_json.items() + contents.items())
succeeded = True
except IOError, error:
_log.error("Failed to read %s: %s" % (source_json_path, error))
except ValueError, error:
_log.error("Failed to parse %s: %s" %
(source_json_path, error))
except TypeError, error:
_log.error("Failed to merge JSON files: %s" % error)
class PerfTestsRunner(object):
_default_branch = "webkit-trunk"
_EXIT_CODE_BAD_BUILD = -1
_EXIT_CODE_BAD_JSON = -2
_EXIT_CODE_FAILED_UPLOADING = -3
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host._initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
perf_option_list = [
optparse.make_option(
"--debug",
action="store_const",
const="Debug",
dest="configuration",
help="Set the configuration to Debug",
),
optparse.make_option(
"--release",
action="store_const",
const="Release",
dest="configuration",
help="Set the configuration to Release",
),
optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option(
"--chromium",
action="store_const",
const="chromium",
dest="platform",
help="Alias for --platform=chromium",
),
optparse.make_option(
"--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2."),
),
optparse.make_option("--build-number", help=("The build number of the builder running this script.")),
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default).",
),
optparse.make_option(
"--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)",
),
optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"),
optparse.make_option(
"--pause-before-testing",
dest="pause_before_testing",
action="store_true",
default=False,
help="Pause before running the tests to let user attach a performance monitor.",
),
optparse.make_option("--output-json-path", help="Filename of the JSON file that summaries the results"),
optparse.make_option(
"--source-json-path",
help="Path to a JSON file to be merged into the JSON file when --output-json-path is present",
),
optparse.make_option(
"--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present",
),
optparse.make_option(
"--webkit-test-runner",
"-2",
action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree.",
),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in [".html", ".svg"]
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set([".svn", "resources"])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(path).replace("\\", "/")
if self._port.skips_perf_test(relative_path):
continue
tests.append(PerfTestFactory.create_perf_test(relative_path, path))
return tests
def run(self):
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self._EXIT_CODE_BAD_BUILD
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected = -1
tests = self._collect_tests()
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
options = self._options
if self._options.output_json_path:
# FIXME: Add --branch or auto-detect the branch we're in
test_results_server = options.test_results_server
branch = self._default_branch if test_results_server else None
build_number = int(options.build_number) if options.build_number else None
if (
not self._generate_json(
self._timestamp,
options.output_json_path,
options.source_json_path,
branch,
options.platform,
options.builder_name,
build_number,
)
and not unexpected
):
return self._EXIT_CODE_BAD_JSON
if test_results_server and not self._upload_json(test_results_server, options.output_json_path):
return self._EXIT_CODE_FAILED_UPLOADING
return unexpected
def _generate_json(
self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number
):
contents = {"timestamp": int(timestamp), "results": self._results}
for (name, path) in self._port.repository_paths():
contents[name + "-revision"] = self._host.scm().svn_revision(path)
for key, value in {
"branch": branch,
"platform": platform,
"builder-name": builder_name,
"build-number": build_number,
}.items():
if value:
contents[key] = value
filesystem = self._host.filesystem
succeeded = False
if source_json_path:
try:
source_json_file = filesystem.open_text_file_for_reading(source_json_path)
source_json = json.load(source_json_file)
contents = dict(source_json.items() + contents.items())
succeeded = True
except IOError, error:
_log.error("Failed to read %s: %s" % (source_json_path, error))
except ValueError, error:
_log.error("Failed to parse %s: %s" % (source_json_path, error))
except TypeError, error:
_log.error("Failed to merge JSON files: %s" % error)
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
_EXIT_CODE_BAD_BUILD = -1
_EXIT_CODE_BAD_JSON = -2
_EXIT_CODE_FAILED_UPLOADING = -3
_EXIT_CODE_BAD_PREPARATION = -4
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host._initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--pause-before-testing", dest="pause_before_testing", action="store_true", default=False,
help="Pause before running the tests to let user attach a performance monitor."),
optparse.make_option("--output-json-path",
help="Filename of the JSON file that summaries the results."),
optparse.make_option("--source-json-path",
help="Path to a JSON file to be merged into the JSON file when --output-json-path is present."),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--webkit-test-runner", "-2", action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
optparse.make_option("--replay", dest="replay", action="store_true", default=False,
help="Run replay tests."),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(path).replace('\\', '/')
if self._port.skips_perf_test(relative_path):
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path)
tests.append(test)
return tests
def run(self):
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self._EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self._EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
options = self._options
if self._options.output_json_path:
# FIXME: Add --branch or auto-detect the branch we're in
test_results_server = options.test_results_server
branch = self._default_branch if test_results_server else None
build_number = int(options.build_number) if options.build_number else None
if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path,
branch, options.platform, options.builder_name, build_number) and not unexpected:
return self._EXIT_CODE_BAD_JSON
if test_results_server and not self._upload_json(test_results_server, options.output_json_path):
return self._EXIT_CODE_FAILED_UPLOADING
return unexpected
def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number):
contents = {'timestamp': int(timestamp), 'results': self._results}
for (name, path) in self._port.repository_paths():
contents[name + '-revision'] = self._host.scm().svn_revision(path)
for key, value in {'branch': branch, 'platform': platform, 'builder-name': builder_name, 'build-number': build_number}.items():
if value:
contents[key] = value
filesystem = self._host.filesystem
succeeded = False
if source_json_path:
try:
source_json_file = filesystem.open_text_file_for_reading(source_json_path)
source_json = json.load(source_json_file)
contents = dict(source_json.items() + contents.items())
succeeded = True
except IOError, error:
_log.error("Failed to read %s: %s" % (source_json_path, error))
except ValueError, error:
_log.error("Failed to parse %s: %s" % (source_json_path, error))
except TypeError, error:
_log.error("Failed to merge JSON files: %s" % error)
def main():
options, args = parse_args()
host = Host()
host._initialize_scm()
port = host.port_factory.get(options.platform, options)
return run(port, options, args)
class PerfTestsRunner(object):
_default_branch = 'webkit-trunk'
EXIT_CODE_BAD_BUILD = -1
EXIT_CODE_BAD_SOURCE_JSON = -2
EXIT_CODE_BAD_MERGE = -3
EXIT_CODE_FAILED_UPLOADING = -4
EXIT_CODE_BAD_PREPARATION = -5
_DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json'
def __init__(self, args=None, port=None):
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._host._initialize_scm()
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
@staticmethod
def _parse_args(args=None):
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--chromium",
action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
optparse.make_option("--builder-name",
help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
optparse.make_option("--build-number",
help=("The build number of the builder running this script.")),
optparse.make_option("--build", dest="build", action="store_true", default=True,
help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
optparse.make_option("--no-build", dest="build", action="store_false",
help="Don't check to see if the DumpRenderTree build is up-to-date."),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=600 * 1000,
help="Set the timeout for each test"),
optparse.make_option("--pause-before-testing", dest="pause_before_testing", action="store_true", default=False,
help="Pause before running the tests to let user attach a performance monitor."),
optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True,
help="Do no generate results JSON and results page."),
optparse.make_option("--output-json-path",
help="Path to generate a JSON file at; may contain previous results if it already exists."),
optparse.make_option("--source-json-path", # FIXME: Rename it to signify the fact it's a slave configuration.
help="Only used on bots. Path to a slave configuration file."),
optparse.make_option("--description",
help="Add a description to the output JSON file if one is generated"),
optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results",
help="Don't launch a browser with results after the tests are done"),
optparse.make_option("--test-results-server",
help="Upload the generated JSON file to the specified server when --output-json-path is present."),
optparse.make_option("--webkit-test-runner", "-2", action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
optparse.make_option("--replay", dest="replay", action="store_true", default=False,
help="Run replay tests."),
optparse.make_option("--force", dest="skipped", action="store_true", default=False,
help="Run all tests, including the ones in the Skipped list."),
]
return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
test_extensions = ['.html', '.svg']
if self._options.replay:
test_extensions.append('.replay')
def _is_test_file(filesystem, dirname, filename):
return filesystem.splitext(filename)[1] in test_extensions
filesystem = self._host.filesystem
paths = []
for arg in self._args:
paths.append(arg)
relpath = filesystem.relpath(arg, self._base_path)
if relpath:
paths.append(relpath)
skipped_directories = set(['.svn', 'resources'])
test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file)
tests = []
for path in test_files:
relative_path = self._port.relative_perf_test_filename(path).replace('\\', '/')
if self._port.skips_perf_test(relative_path) and not self._options.skipped:
continue
test = PerfTestFactory.create_perf_test(self._port, relative_path, path)
tests.append(test)
return tests
def run(self):
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return self.EXIT_CODE_BAD_BUILD
tests = self._collect_tests()
_log.info("Running %d tests" % len(tests))
for test in tests:
if not test.prepare(self._options.time_out_ms):
return self.EXIT_CODE_BAD_PREPARATION
unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port)
if self._options.generate_results:
exit_code = self._generate_and_show_results()
if exit_code:
return exit_code
return unexpected
def _output_json_path(self):
output_json_path = self._options.output_json_path
if output_json_path:
return output_json_path
return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME)
def _generate_and_show_results(self):
options = self._options
output_json_path = self._output_json_path()
output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number)
if options.source_json_path:
output = self._merge_slave_config_json(options.source_json_path, output)
if not output:
return self.EXIT_CODE_BAD_SOURCE_JSON
test_results_server = options.test_results_server
results_page_path = None
if not test_results_server:
output = self._merge_outputs(output_json_path, output)
if not output:
return self.EXIT_CODE_BAD_MERGE
results_page_path = self._host.filesystem.splitext(output_json_path)[0] + '.html'
else:
# FIXME: Remove this code once webkit-perf.appspot.com supported "values".
for result in output['results'].values():
if isinstance(result, dict) and 'values' in result:
del result['values']
self._generate_output_files(output_json_path, results_page_path, output)
if test_results_server:
if not self._upload_json(test_results_server, output_json_path):
return self.EXIT_CODE_FAILED_UPLOADING
elif options.show_results:
self._port.show_results_html_file(results_page_path)
def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number):
contents = {'results': self._results}
if description:
contents['description'] = description
for (name, path) in self._port.repository_paths():
contents[name + '-revision'] = self._host.scm().svn_revision(path)
# FIXME: Add --branch or auto-detect the branch we're in
for key, value in {'timestamp': int(timestamp), 'branch': self._default_branch, 'platform': platform,
'builder-name': builder_name, 'build-number': int(build_number) if build_number else None}.items():
if value:
contents[key] = value
return contents
def _merge_slave_config_json(self, slave_config_json_path, output):
if not self._host.filesystem.isfile(slave_config_json_path):
_log.error("Missing slave configuration JSON file: %s" % slave_config_json_path)
return None
try:
slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path)
slave_config = json.load(slave_config_json)
return dict(slave_config.items() + output.items())
except Exception, error:
_log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error))
return None
class PerfTestsRunner(object):
_perf_tests_base_dir = 'PerformanceTests'
_test_directories_for_chromium_style_tests = ['inspector']
def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None):
self._buildbot_output = buildbot_output
self._options, self._args = self._parse_args(args)
self._host = Host()
self._host._initialize_scm()
self._port = self._host.port_factory.get(self._options.platform, self._options)
self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._host.filesystem.join(self._port.webkit_base(), self._perf_tests_base_dir)
def _parse_args(self, args=None):
print_options = printing.print_options()
perf_option_list = [
optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
help='Set the configuration to Debug'),
optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
help='Set the configuration to Release'),
optparse.make_option("--platform",
help="Specify port/platform being tested (i.e. chromium-mac)"),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
optparse.make_option("--time-out-ms", default=30000,
help="Set the timeout for each test"),
]
option_list = (perf_option_list + print_options)
return optparse.OptionParser(option_list=option_list).parse_args(args)
def _collect_tests(self):
"""Return the list of tests found."""
def _is_test_file(filesystem, dirname, filename):
return filename.endswith('.html')
return find_files.find(self._host.filesystem, self._base_path, paths=self._args, file_filter=_is_test_file)
def run(self):
if self._options.help_printing:
self._printer.help_printing()
self._printer.cleanup()
return 0
if not self._port.check_build(needs_http=False):
_log.error("Build not up to date for %s" % self._port._path_to_driver())
return -1
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected = -1
try:
tests = self._collect_tests()
unexpected = self._run_tests_set(tests, self._port)
finally:
self._printer.cleanup()
return unexpected
def _print_status(self, tests, expected, unexpected):
if len(tests) == expected + unexpected:
status = "Ran %d tests" % len(tests)
else:
status = "Running %d of %d tests" % (expected + unexpected + 1, len(tests))
if unexpected:
status += " (%d didn't run)" % unexpected
self._printer.write(status)
def _run_tests_set(self, tests, port):
result_count = len(tests)
expected = 0
unexpected = 0
driver_need_restart = False
driver = None
for test in tests:
if driver_need_restart:
_log.debug("%s killing driver" % test)
driver.stop()
driver = None
if not driver:
driver = port.create_driver(worker_number=1)
relative_test_path = self._host.filesystem.relpath(test, self._base_path)
self._printer.write('Running %s (%d of %d)' % (relative_test_path, expected + unexpected + 1, len(tests)))
is_chromium_style = self._host.filesystem.split(relative_test_path)[0] in self._test_directories_for_chromium_style_tests
test_failed, driver_need_restart = self._run_single_test(test, driver, is_chromium_style)
if test_failed:
unexpected = unexpected + 1
else:
expected = expected + 1
self._printer.write('')
if driver:
driver.stop()
return unexpected
_inspector_result_regex = re.compile('^RESULT .*$')
def _process_chromium_style_test_result(self, test, output):
test_failed = False
got_a_result = False
for line in re.split('\n', output.text):
if self._inspector_result_regex.match(line):
self._buildbot_output.write("%s\n" % line)
got_a_result = True
elif not len(line) == 0:
test_failed = True
self._printer.write("%s" % line)
return test_failed or not got_a_result
_lines_to_ignore_in_parser_result = [
re.compile(r'^Running \d+ times$'),
re.compile(r'^Ignoring warm-up '),
re.compile(r'^\d+$'),
]
def _should_ignore_line_in_parser_test_result(self, line):
if not line:
return True
for regex in self._lines_to_ignore_in_parser_result:
if regex.match(line):
return True
return False
def _process_parser_test_result(self, test, output):
got_a_result = False
test_failed = False
filesystem = self._host.filesystem
category, test_name = filesystem.split(filesystem.relpath(test, self._base_path))
test_name = filesystem.splitext(test_name)[0]
results = {}
keys = ['avg', 'median', 'stdev', 'min', 'max']
score_regex = re.compile(r'^(' + r'|'.join(keys) + r')\s+([0-9\.]+)')
for line in re.split('\n', output.text):
score = score_regex.match(line)
if score:
results[score.group(1)] = score.group(2)
continue
if not self._should_ignore_line_in_parser_test_result(line):
test_failed = True
self._printer.write("%s" % line)
if test_failed or set(keys) != set(results.keys()):
return True
self._buildbot_output.write('RESULT %s: %s= %s ms\n' % (category, test_name, results['avg']))
self._buildbot_output.write(', '.join(['%s= %s ms' % (key, results[key]) for key in keys[1:]]) + '\n')
return False
def _run_single_test(self, test, driver, is_chromium_style):
test_failed = False
driver_need_restart = False
output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
if output.text == None:
test_failed = True
elif output.timeout:
self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
test_failed = True
driver_need_restart = True
elif output.crash:
self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
driver_need_restart = True
test_failed = True
else:
if is_chromium_style:
test_failed = self._process_chromium_style_test_result(test, output)
else:
test_failed = self._process_parser_test_result(test, output)
if len(output.error):
self._printer.write('error:\n%s' % output.error)
test_failed = True
if test_failed:
self._printer.write('FAILED')
return test_failed, driver_need_restart
