def _prepare_config(self, options, args, tool): results_directory = args[0] host = Host() host.initialize_scm() print 'Parsing full_results.json...' results_json_path = host.filesystem.join(results_directory, 'full_results.json') results_json = json_results_generator.load_json(host.filesystem, results_json_path) port = tool.port_factory.get() layout_tests_directory = port.layout_tests_dir() platforms = host.filesystem.listdir(host.filesystem.join(layout_tests_directory, 'platform')) self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, host) print 'Gathering current baselines...' self._gather_baselines(results_json) return { 'test_config': self._test_config, "results_json": results_json, "platforms_json": { 'platforms': platforms, 'defaultPlatform': port.name(), }, }
def __init__(self, path): MultiCommandTool.__init__(self) Host.__init__(self) self._path = path self.status_server = StatusServer() self.wakeup_event = threading.Event() self._deprecated_port = None
def main(argv, stderr, host=None): parser = optparse.OptionParser(option_list=platform_options( use_globs=True)) parser.add_option('--json', help='Path to JSON output file') options, _ = parser.parse_args(argv) if not host: if options.platform and 'test' in options.platform: # It's a bit lame to import mocks into real code, but this allows the user # to run tests against the test platform interactively, which is useful for # debugging test failures. from webkitpy.common.host_mock import MockHost host = MockHost() else: host = Host() # Need to generate MANIFEST.json since some expectations correspond to WPT # tests that aren't files and only exist in the manifest. _log.info('Generating MANIFEST.json for web-platform-tests ...') WPTManifest.ensure_manifest(host) try: exit_status = run_checks(host, options, stderr) except KeyboardInterrupt: exit_status = exit_codes.INTERRUPTED_EXIT_STATUS except Exception as error: # pylint: disable=broad-except print >> stderr, '\n%s raised: %s' % (error.__class__.__name__, error) traceback.print_exc(file=stderr) exit_status = exit_codes.EXCEPTIONAL_EXIT_STATUS return exit_status
def convert_for_webkit(new_path, filename, reference_support_info, host=Host()): """Converts a file's contents so the Blink layout test runner can run it. Args: new_path: Absolute path where file will be copied to in the Chromium repo. filename: Absolute path to where the file is. reference_support_info: Dict of information about a related reference HTML, if any. Returns: A pair of (list of modified CSS properties, modified text) if the file should be modified; None, if the file is not modified. """ # Conversion is not necessary for any tests in wpt now; see http://crbug.com/654081. if re.search(r'[/\\]imported[/\\]wpt[/\\]', new_path): return None contents = host.filesystem.read_binary_file(filename) converter = _W3CTestConverter(new_path, filename, reference_support_info, host) if filename.endswith('.css'): return converter.add_webkit_prefix_to_unprefixed_properties( contents.decode('utf-8')) else: try: converter.feed(contents.decode('utf-8')) except UnicodeDecodeError: converter.feed(contents.decode('utf-16')) converter.close() return converter.output()
def main(_argv, _stdout, _stderr): options, args = parse_args() import_dir = args[0] if len(args) == 1: repo_dir_parts = [] for part in import_dir.split(os.path.sep): if part in VALID_TEST_STATUSES: break else: repo_dir_parts.append(part) repo_dir = os.path.sep.join(repo_dir_parts) else: repo_dir = args[1] if not os.path.exists(import_dir): sys.exit('Source directory %s not found!' % import_dir) if not os.path.exists(repo_dir): sys.exit('Repository directory %s not found!' % repo_dir) if not repo_dir in import_dir: sys.exit('Repository directory %s must be a parent of %s' % (repo_dir, import_dir)) configure_logging() test_importer = TestImporter(Host(), import_dir, repo_dir, options) test_importer.do_import()
def main(server_constructor, input_fn=None, argv=None, **kwargs): input_fn = input_fn or raw_input option_parser = optparse.OptionParser() option_parser.add_option('--output-dir', dest='output_dir', default=None, help='output directory.') option_parser.add_option('-v', '--verbose', action='store_true') options, args = option_parser.parse_args(argv) logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.DEBUG if options.verbose else logging.INFO) host = Host() port_obj = host.port_factory.get() if not options.output_dir: options.output_dir = port_obj.default_results_directory() server = server_constructor(port_obj, options.output_dir, **kwargs) server.start() try: _ = input_fn('Hit any key to stop the server and exit.') except (KeyboardInterrupt, EOFError) as e: pass server.stop()
def main(argv, _, stderr): parser = optparse.OptionParser(option_list=platform_options( use_globs=True)) parser.add_option('--json', help='Path to JSON output file') options, _ = parser.parse_args(argv) if options.platform and 'test' in options.platform: # It's a bit lame to import mocks into real code, but this allows the user # to run tests against the test platform interactively, which is useful for # debugging test failures. from webkitpy.common.host_mock import MockHost host = MockHost() else: host = Host() try: exit_status = run_checks(host, options, stderr) except KeyboardInterrupt: exit_status = exit_codes.INTERRUPTED_EXIT_STATUS except Exception as error: # pylint: disable=broad-except print >> stderr, '\n%s raised: %s' % (error.__class__.__name__, error) traceback.print_exc(file=stderr) exit_status = exit_codes.EXCEPTIONAL_EXIT_STATUS return exit_status
def __init__(self, new_path, filename, host=Host()): HTMLParser.__init__(self) self._host = host self._filesystem = self._host.filesystem self._webkit_root = WebKitFinder(self._filesystem).webkit_base() self.converted_data = [] self.converted_properties = [] self.in_style_tag = False self.style_data = [] self.filename = filename resources_path = self.path_from_webkit_root('LayoutTests', 'resources') resources_relpath = self._filesystem.relpath(resources_path, new_path) self.resources_relpath = resources_relpath # These settings might vary between WebKit and Blink self._css_property_file = self.path_from_webkit_root( 'Source', 'core', 'css', 'CSSProperties.in') self.prefixed_properties = self.read_webkit_prefixed_css_property_list( ) self.prefixed_properties = self.read_webkit_prefixed_css_property_list( ) prop_regex = '([\s{]|^)(' + "|".join( prop.replace('-webkit-', '') for prop in self.prefixed_properties) + ')(\s+:|:)' self.prop_re = re.compile(prop_regex)
def main(argv, stdout, stderr): options, args = parse_args(argv) host = Host() try: options.webkit_test_runner = True port = host.port_factory.get(options.platform, options) except NotImplementedError as e: print(str(e), file=stderr) return EXCEPTIONAL_EXIT_STATUS # Some platforms do not support API tests does_not_support_api_tests = ['ios-device'] if port.operating_system() in does_not_support_api_tests: print('{} cannot run API tests'.format(port.operating_system()), file=stderr) return EXCEPTIONAL_EXIT_STATUS try: return run(port, options, args, stderr) except KeyboardInterrupt: return INTERRUPT_EXIT_STATUS except BaseException as e: if isinstance(e, Exception): print('\n%s raised: %s' % (e.__class__.__name__, str(e)), file=stderr) traceback.print_exc(file=stderr) return EXCEPTIONAL_EXIT_STATUS
def main(argv, stdout, stderr): options, args = parse_args(argv) if options.platform and 'test' in options.platform and not 'browser_test' in options.platform: # It's a bit lame to import mocks into real code, but this allows the user # to run tests against the test platform interactively, which is useful for # debugging test failures. from webkitpy.common.host_mock import MockHost host = MockHost() else: host = Host() try: port = host.port_factory.get(options.platform, options) except (NotImplementedError, ValueError) as error: # FIXME: is this the best way to handle unsupported port names? print >> stderr, str(error) return test_run_results.UNEXPECTED_ERROR_EXIT_STATUS try: return run(port, options, args, stderr, stdout).exit_code # We need to still handle KeyboardInterrupt, at least for webkitpy unittest cases. except KeyboardInterrupt: return test_run_results.INTERRUPTED_EXIT_STATUS except test_run_results.TestRunException as error: print >> stderr, error.msg return error.code except BaseException as error: if isinstance(error, Exception): print >> stderr, '\n%s raised: %s' % (error.__class__.__name__, error) traceback.print_exc(file=stderr) return test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time() self._needs_http = None self._has_http_lock = False
def run(self): if not self.host: self.host = Host() if not self._running_inline: self._set_up_logging() worker = self._worker exception_msg = "" _log.debug("%s starting" % self.name) try: if hasattr(worker, 'start'): worker.start() while True: message = self._messages_to_worker.get() if message.from_user: worker.handle(message.name, message.src, *message.args) self._yield_to_manager() else: assert message.name == 'stop', 'bad message %s' % repr(message) break _log.debug("%s exiting" % self.name) except Queue.Empty: assert False, '%s: ran out of messages in worker queue.' % self.name except KeyboardInterrupt, e: self._raise(sys.exc_info())
def _prepare_config(self, options, args, tool): results_directory = args[0] scm = self._tool.scm() host = Host() filesystem = host.filesystem print('Parsing full_results.json...') results_json_path = filesystem.join(results_directory, 'full_results.json') results_json = json_results_generator.load_json( filesystem, results_json_path) port = tool.port_factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, scm, host) print('Gathering current baselines...') self._gather_baselines(results_json) return { 'test_config': self._test_config, "results_json": results_json, "platforms_json": { 'platforms': platforms, 'defaultPlatform': port.name(), }, }
def test_import_dir_with_no_tests_and_no_hg(self): # FIXME: Use MockHosts instead. host = Host() host.executive = MockExecutive2(exception=OSError()) importer = TestImporter(host, None, optparse.Values({"overwrite": False})) importer.source_directory = importer.path_from_webkit_root("Tools", "Scripts", "webkitpy", "w3c") importer.destination_directory = tempfile.mkdtemp(prefix='csswg') oc = OutputCapture() oc.capture_output() try: importer.do_import() finally: oc.restore_output() shutil.rmtree(importer.destination_directory, ignore_errors=True)
def main(server_constructor, input_fn=None, argv=None, **kwargs): input_fn = input_fn or raw_input parser = argparse.ArgumentParser() parser.add_argument('--output-dir', type=str, default=None, help='output directory, for log files etc.') parser.add_argument('-v', '--verbose', action='store_true', help='print more information, including port numbers') args = parser.parse_args(argv) logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.DEBUG if args.verbose else logging.INFO) host = Host() port_obj = host.port_factory.get() if not args.output_dir: args.output_dir = port_obj.default_results_directory() # Create the output directory if it doesn't already exist. port_obj.host.filesystem.maybe_make_directory(args.output_dir) server = server_constructor(port_obj, args.output_dir, **kwargs) server.start() try: _ = input_fn('Hit any key to stop the server and exit.') except (KeyboardInterrupt, EOFError) as e: pass server.stop()
def test_import_dir_with_no_tests(self): # FIXME: Use MockHosts instead. host = Host() host.executive = MockExecutive2(exception=ScriptError("abort: no repository found in '/Volumes/Source/src/wk/Tools/Scripts/webkitpy/w3c' (.hg not found)!")) importer = TestImporter(host, None, optparse.Values({"overwrite": False})) importer.source_directory = importer.path_from_webkit_root("Tools", "Scripts", "webkitpy", "w3c") importer.destination_directory = tempfile.mkdtemp(prefix='csswg') oc = OutputCapture() oc.capture_output() try: importer.do_import() finally: oc.restore_output() shutil.rmtree(importer.destination_directory, ignore_errors=True)
def main(_argv, _stdout, _stderr): options, test_paths = parse_args(_argv) configure_logging() test_importer = TestImporter(Host(), test_paths, options) test_importer.do_import()
def main(argv): parser = optparse.OptionParser(usage='%prog [stats.json]') parser.description = "Prints out lists of tests run on each worker as per the stats.json file." options, args = parser.parse_args(argv) if args and args[0]: stats_path = args[0] else: host = Host() stats_path = host.filesystem.join( host.port_factory.get().results_directory(), 'stats.json') with open(stats_path, 'r') as fp: stats_trie = json.load(fp) stats = convert_trie_to_flat_paths(stats_trie) stats_by_worker = {} for test_name, data in stats.items(): worker = "worker/" + str(data["results"][0]) if worker not in stats_by_worker: stats_by_worker[worker] = [] test_number = data["results"][1] stats_by_worker[worker].append({ "name": test_name, "number": test_number }) for worker in sorted(stats_by_worker.keys()): print worker + ':' for test in sorted(stats_by_worker[worker], key=lambda test: test["number"]): print test["name"] print
def convert_for_webkit(new_path, filename, reference_support_info, host=Host(), convert_test_harness_links=True, webkit_test_runner_options=''): """ Converts a file's |contents| so it will function correctly in its |new_path| in Webkit. Returns the list of modified properties and the modified text if the file was modifed, None otherwise.""" contents = host.filesystem.read_text_file(filename) # WebKit does not have a www test domain. contents = contents.replace('{{domains[www]}}', '{{hosts[alt][]}}') converter = _W3CTestConverter(new_path, filename, reference_support_info, host, convert_test_harness_links, webkit_test_runner_options) if filename.endswith('.css'): return converter.add_webkit_prefix_to_unprefixed_properties_and_values( contents) elif filename.endswith('.js'): return ([], [], contents) else: converter.feed(contents) converter.close() return converter.output()
def test_generate_repaint_overlay_html(self): test_name = 'paint/invalidation/repaint-overlay/layers.html' host = Host() port = host.port_factory.get() layer_tree_file = port.expected_filename(test_name, '.txt') if not layer_tree_file or not host.filesystem.exists(layer_tree_file): # This can happen if the scripts are not in the standard blink directory. return layer_tree = str(host.filesystem.read_text_file(layer_tree_file)) self.assertTrue(repaint_overlay.result_contains_repaint_rects(layer_tree)) overlay_html = ( '<!-- Generated by Tools/Scripts/test-webkitpy\n' + ' test case: TestRepaintOverlay.test_generate_repaint_overlay_html. -->\n' + repaint_overlay.generate_repaint_overlay_html(test_name, layer_tree, layer_tree)) results_directory = port.results_directory() host.filesystem.maybe_make_directory(results_directory) actual_overlay_html_file = host.filesystem.join(results_directory, 'layers-overlay.html') host.filesystem.write_text_file(actual_overlay_html_file, overlay_html) overlay_html_file = port.abspath_for_test('paint/invalidation/repaint-overlay/layers-overlay.html') expected = host.filesystem.read_text_file(overlay_html_file) self.assertEquals( expected, overlay_html, 'This failure is probably caused by changed repaint_overlay.py. ' 'Please examine the diffs:\n diff %s %s\n' 'If the diffs are valid, update the file:\n cp %s %s\n' 'then update layers-overlay-expected.html in the same directory if needed,' ' and commit the files together with the changed repaint_overlay.py.' % (overlay_html_file, actual_overlay_html_file, actual_overlay_html_file, overlay_html_file))
def _begin_logging(self): _queue_log_path = self._delegate.queue_log_path() # We are using logging.getLogger("webkitpy") instead of _log since we want to capture all messages logged from webkitpy modules. self._log_handler = logutils.configure_logger_to_log_to_file( logging.getLogger("webkitpy"), _queue_log_path, Host().filesystem) self._queue_log = self._output_tee.add_log(_queue_log_path) self._work_log = None
def __init__(self, port, options, tests=[]): self._options = options self._build_type = "Debug" if self._options.debug else "Release" common.set_build_types((self._build_type, )) self._port = Host().port_factory.get(port) self._driver = self._create_driver() self._programs_path = common.binary_build_path() expectations_file = os.path.join(common.top_level_path(), "Tools", "TestWebKitAPI", "glib", "TestExpectations.json") self._expectations = TestExpectations(self._port.name(), expectations_file, self._build_type) self._tests = self._get_tests(tests) self._disabled_tests = []
def main(_argv, _stdout, _stderr): options = parse_args(_argv) configure_logging() test_exporter = TestExporter(Host(), options) test_exporter.do_export()
def get_test_baselines(test_file, test_config): # FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic. class AllPlatformsPort(Port): def __init__(self, host): super(AllPlatformsPort, self).__init__(host, 'mac') self._platforms_by_directory = dict([ (self._webkit_baseline_path(p), p) for p in test_config.platforms ]) def baseline_search_path(self): return self._platforms_by_directory.keys() def platform_from_directory(self, directory): return self._platforms_by_directory[directory] test_path = test_config.filesystem.join(test_config.layout_tests_directory, test_file) # FIXME: This should get the Host from the test_config to be mockable! host = Host() host.initialize_scm() host.filesystem = test_config.filesystem all_platforms_port = AllPlatformsPort(host) all_test_baselines = {} for baseline_extension in ('.txt', '.checksum', '.png'): test_baselines = test_config.test_port.expected_baselines( test_file, baseline_extension) baselines = all_platforms_port.expected_baselines(test_file, baseline_extension, all_baselines=True) for platform_directory, expected_filename in baselines: if not platform_directory: continue if platform_directory == test_config.layout_tests_directory: platform = 'base' else: platform = all_platforms_port.platform_from_directory( platform_directory) platform_baselines = all_test_baselines.setdefault(platform, {}) was_used_for_test = (platform_directory, expected_filename) in test_baselines platform_baselines[baseline_extension] = was_used_for_test return all_test_baselines
def main(): configure_logging() options = parse_args() host = Host() wpt_github = WPTGitHub(host) test_exporter = TestExporter(host, wpt_github, dry_run=options.dry_run) test_exporter.run()
def do_association_check(self, files, cwd, host=Host()): _log.debug("Running TestExpectations linter") TestExpectationsChecker.lint_test_expectations( files, self._configuration, cwd, self._increment_error_count, host=host)
def main(argv, stdout, stderr): options, args = parse_args(argv) if options.platform and 'test' in options.platform: # It's a bit lame to import mocks into real code, but this allows the user # to run tests against the test platform interactively, which is useful for # debugging test failures. from webkitpy.common.host_mock import MockHost host = MockHost() else: host = Host() if options.lint_test_files: from webkitpy.layout_tests.lint_test_expectations import lint return lint(host, options, stderr) try: port = host.port_factory.get(options.platform, options) except NotImplementedError as e: # FIXME: is this the best way to handle unsupported port names? print(str(e), file=stderr) return EXCEPTIONAL_EXIT_STATUS stack_trace_path = host.filesystem.join(port.results_directory(), 'python_stack_trace.txt') log_stack_trace_on_ctrl_c(output_file=stack_trace_path) log_stack_trace_on_term(output_file=stack_trace_path) if options.print_expectations: return _print_expectations(port, options, args, stderr) try: # Force all tests to use a smaller stack so that stack overflow tests can run faster. stackSizeInBytes = int(1.5 * 1024 * 1024) options.additional_env_var.append('JSC_maxPerThreadStackUsage=' + str(stackSizeInBytes)) options.additional_env_var.append('__XPC_JSC_maxPerThreadStackUsage=' + str(stackSizeInBytes)) options.additional_env_var.append('JSC_useSharedArrayBuffer=1') options.additional_env_var.append('__XPC_JSC_useSharedArrayBuffer=1') run_details = run(port, options, args, stderr) if run_details.exit_code != -1 and run_details.skipped_all_tests: return run_details.exit_code if run_details.exit_code != -1 and not run_details.initial_results.keyboard_interrupted: bot_printer = buildbot_results.BuildBotPrinter( stdout, options.debug_rwt_logging) bot_printer.print_results(run_details) return run_details.exit_code # We still need to handle KeyboardInterrupt, at least for webkitpy unittest cases. except KeyboardInterrupt: return INTERRUPTED_EXIT_STATUS except BaseException as e: if isinstance(e, Exception): print('\n%s raised: %s' % (e.__class__.__name__, str(e)), file=stderr) traceback.print_exc(file=stderr) return EXCEPTIONAL_EXIT_STATUS
def _set_up_host_and_port(self): options = self._options if options.platform and 'test' in options.platform: # It is lame to import mocks into real code, but this allows us to use the test port in multi-process tests as well. from webkitpy.common.host_mock import MockHost host = MockHost() else: host = Host() self._port = host.port_factory.get(options.platform, options)
def test_import_dir_with_no_tests_and_no_hg(self): # FIXME: Use MockHosts instead. host = Host() host.executive = MockExecutive2(exception=OSError()) importer = TestImporter(host, DUMMY_SOURCE_DIR, DUMMY_REPO_DIR, optparse.Values({"overwrite": False})) importer.source_directory = importer.path_from_webkit_root( "Tools", "Scripts", "webkitpy", "w3c") importer.destination_directory = tempfile.mkdtemp(prefix='csswg') oc = OutputCapture() oc.capture_output() try: importer.do_import() finally: oc.restore_output() shutil.rmtree(importer.destination_directory, ignore_errors=True)
def __init__(self, perf_tests_dir, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None): self._perf_tests_dir = perf_tests_dir self._buildbot_output = buildbot_output self._options, self._args = self._parse_args(args) self._host = Host() self._host._initialize_scm() self._port = self._host.port_factory.get(self._options.platform, self._options) self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False) self._webkit_base_dir_len = len(self._port.webkit_base())
def __init__(self, options, filename, host=Host()): self.options = options self.filename = filename self.host = host self.filesystem = self.host.filesystem self.test_doc = None self.ref_doc = None self.load_file(filename)
def _generate_testing_host(self, files={}): host = Host() expectation_files = files host.filesystem = MockFileSystem(dirs=['/mock-checkout/LayoutTests']) options = optparse.Values() setattr(options, 'layout_tests_dir', '/mock-checkout/LayoutTests') all_ports = [host.port_factory.get(name, options=options) for name in host.port_factory.all_port_names()] for port in all_ports: for path in port.expectations_files(): if path not in expectation_files: expectation_files[path] = '# Empty expectation file\n' expectation_files['/mock-checkout/LayoutTests/css1/test.html'] = 'Test' expectation_files['/mock-checkout/LayoutTests/css1/test-expected.txt'] = 'Test Expectation' host.filesystem = MockFileSystem(files=expectation_files) return host
def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) # The GTK+ and EFL ports only supports WebKit2, so they always use WKTR. if self._port.name().startswith("gtk") or self._port.name().startswith("efl"): self._options.webkit_test_runner = True self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._timestamp = time.time() self._utc_timestamp = datetime.datetime.utcnow()
def _new_or_modified_tests(self): touched_files = self._tool.scm().changed_files() touched_files.extend(self._tool.scm().untracked_files()) if not touched_files: return None configuration = "Debug" if (self._options.build_style == "debug") else "Release" port = Host().port_factory.get(self._tool.deprecated_port().port_flag_name, optparse.Values({'configuration': configuration})) return LayoutTestFinder(port, optparse.Values({'skipped': 'always', 'skip_failing_tests': False, 'http': True})).find_touched_tests(touched_files)
def run(self): options = self._options # FIXME: This should get the Host from the owner of this object # so this function can be properly mocked! host = Host() host._initialize_scm() port_obj = host.port_factory.get(self._platform_name, options) # The unix multiprocessing implementation clones the # log handler configuration into the child processes, # but the win implementation doesn't. configure_logging = (sys.platform == 'win32') # FIXME: this won't work if the calling process is logging # somewhere other than sys.stderr and sys.stdout, but I'm not sure # if this will be an issue in practice. printer = printing.Printer(port_obj, options, sys.stderr, sys.stdout, configure_logging) self._client.run(port_obj) printer.cleanup()
def __init__(self): self._host = Host() self._filesystem = self._host.filesystem self._host.initialize_scm() self._webkit_root = self._host.scm().checkout_root # These settings might vary between WebKit and Blink self._css_property_file = self.path_from_webkit_root('Source', 'core', 'css', 'CSSPropertyNames.in') self._css_property_split_string = 'alias_for=' self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
def main(args): """Bootstrap function that sets up the object references we need and calls real_main().""" options, target_options = parse_options(args) logger = logging.getLogger() logger.setLevel(logging.INFO) if options.verbose: log_level = logging.DEBUG log_handler = DebugLogHandler() else: log_level = logging.INFO log_handler = NormalLogHandler() logger = logging.getLogger() logger.setLevel(log_level) logger.addHandler(log_handler) host = Host() host._initialize_scm() target_port_obj = host.port_factory.get(None, target_options) host_port_obj = get_host_port_object(host.port_factory, options) if not host_port_obj or not target_port_obj: return 1 url_fetcher = urlfetcher.UrlFetcher(host_port_obj._filesystem) # We use the default zip factory method. zip_factory = None # FIXME: SCM module doesn't handle paths that aren't relative to the checkout_root consistently. host_port_obj._filesystem.chdir(host.scm().checkout_root) ret_code = real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher, zip_factory, host.scm()) if not ret_code and log_handler.num_failures: ret_code = 1 print "" if ret_code: print "Rebaselining failed." else: print "Rebaselining succeeded." return ret_code
def main(self): args = sys.argv[1:] host = Host() host.initialize_scm() stderr = self._engage_awesome_stderr_hacks() # Checking for the verbose flag before calling check_webkit_style_parser() # lets us enable verbose logging earlier. is_verbose = "-v" in args or "--verbose" in args checker.configure_logging(stream=stderr, is_verbose=is_verbose) _log.debug("Verbose logging enabled.") parser = checker.check_webkit_style_parser() (paths, options) = parser.parse(args) configuration = checker.check_webkit_style_configuration(options) paths = change_directory(host.filesystem, checkout_root=host.scm().checkout_root, paths=paths) style_processor = StyleProcessor(configuration) file_reader = TextFileReader(host.filesystem, style_processor) if paths and not options.diff_files: file_reader.process_paths(paths) else: changed_files = paths if options.diff_files else None patch = host.scm().create_patch(options.git_commit, changed_files=changed_files) patch_checker = PatchReader(file_reader) patch_checker.check(patch) error_count = style_processor.error_count file_count = file_reader.file_count delete_only_file_count = file_reader.delete_only_file_count _log.info("Total errors found: %d in %d files" % (error_count, file_count)) # We fail when style errors are found or there are no checked files. return error_count > 0
def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time()
def get_test_baselines(test_file, test_config): # FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic. class AllPlatformsPort(Port): def __init__(self, host): super(AllPlatformsPort, self).__init__(host, 'mac') self._platforms_by_directory = dict([(self._webkit_baseline_path(p), p) for p in test_config.platforms]) def baseline_search_path(self): return self._platforms_by_directory.keys() def platform_from_directory(self, directory): return self._platforms_by_directory[directory] test_path = test_config.filesystem.join(test_config.layout_tests_directory, test_file) # FIXME: This should get the Host from the test_config to be mockable! host = Host() host.initialize_scm() host.filesystem = test_config.filesystem all_platforms_port = AllPlatformsPort(host) all_test_baselines = {} for baseline_extension in ('.txt', '.checksum', '.png'): test_baselines = test_config.test_port.expected_baselines(test_file, baseline_extension) baselines = all_platforms_port.expected_baselines(test_file, baseline_extension, all_baselines=True) for platform_directory, expected_filename in baselines: if not platform_directory: continue if platform_directory == test_config.layout_tests_directory: platform = 'base' else: platform = all_platforms_port.platform_from_directory(platform_directory) platform_baselines = all_test_baselines.setdefault(platform, {}) was_used_for_test = (platform_directory, expected_filename) in test_baselines platform_baselines[baseline_extension] = was_used_for_test return all_test_baselines
def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None, port=None): self._buildbot_output = buildbot_output self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host._initialize_scm() self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False) self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time()
class PerfTestsRunner(object): _perf_tests_base_dir = 'PerformanceTests' _test_directories_for_chromium_style_tests = ['inspector'] def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None): self._buildbot_output = buildbot_output self._options, self._args = self._parse_args(args) self._host = Host() self._host._initialize_scm() self._port = self._host.port_factory.get(self._options.platform, self._options) self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False) self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._host.filesystem.join(self._port.webkit_base(), self._perf_tests_base_dir) def _parse_args(self, args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=30000, help="Set the timeout for each test"), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args) def _collect_tests(self): """Return the list of tests found.""" def _is_test_file(filesystem, dirname, filename): return filename.endswith('.html') return find_files.find(self._host.filesystem, self._base_path, paths=self._args, file_filter=_is_test_file) def run(self): if self._options.help_printing: self._printer.help_printing() self._printer.cleanup() return 0 if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return -1 # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected = -1 try: tests = self._collect_tests() unexpected = self._run_tests_set(tests, self._port) finally: self._printer.cleanup() return unexpected def _print_status(self, tests, expected, unexpected): if len(tests) == expected + unexpected: status = "Ran %d tests" % len(tests) else: status = "Running %d of %d tests" % (expected + unexpected + 1, len(tests)) if unexpected: status += " (%d didn't run)" % unexpected self._printer.write(status) def _run_tests_set(self, tests, port): result_count = len(tests) expected = 0 unexpected = 0 driver_need_restart = False driver = None for test in tests: if driver_need_restart: _log.debug("%s killing driver" % test) driver.stop() driver = None if not driver: driver = port.create_driver(worker_number=1) relative_test_path = self._host.filesystem.relpath(test, self._base_path) self._printer.write('Running %s (%d of %d)' % (relative_test_path, expected + unexpected + 1, len(tests))) is_chromium_style = self._host.filesystem.split(relative_test_path)[0] in self._test_directories_for_chromium_style_tests test_failed, driver_need_restart = self._run_single_test(test, driver, is_chromium_style) if test_failed: unexpected = unexpected + 1 else: expected = expected + 1 self._printer.write('') if driver: driver.stop() return unexpected _inspector_result_regex = re.compile('^RESULT .*$') def _process_chromium_style_test_result(self, test, output): test_failed = False got_a_result = False for line in re.split('\n', output.text): if self._inspector_result_regex.match(line): self._buildbot_output.write("%s\n" % line) got_a_result = True elif not len(line) == 0: test_failed = True self._printer.write("%s" % line) return test_failed or not got_a_result _lines_to_ignore_in_parser_result = [ re.compile(r'^Running \d+ times$'), re.compile(r'^Ignoring warm-up '), re.compile(r'^\d+$'), ] def _should_ignore_line_in_parser_test_result(self, line): if not line: return True for regex in self._lines_to_ignore_in_parser_result: if regex.match(line): return True return False def _process_parser_test_result(self, test, output): got_a_result = False test_failed = False filesystem = self._host.filesystem category, test_name = filesystem.split(filesystem.relpath(test, self._base_path)) test_name = filesystem.splitext(test_name)[0] results = {} keys = ['avg', 'median', 'stdev', 'min', 'max'] score_regex = re.compile(r'^(' + r'|'.join(keys) + r')\s+([0-9\.]+)') for line in re.split('\n', output.text): score = score_regex.match(line) if score: results[score.group(1)] = score.group(2) continue if not self._should_ignore_line_in_parser_test_result(line): test_failed = True self._printer.write("%s" % line) if test_failed or set(keys) != set(results.keys()): return True self._buildbot_output.write('RESULT %s: %s= %s ms\n' % (category, test_name, results['avg'])) self._buildbot_output.write(', '.join(['%s= %s ms' % (key, results[key]) for key in keys[1:]]) + '\n') return False def _run_single_test(self, test, driver, is_chromium_style): test_failed = False driver_need_restart = False output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False)) if output.text == None: test_failed = True elif output.timeout: self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:]) test_failed = True driver_need_restart = True elif output.crash: self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:]) driver_need_restart = True test_failed = True else: if is_chromium_style: test_failed = self._process_chromium_style_test_result(test, output) else: test_failed = self._process_parser_test_result(test, output) if len(output.error): self._printer.write('error:\n%s' % output.error) test_failed = True if test_failed: self._printer.write('FAILED') return test_failed, driver_need_restart
class PerfTestsRunner(object): _default_branch = "webkit-trunk" _EXIT_CODE_BAD_BUILD = -1 _EXIT_CODE_BAD_JSON = -2 _EXIT_CODE_FAILED_UPLOADING = -3 def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host._initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time() @staticmethod def _parse_args(args=None): perf_option_list = [ optparse.make_option( "--debug", action="store_const", const="Debug", dest="configuration", help="Set the configuration to Debug", ), optparse.make_option( "--release", action="store_const", const="Release", dest="configuration", help="Set the configuration to Release", ), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option( "--chromium", action="store_const", const="chromium", dest="platform", help="Alias for --platform=chromium", ), optparse.make_option( "--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2."), ), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default).", ), optparse.make_option( "--build-directory", help="Path to the directory under which build files are kept (should not include configuration)", ), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option( "--pause-before-testing", dest="pause_before_testing", action="store_true", default=False, help="Pause before running the tests to let user attach a performance monitor.", ), optparse.make_option("--output-json-path", help="Filename of the JSON file that summaries the results"), optparse.make_option( "--source-json-path", help="Path to a JSON file to be merged into the JSON file when --output-json-path is present", ), optparse.make_option( "--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present", ), optparse.make_option( "--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree.", ), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): """Return the list of tests found.""" def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in [".html", ".svg"] filesystem = self._host.filesystem paths = [] for arg in self._args: paths.append(arg) relpath = filesystem.relpath(arg, self._base_path) if relpath: paths.append(relpath) skipped_directories = set([".svn", "resources"]) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] for path in test_files: relative_path = self._port.relative_perf_test_filename(path).replace("\\", "/") if self._port.skips_perf_test(relative_path): continue tests.append(PerfTestFactory.create_perf_test(relative_path, path)) return tests def run(self): if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self._EXIT_CODE_BAD_BUILD # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected = -1 tests = self._collect_tests() unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port) options = self._options if self._options.output_json_path: # FIXME: Add --branch or auto-detect the branch we're in test_results_server = options.test_results_server branch = self._default_branch if test_results_server else None build_number = int(options.build_number) if options.build_number else None if ( not self._generate_json( self._timestamp, options.output_json_path, options.source_json_path, branch, options.platform, options.builder_name, build_number, ) and not unexpected ): return self._EXIT_CODE_BAD_JSON if test_results_server and not self._upload_json(test_results_server, options.output_json_path): return self._EXIT_CODE_FAILED_UPLOADING return unexpected def _generate_json( self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number ): contents = {"timestamp": int(timestamp), "results": self._results} for (name, path) in self._port.repository_paths(): contents[name + "-revision"] = self._host.scm().svn_revision(path) for key, value in { "branch": branch, "platform": platform, "builder-name": builder_name, "build-number": build_number, }.items(): if value: contents[key] = value filesystem = self._host.filesystem succeeded = False if source_json_path: try: source_json_file = filesystem.open_text_file_for_reading(source_json_path) source_json = json.load(source_json_file) contents = dict(source_json.items() + contents.items()) succeeded = True except IOError, error: _log.error("Failed to read %s: %s" % (source_json_path, error)) except ValueError, error: _log.error("Failed to parse %s: %s" % (source_json_path, error)) except TypeError, error: _log.error("Failed to merge JSON files: %s" % error)
def __init__(self, path): MultiCommandTool.__init__(self) Host.__init__(self) self._path = path
class PerfTestsRunner(object): _default_branch = "webkit-trunk" EXIT_CODE_BAD_BUILD = -1 EXIT_CODE_BAD_SOURCE_JSON = -2 EXIT_CODE_BAD_MERGE = -3 EXIT_CODE_FAILED_UPLOADING = -4 EXIT_CODE_BAD_PREPARATION = -5 _DEFAULT_JSON_FILENAME = "PerformanceTestsResults.json" def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._timestamp = time.time() self._utc_timestamp = datetime.datetime.utcnow() @staticmethod def _parse_args(args=None): def _expand_path(option, opt_str, value, parser): path = os.path.expandvars(os.path.expanduser(value)) setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option( "--debug", action="store_const", const="Debug", dest="configuration", help="Set the configuration to Debug", ), optparse.make_option( "--release", action="store_const", const="Release", dest="configuration", help="Set the configuration to Release", ), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option( "--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2."), ), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default).", ), optparse.make_option( "--no-build", dest="build", action="store_false", help="Don't check to see if the DumpRenderTree build is up-to-date.", ), optparse.make_option( "--build-directory", help="Path to the directory under which build files are kept (should not include configuration)", ), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option( "--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page.", ), optparse.make_option( "--output-json-path", action="callback", callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists.", ), optparse.make_option( "--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results.", ), optparse.make_option( "--slave-config-json-path", action="callback", callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file.", ), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests are done", ), optparse.make_option( "--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present.", ), optparse.make_option( "--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree.", ), optparse.make_option( "--replay", dest="replay", action="store_true", default=False, help="Run replay tests." ), optparse.make_option( "--force", dest="use_skipped_list", action="store_false", default=True, help="Run all tests, including the ones in the Skipped list.", ), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option( "--profiler", action="store", help="Output per-test profile information, using the specified profiler." ), optparse.make_option( "--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags.", ), optparse.make_option("--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option( "--repeat", default=1, type="int", help="Specify number of times to run test set (default: 1)." ), optparse.make_option( "--test-runner-count", default=DEFAULT_TEST_RUNNER_COUNT, type="int", help="Specify number of times to invoke test runner for each performance test.", ), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): test_extensions = [".html", ".svg"] if self._options.replay: test_extensions.append(".replay") def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in test_extensions filesystem = self._host.filesystem paths = [] for arg in self._args: if filesystem.exists(filesystem.join(self._base_path, arg)): paths.append(arg) else: relpath = filesystem.relpath(arg, self._base_path) if filesystem.exists(filesystem.join(self._base_path, relpath)): paths.append(filesystem.normpath(relpath)) else: _log.warn("Path was not found:" + arg) skipped_directories = set([".svn", "resources"]) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] for path in test_files: relative_path = filesystem.relpath(path, self._base_path).replace("\\", "/") if ( self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths ): continue test = PerfTestFactory.create_perf_test( self._port, relative_path, path, test_runner_count=self._options.test_runner_count ) tests.append(test) return tests def _start_http_servers(self): self._port.acquire_http_lock() self._port.start_http_server(number_of_servers=2) def _stop_http_servers(self): self._port.stop_http_server() self._port.release_http_lock() def run(self): needs_http = self._port.requires_http_server() if not self._port.check_build(needs_http=needs_http): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self.EXIT_CODE_BAD_BUILD run_count = 0 repeat = self._options.repeat while run_count < repeat: run_count += 1 tests = self._collect_tests() runs = " (Run %d of %d)" % (run_count, repeat) if repeat > 1 else "" _log.info("Running %d tests%s" % (len(tests), runs)) for test in tests: if not test.prepare(self._options.time_out_ms): return self.EXIT_CODE_BAD_PREPARATION try: if needs_http: self._start_http_servers() unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name())) finally: if needs_http: self._stop_http_servers() if self._options.generate_results and not self._options.profile: exit_code = self._generate_results() if exit_code: return exit_code if self._options.generate_results and not self._options.profile: test_results_server = self._options.test_results_server if test_results_server and not self._upload_json(test_results_server, self._output_json_path()): return self.EXIT_CODE_FAILED_UPLOADING if self._options.show_results: self._port.show_results_html_file(self._results_page_path()) return unexpected def _output_json_path(self): output_json_path = self._options.output_json_path if output_json_path: return output_json_path return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) def _results_page_path(self): return self._host.filesystem.splitext(self._output_json_path())[0] + ".html" def _generate_results(self): options = self._options output_json_path = self._output_json_path() output = self._generate_results_dict( self._timestamp, options.description, options.platform, options.builder_name, options.build_number ) if options.slave_config_json_path: output = self._merge_slave_config_json(options.slave_config_json_path, output) if not output: return self.EXIT_CODE_BAD_SOURCE_JSON output = self._merge_outputs_if_needed(output_json_path, output) if not output: return self.EXIT_CODE_BAD_MERGE filesystem = self._host.filesystem json_output = json.dumps(output) filesystem.write_text_file(output_json_path, json_output) template_path = filesystem.join(self._port.perf_tests_dir(), "resources/results-template.html") template = filesystem.read_text_file(template_path) absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir()) results_page = template.replace("%AbsolutePathToWebKitTrunk%", absolute_path_to_trunk) results_page = results_page.replace("%PeformanceTestsResultsJSON%", json_output) filesystem.write_text_file(self._results_page_path(), results_page) def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): revisions = {} for (name, path) in self._port.repository_paths(): scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() revision = scm.svn_revision(path) revisions[name] = {"revision": revision, "timestamp": scm.timestamp_of_revision(path, revision)} meta_info = { "description": description, "buildTime": self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp), "platform": platform, "revisions": revisions, "builderName": builder_name, "buildNumber": int(build_number) if build_number else None, } contents = {"tests": {}} for key, value in meta_info.items(): if value: contents[key] = value for test, metrics in self._results: for metric_name, iteration_values in metrics.iteritems(): if not isinstance(iteration_values, list): # We can't reports results without individual measurements. continue tests = contents["tests"] path = test.test_name_without_file_extension().split("/") for i in range(0, len(path)): is_last_token = i + 1 == len(path) url = view_source_url( "PerformanceTests/" + (test.test_name() if is_last_token else "/".join(path[0 : i + 1])) ) tests.setdefault(path[i], {"url": url}) current_test = tests[path[i]] if is_last_token: current_test.setdefault("metrics", {}) assert metric_name not in current_test["metrics"] current_test["metrics"][metric_name] = {"current": iteration_values} else: current_test.setdefault("tests", {}) tests = current_test["tests"] return contents @staticmethod def _datetime_in_ES5_compatible_iso_format(datetime): return datetime.strftime("%Y-%m-%dT%H:%M:%S.%f") def _merge_slave_config_json(self, slave_config_json_path, contents): if not self._host.filesystem.isfile(slave_config_json_path): _log.error("Missing slave configuration JSON file: %s" % slave_config_json_path) return None try: slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path) slave_config = json.load(slave_config_json) for key in slave_config: contents["builder" + key.capitalize()] = slave_config[key] return contents except Exception, error: _log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error)) return None
test_name = self._port.uri_to_test_name(test_input.uri) actual_text = port.expected_text(test_name) actual_image = '' actual_checksum = '' if self._options.pixel_tests and test_input.checksum: actual_checksum = port.expected_checksum(test_name) if actual_checksum != test_input.checksum: actual_image = port.expected_image(test_name) self._stdout.write("#URL:%s\n" % test_input.uri) if self._options.pixel_tests and test_input.checksum: self._stdout.write("#MD5:%s\n" % actual_checksum) self._host.filesystem.write_binary_file(self._options.pixel_path, actual_image) self._stdout.write(actual_text) # FIXME: (See above FIXME as well). Chromium DRT appears to always # ensure the text output has a trailing newline. Mac DRT does not. if not actual_text.endswith('\n'): self._stdout.write('\n') self._stdout.write('#EOF\n') self._stdout.flush() if __name__ == '__main__': # FIXME: Why is this using a real Host object instead of MockHost? host = Host() host._initialize_scm() sys.exit(main(sys.argv[1:], host, sys.stdin, sys.stdout, sys.stderr))
class PerfTestsRunner(object): _perf_tests_base_dir = 'PerformanceTests' _result_regex = re.compile('^RESULT .*$') def __init__(self, perf_tests_dir, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None): self._perf_tests_dir = perf_tests_dir self._buildbot_output = buildbot_output self._options, self._args = self._parse_args(args) self._host = Host() self._host._initialize_scm() self._port = self._host.port_factory.get(self._options.platform, self._options) self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False) self._webkit_base_dir_len = len(self._port.webkit_base()) def _parse_args(self, args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=30000, help="Set the timeout for each test"), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args) def _collect_tests(self, webkit_base, filesystem=None): """Return the list of tests found.""" def _is_test_file(filesystem, dirname, filename): return filename.endswith('.html') filesystem = filesystem or self._host.filesystem base_dir = filesystem.join(webkit_base, self._perf_tests_base_dir, self._perf_tests_dir) return find_files.find(filesystem, base_dir, paths=self._args, file_filter=_is_test_file) def run(self): if self._options.help_printing: self._printer.help_printing() self._printer.cleanup() return 0 if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return -1 # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected = -1 try: tests = self._collect_tests(self._port.webkit_base()) unexpected = self._run_tests_set(tests, self._port) finally: self._printer.cleanup() return unexpected def _run_tests_set(self, tests, port): result_count = len(tests) expected = 0 unexpected = 0 self._printer.print_one_line_summary(result_count, 0, 0) driver_need_restart = False driver = None for test in tests: if driver_need_restart: _log.debug("%s killing driver" % test) driver.stop() driver = None if not driver: driver = port.create_driver(worker_number=1) test_failed, driver_need_restart = self._run_single_test(test, driver) if test_failed: unexpected = unexpected + 1 else: expected = expected + 1 self._printer.print_one_line_summary(result_count, expected, unexpected) if driver: driver.stop() return unexpected def _run_single_test(self, test, driver): test_failed = False driver_need_restart = False output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False)) if output.text == None: test_failed = True elif output.timeout: self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:]) test_failed = True driver_need_restart = True elif output.crash: self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:]) driver_need_restart = True test_failed = True else: got_a_result = False for line in re.split('\n', output.text): if self._result_regex.match(line): self._buildbot_output.write("%s\n" % line) got_a_result = True elif not len(line) == 0: test_failed = True self._printer.write("%s" % line) test_failed = test_failed or not got_a_result if len(output.error): self._printer.write('error:\n%s' % output.error) test_failed = True return test_failed, driver_need_restart
def __init__(self, path): MultiCommandTool.__init__(self) Host.__init__(self) self._path = path self.wakeup_event = threading.Event()
def __init__(self): self._host = Host() self._filesystem = self._host.filesystem self._host.initialize_scm() self._webkit_root = self._host.scm().checkout_root self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
def fake_dir_path(self, dirname): filesystem = Host().filesystem webkit_root = WebKitFinder(filesystem).webkit_base() return filesystem.abspath(filesystem.join(webkit_root, "LayoutTests", "css", dirname))
class PerfTestsRunner(object): _default_branch = "webkit-trunk" EXIT_CODE_BAD_BUILD = -1 EXIT_CODE_BAD_SOURCE_JSON = -2 EXIT_CODE_BAD_MERGE = -3 EXIT_CODE_FAILED_UPLOADING = -4 EXIT_CODE_BAD_PREPARATION = -5 _DEFAULT_JSON_FILENAME = "PerformanceTestsResults.json" def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time() @staticmethod def _parse_args(args=None): def _expand_path(option, opt_str, value, parser): path = os.path.expandvars(os.path.expanduser(value)) setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option( "--debug", action="store_const", const="Debug", dest="configuration", help="Set the configuration to Debug", ), optparse.make_option( "--release", action="store_const", const="Release", dest="configuration", help="Set the configuration to Release", ), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option( "--chromium", action="store_const", const="chromium", dest="platform", help="Alias for --platform=chromium", ), optparse.make_option( "--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2."), ), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default).", ), optparse.make_option( "--no-build", dest="build", action="store_false", help="Don't check to see if the DumpRenderTree build is up-to-date.", ), optparse.make_option( "--build-directory", help="Path to the directory under which build files are kept (should not include configuration)", ), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option( "--pause-before-testing", dest="pause_before_testing", action="store_true", default=False, help="Pause before running the tests to let user attach a performance monitor.", ), optparse.make_option( "--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page.", ), optparse.make_option( "--output-json-path", action="callback", callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists.", ), optparse.make_option( "--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results.", ), optparse.make_option( "--slave-config-json-path", action="callback", callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file.", ), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests are done", ), optparse.make_option( "--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present.", ), optparse.make_option( "--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree.", ), optparse.make_option( "--replay", dest="replay", action="store_true", default=False, help="Run replay tests." ), optparse.make_option( "--force", dest="skipped", action="store_true", default=False, help="Run all tests, including the ones in the Skipped list.", ), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): """Return the list of tests found.""" test_extensions = [".html", ".svg"] if self._options.replay: test_extensions.append(".replay") def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in test_extensions filesystem = self._host.filesystem paths = [] for arg in self._args: paths.append(arg) relpath = filesystem.relpath(arg, self._base_path) if relpath: paths.append(relpath) skipped_directories = set([".svn", "resources"]) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] for path in test_files: relative_path = self._port.relative_perf_test_filename(path).replace("\\", "/") if self._port.skips_perf_test(relative_path) and not self._options.skipped: continue test = PerfTestFactory.create_perf_test(self._port, relative_path, path) tests.append(test) return tests def run(self): if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self.EXIT_CODE_BAD_BUILD tests = self._collect_tests() _log.info("Running %d tests" % len(tests)) for test in tests: if not test.prepare(self._options.time_out_ms): return self.EXIT_CODE_BAD_PREPARATION unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port) if self._options.generate_results: exit_code = self._generate_and_show_results() if exit_code: return exit_code return unexpected def _output_json_path(self): output_json_path = self._options.output_json_path if output_json_path: return output_json_path return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) def _generate_and_show_results(self): options = self._options output_json_path = self._output_json_path() output = self._generate_results_dict( self._timestamp, options.description, options.platform, options.builder_name, options.build_number ) if options.slave_config_json_path: output = self._merge_slave_config_json(options.slave_config_json_path, output) if not output: return self.EXIT_CODE_BAD_SOURCE_JSON output = self._merge_outputs_if_needed(output_json_path, output) if not output: return self.EXIT_CODE_BAD_MERGE results_page_path = self._host.filesystem.splitext(output_json_path)[0] + ".html" self._generate_output_files(output_json_path, results_page_path, output) if options.test_results_server: if not self._upload_json(options.test_results_server, output_json_path): return self.EXIT_CODE_FAILED_UPLOADING if options.show_results: self._port.show_results_html_file(results_page_path) def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): contents = {"results": self._results} if description: contents["description"] = description for (name, path) in self._port.repository_paths(): scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() contents[name + "-revision"] = scm.svn_revision(path) # FIXME: Add --branch or auto-detect the branch we're in for key, value in { "timestamp": int(timestamp), "branch": self._default_branch, "platform": platform, "builder-name": builder_name, "build-number": int(build_number) if build_number else None, }.items(): if value: contents[key] = value return contents def _merge_slave_config_json(self, slave_config_json_path, output): if not self._host.filesystem.isfile(slave_config_json_path): _log.error("Missing slave configuration JSON file: %s" % slave_config_json_path) return None try: slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path) slave_config = json.load(slave_config_json) return dict(slave_config.items() + output.items()) except Exception, error: _log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error)) return None
class PerfTestsRunner(object): _perf_tests_base_dir = 'PerformanceTests' _test_directories_for_chromium_style_tests = ['inspector'] _default_branch = 'webkit-trunk' _EXIT_CODE_BAD_BUILD = -1 _EXIT_CODE_BAD_JSON = -2 _EXIT_CODE_FAILED_UPLOADING = -3 def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None, port=None): self._buildbot_output = buildbot_output self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host._initialize_scm() self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False) self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time() @staticmethod def _parse_args(args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default)."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option("--output-json-path", help="Filename of the JSON file that summaries the results"), optparse.make_option("--source-json-path", help="Path to a JSON file to be merged into the JSON file when --output-json-path is present"), optparse.make_option("--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present"), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args) def _collect_tests(self): """Return the list of tests found.""" def _is_test_file(filesystem, dirname, filename): return filename.endswith('.html') skipped_directories = set(['.svn', 'resources']) tests = find_files.find(self._host.filesystem, self._base_path, self._args, skipped_directories, _is_test_file) return [test for test in tests if not self._port.skips_perf_test(self._port.relative_perf_test_filename(test))] def run(self): if self._options.help_printing: self._printer.help_printing() self._printer.cleanup() return 0 if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self._EXIT_CODE_BAD_BUILD # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected = -1 try: tests = self._collect_tests() unexpected = self._run_tests_set(sorted(list(tests)), self._port) finally: self._printer.cleanup() options = self._options if self._options.output_json_path: # FIXME: Add --branch or auto-detect the branch we're in test_results_server = options.test_results_server branch = self._default_branch if test_results_server else None build_number = int(options.build_number) if options.build_number else None if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path, branch, options.platform, options.builder_name, build_number) and not unexpected: return self._EXIT_CODE_BAD_JSON if test_results_server and not self._upload_json(test_results_server, options.output_json_path): return self._EXIT_CODE_FAILED_UPLOADING return unexpected def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number): contents = {'timestamp': int(timestamp), 'results': self._results} for (name, path) in self._port.repository_paths(): contents[name + '-revision'] = self._host.scm().svn_revision(path) for key, value in {'branch': branch, 'platform': platform, 'builder-name': builder_name, 'build-number': build_number}.items(): if value: contents[key] = value filesystem = self._host.filesystem succeeded = False if source_json_path: try: source_json_file = filesystem.open_text_file_for_reading(source_json_path) source_json = json.load(source_json_file) contents = dict(source_json.items() + contents.items()) succeeded = True except IOError, error: _log.error("Failed to read %s: %s" % (source_json_path, error)) except ValueError, error: _log.error("Failed to parse %s: %s" % (source_json_path, error)) except TypeError, error: _log.error("Failed to merge JSON files: %s" % error)
class PerfTestsRunner(object): _default_branch = 'webkit-trunk' EXIT_CODE_BAD_BUILD = -1 EXIT_CODE_BAD_SOURCE_JSON = -2 EXIT_CODE_BAD_MERGE = -3 EXIT_CODE_FAILED_UPLOADING = -4 EXIT_CODE_BAD_PREPARATION = -5 _DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json' def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) # The GTK+ and EFL ports only supports WebKit2, so they always use WKTR. if self._port.name().startswith("gtk") or self._port.name().startswith("efl"): self._options.webkit_test_runner = True self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._timestamp = time.time() self._utc_timestamp = datetime.datetime.utcnow() @staticmethod def _parse_args(args=None): def _expand_path(option, opt_str, value, parser): path = os.path.expandvars(os.path.expanduser(value)) setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the DumpRenderTree build is up-to-date."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page."), optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists."), optparse.make_option("--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results."), optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file."), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests are done"), optparse.make_option("--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present."), optparse.make_option("--dump-render-tree", "-1", action="store_false", default=True, dest="webkit_test_runner", help="Use DumpRenderTree rather than WebKitTestRunner."), optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True, help="Run all tests, including the ones in the Skipped list."), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option("--profiler", action="store", help="Output per-test profile information, using the specified profiler."), optparse.make_option("--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option("--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option("--repeat", default=1, type="int", help="Specify number of times to run test set (default: 1)."), optparse.make_option("--test-runner-count", default=-1, type="int", help="Specify number of times to invoke test runner for each performance test."), optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree or WebKitTestRunner; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): test_extensions = ['.html', '.svg'] def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in test_extensions filesystem = self._host.filesystem paths = [] for arg in self._args: if filesystem.exists(filesystem.join(self._base_path, arg)): paths.append(arg) else: relpath = filesystem.relpath(arg, self._base_path) if filesystem.exists(filesystem.join(self._base_path, relpath)): paths.append(filesystem.normpath(relpath)) else: _log.warn('Path was not found:' + arg) skipped_directories = set(['.svn', 'resources']) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] test_runner_count = DEFAULT_TEST_RUNNER_COUNT if self._options.test_runner_count > 0: test_runner_count = self._options.test_runner_count elif self._options.profile: test_runner_count = 1 for path in test_files: relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/') if self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths: continue test = PerfTestFactory.create_perf_test(self._port, relative_path, path, test_runner_count=test_runner_count) tests.append(test) return tests def run(self): if "Debug" == self._port.get_option("configuration"): _log.warning("""**************************************************** * WARNING: run-perf-tests is running in DEBUG mode * ****************************************************""") if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self.EXIT_CODE_BAD_BUILD # Check that the system dependencies (themes, fonts, ...) are correct. if not self._port.check_sys_deps(needs_http=False): _log.error("Failed to check system dependencies.") self._port.stop_helper() return self.EXIT_CODE_BAD_PREPARATION run_count = 0 repeat = self._options.repeat while (run_count < repeat): run_count += 1 tests = self._collect_tests() runs = ' (Run %d of %d)' % (run_count, repeat) if repeat > 1 else '' _log.info("Running %d tests%s" % (len(tests), runs)) for test in tests: if not test.prepare(self._options.time_out_ms): return self.EXIT_CODE_BAD_PREPARATION unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name())) if self._options.generate_results and not self._options.profile: exit_code = self._generate_results() if exit_code: return exit_code if self._options.generate_results and not self._options.profile: test_results_server = self._options.test_results_server if test_results_server and not self._upload_json(test_results_server, self._output_json_path()): return self.EXIT_CODE_FAILED_UPLOADING if self._options.show_results: self._port.show_results_html_file(self._results_page_path()) return unexpected def _output_json_path(self): output_json_path = self._options.output_json_path if output_json_path: return output_json_path return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) def _results_page_path(self): return self._host.filesystem.splitext(self._output_json_path())[0] + '.html' def _generate_results(self): options = self._options output_json_path = self._output_json_path() output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) if options.slave_config_json_path: output = self._merge_slave_config_json(options.slave_config_json_path, output) if not output: return self.EXIT_CODE_BAD_SOURCE_JSON output = self._merge_outputs_if_needed(output_json_path, output) if not output: return self.EXIT_CODE_BAD_MERGE filesystem = self._host.filesystem json_output = json.dumps(output) filesystem.write_text_file(output_json_path, json_output) template_path = filesystem.join(self._port.perf_tests_dir(), 'resources/results-template.html') template = filesystem.read_text_file(template_path) absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir()) results_page = template.replace('%AbsolutePathToWebKitTrunk%', absolute_path_to_trunk) results_page = results_page.replace('%PeformanceTestsResultsJSON%', json_output) filesystem.write_text_file(self._results_page_path(), results_page) def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): revisions = {} for (name, path) in self._port.repository_paths(): scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() revision = scm.svn_revision(path) revisions[name] = {'revision': revision, 'timestamp': scm.timestamp_of_revision(path, revision)} meta_info = { 'description': description, 'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp), 'platform': platform, 'revisions': revisions, 'builderName': builder_name, 'buildNumber': int(build_number) if build_number else None} contents = {'tests': {}} for key, value in meta_info.items(): if value: contents[key] = value for metric in self._results: tests = contents['tests'] path = metric.path() for i in range(0, len(path)): is_last_token = i + 1 == len(path) url = view_source_url('PerformanceTests/' + '/'.join(path[0:i + 1])) test_name = path[i] # FIXME: This is a temporary workaround for the fact perf dashboard doesn't support renaming tests. if test_name == 'Speedometer': test_name = 'DoYouEvenBench' tests.setdefault(test_name, {'url': url}) current_test = tests[test_name] if is_last_token: current_test['url'] = view_source_url('PerformanceTests/' + metric.test_file_name()) current_test.setdefault('metrics', {}) assert metric.name() not in current_test['metrics'] test_results = {'current': metric.grouped_iteration_values()} if metric.aggregator(): test_results['aggregators'] = [metric.aggregator()] current_test['metrics'][metric.name()] = test_results else: current_test.setdefault('tests', {}) tests = current_test['tests'] return contents @staticmethod def _datetime_in_ES5_compatible_iso_format(datetime): return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f') def _merge_slave_config_json(self, slave_config_json_path, contents): if not self._host.filesystem.isfile(slave_config_json_path): _log.error("Missing slave configuration JSON file: %s" % slave_config_json_path) return None try: slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path) slave_config = json.load(slave_config_json) for key in slave_config: contents['builder' + key.capitalize()] = slave_config[key] return contents except Exception, error: _log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error)) return None
class PerfTestsRunner(object): _default_branch = 'webkit-trunk' EXIT_CODE_BAD_BUILD = -1 EXIT_CODE_BAD_SOURCE_JSON = -2 EXIT_CODE_BAD_MERGE = -3 EXIT_CODE_FAILED_UPLOADING = -4 EXIT_CODE_BAD_PREPARATION = -5 _DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json' def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._timestamp = time.time() self._utc_timestamp = datetime.datetime.utcnow() @staticmethod def _parse_args(args=None): def _expand_path(option, opt_str, value, parser): path = os.path.expandvars(os.path.expanduser(value)) setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option('-t', '--target', dest='configuration', help='Specify the target build subdirectory under src/out/'), optparse.make_option("--platform", help="Specify port/platform being tested (e.g. mac)"), optparse.make_option("--chromium", action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'), optparse.make_option("--android", action="store_const", const='android', dest='platform', help='Alias for --platform=android'), optparse.make_option("--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up to date (default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the DumpRenderTree build is up to date."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page."), optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists."), optparse.make_option("--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results."), optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file."), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests are done"), optparse.make_option("--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present."), optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True, help="Run all tests, including the ones in the Skipped list."), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option("--profiler", action="store", help="Output per-test profile information, using the specified profiler."), optparse.make_option("--additional-driver-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option("--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option("--content-shell", action="store_true", help="Use Content Shell instead of DumpRenderTree"), optparse.make_option("--repeat", default=1, type="int", help="Specify number of times to run test set (default: 1)."), optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUNNER_COUNT, type="int", help="Specify number of times to invoke test runner for each performance test."), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): test_extensions = ['.html', '.svg'] def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in test_extensions filesystem = self._host.filesystem paths = [] for arg in self._args: if filesystem.exists(filesystem.join(self._base_path, arg)): paths.append(arg) else: relpath = filesystem.relpath(arg, self._base_path) if filesystem.exists(filesystem.join(self._base_path, relpath)): paths.append(filesystem.normpath(relpath)) else: _log.warning('Path was not found:' + arg) skipped_directories = set(['.svn', 'resources']) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] for path in test_files: relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/') if self._options.use_skipped_list and self._port.skips_perf_test( relative_path) and filesystem.normpath(relative_path) not in paths: continue test = PerfTestFactory.create_perf_test(self._port, relative_path, path, test_runner_count=self._options.test_runner_count) tests.append(test) return tests def _start_http_servers(self): self._port.acquire_http_lock() self._port.start_http_server(number_of_servers=2) def _stop_http_servers(self): self._port.stop_http_server() self._port.release_http_lock() def run(self): needs_http = self._port.requires_http_server() class FakePrinter(object): def write_update(self, msg): print msg def write_throttled_update(self, msg): pass if self._port.check_build(needs_http=needs_http, printer=FakePrinter()): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self.EXIT_CODE_BAD_BUILD run_count = 0 repeat = self._options.repeat while run_count < repeat: run_count += 1 tests = self._collect_tests() runs = ' (Run %d of %d)' % (run_count, repeat) if repeat > 1 else '' _log.info("Running %d tests%s", len(tests), runs) try: if needs_http: self._start_http_servers() unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name())) finally: if needs_http: self._stop_http_servers() if self._options.generate_results and not self._options.profile: exit_code = self._generate_results() if exit_code: return exit_code if self._options.generate_results and not self._options.profile: test_results_server = self._options.test_results_server if test_results_server and not self._upload_json(test_results_server, self._output_json_path()): return self.EXIT_CODE_FAILED_UPLOADING if self._options.show_results: self._port.show_results_html_file(self._results_page_path()) return unexpected def _output_json_path(self): output_json_path = self._options.output_json_path if output_json_path: return output_json_path return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) def _results_page_path(self): return self._host.filesystem.splitext(self._output_json_path())[0] + '.html' def _generate_results(self): options = self._options output_json_path = self._output_json_path() output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) if options.slave_config_json_path: output = self._merge_slave_config_json(options.slave_config_json_path, output) if not output: return self.EXIT_CODE_BAD_SOURCE_JSON output = self._merge_outputs_if_needed(output_json_path, output) if not output: return self.EXIT_CODE_BAD_MERGE filesystem = self._host.filesystem json_output = json.dumps(output) filesystem.write_text_file(output_json_path, json_output) template_path = filesystem.join(self._port.perf_tests_dir(), 'resources/results-template.html') template = filesystem.read_text_file(template_path) absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir()) results_page = template.replace('%AbsolutePathToWebKitTrunk%', absolute_path_to_trunk) results_page = results_page.replace('%PeformanceTestsResultsJSON%', json_output) filesystem.write_text_file(self._results_page_path(), results_page) def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): revisions = {} path = self._port.repository_path() scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() revision = str(scm.commit_position(path)) revisions['chromium'] = {'revision': revision, 'timestamp': scm.timestamp_of_revision(path, revision)} meta_info = { 'description': description, 'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp), 'platform': platform, 'revisions': revisions, 'builderName': builder_name, 'buildNumber': int(build_number) if build_number else None} contents = {'tests': {}} for key, value in meta_info.items(): if value: contents[key] = value for test, metrics in self._results: for metric_name, iteration_values in metrics.iteritems(): if not isinstance(iteration_values, list): # We can't reports results without individual measurements. continue tests = contents['tests'] path = test.test_name_without_file_extension().split('/') for i in range(0, len(path)): is_last_token = i + 1 == len(path) url = self.view_source_url( 'PerformanceTests/' + (test.test_name() if is_last_token else '/'.join(path[0:i + 1]))) tests.setdefault(path[i], {'url': url}) current_test = tests[path[i]] if is_last_token: current_test.setdefault('metrics', {}) assert metric_name not in current_test['metrics'] current_test['metrics'][metric_name] = {'current': iteration_values} else: current_test.setdefault('tests', {}) tests = current_test['tests'] return contents @staticmethod def view_source_url(path_from_blink): return 'https://chromium.googlesource.com/chromium/src/+/master/third_party/WebKit/%s' % path_from_blink @staticmethod def _datetime_in_ES5_compatible_iso_format(datetime): return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f') def _merge_slave_config_json(self, slave_config_json_path, contents): if not self._host.filesystem.isfile(slave_config_json_path): _log.error("Missing slave configuration JSON file: %s", slave_config_json_path) return None try: slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path) slave_config = json.load(slave_config_json) for key in slave_config: contents['builder' + key.capitalize()] = slave_config[key] return contents except Exception as error: _log.error("Failed to merge slave configuration JSON file %s: %s", slave_config_json_path, error) return None def _merge_outputs_if_needed(self, output_json_path, output): if self._options.reset_results or not self._host.filesystem.isfile(output_json_path): return [output] try: existing_outputs = json.loads(self._host.filesystem.read_text_file(output_json_path)) return existing_outputs + [output] except Exception as error: _log.error("Failed to merge output JSON file %s: %s", output_json_path, error) return None def _upload_json(self, test_results_server, json_path, host_path="/api/report", file_uploader=FileUploader): url = "https://%s%s" % (test_results_server, host_path) uploader = file_uploader(url, 120) try: response = uploader.upload_single_text_file(self._host.filesystem, 'application/json', json_path) except Exception as error: _log.error("Failed to upload JSON file to %s in 120s: %s", url, error) return False response_body = [line.strip('\n') for line in response] if response_body != ['OK']: try: parsed_response = json.loads('\n'.join(response_body)) except: _log.error("Uploaded JSON to %s but got a bad response:", url) for line in response_body: _log.error(line) return False if parsed_response.get('status') != 'OK': _log.error("Uploaded JSON to %s but got an error:", url) _log.error(json.dumps(parsed_response, indent=4)) return False _log.info("JSON file uploaded to %s.", url) return True def _run_tests_set(self, tests): failures = 0 self._results = [] for i, test in enumerate(tests): _log.info('Running %s (%d of %d)', test.test_name(), i + 1, len(tests)) start_time = time.time() metrics = test.run(self._options.time_out_ms) if metrics: self._results.append((test, metrics)) else: failures += 1 _log.error('FAILED') _log.info('Finished: %f s', time.time() - start_time) _log.info('') return failures
class PerfTestsRunner(object): _default_branch = 'webkit-trunk' EXIT_CODE_BAD_BUILD = -1 EXIT_CODE_BAD_SOURCE_JSON = -2 EXIT_CODE_BAD_MERGE = -3 EXIT_CODE_FAILED_UPLOADING = -4 EXIT_CODE_BAD_PREPARATION = -5 _DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json' def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time() self._utc_timestamp = datetime.datetime.utcnow() @staticmethod def _parse_args(args=None): def _expand_path(option, opt_str, value, parser): path = os.path.expandvars(os.path.expanduser(value)) setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--chromium", action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'), optparse.make_option("--chromium-android", action="store_const", const='chromium-android', dest='platform', help='Alias for --platform=chromium-android'), optparse.make_option("--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the DumpRenderTree build is up-to-date."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page."), optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists."), optparse.make_option("--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results."), optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file."), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests are done"), optparse.make_option("--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present."), optparse.make_option("--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), optparse.make_option("--replay", dest="replay", action="store_true", default=False, help="Run replay tests."), optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True, help="Run all tests, including the ones in the Skipped list."), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option("--profiler", action="store", help="Output per-test profile information, using the specified profiler."), optparse.make_option("--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): test_extensions = ['.html', '.svg'] if self._options.replay: test_extensions.append('.replay') def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in test_extensions filesystem = self._host.filesystem paths = [] for arg in self._args: if filesystem.exists(filesystem.join(self._base_path, arg)): paths.append(arg) else: relpath = filesystem.relpath(arg, self._base_path) if filesystem.exists(filesystem.join(self._base_path, relpath)): paths.append(filesystem.normpath(relpath)) else: _log.warn('Path was not found:' + arg) skipped_directories = set(['.svn', 'resources']) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] for path in test_files: relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/') if self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths: continue test = PerfTestFactory.create_perf_test(self._port, relative_path, path) tests.append(test) return tests def _start_http_servers(self): self._port.acquire_http_lock() self._port.start_http_server(number_of_servers=2) def _stop_http_servers(self): self._port.stop_http_server() self._port.release_http_lock() def run(self): needs_http = self._port.requires_http_server() if not self._port.check_build(needs_http=needs_http): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self.EXIT_CODE_BAD_BUILD tests = self._collect_tests() _log.info("Running %d tests" % len(tests)) for test in tests: if not test.prepare(self._options.time_out_ms): return self.EXIT_CODE_BAD_PREPARATION try: if needs_http: self._start_http_servers() unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port) finally: if needs_http: self._stop_http_servers() if self._options.generate_results and not self._options.profile: exit_code = self._generate_and_show_results() if exit_code: return exit_code return unexpected def _output_json_path(self): output_json_path = self._options.output_json_path if output_json_path: return output_json_path return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) def _generate_and_show_results(self): options = self._options output_json_path = self._output_json_path() output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) if options.slave_config_json_path: output = self._merge_slave_config_json(options.slave_config_json_path, output) if not output: return self.EXIT_CODE_BAD_SOURCE_JSON output = self._merge_outputs_if_needed(output_json_path, output) if not output: return self.EXIT_CODE_BAD_MERGE results_page_path = self._host.filesystem.splitext(output_json_path)[0] + '.html' self._generate_output_files(output_json_path, results_page_path, output) if options.test_results_server: if options.test_results_server == 'webkit-perf.appspot.com': options.test_results_server = 'perf.webkit.org' if not self._upload_json(options.test_results_server, output_json_path): return self.EXIT_CODE_FAILED_UPLOADING if options.show_results: self._port.show_results_html_file(results_page_path) def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): contents = {'tests': {}} if description: contents['description'] = description revisions = {} for (name, path) in self._port.repository_paths(): scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() revision = scm.svn_revision(path) revisions[name] = {'revision': str(revision), 'timestamp': scm.timestamp_of_latest_commit(path, revision)} meta_info = { 'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp), 'platform': platform, 'revisions': revisions, 'builderName': builder_name, 'buildNumber': int(build_number) if build_number else None} for key, value in meta_info.items(): if value: contents[key] = value # FIXME: Make this function shorter once we've transitioned to use perf.webkit.org. for metric_full_name, result in self._results.iteritems(): if not isinstance(result, dict): # We can't reports results without indivisual measurements. continue assert metric_full_name.count(':') <= 1 test_full_name, _, metric = metric_full_name.partition(':') if not metric: metric = {'fps': 'FrameRate', 'runs/s': 'Runs', 'ms': 'Time'}[result['unit']] tests = contents['tests'] path = test_full_name.split('/') for i in range(0, len(path)): # FIXME: We shouldn't assume HTML extension. is_last_token = i + 1 == len(path) url = 'http://trac.webkit.org/browser/trunk/PerformanceTests/' + '/'.join(path[0:i + 1]) if is_last_token: url += '.html' tests.setdefault(path[i], {'url': url}) current_test = tests[path[i]] if is_last_token: current_test.setdefault('metrics', {}) assert metric not in current_test['metrics'] current_test['metrics'][metric] = {'current': result['values']} else: current_test.setdefault('tests', {}) tests = current_test['tests'] return contents @staticmethod def _datetime_in_ES5_compatible_iso_format(datetime): return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f') def _merge_slave_config_json(self, slave_config_json_path, contents): if not self._host.filesystem.isfile(slave_config_json_path): _log.error("Missing slave configuration JSON file: %s" % slave_config_json_path) return None try: slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path) slave_config = json.load(slave_config_json) for key in slave_config: contents['builder' + key.capitalize()] = slave_config[key] return contents except Exception, error: _log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error)) return None
class PerfTestsRunner(object): _default_branch = 'webkit-trunk' _EXIT_CODE_BAD_BUILD = -1 _EXIT_CODE_BAD_JSON = -2 _EXIT_CODE_FAILED_UPLOADING = -3 _EXIT_CODE_BAD_PREPARATION = -4 def __init__(self, args=None, port=None): self._options, self._args = PerfTestsRunner._parse_args(args) if port: self._port = port self._host = self._port.host else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) self._host._initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} self._timestamp = time.time() @staticmethod def _parse_args(args=None): perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--chromium", action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'), optparse.make_option("--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the DumpRenderTree build is up-to-date."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option("--pause-before-testing", dest="pause_before_testing", action="store_true", default=False, help="Pause before running the tests to let user attach a performance monitor."), optparse.make_option("--output-json-path", help="Filename of the JSON file that summaries the results."), optparse.make_option("--source-json-path", help="Path to a JSON file to be merged into the JSON file when --output-json-path is present."), optparse.make_option("--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present."), optparse.make_option("--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), optparse.make_option("--replay", dest="replay", action="store_true", default=False, help="Run replay tests."), optparse.make_option("--force", dest="skipped", action="store_true", default=False, help="Run all tests, including the ones in the Skipped list."), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) def _collect_tests(self): """Return the list of tests found.""" test_extensions = ['.html', '.svg'] if self._options.replay: test_extensions.append('.replay') def _is_test_file(filesystem, dirname, filename): return filesystem.splitext(filename)[1] in test_extensions filesystem = self._host.filesystem paths = [] for arg in self._args: paths.append(arg) relpath = filesystem.relpath(arg, self._base_path) if relpath: paths.append(relpath) skipped_directories = set(['.svn', 'resources']) test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) tests = [] for path in test_files: relative_path = self._port.relative_perf_test_filename(path).replace('\\', '/') if self._port.skips_perf_test(relative_path) and not self._options.skipped: continue test = PerfTestFactory.create_perf_test(self._port, relative_path, path) tests.append(test) return tests def run(self): if not self._port.check_build(needs_http=False): _log.error("Build not up to date for %s" % self._port._path_to_driver()) return self._EXIT_CODE_BAD_BUILD tests = self._collect_tests() _log.info("Running %d tests" % len(tests)) for test in tests: if not test.prepare(self._options.time_out_ms): return self._EXIT_CODE_BAD_PREPARATION unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name()), self._port) options = self._options if self._options.output_json_path: # FIXME: Add --branch or auto-detect the branch we're in test_results_server = options.test_results_server branch = self._default_branch if test_results_server else None build_number = int(options.build_number) if options.build_number else None if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path, branch, options.platform, options.builder_name, build_number) and not unexpected: return self._EXIT_CODE_BAD_JSON if test_results_server and not self._upload_json(test_results_server, options.output_json_path): return self._EXIT_CODE_FAILED_UPLOADING return unexpected def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number): contents = {'timestamp': int(timestamp), 'results': self._results} for (name, path) in self._port.repository_paths(): contents[name + '-revision'] = self._host.scm().svn_revision(path) for key, value in {'branch': branch, 'platform': platform, 'builder-name': builder_name, 'build-number': build_number}.items(): if value: contents[key] = value filesystem = self._host.filesystem succeeded = False if source_json_path: try: source_json_file = filesystem.open_text_file_for_reading(source_json_path) source_json = json.load(source_json_file) contents = dict(source_json.items() + contents.items()) succeeded = True except IOError, error: _log.error("Failed to read %s: %s" % (source_json_path, error)) except ValueError, error: _log.error("Failed to parse %s: %s" % (source_json_path, error)) except TypeError, error: _log.error("Failed to merge JSON files: %s" % error)
class W3CTestConverter(object): def __init__(self): self._host = Host() self._filesystem = self._host.filesystem self._host.initialize_scm() self._webkit_root = self._host.scm().checkout_root # These settings might vary between WebKit and Blink self._css_property_file = self.path_from_webkit_root('Source', 'core', 'css', 'CSSPropertyNames.in') self._css_property_split_string = 'alias_for=' self.prefixed_properties = self.read_webkit_prefixed_css_property_list() def path_from_webkit_root(self, *comps): return self._filesystem.abspath(self._filesystem.join(self._webkit_root, *comps)) def read_webkit_prefixed_css_property_list(self): prefixed_properties = [] contents = self._filesystem.read_text_file(self._css_property_file) for line in contents.splitlines(): # Find lines starting with the -webkit- prefix. match = re.match('-webkit-[\w|-]*', line) if match: # Ignore lines where both the prefixed and non-prefixed property # are supported - denoted by -webkit-some-property = some-property. fields = line.split(self._css_property_split_string) if len(fields) == 2 and fields[1].strip() in fields[0].strip(): continue prefixed_properties.append(match.group(0)) return prefixed_properties def convert_for_webkit(self, new_path, filename): """ Converts a file's |contents| so it will function correctly in its |new_path| in Webkit. Returns the list of modified properties and the modified text if the file was modifed, None otherwise.""" contents = self._filesystem.read_binary_file(filename) if filename.endswith('.css'): return self.convert_css(contents, filename) return self.convert_html(new_path, contents, filename) def convert_css(self, contents, filename): return self.add_webkit_prefix_to_unprefixed_properties(contents, filename) def convert_html(self, new_path, contents, filename): doc = BeautifulSoup(contents) did_modify_paths = self.convert_testharness_paths(doc, new_path, filename) converted_properties_and_content = self.convert_prefixed_properties(doc, filename) return converted_properties_and_content if (did_modify_paths or converted_properties_and_content[0]) else None def convert_testharness_paths(self, doc, new_path, filename): """ Update links to testharness.js in the BeautifulSoup |doc| to point to the copy in |new_path|. Returns whether the document was modified.""" # Look for the W3C-style path to any testharness files - scripts (.js) or links (.css) pattern = re.compile('/resources/testharness') script_tags = doc.findAll(src=pattern) link_tags = doc.findAll(href=pattern) testharness_tags = script_tags + link_tags if not testharness_tags: return False resources_path = self.path_from_webkit_root('LayoutTests', 'resources') resources_relpath = self._filesystem.relpath(resources_path, new_path) for tag in testharness_tags: # FIXME: We need to handle img, audio, video tags also. attr = 'src' if tag.name != 'script': attr = 'href' if not attr in tag.attrMap: # FIXME: Figure out what to do w/ invalid tags. For now, we return False # and leave the document unmodified, which means that it'll probably fail to run. _log.error("Missing an attr in %s" % filename) return False old_path = tag[attr] new_tag = Tag(doc, tag.name, tag.attrs) new_tag[attr] = re.sub(pattern, resources_relpath + '/testharness', old_path) self.replace_tag(tag, new_tag) return True def convert_prefixed_properties(self, doc, filename): """ Searches a BeautifulSoup |doc| for any CSS properties requiring the -webkit- prefix and converts them. Returns the list of converted properties and the modified document as a string """ converted_properties = [] # Look for inline and document styles. inline_styles = doc.findAll(style=re.compile('.*')) style_tags = doc.findAll('style') all_styles = inline_styles + style_tags for tag in all_styles: # Get the text whether in a style tag or style attribute. style_text = '' if tag.name == 'style': if not tag.contents: continue style_text = tag.contents[0] else: style_text = tag['style'] updated_style_text = self.add_webkit_prefix_to_unprefixed_properties(style_text, filename) # Rewrite tag only if changes were made. if updated_style_text[0]: converted_properties.extend(updated_style_text[0]) new_tag = Tag(doc, tag.name, tag.attrs) new_tag.insert(0, updated_style_text[1]) self.replace_tag(tag, new_tag) return (converted_properties, doc.prettify()) def add_webkit_prefix_to_unprefixed_properties(self, text, filename): """ Searches |text| for instances of properties requiring the -webkit- prefix and adds the prefix to them. Returns the list of converted properties and the modified text.""" converted_properties = [] for prefixed_property in self.prefixed_properties: # FIXME: add in both the prefixed and unprefixed versions, rather than just replacing them? # That might allow the imported test to work in other browsers more easily. unprefixed_property = prefixed_property.replace('-webkit-', '') # Look for the various ways it might be in the CSS # Match the the property preceded by either whitespace or left curly brace # or at the beginning of the string (for inline style attribute) pattern = '([\s{]|^)' + unprefixed_property + '(\s+:|:)' if re.search(pattern, text): _log.info('converting %s -> %s' % (unprefixed_property, prefixed_property)) converted_properties.append(prefixed_property) text = re.sub(pattern, prefixed_property + ':', text) # FIXME: Handle the JS versions of these properties and GetComputedStyle, too. return (converted_properties, text) def replace_tag(self, old_tag, new_tag): index = old_tag.parent.contents.index(old_tag) old_tag.parent.insert(index, new_tag) old_tag.extract()
def main(): options, args = parse_args() host = Host() host._initialize_scm() port = host.port_factory.get(options.platform, options) return run(port, options, args)