def _prepare_config(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() scm = self._tool.scm() print 'Parsing full_results.json...' results_json_path = filesystem.join(results_directory, 'full_results.json') results_json = json_results_generator.load_json( filesystem, results_json_path) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, filesystem, scm) print 'Gathering current baselines...' self._gather_baselines(results_json) return { 'test_config': self._test_config, "results_json": results_json, "platforms_json": { 'platforms': platforms, 'defaultPlatform': port.name(), }, }
def _baseline_search_hypergraph(fs): hypergraph = {} # These edges in the hypergraph aren't visible on build.webkit.org, # but they impose constraints on how we optimize baselines. hypergraph['mac-future'] = [ 'LayoutTests/platform/mac-future', 'LayoutTests/platform/mac', 'LayoutTests' ] hypergraph['qt-unknown'] = [ 'LayoutTests/platform/qt-unknown', 'LayoutTests/platform/qt', 'LayoutTests' ] # FIXME: Should we get this constant from somewhere? fallback_path = ['LayoutTests'] for port_name in port_factory.all_port_names(): port = port_factory.get(port_name) webkit_base = port.webkit_base() search_path = port.baseline_search_path() if search_path: hypergraph[port_name] = [ fs.relpath(path, webkit_base) for path in search_path ] + fallback_path return hypergraph
def _update_expectations_file(self, port_name): if not self._is_supported_port(port_name): return port = factory.get(port_name) expectations = self._expectations(port) path = port.path_to_test_expectations_file() self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures()))
def execute(self, options, args, tool): self._baseline_optimizer = BaselineOptimizer(tool.scm(), tool.filesystem) self._port = factory.get("chromium-win-win7") # FIXME: This should be selectable. for test_name in map(self._to_test_name, test_files.find(self._port, args)): print "Optimizing %s." % test_name self._optimize_baseline(test_name)
def execute(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() print 'Parsing unexpected_results.json...' results_json_path = filesystem.join( results_directory, 'unexpected_results.json') with codecs.open(results_json_path, "r") as results_json_file: results_json_file = file(results_json_path) results_json = simplejson.load(results_json_file) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) print 'Gathering current baselines...' for test_file, test_json in results_json['tests'].items(): test_json['state'] = STATE_NEEDS_REBASELINE test_json['baselines'] = _get_test_baselines( test_file, layout_tests_directory, platforms, filesystem) print "Starting server at http://localhost:%d/" % options.httpd_port print ("Use the 'Exit' link in the UI, http://localhost:%d/" "quitquitquit or Ctrl-C to stop") % options.httpd_port httpd = RebaselineHTTPServer( httpd_port=options.httpd_port, results_directory=results_directory, results_json=results_json, platforms_json={ 'platforms': platforms, 'defaultPlatform': port.name(), }) httpd.serve_forever()
def assertTest(self, test_name, pixel_tests, expected_checksum=None, drt_output=None, filesystem=None): platform = 'test' filesystem = filesystem or test.unit_test_filesystem() port = factory.get(platform, filesystem=filesystem) drt_input, drt_output = self.make_input_output(port, test_name, pixel_tests, expected_checksum, drt_output) args = ['--platform', 'test'] + self.extra_args(pixel_tests) stdin = newstringio.StringIO(drt_input) stdout = newstringio.StringIO() stderr = newstringio.StringIO() options, args = mock_drt.parse_options(args) drt = self.make_drt(options, args, filesystem, stdin, stdout, stderr) res = drt.run() self.assertEqual(res, 0) # We use the StringIO.buflist here instead of getvalue() because # the StringIO might be a mix of unicode/ascii and 8-bit strings. self.assertEqual(stdout.buflist, drt_output) self.assertEqual(stderr.getvalue(), '')
def execute(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() scm = self._tool.scm() if options.dry_run: def no_op_copyfile(src, dest): pass def no_op_add(path, return_exit_code=False): if return_exit_code: return 0 filesystem.copyfile = no_op_copyfile scm.add = no_op_add print 'Parsing unexpected_results.json...' results_json_path = filesystem.join( results_directory, 'unexpected_results.json') with codecs.open(results_json_path, "r") as results_json_file: results_json_file = file(results_json_path) results_json = simplejson.load(results_json_file) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) test_config = TestConfig( port, layout_tests_directory, results_directory, platforms, filesystem, scm) print 'Gathering current baselines...' for test_file, test_json in results_json['tests'].items(): test_json['state'] = STATE_NEEDS_REBASELINE test_path = filesystem.join(layout_tests_directory, test_file) test_json['baselines'] = _get_test_baselines(test_file, test_config) server_url = "http://localhost:%d/" % options.httpd_port print "Starting server at %s" % server_url print ("Use the 'Exit' link in the UI, %squitquitquit " "or Ctrl-C to stop") % server_url threading.Timer( .1, lambda: self._tool.user.open_url(server_url)).start() httpd = RebaselineHTTPServer( httpd_port=options.httpd_port, test_config=test_config, results_json=results_json, platforms_json={ 'platforms': platforms, 'defaultPlatform': port.name(), }) httpd.serve_forever()
def execute(self, options, args, tool): port = factory.get("chromium-win-win7") # FIXME: This should be selectable. expectation_lines = TestExpectationParser.tokenize_list(port.test_expectations()) parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False) for expectation_line in expectation_lines: parser.parse(expectation_line) converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros()) tool.filesystem.write_text_file(port.path_to_test_expectations_file(), TestExpectationSerializer.list_to_string(expectation_lines, converter))
def __init__(self, **kwargs): pfx = 'dryrun-' if 'port_name' in kwargs: if kwargs['port_name'].startswith(pfx): kwargs['port_name'] = kwargs['port_name'][len(pfx):] else: kwargs['port_name'] = None self.__delegate = factory.get(**kwargs)
def execute(self, options, args, tool): self._baseline_optimizer = BaselineOptimizer(tool.scm(), tool.filesystem) self._port = factory.get( "chromium-win-win7") # FIXME: This should be selectable. for test_name in map(self._to_test_name, test_files.find(self._port, args)): self._analyze_baseline(test_name)
def _update_expectations_file(self, port_name): if not self._is_supported_port(port_name): return port = factory.get(port_name) expectations = self._expectations(port) path = port.path_to_test_expectations_file() self._tool.filesystem.write_text_file( path, expectations.remove_rebaselined_tests( expectations.get_rebaselining_failures()))
def execute(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() scm = self._tool.scm() if options.dry_run: def no_op_copyfile(src, dest): pass def no_op_add(path, return_exit_code=False): if return_exit_code: return 0 filesystem.copyfile = no_op_copyfile scm.add = no_op_add print 'Parsing unexpected_results.json...' results_json_path = filesystem.join(results_directory, 'unexpected_results.json') results_json = json_results_generator.load_json( filesystem, results_json_path) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, filesystem, scm) print 'Gathering current baselines...' for test_file, test_json in results_json['tests'].items(): test_json['state'] = STATE_NEEDS_REBASELINE test_path = filesystem.join(layout_tests_directory, test_file) test_json['baselines'] = _get_test_baselines( test_file, test_config) server_url = "http://localhost:%d/" % options.httpd_port print "Starting server at %s" % server_url print( "Use the 'Exit' link in the UI, %squitquitquit " "or Ctrl-C to stop") % server_url threading.Timer(.1, lambda: self._tool.user.open_url(server_url)).start() httpd = RebaselineHTTPServer(httpd_port=options.httpd_port, test_config=test_config, results_json=results_json, platforms_json={ 'platforms': platforms, 'defaultPlatform': port.name(), }) httpd.serve_forever()
def _rebaseline_port(self, port_name): if not self._is_supported_port(port_name): return builder_name = builders.builder_name_for_port_name(port_name) if not builder_name: return print "Retrieving results for %s from %s." % (port_name, builder_name) for test_name in self._tests_to_rebaseline(factory.get(port_name)): self._touched_test_names.add(test_name) print " %s" % test_name self._run_webkit_patch(['rebaseline-test', builder_name, test_name])
def test_extrapolate(self): # FIXME: Make this test not rely on actual (not mock) port objects. port = factory.get('chromium-win-win7', None) converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros()) extrapolator = BuildCoverageExtrapolator(converter) self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win"), set([TestConfiguration(version='xp', architecture='x86', build_type='release', graphics_type='cpu')])) self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Vista"), set([ TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='cpu'), TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='gpu'), TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='gpu'), TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='cpu')])) self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato")
def _rebaseline_port(self, port_name): if not self._is_supported_port(port_name): return builder_name = builders.builder_name_for_port_name(port_name) if not builder_name: return print "Retrieving results for %s from %s." % (port_name, builder_name) for test_name in self._tests_to_rebaseline(factory.get(port_name)): self._touched_test_names.add(test_name) print " %s" % test_name self._run_webkit_patch( ['rebaseline-test', builder_name, test_name])
def __init__(self, options, args, filesystem, stdin, stdout, stderr): self._options = options self._args = args self._filesystem = filesystem self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = factory.get(port_name, options=options, filesystem=filesystem)
def execute(self, options, args, tool): port = factory.get( "chromium-win-win7") # FIXME: This should be selectable. expectation_lines = TestExpectationParser.tokenize_list( port.test_expectations()) parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False) for expectation_line in expectation_lines: parser.parse(expectation_line) converter = TestConfigurationConverter( port.all_test_configurations(), port.configuration_specifier_macros()) tool.filesystem.write_text_file( port.path_to_test_expectations_file(), TestExpectationSerializer.list_to_string(expectation_lines, converter))
def port_fallbacks(): """Get the port fallback information. Returns: A dictionary mapping platform name to a list of other platforms to fall back on. All platforms fall back on 'base'. """ fallbacks = {_BASE_PLATFORM: []} for port_name in os.listdir(os.path.join('LayoutTests', 'platform')): try: platforms = port_factory.get(port_name).baseline_search_path() except NotImplementedError: _log.error("'%s' lacks baseline_search_path(), please fix." % port_name) fallbacks[port_name] = [_BASE_PLATFORM] continue fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:] fallbacks[port_name].append(_BASE_PLATFORM) return fallbacks
def _baseline_search_hypergraph(fs): hypergraph = {} # These edges in the hypergraph aren't visible on build.webkit.org, # but they impose constraints on how we optimize baselines. hypergraph['mac-future'] = ['LayoutTests/platform/mac-future', 'LayoutTests/platform/mac', 'LayoutTests'] hypergraph['qt-unknown'] = ['LayoutTests/platform/qt-unknown', 'LayoutTests/platform/qt', 'LayoutTests'] # FIXME: Should we get this constant from somewhere? fallback_path = ['LayoutTests'] for port_name in port_factory.all_port_names(): port = port_factory.get(port_name) webkit_base = port.webkit_base() search_path = port.baseline_search_path() if search_path: hypergraph[port_name] = [fs.relpath(path, webkit_base) for path in search_path] + fallback_path return hypergraph
def _prepare_config(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() scm = self._tool.scm() print "Parsing full_results.json..." results_json_path = filesystem.join(results_directory, "full_results.json") results_json = json_results_generator.load_json(filesystem, results_json_path) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir(filesystem.join(layout_tests_directory, "platform")) self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, filesystem, scm) print "Gathering current baselines..." self._gather_baselines(results_json) return { "test_config": self._test_config, "results_json": results_json, "platforms_json": {"platforms": platforms, "defaultPlatform": port.name()}, }
def test_extrapolate(self): # FIXME: Make this test not rely on actual (not mock) port objects. port = factory.get('chromium-win-win7', None) converter = TestConfigurationConverter( port.all_test_configurations(), port.configuration_specifier_macros()) extrapolator = BuildCoverageExtrapolator(converter) self.assertEquals( extrapolator.extrapolate_test_configurations("Webkit Win"), set([ TestConfiguration(version='xp', architecture='x86', build_type='release', graphics_type='cpu') ])) self.assertEquals( extrapolator.extrapolate_test_configurations("Webkit Vista"), set([ TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='cpu'), TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='gpu'), TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='gpu'), TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='cpu') ])) self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato")
def test_chromium_gpu__snowleopard(self): port = factory.get('chromium-gpu-mac-snowleopard') self.assertEquals(port.name(), 'chromium-gpu-mac-snowleopard') self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-mac'))
def test_chromium_gpu__win7(self): port = factory.get('chromium-gpu-win-win7') self.assertEquals(port.name(), 'chromium-gpu-win-win7') self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-win'))
def port_for_builder(self, builder_name): port_name = self._port_name_for_builder_name(builder_name) assert(port_name) # Need to update _builder_name_to_port_name port = factory.get(port_name) assert(port) # Need to update _builder_name_to_port_name return port
def _expectations_updater(self): # FIXME: Should split failure_info_list into lists per port, then edit each expectations file separately. # For now, assume Chromium port. port = factory.get("chromium-win-win7") return GardeningExpectationsUpdater(self.server.tool, port)
def test_graphics_type(self): port = factory.get('chromium-gpu-cg-mac') self.assertEquals('gpu-cg', port.graphics_type()) port = factory.get('chromium-gpu-mac') self.assertEquals('gpu', port.graphics_type())
def create(cls): return factory.get("test-win-xp")
def __init__(self, **kwargs): prefix = 'mock-' if 'port_name' in kwargs: kwargs['port_name'] = kwargs['port_name'][len(prefix):] self.__delegate = factory.get(**kwargs) self.__real_name = prefix + self.__delegate.name()
def test_chromium_gpu__vista(self): port = factory.get('chromium-gpu-win-vista') self.assertEquals(port.name(), 'chromium-gpu-win-vista') self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-win'))
def execute(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() scm = self._tool.scm() if options.dry_run: def no_op_copyfile(src, dest): pass def no_op_add(path, return_exit_code=False): if return_exit_code: return 0 filesystem.copyfile = no_op_copyfile scm.add = no_op_add print 'Parsing unexpected_results.json...' results_json_path = filesystem.join(results_directory, 'unexpected_results.json') results_json = json_results_generator.load_json(filesystem, results_json_path) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) test_config = TestConfig( port, layout_tests_directory, results_directory, platforms, filesystem, scm) print 'Gathering current baselines...' # Rebaseline server and it's associated JavaScript expected the tests subtree to # be key-value pairs instead of hierarchical. # FIXME: make the rebaseline server use the hierarchical tree. new_tests_subtree = {} def gather_baselines(test, result): result['state'] = STATE_NEEDS_REBASELINE result['baselines'] = _get_test_baselines(test, test_config) new_tests_subtree[test] = result resultsjsonparser.for_each_test(results_json['tests'], gather_baselines) results_json['tests'] = new_tests_subtree server_url = "http://localhost:%d/" % options.httpd_port print "Starting server at %s" % server_url print ("Use the 'Exit' link in the UI, %squitquitquit " "or Ctrl-C to stop") % server_url threading.Timer( .1, lambda: self._tool.user.open_url(server_url)).start() httpd = RebaselineHTTPServer( httpd_port=options.httpd_port, test_config=test_config, results_json=results_json, platforms_json={ 'platforms': platforms, 'defaultPlatform': port.name(), }) httpd.serve_forever()
def __init__(self, **kwargs): prefix = 'mock-' if 'port_name' in kwargs: kwargs['port_name'] = kwargs['port_name'][len(prefix):] self.__delegate = factory.get(**kwargs)