def _prepare_config(self, options, args, tool):
results_directory = args[0]
filesystem = system.filesystem.FileSystem()
scm = self._tool.scm()
print 'Parsing full_results.json...'
results_json_path = filesystem.join(results_directory,
'full_results.json')
results_json = json_results_generator.load_json(
filesystem, results_json_path)
port = factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = filesystem.listdir(
filesystem.join(layout_tests_directory, 'platform'))
self._test_config = TestConfig(port, layout_tests_directory,
results_directory, platforms,
filesystem, scm)
print 'Gathering current baselines...'
self._gather_baselines(results_json)
return {
'test_config': self._test_config,
"results_json": results_json,
"platforms_json": {
'platforms': platforms,
'defaultPlatform': port.name(),
},
}
def _baseline_search_hypergraph(fs):
hypergraph = {}
# These edges in the hypergraph aren't visible on build.webkit.org,
# but they impose constraints on how we optimize baselines.
hypergraph['mac-future'] = [
'LayoutTests/platform/mac-future', 'LayoutTests/platform/mac',
'LayoutTests'
]
hypergraph['qt-unknown'] = [
'LayoutTests/platform/qt-unknown', 'LayoutTests/platform/qt',
'LayoutTests'
]
# FIXME: Should we get this constant from somewhere?
fallback_path = ['LayoutTests']
for port_name in port_factory.all_port_names():
port = port_factory.get(port_name)
webkit_base = port.webkit_base()
search_path = port.baseline_search_path()
if search_path:
hypergraph[port_name] = [
fs.relpath(path, webkit_base) for path in search_path
] + fallback_path
return hypergraph
def _update_expectations_file(self, port_name):
if not self._is_supported_port(port_name):
return
port = factory.get(port_name)
expectations = self._expectations(port)
path = port.path_to_test_expectations_file()
self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures()))
def execute(self, options, args, tool):
self._baseline_optimizer = BaselineOptimizer(tool.scm(), tool.filesystem)
self._port = factory.get("chromium-win-win7") # FIXME: This should be selectable.
for test_name in map(self._to_test_name, test_files.find(self._port, args)):
print "Optimizing %s." % test_name
self._optimize_baseline(test_name)
def execute(self, options, args, tool):
results_directory = args[0]
filesystem = system.filesystem.FileSystem()
print 'Parsing unexpected_results.json...'
results_json_path = filesystem.join(
results_directory, 'unexpected_results.json')
with codecs.open(results_json_path, "r") as results_json_file:
results_json_file = file(results_json_path)
results_json = simplejson.load(results_json_file)
port = factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = filesystem.listdir(
filesystem.join(layout_tests_directory, 'platform'))
print 'Gathering current baselines...'
for test_file, test_json in results_json['tests'].items():
test_json['state'] = STATE_NEEDS_REBASELINE
test_json['baselines'] = _get_test_baselines(
test_file, layout_tests_directory, platforms, filesystem)
print "Starting server at http://localhost:%d/" % options.httpd_port
print ("Use the 'Exit' link in the UI, http://localhost:%d/"
"quitquitquit or Ctrl-C to stop") % options.httpd_port
httpd = RebaselineHTTPServer(
httpd_port=options.httpd_port,
results_directory=results_directory,
results_json=results_json,
platforms_json={
'platforms': platforms,
'defaultPlatform': port.name(),
})
httpd.serve_forever()
def assertTest(self,
test_name,
pixel_tests,
expected_checksum=None,
drt_output=None,
filesystem=None):
platform = 'test'
filesystem = filesystem or test.unit_test_filesystem()
port = factory.get(platform, filesystem=filesystem)
drt_input, drt_output = self.make_input_output(port, test_name,
pixel_tests,
expected_checksum,
drt_output)
args = ['--platform', 'test'] + self.extra_args(pixel_tests)
stdin = newstringio.StringIO(drt_input)
stdout = newstringio.StringIO()
stderr = newstringio.StringIO()
options, args = mock_drt.parse_options(args)
drt = self.make_drt(options, args, filesystem, stdin, stdout, stderr)
res = drt.run()
self.assertEqual(res, 0)
# We use the StringIO.buflist here instead of getvalue() because
# the StringIO might be a mix of unicode/ascii and 8-bit strings.
self.assertEqual(stdout.buflist, drt_output)
self.assertEqual(stderr.getvalue(), '')
def execute(self, options, args, tool):
results_directory = args[0]
filesystem = system.filesystem.FileSystem()
scm = self._tool.scm()
if options.dry_run:
def no_op_copyfile(src, dest):
pass
def no_op_add(path, return_exit_code=False):
if return_exit_code:
return 0
filesystem.copyfile = no_op_copyfile
scm.add = no_op_add
print 'Parsing unexpected_results.json...'
results_json_path = filesystem.join(
results_directory, 'unexpected_results.json')
with codecs.open(results_json_path, "r") as results_json_file:
results_json_file = file(results_json_path)
results_json = simplejson.load(results_json_file)
port = factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = filesystem.listdir(
filesystem.join(layout_tests_directory, 'platform'))
test_config = TestConfig(
port,
layout_tests_directory,
results_directory,
platforms,
filesystem,
scm)
print 'Gathering current baselines...'
for test_file, test_json in results_json['tests'].items():
test_json['state'] = STATE_NEEDS_REBASELINE
test_path = filesystem.join(layout_tests_directory, test_file)
test_json['baselines'] = _get_test_baselines(test_file, test_config)
server_url = "http://localhost:%d/" % options.httpd_port
print "Starting server at %s" % server_url
print ("Use the 'Exit' link in the UI, %squitquitquit "
"or Ctrl-C to stop") % server_url
threading.Timer(
.1, lambda: self._tool.user.open_url(server_url)).start()
httpd = RebaselineHTTPServer(
httpd_port=options.httpd_port,
test_config=test_config,
results_json=results_json,
platforms_json={
'platforms': platforms,
'defaultPlatform': port.name(),
})
httpd.serve_forever()
def execute(self, options, args, tool):
port = factory.get("chromium-win-win7") # FIXME: This should be selectable.
expectation_lines = TestExpectationParser.tokenize_list(port.test_expectations())
parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False)
for expectation_line in expectation_lines:
parser.parse(expectation_line)
converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
tool.filesystem.write_text_file(port.path_to_test_expectations_file(), TestExpectationSerializer.list_to_string(expectation_lines, converter))
def __init__(self, **kwargs):
pfx = 'dryrun-'
if 'port_name' in kwargs:
if kwargs['port_name'].startswith(pfx):
kwargs['port_name'] = kwargs['port_name'][len(pfx):]
else:
kwargs['port_name'] = None
self.__delegate = factory.get(**kwargs)
def execute(self, options, args, tool):
self._baseline_optimizer = BaselineOptimizer(tool.scm(),
tool.filesystem)
self._port = factory.get(
"chromium-win-win7") # FIXME: This should be selectable.
for test_name in map(self._to_test_name,
test_files.find(self._port, args)):
self._analyze_baseline(test_name)
def _update_expectations_file(self, port_name):
if not self._is_supported_port(port_name):
return
port = factory.get(port_name)
expectations = self._expectations(port)
path = port.path_to_test_expectations_file()
self._tool.filesystem.write_text_file(
path,
expectations.remove_rebaselined_tests(
expectations.get_rebaselining_failures()))
def execute(self, options, args, tool):
results_directory = args[0]
filesystem = system.filesystem.FileSystem()
scm = self._tool.scm()
if options.dry_run:
def no_op_copyfile(src, dest):
pass
def no_op_add(path, return_exit_code=False):
if return_exit_code:
return 0
filesystem.copyfile = no_op_copyfile
scm.add = no_op_add
print 'Parsing unexpected_results.json...'
results_json_path = filesystem.join(results_directory,
'unexpected_results.json')
results_json = json_results_generator.load_json(
filesystem, results_json_path)
port = factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = filesystem.listdir(
filesystem.join(layout_tests_directory, 'platform'))
test_config = TestConfig(port, layout_tests_directory,
results_directory, platforms, filesystem, scm)
print 'Gathering current baselines...'
for test_file, test_json in results_json['tests'].items():
test_json['state'] = STATE_NEEDS_REBASELINE
test_path = filesystem.join(layout_tests_directory, test_file)
test_json['baselines'] = _get_test_baselines(
test_file, test_config)
server_url = "http://localhost:%d/" % options.httpd_port
print "Starting server at %s" % server_url
print(
"Use the 'Exit' link in the UI, %squitquitquit "
"or Ctrl-C to stop") % server_url
threading.Timer(.1,
lambda: self._tool.user.open_url(server_url)).start()
httpd = RebaselineHTTPServer(httpd_port=options.httpd_port,
test_config=test_config,
results_json=results_json,
platforms_json={
'platforms': platforms,
'defaultPlatform': port.name(),
})
httpd.serve_forever()
def _rebaseline_port(self, port_name):
if not self._is_supported_port(port_name):
return
builder_name = builders.builder_name_for_port_name(port_name)
if not builder_name:
return
print "Retrieving results for %s from %s." % (port_name, builder_name)
for test_name in self._tests_to_rebaseline(factory.get(port_name)):
self._touched_test_names.add(test_name)
print " %s" % test_name
self._run_webkit_patch(['rebaseline-test', builder_name, test_name])
def test_extrapolate(self):
# FIXME: Make this test not rely on actual (not mock) port objects.
port = factory.get('chromium-win-win7', None)
converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
extrapolator = BuildCoverageExtrapolator(converter)
self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win"), set([TestConfiguration(version='xp', architecture='x86', build_type='release', graphics_type='cpu')]))
self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Vista"), set([
TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='cpu'),
TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='gpu'),
TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='gpu'),
TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='cpu')]))
self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato")
def _rebaseline_port(self, port_name):
if not self._is_supported_port(port_name):
return
builder_name = builders.builder_name_for_port_name(port_name)
if not builder_name:
return
print "Retrieving results for %s from %s." % (port_name, builder_name)
for test_name in self._tests_to_rebaseline(factory.get(port_name)):
self._touched_test_names.add(test_name)
print " %s" % test_name
self._run_webkit_patch(
['rebaseline-test', builder_name, test_name])
def __init__(self, options, args, filesystem, stdin, stdout, stderr):
self._options = options
self._args = args
self._filesystem = filesystem
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = factory.get(port_name, options=options, filesystem=filesystem)
def __init__(self, options, args, filesystem, stdin, stdout, stderr):
self._options = options
self._args = args
self._filesystem = filesystem
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = factory.get(port_name, options=options, filesystem=filesystem)
def execute(self, options, args, tool):
port = factory.get(
"chromium-win-win7") # FIXME: This should be selectable.
expectation_lines = TestExpectationParser.tokenize_list(
port.test_expectations())
parser = TestExpectationParser(port, [],
allow_rebaseline_modifier=False)
for expectation_line in expectation_lines:
parser.parse(expectation_line)
converter = TestConfigurationConverter(
port.all_test_configurations(),
port.configuration_specifier_macros())
tool.filesystem.write_text_file(
port.path_to_test_expectations_file(),
TestExpectationSerializer.list_to_string(expectation_lines,
converter))
def port_fallbacks():
"""Get the port fallback information.
Returns:
A dictionary mapping platform name to a list of other platforms to fall
back on. All platforms fall back on 'base'.
"""
fallbacks = {_BASE_PLATFORM: []}
for port_name in os.listdir(os.path.join('LayoutTests', 'platform')):
try:
platforms = port_factory.get(port_name).baseline_search_path()
except NotImplementedError:
_log.error("'%s' lacks baseline_search_path(), please fix." % port_name)
fallbacks[port_name] = [_BASE_PLATFORM]
continue
fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:]
fallbacks[port_name].append(_BASE_PLATFORM)
return fallbacks
def _baseline_search_hypergraph(fs):
hypergraph = {}
# These edges in the hypergraph aren't visible on build.webkit.org,
# but they impose constraints on how we optimize baselines.
hypergraph['mac-future'] = ['LayoutTests/platform/mac-future', 'LayoutTests/platform/mac', 'LayoutTests']
hypergraph['qt-unknown'] = ['LayoutTests/platform/qt-unknown', 'LayoutTests/platform/qt', 'LayoutTests']
# FIXME: Should we get this constant from somewhere?
fallback_path = ['LayoutTests']
for port_name in port_factory.all_port_names():
port = port_factory.get(port_name)
webkit_base = port.webkit_base()
search_path = port.baseline_search_path()
if search_path:
hypergraph[port_name] = [fs.relpath(path, webkit_base) for path in search_path] + fallback_path
return hypergraph
def port_fallbacks():
"""Get the port fallback information.
Returns:
A dictionary mapping platform name to a list of other platforms to fall
back on. All platforms fall back on 'base'.
"""
fallbacks = {_BASE_PLATFORM: []}
for port_name in os.listdir(os.path.join('LayoutTests', 'platform')):
try:
platforms = port_factory.get(port_name).baseline_search_path()
except NotImplementedError:
_log.error("'%s' lacks baseline_search_path(), please fix." %
port_name)
fallbacks[port_name] = [_BASE_PLATFORM]
continue
fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:]
fallbacks[port_name].append(_BASE_PLATFORM)
return fallbacks
def _prepare_config(self, options, args, tool):
results_directory = args[0]
filesystem = system.filesystem.FileSystem()
scm = self._tool.scm()
print "Parsing full_results.json..."
results_json_path = filesystem.join(results_directory, "full_results.json")
results_json = json_results_generator.load_json(filesystem, results_json_path)
port = factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = filesystem.listdir(filesystem.join(layout_tests_directory, "platform"))
self._test_config = TestConfig(port, layout_tests_directory, results_directory, platforms, filesystem, scm)
print "Gathering current baselines..."
self._gather_baselines(results_json)
return {
"test_config": self._test_config,
"results_json": results_json,
"platforms_json": {"platforms": platforms, "defaultPlatform": port.name()},
}
def test_extrapolate(self):
# FIXME: Make this test not rely on actual (not mock) port objects.
port = factory.get('chromium-win-win7', None)
converter = TestConfigurationConverter(
port.all_test_configurations(),
port.configuration_specifier_macros())
extrapolator = BuildCoverageExtrapolator(converter)
self.assertEquals(
extrapolator.extrapolate_test_configurations("Webkit Win"),
set([
TestConfiguration(version='xp',
architecture='x86',
build_type='release',
graphics_type='cpu')
]))
self.assertEquals(
extrapolator.extrapolate_test_configurations("Webkit Vista"),
set([
TestConfiguration(version='vista',
architecture='x86',
build_type='debug',
graphics_type='cpu'),
TestConfiguration(version='vista',
architecture='x86',
build_type='debug',
graphics_type='gpu'),
TestConfiguration(version='vista',
architecture='x86',
build_type='release',
graphics_type='gpu'),
TestConfiguration(version='vista',
architecture='x86',
build_type='release',
graphics_type='cpu')
]))
self.assertRaises(KeyError,
extrapolator.extrapolate_test_configurations,
"Potato")
def assertTest(self, test_name, pixel_tests, expected_checksum=None,
drt_output=None, filesystem=None):
platform = 'test'
filesystem = filesystem or test.unit_test_filesystem()
port = factory.get(platform, filesystem=filesystem)
drt_input, drt_output = self.make_input_output(port, test_name,
pixel_tests, expected_checksum, drt_output)
args = ['--platform', 'test'] + self.extra_args(pixel_tests)
stdin = newstringio.StringIO(drt_input)
stdout = newstringio.StringIO()
stderr = newstringio.StringIO()
options, args = mock_drt.parse_options(args)
drt = self.make_drt(options, args, filesystem, stdin, stdout, stderr)
res = drt.run()
self.assertEqual(res, 0)
# We use the StringIO.buflist here instead of getvalue() because
# the StringIO might be a mix of unicode/ascii and 8-bit strings.
self.assertEqual(stdout.buflist, drt_output)
self.assertEqual(stderr.getvalue(), '')
def test_chromium_gpu__snowleopard(self):
port = factory.get('chromium-gpu-mac-snowleopard')
self.assertEquals(port.name(), 'chromium-gpu-mac-snowleopard')
self.assertEquals(port.baseline_path(),
port._webkit_baseline_path('chromium-gpu-mac'))
def test_chromium_gpu__win7(self):
port = factory.get('chromium-gpu-win-win7')
self.assertEquals(port.name(), 'chromium-gpu-win-win7')
self.assertEquals(port.baseline_path(),
port._webkit_baseline_path('chromium-gpu-win'))
def port_for_builder(self, builder_name):
port_name = self._port_name_for_builder_name(builder_name)
assert(port_name) # Need to update _builder_name_to_port_name
port = factory.get(port_name)
assert(port) # Need to update _builder_name_to_port_name
return port
def _expectations_updater(self):
# FIXME: Should split failure_info_list into lists per port, then edit each expectations file separately.
# For now, assume Chromium port.
port = factory.get("chromium-win-win7")
return GardeningExpectationsUpdater(self.server.tool, port)
def _expectations_updater(self):
# FIXME: Should split failure_info_list into lists per port, then edit each expectations file separately.
# For now, assume Chromium port.
port = factory.get("chromium-win-win7")
return GardeningExpectationsUpdater(self.server.tool, port)
def test_graphics_type(self):
port = factory.get('chromium-gpu-cg-mac')
self.assertEquals('gpu-cg', port.graphics_type())
port = factory.get('chromium-gpu-mac')
self.assertEquals('gpu', port.graphics_type())
def create(cls):
return factory.get("test-win-xp")
def port_for_builder(self, builder_name):
port_name = self._port_name_for_builder_name(builder_name)
assert(port_name) # Need to update _builder_name_to_port_name
port = factory.get(port_name)
assert(port) # Need to update _builder_name_to_port_name
return port
def __init__(self, **kwargs):
prefix = 'mock-'
if 'port_name' in kwargs:
kwargs['port_name'] = kwargs['port_name'][len(prefix):]
self.__delegate = factory.get(**kwargs)
self.__real_name = prefix + self.__delegate.name()
def test_chromium_gpu__vista(self):
port = factory.get('chromium-gpu-win-vista')
self.assertEquals(port.name(), 'chromium-gpu-win-vista')
self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-win'))
def test_chromium_gpu__snowleopard(self):
port = factory.get('chromium-gpu-mac-snowleopard')
self.assertEquals(port.name(), 'chromium-gpu-mac-snowleopard')
self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-mac'))
def execute(self, options, args, tool):
results_directory = args[0]
filesystem = system.filesystem.FileSystem()
scm = self._tool.scm()
if options.dry_run:
def no_op_copyfile(src, dest):
pass
def no_op_add(path, return_exit_code=False):
if return_exit_code:
return 0
filesystem.copyfile = no_op_copyfile
scm.add = no_op_add
print 'Parsing unexpected_results.json...'
results_json_path = filesystem.join(results_directory, 'unexpected_results.json')
results_json = json_results_generator.load_json(filesystem, results_json_path)
port = factory.get()
layout_tests_directory = port.layout_tests_dir()
platforms = filesystem.listdir(
filesystem.join(layout_tests_directory, 'platform'))
test_config = TestConfig(
port,
layout_tests_directory,
results_directory,
platforms,
filesystem,
scm)
print 'Gathering current baselines...'
# Rebaseline server and it's associated JavaScript expected the tests subtree to
# be key-value pairs instead of hierarchical.
# FIXME: make the rebaseline server use the hierarchical tree.
new_tests_subtree = {}
def gather_baselines(test, result):
result['state'] = STATE_NEEDS_REBASELINE
result['baselines'] = _get_test_baselines(test, test_config)
new_tests_subtree[test] = result
resultsjsonparser.for_each_test(results_json['tests'], gather_baselines)
results_json['tests'] = new_tests_subtree
server_url = "http://localhost:%d/" % options.httpd_port
print "Starting server at %s" % server_url
print ("Use the 'Exit' link in the UI, %squitquitquit "
"or Ctrl-C to stop") % server_url
threading.Timer(
.1, lambda: self._tool.user.open_url(server_url)).start()
httpd = RebaselineHTTPServer(
httpd_port=options.httpd_port,
test_config=test_config,
results_json=results_json,
platforms_json={
'platforms': platforms,
'defaultPlatform': port.name(),
})
httpd.serve_forever()
def __init__(self, **kwargs):
prefix = 'mock-'
if 'port_name' in kwargs:
kwargs['port_name'] = kwargs['port_name'][len(prefix):]
self.__delegate = factory.get(**kwargs)
def create(cls):
return factory.get("test-win-xp")
