def __init__(self, host, **kwargs):
prefix = 'mock-'
if 'port_name' in kwargs:
kwargs['port_name'] = kwargs['port_name'][len(prefix):]
self._host = host
self.__delegate = PortFactory(host).get(**kwargs)
self.__real_name = prefix + self.__delegate.name()
def test_basic(self):
cmd = [
sys.executable, '-c',
'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"'
]
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
self.assertEquals(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEquals(line, None)
line = proc.read_stdout_line(now + 1.0)
self.assertEquals(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
self.assertEquals(line.strip(), "stderr")
proc.stop()
def test_basic(self):
cmd = [sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"']
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
self.assertEqual(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEqual(line, None)
# FIXME: This part appears to be flaky. line should always be non-None.
# FIXME: https://bugs.webkit.org/show_bug.cgi?id=88280
line = proc.read_stdout_line(now + 1.0)
if line:
self.assertEqual(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
if line:
self.assertEqual(line.strip(), "stderr")
proc.stop(0)
def test_basic(self):
cmd = [
sys.executable, '-c',
'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"'
]
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
self.assertEquals(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEquals(line, None)
# FIXME: This part appears to be flaky. line should always be non-None.
# FIXME: https://bugs.webkit.org/show_bug.cgi?id=88280
line = proc.read_stdout_line(now + 1.0)
if line:
self.assertEquals(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
if line:
self.assertEquals(line.strip(), "stderr")
proc.stop(0)
def test_path_to_expectations(self):
port_factory = PortFactory(MockSystemHost())
for port_name in ("google-chrome-linux32", "google-chrome-linux64", "google-chrome-mac", "google-chrome-win"):
self.assertTrue(
port_factory.get(port_name)
.path_to_test_expectations_file()
.endswith("platform/chromium/TestExpectations")
)
def _verify_baseline_search_path_startswith(self, port_name,
expected_platform_dirs):
port = PortFactory(MockSystemHost()).get(port_name=port_name)
actual_platform_dirs = [
port._filesystem.basename(path)
for path in port.baseline_search_path()
]
self.assertEqual(expected_platform_dirs,
actual_platform_dirs[0:len(expected_platform_dirs)])
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name, options=options)
self._driver = self._port.create_driver(0)
def __init__(self,
log_executive=False,
executive_throws_when_run=None,
initialize_scm_by_default=True,
web=None,
scm=None,
os_name=None,
os_version=None):
MockSystemHost.__init__(self,
log_executive,
executive_throws_when_run,
os_name=os_name,
os_version=os_version)
add_unit_tests_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._scm = scm
# FIXME: we should never initialize the SCM by default, since the real
# object doesn't either. This has caused at least one bug (see bug 89498).
if initialize_scm_by_default:
self.initialize_scm()
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = Builders()
def assertTest(self,
test_name,
pixel_tests,
expected_checksum=None,
drt_output=None,
host=None,
expected_text=None):
port_name = 'test'
host = host or MockSystemHost()
test.add_unit_tests_to_mock_filesystem(host.filesystem)
port = PortFactory(host).get(port_name)
drt_input, drt_output = self.make_input_output(
port,
test_name,
pixel_tests,
expected_checksum,
drt_output,
drt_input=None,
expected_text=expected_text)
args = ['--dump-render-tree', '--platform', port_name, '-']
stdin = newstringio.StringIO(drt_input)
stdout = newstringio.StringIO()
stderr = newstringio.StringIO()
options, args = mock_drt.parse_options(args)
drt = self.make_drt(options, args, host, stdin, stdout, stderr)
res = drt.run()
self.assertEqual(res, 0)
# We use the StringIO.buflist here instead of getvalue() because
# the StringIO might be a mix of unicode/ascii and 8-bit strings.
self.assertEqual(stdout.buflist, drt_output)
self.assertEqual(stderr.getvalue(), '#EOF\n')
def __init__(self,
log_executive=False,
initialize_scm_by_default=True,
web=None,
scm=None,
os_name=None,
os_version=None,
time_return_val=123):
super(MockHost, self).__init__(log_executive=log_executive,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._scm = scm
# TODO(qyearsley): we should never initialize the SCM by default, since
# the real object doesn't either. This has caused at least one bug
# (see bug 89498).
if initialize_scm_by_default:
self.initialize_scm()
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList(BUILDERS)
def assertTest(self,
test_name,
pixel_tests,
expected_checksum=None,
drt_output=None,
host=None,
expected_text=None):
port_name = 'test'
host = host or MockSystemHost()
test.add_unit_tests_to_mock_filesystem(host.filesystem)
port = PortFactory(host).get(port_name)
drt_input, drt_output = self.make_input_output(
port,
test_name,
pixel_tests,
expected_checksum,
drt_output,
drt_input=None,
expected_text=expected_text)
args = ['--run-layout-test', '--platform', port_name, '-']
stdin = io.BytesIO(drt_input)
stdout = io.BytesIO()
stderr = io.BytesIO()
options, args = mock_drt.parse_options(args)
drt = self.make_drt(options, args, host, stdin, stdout, stderr)
res = drt.run()
self.assertEqual(res, 0)
self.assertEqual(stdout.getvalue(), ''.join(drt_output))
self.assertEqual(stderr.getvalue(), '#EOF\n')
def __init__(self, host, **kwargs):
prefix = "mock-"
if "port_name" in kwargs:
kwargs["port_name"] = kwargs["port_name"][len(prefix) :]
self._host = host
self.__delegate = PortFactory(host).get(**kwargs)
self.__real_name = prefix + self.__delegate.name()
def _verify_expectations_overrides(self, port_name):
host = MockSystemHost()
chromium_port = PortFactory(host).get("chromium-mac-leopard")
chromium_base = chromium_port.path_from_chromium_base()
port = PortFactory(host).get(port_name=port_name, options=None)
expected_chromium_overrides = '// chromium overrides\n'
expected_chrome_overrides = '// chrome overrides\n'
chromium_path = host.filesystem.join(chromium_base, 'webkit', 'tools',
'layout_tests',
'test_expectations.txt')
chrome_path = host.filesystem.join(chromium_base, 'webkit', 'tools',
'layout_tests',
'test_expectations_chrome.txt')
host.filesystem.files[chromium_path] = expected_chromium_overrides
host.filesystem.files[chrome_path] = None
actual_chrome_overrides = port.test_expectations_overrides()
self.assertEqual(expected_chromium_overrides, actual_chrome_overrides)
host.filesystem.files[chrome_path] = expected_chrome_overrides
actual_chrome_overrides = port.test_expectations_overrides()
self.assertEqual(
actual_chrome_overrides,
expected_chromium_overrides + expected_chrome_overrides)
def _verify_expectations_overrides(self, port_name):
host = MockSystemHost()
chromium_port = PortFactory(host).get("chromium-mac-leopard")
chromium_base = chromium_port.path_from_chromium_base()
port = PortFactory(host).get(port_name=port_name, options=None)
expected_chromium_overrides = '// chromium overrides\n'
expected_chrome_overrides = '// chrome overrides\n'
chromium_path = host.filesystem.join(chromium_base, 'webkit', 'tools', 'layout_tests', 'test_expectations.txt')
chrome_path = host.filesystem.join(chromium_base, 'webkit', 'tools', 'layout_tests', 'test_expectations_chrome.txt')
host.filesystem.files[chromium_path] = expected_chromium_overrides
host.filesystem.files[chrome_path] = None
actual_chrome_overrides = port.test_expectations_overrides()
self.assertEqual(expected_chromium_overrides, actual_chrome_overrides)
host.filesystem.files[chrome_path] = expected_chrome_overrides
actual_chrome_overrides = port.test_expectations_overrides()
self.assertEqual(actual_chrome_overrides, expected_chromium_overrides + expected_chrome_overrides)
class Host(SystemHost):
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
# FIXME: Checkout should own the scm object.
self._scm = None
self._checkout = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.bugs = bugzilla.Bugzilla()
self.buildbot = buildbot.BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self._engage_awesome_locale_hacks()
# We call this from the Host constructor, as it's one of the
# earliest calls made for all webkitpy-based programs.
def _engage_awesome_locale_hacks(self):
# To make life easier on our non-english users, we override
# the locale environment variables inside webkitpy.
# If we don't do this, programs like SVN will output localized
# messages and svn.py will fail to parse them.
# FIXME: We should do these overrides *only* for the subprocesses we know need them!
# This hack only works in unix environments.
os.environ['LANGUAGE'] = 'en'
os.environ['LANG'] = 'en_US.UTF-8'
os.environ['LC_MESSAGES'] = 'en_US.UTF-8'
os.environ['LC_ALL'] = ''
def initialize_scm(self, patch_directories=None):
detector = SCMDetector(self.filesystem, self.executive)
self._scm = detector.default_scm(patch_directories)
self._checkout = Checkout(self.scm())
def scm(self):
return self._scm
def checkout(self):
return self._checkout
def buildbot_for_builder_name(self, name):
if self.port_factory.get_from_builder_name(name).is_chromium():
return self.chromium_buildbot()
return self.buildbot
@memoized
def watch_list(self):
return WatchListLoader(self.filesystem).load()
def assert_port_works(self, port_name, input_name=None, platform=None):
host = MockSystemHost()
host.filesystem = FileSystem() # FIXME: This test should not use a real filesystem!
# test that we got the right port
mock_options = MockOptions(accelerated_2d_canvas=None,
accelerated_video=None,
builder_name='foo',
child_processes=None)
if input_name and platform:
port = PortFactory(host).get(host, platform=platform, port_name=input_name, options=mock_options)
else:
port = PortFactory(host).get(host, port_name=port_name, options=mock_options)
self.assertTrue(port._options.accelerated_2d_canvas)
self.assertTrue(port._options.accelerated_video)
self.assertTrue(port._options.experimental_fully_parallel)
self.assertEqual(port._options.builder_name, 'foo - GPU')
self.assertTrue(port.name().startswith(port_name))
# test that it has the right directories in front of the search path.
paths = port.baseline_search_path()
self.assertEqual(port._webkit_baseline_path(port_name), paths[0])
if port_name == 'chromium-gpu-linux':
self.assertEqual(port._webkit_baseline_path('chromium-gpu-win'), paths[1])
self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[2])
else:
self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[1])
# Test that we're limiting to the correct directories.
# These two tests are picked mostly at random, but we make sure they
# exist separately from being filtered out by the port.
# Note that this is using a real filesystem.
files = port.tests(None)
path = 'fast/html/keygen.html'
self.assertTrue(port._filesystem.exists(port.abspath_for_test(path)))
self.assertFalse(path in files)
def test_pixeltest__fails(self):
host = MockSystemHost()
url = '#URL:file://'
url = url + '%s/failures/expected/image_checksum.html' % PortFactory(host).get('test').layout_tests_dir()
self.assertTest('failures/expected/image_checksum.html', pixel_tests=True,
expected_checksum='image_checksum',
drt_output=[url + '\n',
'#MD5:image_checksum-checksum\n',
'image_checksum-txt',
'\n',
'#EOF\n'],
host=host)
self.assertEqual(host.filesystem.written_files,
{'/tmp/png_result0.png': 'image_checksum\x8a-pngtEXtchecksum\x00image_checksum-checksum'})
def test_basic(self):
cmd = [sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"']
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
self.assertEquals(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEquals(line, None)
line = proc.read_stdout_line(now + 1.0)
self.assertEquals(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
self.assertEquals(line.strip(), "stderr")
proc.stop()
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
self._git = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.buildbot = BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, git, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self.builders = BuilderList.load_default_builder_list(self.filesystem)
def __init__(self,
log_executive=False,
web=None,
git=None,
os_name=None,
os_version=None,
time_return_val=123):
super(MockHost, self).__init__(log_executive=log_executive,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self._add_base_manifest_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._git = git
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Linux (dbg)': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Debug']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'Fake Test Linux Try Bot': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
})
def __init__(self):
self.bugs = bugzilla.Bugzilla()
self.buildbot = buildbot.BuildBot()
self.executive = executive.Executive()
self.web = web.Web()
self._irc = None
self.filesystem = filesystem.FileSystem()
self.workspace = workspace.Workspace(self.filesystem, self.executive)
self._port = None
self.user = user.User()
self._scm = None
self._checkout = None
self.status_server = statusserver.StatusServer()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem
# so for now we just pass along the whole Host object.
self.port_factory = PortFactory(self)
self.platform = platforminfo.PlatformInfo()
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
# FIXME: Checkout should own the scm object.
self._scm = None
self._checkout = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.bugs = bugzilla.Bugzilla()
self.buildbot = buildbot.BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self._engage_awesome_locale_hacks()
def __init__(
self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm, logged_before=False
):
"""
Args:
running_port: the Port the script is running on.
target_port: the Port the script uses to find port-specific
configuration information like the test_expectations.txt
file location and the list of test platforms.
platform: the test platform to rebaseline
options: the command-line options object.
url_fetcher: object that can fetch objects from URLs
zip_factory: optional object that can fetch zip files from URLs
scm: scm object for adding new baselines
logged_before: whether the previous running port logged anything.
"""
self._platform = platform
self._options = options
self._port = running_port
self._filesystem = running_port._filesystem
self._target_port = target_port
# FIXME: This should get its PortFactory from a Host object.
# Note: using running_port.executive, running_port.user since we can't get them from a host.
self._rebaseline_port = PortFactory().get(
platform, options, filesystem=self._filesystem, executive=running_port.executive, user=running_port.user
)
self._rebaselining_tests = set()
self._rebaselined_tests = []
self._logged_before = logged_before
self.did_log = False
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
expectations_str = self._rebaseline_port.test_expectations()
self._test_expectations = test_expectations.TestExpectations(
self._rebaseline_port, None, expectations_str, self._rebaseline_port.test_configuration(), False
)
self._url_fetcher = url_fetcher
self._zip_factory = zip_factory
self._scm = scm
def __init__(self, log_executive=False, executive_throws_when_run=None, initialize_scm_by_default=True):
MockSystemHost.__init__(self, log_executive, executive_throws_when_run)
add_unit_tests_to_mock_filesystem(self.filesystem)
self.web = MockWeb()
self._checkout = MockCheckout()
self._scm = None
# FIXME: we should never initialize the SCM by default, since the real
# object doesn't either. This has caused at least one bug (see bug 89498).
if initialize_scm_by_default:
self.initialize_scm()
self.bugs = MockBugzilla()
self.buildbot = MockBuildBot()
self._chromium_buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self._watch_list = MockWatchList()
def test_pixeltest__fails(self):
host = MockSystemHost()
url = '#URL:file://'
if sys.platform == 'win32':
host = MockSystemHost(os_name='win', os_version='xp')
url = '#URL:file:///'
url = url + '%s/failures/expected/checksum.html' % PortFactory(
host).get('test').layout_tests_dir()
self.assertTest('failures/expected/checksum.html',
pixel_tests=True,
expected_checksum='wrong-checksum',
drt_output=[
url + '\n', '#MD5:checksum-checksum\n',
'checksum-txt', '\n', '#EOF\n'
],
host=host)
self.assertEquals(
host.filesystem.written_files, {
'/tmp/png_result0.png':
'checksum\x8a-pngtEXtchecksum\x00checksum-checksum'
})
def __init__(self, log_executive=False, executive_throws_when_run=None):
MockSystemHost.__init__(self, log_executive, executive_throws_when_run)
add_unit_tests_to_mock_filesystem(self.filesystem)
self.web = MockWeb()
self._checkout = MockCheckout()
self._scm = MockSCM(filesystem=self.filesystem,
executive=self.executive)
# Various pieces of code (wrongly) call filesystem.chdir(checkout_root).
# Making the checkout_root exist in the mock filesystem makes that chdir not raise.
self.filesystem.maybe_make_directory(self._scm.checkout_root)
self.bugs = MockBugzilla()
self.buildbot = MockBuildBot()
self._chromium_buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self._watch_list = MockWatchList()
def __init__(self,
log_executive=False,
web=None,
git=None,
os_name=None,
os_version=None,
time_return_val=123):
super(MockHost, self).__init__(log_executive=log_executive,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self._add_base_manifest_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._git = git
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList(BUILDERS)
class MockDRT(object):
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name, options=options)
self._driver = self._port.create_driver(0)
def run(self):
while True:
line = self._stdin.readline()
if not line:
return 0
driver_input = self.input_from_line(line)
dirname, basename = self._port.split_test(driver_input.test_name)
is_reftest = (self._port.reference_files(driver_input.test_name) or
self._port.is_reference_html_file(self._port._filesystem, dirname, basename))
output = self.output_for_test(driver_input, is_reftest)
self.write_test_output(driver_input, output, is_reftest)
def input_from_line(self, line):
vals = line.strip().split("'")
uri = vals[0]
checksum = None
should_run_pixel_tests = False
if len(vals) == 2 and vals[1] == '--pixel-test':
should_run_pixel_tests = True
elif len(vals) == 3 and vals[1] == '--pixel-test':
should_run_pixel_tests = True
checksum = vals[2]
elif len(vals) != 1:
raise NotImplementedError
if uri.startswith('http://') or uri.startswith('https://'):
test_name = self._driver.uri_to_test(uri)
else:
test_name = self._port.relative_test_filename(uri)
return DriverInput(test_name, 0, checksum, should_run_pixel_tests)
def output_for_test(self, test_input, is_reftest):
port = self._port
if self._options.virtual_test_suite_name:
test_input.test_name = test_input.test_name.replace(self._options.virtual_test_suite_base, self._options.virtual_test_suite_name)
actual_text = port.expected_text(test_input.test_name)
actual_audio = port.expected_audio(test_input.test_name)
actual_image = None
actual_checksum = None
if is_reftest:
# Make up some output for reftests.
actual_text = 'reference text\n'
actual_checksum = 'mock-checksum'
actual_image = 'blank'
if test_input.test_name.endswith('-mismatch.html'):
actual_text = 'not reference text\n'
actual_checksum = 'not-mock-checksum'
actual_image = 'not blank'
elif test_input.should_run_pixel_test and test_input.image_hash:
actual_checksum = port.expected_checksum(test_input.test_name)
actual_image = port.expected_image(test_input.test_name)
if self._options.actual_directory:
actual_path = port._filesystem.join(self._options.actual_directory, test_input.test_name)
root, _ = port._filesystem.splitext(actual_path)
text_path = root + '-actual.txt'
if port._filesystem.exists(text_path):
actual_text = port._filesystem.read_binary_file(text_path)
audio_path = root + '-actual.wav'
if port._filesystem.exists(audio_path):
actual_audio = port._filesystem.read_binary_file(audio_path)
image_path = root + '-actual.png'
if port._filesystem.exists(image_path):
actual_image = port._filesystem.read_binary_file(image_path)
with port._filesystem.open_binary_file_for_reading(image_path) as filehandle:
actual_checksum = read_checksum_from_png.read_checksum(filehandle)
return DriverOutput(actual_text, actual_image, actual_checksum, actual_audio)
def write_test_output(self, test_input, output, is_reftest):
if output.audio:
self._stdout.write('Content-Type: audio/wav\n')
self._stdout.write('Content-Transfer-Encoding: base64\n')
self._stdout.write(base64.b64encode(output.audio))
self._stdout.write('\n')
else:
self._stdout.write('Content-Type: text/plain\n')
# FIXME: Note that we don't ensure there is a trailing newline!
# This mirrors actual (Mac) DRT behavior but is a bug.
if output.text:
self._stdout.write(output.text)
self._stdout.write('#EOF\n')
if test_input.should_run_pixel_test and output.image_hash:
self._stdout.write('\n')
self._stdout.write('ActualHash: %s\n' % output.image_hash)
self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash)
if output.image_hash != test_input.image_hash:
self._stdout.write('Content-Type: image/png\n')
self._stdout.write('Content-Length: %s\n' % len(output.image))
self._stdout.write(output.image)
self._stdout.write('#EOF\n')
self._stdout.flush()
self._stderr.write('#EOF\n')
self._stderr.flush()
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''),
**kwargs)
self.__delegate_driver_class = self.__delegate._driver_class
self.__delegate._driver_class = types.MethodType(
self._driver_class, self.__delegate)
def test_graphics_type(self):
port = PortFactory(MockSystemHost()).get('chromium-gpu-mac')
self.assertEquals('gpu', port.graphics_type())
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs)
def _verify_baseline_path(self, expected_path, port_name):
port = PortFactory(MockSystemHost()).get(port_name=port_name)
path = port.baseline_search_path()[0]
self.assertEqual(expected_path, port._filesystem.basename(path))
class MockDRTPort(object):
"""MockPort implementation of the Port interface."""
def __init__(self, host, **kwargs):
prefix = 'mock-'
if 'port_name' in kwargs:
kwargs['port_name'] = kwargs['port_name'][len(prefix):]
self._host = host
self.__delegate = PortFactory(host).get(**kwargs)
self.__real_name = prefix + self.__delegate.name()
def real_name(self):
return self.__real_name
def __getattr__(self, name):
return getattr(self.__delegate, name)
def check_build(self, needs_http):
return True
def check_sys_deps(self, needs_http):
return True
def driver_cmd_line(self):
driver = self.create_driver(0)
return driver.cmd_line()
def _path_to_driver(self):
return self._host.filesystem.abspath(__file__)
def create_driver(self, worker_number):
# We need to create a driver object as the delegate would, but
# overwrite the path to the driver binary in its command line. We do
# this by actually overwriting its cmd_line() method with a proxy
# method that splices in the mock_drt path and command line arguments
# in place of the actual path to the driver binary.
def overriding_cmd_line():
cmd = self.__original_driver_cmd_line()
index = cmd.index(self.__delegate._path_to_driver())
# FIXME: Why does this need to use sys.executable (instead of something mockable)?
cmd[index:index + 1] = [
sys.executable,
self._path_to_driver(), '--platform',
self.name()
]
return cmd
delegated_driver = self.__delegate.create_driver(worker_number)
self.__original_driver_cmd_line = delegated_driver.cmd_line
delegated_driver.cmd_line = overriding_cmd_line
return delegated_driver
def start_helper(self):
pass
def start_http_server(self):
pass
def start_websocket_server(self):
pass
def acquire_http_lock(self):
pass
def stop_helper(self):
pass
def stop_http_server(self):
pass
def stop_websocket_server(self):
pass
def release_http_lock(self):
pass
class MockDRT(object):
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name, options=options)
self._driver = self._port.create_driver(0)
def run(self):
while True:
line = self._stdin.readline()
if not line:
return 0
driver_input = self.input_from_line(line)
dirname, basename = self._port.split_test(driver_input.test_name)
is_reftest = (self._port.reference_files(driver_input.test_name) or
self._port.is_reference_html_file(self._port._filesystem, dirname, basename))
output = self.output_for_test(driver_input, is_reftest)
self.write_test_output(driver_input, output, is_reftest)
def input_from_line(self, line):
vals = line.strip().split("'")
if len(vals) == 1:
uri = vals[0]
checksum = None
else:
uri = vals[0]
checksum = vals[1]
if uri.startswith('http://') or uri.startswith('https://'):
test_name = self._driver.uri_to_test(uri)
else:
test_name = self._port.relative_test_filename(uri)
return DriverInput(test_name, 0, checksum, self._options.pixel_tests)
def output_for_test(self, test_input, is_reftest):
port = self._port
actual_text = port.expected_text(test_input.test_name)
actual_audio = port.expected_audio(test_input.test_name)
actual_image = None
actual_checksum = None
if is_reftest:
# Make up some output for reftests.
actual_text = 'reference text\n'
actual_checksum = 'mock-checksum'
actual_image = 'blank'
if test_input.test_name.endswith('-mismatch.html'):
actual_text = 'not reference text\n'
actual_checksum = 'not-mock-checksum'
actual_image = 'not blank'
elif self._options.pixel_tests and test_input.image_hash:
actual_checksum = port.expected_checksum(test_input.test_name)
actual_image = port.expected_image(test_input.test_name)
return DriverOutput(actual_text, actual_image, actual_checksum, actual_audio)
def write_test_output(self, test_input, output, is_reftest):
if output.audio:
self._stdout.write('Content-Type: audio/wav\n')
self._stdout.write('Content-Transfer-Encoding: base64\n')
self._stdout.write(base64.b64encode(output.audio))
else:
self._stdout.write('Content-Type: text/plain\n')
# FIXME: Note that we don't ensure there is a trailing newline!
# This mirrors actual (Mac) DRT behavior but is a bug.
if output.text:
self._stdout.write(output.text)
self._stdout.write('#EOF\n')
if self._options.pixel_tests and output.image_hash:
self._stdout.write('\n')
self._stdout.write('ActualHash: %s\n' % output.image_hash)
self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash)
if output.image_hash != test_input.image_hash:
self._stdout.write('Content-Type: image/png\n')
self._stdout.write('Content-Length: %s\n' % len(output.image))
self._stdout.write(output.image)
self._stdout.write('#EOF\n')
self._stdout.flush()
self._stderr.write('#EOF\n')
self._stderr.flush()
def _verify_expectations_overrides(self, port_name):
host = MockSystemHost()
port = PortFactory(host).get(port_name=port_name, options=None)
self.assertTrue('TestExpectations' in port.expectations_files()[0])
self.assertTrue('skia_test_expectations.txt' in port.expectations_files()[1])
self.assertTrue('test_expectations_chrome.txt' in port.expectations_files()[-1])
class Rebaseliner(object):
"""Class to produce new baselines for a given platform."""
REVISION_REGEX = r"<a href=\"(\d+)/\">"
def __init__(
self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm, logged_before=False
):
"""
Args:
running_port: the Port the script is running on.
target_port: the Port the script uses to find port-specific
configuration information like the test_expectations.txt
file location and the list of test platforms.
platform: the test platform to rebaseline
options: the command-line options object.
url_fetcher: object that can fetch objects from URLs
zip_factory: optional object that can fetch zip files from URLs
scm: scm object for adding new baselines
logged_before: whether the previous running port logged anything.
"""
self._platform = platform
self._options = options
self._port = running_port
self._filesystem = running_port._filesystem
self._target_port = target_port
# FIXME: This should get its PortFactory from a Host object.
# Note: using running_port.executive, running_port.user since we can't get them from a host.
self._rebaseline_port = PortFactory().get(
platform, options, filesystem=self._filesystem, executive=running_port.executive, user=running_port.user
)
self._rebaselining_tests = set()
self._rebaselined_tests = []
self._logged_before = logged_before
self.did_log = False
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
expectations_str = self._rebaseline_port.test_expectations()
self._test_expectations = test_expectations.TestExpectations(
self._rebaseline_port, None, expectations_str, self._rebaseline_port.test_configuration(), False
)
self._url_fetcher = url_fetcher
self._zip_factory = zip_factory
self._scm = scm
def run(self):
"""Run rebaseline process."""
log_dashed_string("Compiling rebaselining tests", self._platform, logging.DEBUG)
if not self._compile_rebaselining_tests():
return False
if not self._rebaselining_tests:
return True
self.did_log = True
log_dashed_string("Downloading archive", self._platform, logging.DEBUG)
archive_file = self._download_buildbot_archive()
_log.debug("")
if not archive_file:
_log.error("No archive found.")
return False
log_dashed_string("Extracting and adding new baselines", self._platform, logging.DEBUG)
self._extract_and_add_new_baselines(archive_file)
archive_file.close()
log_dashed_string("Updating rebaselined tests in file", self._platform)
if len(self._rebaselining_tests) != len(self._rebaselined_tests):
_log.debug("")
_log.debug("NOT ALL TESTS WERE REBASELINED.")
_log.debug(" Number marked for rebaselining: %d", len(self._rebaselining_tests))
_log.debug(" Number actually rebaselined: %d", len(self._rebaselined_tests))
_log.info("")
return False
_log.debug(" All tests needing rebaselining were successfully rebaselined.")
_log.info("")
return True
def remove_rebaselining_expectations(self, tests, backup):
"""if backup is True, we backup the original test expectations file."""
new_expectations = self._test_expectations.remove_rebaselined_tests(tests)
path = self._target_port.path_to_test_expectations_file()
if backup:
date_suffix = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
backup_file = "%s.orig.%s" % (path, date_suffix)
if self._filesystem.exists(backup_file):
self._filesystem.remove(backup_file)
_log.debug('Saving original file to "%s"', backup_file)
self._filesystem.move(path, backup_file)
self._filesystem.write_text_file(path, new_expectations)
# self._scm.add(path)
def get_rebaselined_tests(self):
return self._rebaselined_tests
def _compile_rebaselining_tests(self):
"""Compile list of tests that need rebaselining for the platform.
Returns:
False if reftests are wrongly marked as 'needs rebaselining' or True
"""
self._rebaselining_tests = self._test_expectations.get_rebaselining_failures()
if not self._rebaselining_tests:
_log.info("%s: No tests to rebaseline.", self._platform)
return True
fs = self._target_port._filesystem
for test in self._rebaselining_tests:
test_abspath = self._target_port.abspath_for_test(test)
if fs.exists(self._target_port.reftest_expected_filename(test_abspath)) or fs.exists(
self._target_port.reftest_expected_mismatch_filename(test_abspath)
):
_log.error("%s seems to be a reftest. We can not rebase for reftests.", test)
self._rebaselining_tests = set()
return False
if not self._logged_before:
_log.info("")
_log.info("%s: Rebaselining %d tests:", self._platform, len(self._rebaselining_tests))
test_no = 1
for test in self._rebaselining_tests:
_log.debug(" %d: %s", test_no, test)
test_no += 1
return True
def _get_latest_revision(self, url):
"""Get the latest layout test revision number from buildbot.
Args:
url: Url to retrieve layout test revision numbers.
Returns:
latest revision or
None on failure.
"""
_log.debug('Url to retrieve revision: "%s"', url)
content = self._url_fetcher.fetch(url)
revisions = re.findall(self.REVISION_REGEX, content)
if not revisions:
_log.error('Failed to find revision, content: "%s"', content)
return None
revisions.sort(key=int)
_log.debug(" Latest revision: %s", revisions[len(revisions) - 1])
return revisions[len(revisions) - 1]
def _get_archive_dir_name(self, platform):
"""Get name of the layout test archive directory.
Returns:
Directory name or
None on failure
"""
if platform in ARCHIVE_DIR_NAME_DICT:
return ARCHIVE_DIR_NAME_DICT[platform]
else:
_log.error("Cannot find platform key %s in archive " "directory name dictionary", platform)
return None
def _get_archive_url(self):
"""Generate the url to download latest layout test archive.
Returns:
Url to download archive or
None on failure
"""
if self._options.force_archive_url:
return self._options.force_archive_url
dir_name = self._get_archive_dir_name(self._platform)
if not dir_name:
return None
_log.debug('Buildbot platform dir name: "%s"', dir_name)
url_base = "%s/%s/" % (self._options.archive_url, dir_name)
latest_revision = self._get_latest_revision(url_base)
if latest_revision is None or latest_revision <= 0:
return None
archive_url = "%s%s/layout-test-results.zip" % (url_base, latest_revision)
_log.info(" Using %s", archive_url)
return archive_url
def _download_buildbot_archive(self):
"""Download layout test archive file from buildbot and return a handle to it."""
url = self._get_archive_url()
if url is None:
return None
archive_file = zipfileset.ZipFileSet(url, filesystem=self._filesystem, zip_factory=self._zip_factory)
_log.debug("Archive downloaded")
return archive_file
def _extract_and_add_new_baselines(self, zip_file):
"""Extract new baselines from the zip file and add them to SVN repository.
Returns:
List of tests that have been rebaselined or None on failure."""
zip_namelist = zip_file.namelist()
_log.debug("zip file namelist:")
for name in zip_namelist:
_log.debug(" " + name)
_log.debug('Platform dir: "%s"', self._platform)
self._rebaselined_tests = []
for test_no, test in enumerate(self._rebaselining_tests):
_log.debug("Test %d: %s", test_no + 1, test)
self._extract_and_add_new_baseline(test, zip_file)
def _extract_and_add_new_baseline(self, test, zip_file):
found = False
scm_error = False
test_basename = self._filesystem.splitext(test)[0]
for suffix in BASELINE_SUFFIXES:
archive_test_name = "layout-test-results/%s-actual%s" % (test_basename, suffix)
_log.debug(' Archive test file name: "%s"', archive_test_name)
if not archive_test_name in zip_file.namelist():
_log.debug(" %s file not in archive.", suffix)
continue
found = True
_log.debug(" %s file found in archive.", suffix)
temp_name = self._extract_from_zip_to_tempfile(zip_file, archive_test_name)
expected_filename = "%s-expected%s" % (test_basename, suffix)
expected_fullpath = self._filesystem.join(self._rebaseline_port.baseline_path(), expected_filename)
expected_fullpath = self._filesystem.normpath(expected_fullpath)
_log.debug(' Expected file full path: "%s"', expected_fullpath)
relpath = self._filesystem.relpath(expected_fullpath, self._target_port.layout_tests_dir())
# TODO(victorw): for now, the rebaselining tool checks whether
# or not THIS baseline is duplicate and should be skipped.
# We could improve the tool to check all baselines in upper
# and lower levels and remove all duplicated baselines.
if self._is_dup_baseline(temp_name, expected_fullpath, test, suffix, self._platform):
self._filesystem.remove(temp_name)
if self._filesystem.exists(expected_fullpath):
_log.info(" Removing %s" % relpath)
self._delete_baseline(expected_fullpath)
_log.debug(" %s is a duplicate" % relpath)
# FIXME: We consider a duplicate baseline a success in the normal case.
# FIXME: This may not be what you want sometimes; should this be
# FIXME: controllable?
self._rebaselined_tests.append(test)
continue
if suffix == ".checksum" and self._png_has_same_checksum(temp_name, test, expected_fullpath):
self._filesystem.remove(temp_name)
# If an old checksum exists, delete it.
self._delete_baseline(expected_fullpath)
continue
self._filesystem.maybe_make_directory(self._filesystem.dirname(expected_fullpath))
self._filesystem.move(temp_name, expected_fullpath)
path_from_base = self._filesystem.relpath(expected_fullpath)
if self._scm.exists(path_from_base):
_log.info(" Updating %s" % relpath)
else:
_log.info(" Adding %s" % relpath)
if self._scm.add(expected_fullpath, return_exit_code=True):
# FIXME: print detailed diagnose messages
scm_error = True
elif suffix != ".checksum":
self._create_html_baseline_files(expected_fullpath)
if not found:
_log.warn("No results in archive for %s" % test)
elif scm_error:
_log.warn("Failed to add baselines to your repository.")
else:
_log.debug(" Rebaseline succeeded.")
self._rebaselined_tests.append(test)
def _extract_from_zip_to_tempfile(self, zip_file, filename):
"""Extracts |filename| from |zip_file|, a ZipFileSet. Returns the full
path name to the extracted file."""
data = zip_file.read(filename)
suffix = self._filesystem.splitext(filename)[1]
tempfile, temp_name = self._filesystem.open_binary_tempfile(suffix)
tempfile.write(data)
tempfile.close()
return temp_name
def _png_has_same_checksum(self, checksum_path, test, checksum_expected_fullpath):
"""Returns True if the fallback png for |checksum_expected_fullpath|
contains the same checksum."""
fs = self._filesystem
png_fullpath = self._first_fallback_png_for_test(test)
if not fs.exists(png_fullpath):
_log.error(" Checksum without png file found! Expected %s to exist." % png_fullpath)
return False
with fs.open_binary_file_for_reading(png_fullpath) as filehandle:
checksum_in_png = read_checksum_from_png.read_checksum(filehandle)
checksum_in_text_file = fs.read_text_file(checksum_path)
if checksum_in_png and checksum_in_png != checksum_in_text_file:
_log.error(
" checksum in %s and %s don't match! Continuing"
" to copy but please investigate." % (checksum_expected_fullpath, png_fullpath)
)
return checksum_in_text_file == checksum_in_png
def _first_fallback_png_for_test(self, test):
all_baselines = self._rebaseline_port.expected_baselines(test, ".png", True)
return self._filesystem.join(all_baselines[0][0], all_baselines[0][1])
def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix, platform):
"""Check whether a baseline is duplicate and can fallback to same
baseline for another platform. For example, if a test has same
baseline on linux and windows, then we only store windows
baseline and linux baseline will fallback to the windows version.
Args:
new_baseline: temp filename containing the new baseline results
baseline_path: baseline expectation file name.
test: test name.
suffix: file suffix of the expected results, including dot;
e.g. '.txt' or '.png'.
platform: baseline platform 'mac', 'win' or 'linux'.
Returns:
True if the baseline is unnecessary.
False otherwise.
"""
all_baselines = self._rebaseline_port.expected_baselines(test, suffix, True)
for fallback_dir, fallback_file in all_baselines:
if not fallback_dir or not fallback_file:
continue
fallback_fullpath = self._filesystem.normpath(self._filesystem.join(fallback_dir, fallback_file))
if fallback_fullpath.lower() == baseline_path.lower():
continue
fallback_dir_relpath = self._filesystem.relpath(fallback_dir, self._target_port.layout_tests_dir())
if fallback_dir_relpath == "":
fallback_dir_relpath = "<generic>"
new_output = self._filesystem.read_binary_file(new_baseline)
fallback_output = self._filesystem.read_binary_file(fallback_fullpath)
is_image = baseline_path.lower().endswith(".png")
if not self._diff_baselines(new_output, fallback_output, is_image):
_log.info(" Skipping %s (matches %s)", test, fallback_dir_relpath)
return True
return False
return False
def _diff_baselines(self, output1, output2, is_image):
"""Check whether two baselines are different.
Args:
output1, output2: contents of the baselines to compare.
Returns:
True if two files are different or have different extensions.
False otherwise.
"""
if is_image:
return self._port.diff_image(output1, output2)[0]
return self._port.compare_text(output1, output2)
def _delete_baseline(self, filename):
"""Remove the file from repository and delete it from disk.
Args:
filename: full path of the file to delete.
"""
if not filename or not self._filesystem.isfile(filename):
return
self._scm.delete(filename)
def _create_html_baseline_files(self, baseline_fullpath):
"""Create baseline files (old, new and diff) in html directory.
The files are used to compare the rebaselining results.
Args:
baseline_fullpath: full path of the expected baseline file.
"""
baseline_relpath = self._filesystem.relpath(baseline_fullpath)
_log.debug(' Html: create baselines for "%s"', baseline_relpath)
if not baseline_fullpath or not self._filesystem.exists(baseline_fullpath):
_log.debug(' Html: Does not exist: "%s"', baseline_fullpath)
return
if not self._scm.exists(baseline_relpath):
_log.debug(' Html: Does not exist in scm: "%s"', baseline_relpath)
return
# Copy the new baseline to html directory for result comparison.
baseline_filename = self._filesystem.basename(baseline_fullpath)
new_file = get_result_file_fullpath(
self._filesystem, self._options.html_directory, baseline_filename, self._platform, "new"
)
self._filesystem.copyfile(baseline_fullpath, new_file)
_log.debug(' Html: copied new baseline file from "%s" to "%s".', baseline_fullpath, new_file)
# Get the old baseline from the repository and save to the html directory.
try:
output = self._scm.show_head(baseline_relpath)
except ScriptError, e:
_log.warning(e)
output = ""
if (not output) or (output.upper().rstrip().endswith("NO SUCH FILE OR DIRECTORY")):
_log.warning(' No base file: "%s"', baseline_fullpath)
return
base_file = get_result_file_fullpath(
self._filesystem, self._options.html_directory, baseline_filename, self._platform, "old"
)
if base_file.upper().endswith(".PNG"):
self._filesystem.write_binary_file(base_file, output)
else:
self._filesystem.write_text_file(base_file, output)
_log.debug(' Html: created old baseline file: "%s".', base_file)
# Get the diff between old and new baselines and save to the html dir.
diff_file = get_result_file_fullpath(
self._filesystem, self._options.html_directory, baseline_filename, self._platform, "diff"
)
has_diff = False
if baseline_filename.upper().endswith(".TXT"):
output = self._scm.diff_for_file(baseline_relpath, log=_log)
if output:
self._filesystem.write_text_file(diff_file, output)
has_diff = True
elif baseline_filename.upper().endswith(".PNG"):
old_file = get_result_file_fullpath(
self._filesystem, self._options.html_directory, baseline_filename, self._platform, "old"
)
new_file = get_result_file_fullpath(
self._filesystem, self._options.html_directory, baseline_filename, self._platform, "new"
)
_log.debug(' Html: diffing "%s" and "%s"', old_file, new_file)
old_output = self._filesystem.read_binary_file(old_file)
new_output = self._filesystem.read_binary_file(new_file)
image_diff = self._port.diff_image(old_output, new_output)[0]
self._filesystem.write_binary_file(diff_file, image_diff)
if has_diff:
_log.debug(' Html: created baseline diff file: "%s".', diff_file)
class MockDRT(object):
@classmethod
def determine_full_port_name(cls, host, options, port_name):
"""Return a fully-specified port name that can be used to construct objects."""
# Subclasses will usually override this.
return cls.port_name
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name, options=options)
self._driver = self._port.create_driver(0)
def run(self):
while True:
line = self._stdin.readline()
if not line:
break
self.run_one_test(self.parse_input(line))
return 0
def parse_input(self, line):
return _DRTInput(line)
def run_one_test(self, test_input):
port = self._port
if test_input.uri.startswith('http://') or test_input.uri.startswith('https://'):
test_name = self._driver.uri_to_test(test_input.uri)
else:
test_name = port.relative_test_filename(test_input.uri)
actual_text = port.expected_text(test_name)
actual_audio = port.expected_audio(test_name)
if self._options.pixel_tests and test_input.checksum:
actual_checksum = port.expected_checksum(test_name)
actual_image = port.expected_image(test_name)
if actual_audio:
self._stdout.write('Content-Type: audio/wav\n')
self._stdout.write('Content-Transfer-Encoding: base64\n')
output = base64.b64encode(actual_audio)
self._stdout.write(output)
else:
self._stdout.write('Content-Type: text/plain\n')
# FIXME: Note that we don't ensure there is a trailing newline!
# This mirrors actual (Mac) DRT behavior but is a bug.
self._stdout.write(actual_text)
self._stdout.write('#EOF\n')
if self._options.pixel_tests and test_input.checksum:
self._stdout.write('\n')
self._stdout.write('ActualHash: %s\n' % actual_checksum)
self._stdout.write('ExpectedHash: %s\n' % test_input.checksum)
if actual_checksum != test_input.checksum:
self._stdout.write('Content-Type: image/png\n')
self._stdout.write('Content-Length: %s\n' % len(actual_image))
self._stdout.write(actual_image)
self._stdout.write('#EOF\n')
self._stdout.flush()
self._stderr.flush()
def _verify_baseline_path(self, expected_path, port_name):
port = PortFactory(MockSystemHost()).get(port_name=port_name)
path = port.baseline_search_path()[0]
self.assertEqual(expected_path, port._filesystem.basename(path))
class MockDRTPort(object):
"""MockPort implementation of the Port interface."""
port_name = 'mock'
@classmethod
def determine_full_port_name(cls, host, options, port_name):
"""Return a fully-specified port name that can be used to construct objects."""
# Subclasses will usually override this.
return port_name
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs)
self.__real_name = port_name
self._host = host
def real_name(self):
return self.__real_name
def __getattr__(self, name):
return getattr(self.__delegate, name)
def check_build(self, needs_http):
return True
def check_sys_deps(self, needs_http):
return True
def driver_cmd_line(self):
driver = self.create_driver(0)
return driver.cmd_line()
def _path_to_driver(self):
return self._host.filesystem.abspath(__file__)
def create_driver(self, worker_number):
# We need to create a driver object as the delegate would, but
# overwrite the path to the driver binary in its command line. We do
# this by actually overwriting its cmd_line() method with a proxy
# method that splices in the mock_drt path and command line arguments
# in place of the actual path to the driver binary.
def overriding_cmd_line():
cmd = self.__original_driver_cmd_line()
index = cmd.index(self.__delegate._path_to_driver())
# FIXME: Why does this need to use sys.executable (instead of something mockable)?
cmd[index:index + 1] = [sys.executable, self._path_to_driver(), '--platform', self.name()]
return cmd
delegated_driver = self.__delegate.create_driver(worker_number)
self.__original_driver_cmd_line = delegated_driver.cmd_line
delegated_driver.cmd_line = overriding_cmd_line
return delegated_driver
def start_helper(self):
pass
def start_http_server(self):
pass
def start_websocket_server(self):
pass
def acquire_http_lock(self):
pass
def stop_helper(self):
pass
def stop_http_server(self):
pass
def stop_websocket_server(self):
pass
def release_http_lock(self):
pass
def _verify_baseline_search_path_startswith(self, port_name, expected_platform_dirs):
port = PortFactory(MockSystemHost()).get(port_name=port_name)
actual_platform_dirs = [port._filesystem.basename(path) for path in port.baseline_search_path()]
self.assertEqual(expected_platform_dirs, actual_platform_dirs[0:len(expected_platform_dirs)])
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs)
self.__real_name = port_name
self._host = host
class MockDRTPort(object):
port_name = 'mock'
@classmethod
def determine_full_port_name(cls, host, options, port_name):
return port_name
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs)
self.__delegate_driver_class = self.__delegate._driver_class
self.__delegate._driver_class = types.MethodType(self._driver_class, self.__delegate)
def __getattr__(self, name):
return getattr(self.__delegate, name)
def check_build(self, needs_http, printer):
return True
def check_sys_deps(self, needs_http):
return True
def _driver_class(self, delegate):
return self._mocked_driver_maker
def _mocked_driver_maker(self, port, worker_number, pixel_tests, no_timeout=False):
path_to_this_file = self.host.filesystem.abspath(__file__.replace('.pyc', '.py'))
driver = self.__delegate_driver_class()(self, worker_number, pixel_tests, no_timeout)
driver.cmd_line = self._overriding_cmd_line(driver.cmd_line,
self.__delegate._path_to_driver(),
sys.executable,
path_to_this_file,
self.__delegate.name())
return driver
@staticmethod
def _overriding_cmd_line(original_cmd_line, driver_path, python_exe, this_file, port_name):
def new_cmd_line(pixel_tests, per_test_args):
cmd_line = original_cmd_line(pixel_tests, per_test_args)
index = cmd_line.index(driver_path)
cmd_line[index:index + 1] = [python_exe, this_file, '--platform', port_name]
return cmd_line
return new_cmd_line
def start_helper(self):
pass
def start_http_server(self, number_of_servers):
pass
def start_websocket_server(self):
pass
def acquire_http_lock(self):
pass
def stop_helper(self):
pass
def stop_http_server(self):
pass
def stop_websocket_server(self):
pass
def release_http_lock(self):
pass
def _make_wdiff_available(self):
self.__delegate._wdiff_available = True
def setup_environ_for_server(self, server_name):
env = self.__delegate.setup_environ_for_server()
# We need to propagate PATH down so the python code can find the checkout.
env['PATH'] = os.environ['PATH']
return env
def lookup_virtual_test_args(self, test_name):
suite = self.__delegate.lookup_virtual_suite(test_name)
return suite.args + ['--virtual-test-suite-name', suite.name, '--virtual-test-suite-base', suite.base]
def assert_port_works(self, port_name, input_name=None, platform=None):
host = MockSystemHost()
host.filesystem = FileSystem(
) # FIXME: This test should not use a real filesystem!
# test that we got the right port
mock_options = MockOptions(accelerated_2d_canvas=None,
accelerated_video=None,
builder_name='foo',
child_processes=None)
if input_name and platform:
port = PortFactory(host).get(host,
platform=platform,
port_name=input_name,
options=mock_options)
else:
port = PortFactory(host).get(host,
port_name=port_name,
options=mock_options)
self.assertTrue(port._options.accelerated_2d_canvas)
self.assertTrue(port._options.accelerated_video)
self.assertTrue(port._options.experimental_fully_parallel)
self.assertEqual(port._options.builder_name, 'foo - GPU')
self.assertTrue(port.name().startswith(port_name))
# test that it has the right directories in front of the search path.
paths = port.baseline_search_path()
self.assertEqual(port._webkit_baseline_path(port_name), paths[0])
if port_name == 'chromium-gpu-linux':
self.assertEqual(port._webkit_baseline_path('chromium-gpu-win'),
paths[1])
self.assertEqual(port._webkit_baseline_path('chromium-gpu'),
paths[2])
else:
self.assertEqual(port._webkit_baseline_path('chromium-gpu'),
paths[1])
# Test that we're limiting to the correct directories.
# These two tests are picked mostly at random, but we make sure they
# exist separately from being filtered out by the port.
# Note that this is using a real filesystem.
files = port.tests(None)
path = 'fast/html/keygen.html'
self.assertTrue(port._filesystem.exists(port.abspath_for_test(path)))
self.assertFalse(path in files)
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs)
self.__delegate_driver_class = self.__delegate._driver_class
self.__delegate._driver_class = types.MethodType(self._driver_class, self.__delegate)
class MockDRT(object):
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name, options=options)
self._driver = self._port.create_driver(0)
def run(self):
while True:
line = self._stdin.readline()
if not line:
break
self.run_one_test(self.parse_input(line))
return 0
def parse_input(self, line):
return _DRTInput(line)
def run_one_test(self, test_input):
port = self._port
if test_input.uri.startswith("http://") or test_input.uri.startswith("https://"):
test_name = self._driver.uri_to_test(test_input.uri)
else:
test_name = port.relative_test_filename(test_input.uri)
actual_text = port.expected_text(test_name)
actual_audio = port.expected_audio(test_name)
if self._options.pixel_tests and test_input.checksum:
actual_checksum = port.expected_checksum(test_name)
actual_image = port.expected_image(test_name)
if actual_audio:
self._stdout.write("Content-Type: audio/wav\n")
self._stdout.write("Content-Transfer-Encoding: base64\n")
output = base64.b64encode(actual_audio)
self._stdout.write(output)
else:
self._stdout.write("Content-Type: text/plain\n")
# FIXME: Note that we don't ensure there is a trailing newline!
# This mirrors actual (Mac) DRT behavior but is a bug.
self._stdout.write(actual_text)
self._stdout.write("#EOF\n")
if self._options.pixel_tests and test_input.checksum:
self._stdout.write("\n")
self._stdout.write("ActualHash: %s\n" % actual_checksum)
self._stdout.write("ExpectedHash: %s\n" % test_input.checksum)
if actual_checksum != test_input.checksum:
self._stdout.write("Content-Type: image/png\n")
self._stdout.write("Content-Length: %s\n" % len(actual_image))
self._stdout.write(actual_image)
self._stdout.write("#EOF\n")
self._stdout.flush()
self._stderr.flush()
class MockDRTPort(object):
"""MockPort implementation of the Port interface."""
port_name = 'mock'
@classmethod
def determine_full_port_name(cls, host, options, port_name):
"""Return a fully-specified port name that can be used to construct objects."""
# Subclasses will usually override this.
return port_name
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''),
**kwargs)
self.__real_name = port_name
self._host = host
def real_name(self):
return self.__real_name
def __getattr__(self, name):
return getattr(self.__delegate, name)
def check_build(self, needs_http):
return True
def check_sys_deps(self, needs_http):
return True
def driver_cmd_line(self):
driver = self.create_driver(0)
return driver.cmd_line()
def _path_to_driver(self):
return self._host.filesystem.abspath(__file__)
def create_driver(self, worker_number):
# We need to create a driver object as the delegate would, but
# overwrite the path to the driver binary in its command line. We do
# this by actually overwriting its cmd_line() method with a proxy
# method that splices in the mock_drt path and command line arguments
# in place of the actual path to the driver binary.
def overriding_cmd_line():
cmd = self.__original_driver_cmd_line()
index = cmd.index(self.__delegate._path_to_driver())
# FIXME: Why does this need to use sys.executable (instead of something mockable)?
cmd[index:index + 1] = [
sys.executable,
self._path_to_driver(), '--platform',
self.name()
]
return cmd
delegated_driver = self.__delegate.create_driver(worker_number)
self.__original_driver_cmd_line = delegated_driver.cmd_line
delegated_driver.cmd_line = overriding_cmd_line
return delegated_driver
def start_helper(self):
pass
def start_http_server(self):
pass
def start_websocket_server(self):
pass
def acquire_http_lock(self):
pass
def stop_helper(self):
pass
def stop_http_server(self):
pass
def stop_websocket_server(self):
pass
def release_http_lock(self):
pass
class MockDRT(object):
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name,
options=options)
self._driver = self._port.create_driver(0)
def run(self):
self._stdout.write("#READY\n")
self._stdout.flush()
while True:
line = self._stdin.readline()
if not line:
return 0
driver_input = self.input_from_line(line)
dirname, basename = self._port.split_test(driver_input.test_name)
is_reftest = (self._port.reference_files(driver_input.test_name)
or self._port.is_reference_html_file(
self._port._filesystem, dirname, basename))
output = self.output_for_test(driver_input, is_reftest)
self.write_test_output(driver_input, output, is_reftest)
def input_from_line(self, line):
vals = line.strip().split("'")
uri = vals[0]
checksum = None
should_run_pixel_tests = False
if len(vals) == 2 and vals[1] == '--pixel-test':
should_run_pixel_tests = True
elif len(vals) == 3 and vals[1] == '--pixel-test':
should_run_pixel_tests = True
checksum = vals[2]
elif len(vals) != 1:
raise NotImplementedError
if uri.startswith('http://') or uri.startswith('https://'):
test_name = self._driver.uri_to_test(uri)
else:
test_name = self._port.relative_test_filename(uri)
return DriverInput(test_name,
0,
checksum,
should_run_pixel_tests,
args=[])
def output_for_test(self, test_input, is_reftest):
port = self._port
if self._options.virtual_test_suite_name:
test_input.test_name = test_input.test_name.replace(
self._options.virtual_test_suite_base,
self._options.virtual_test_suite_name)
actual_text = port.expected_text(test_input.test_name)
actual_audio = port.expected_audio(test_input.test_name)
actual_image = None
actual_checksum = None
if is_reftest:
# Make up some output for reftests.
actual_text = 'reference text\n'
actual_checksum = 'mock-checksum'
actual_image = 'blank'
if test_input.test_name.endswith('-mismatch.html'):
actual_text = 'not reference text\n'
actual_checksum = 'not-mock-checksum'
actual_image = 'not blank'
elif test_input.should_run_pixel_test and test_input.image_hash:
actual_checksum = port.expected_checksum(test_input.test_name)
actual_image = port.expected_image(test_input.test_name)
if self._options.actual_directory:
actual_path = port._filesystem.join(self._options.actual_directory,
test_input.test_name)
root, _ = port._filesystem.splitext(actual_path)
text_path = root + '-actual.txt'
if port._filesystem.exists(text_path):
actual_text = port._filesystem.read_binary_file(text_path)
audio_path = root + '-actual.wav'
if port._filesystem.exists(audio_path):
actual_audio = port._filesystem.read_binary_file(audio_path)
image_path = root + '-actual.png'
if port._filesystem.exists(image_path):
actual_image = port._filesystem.read_binary_file(image_path)
with port._filesystem.open_binary_file_for_reading(
image_path) as filehandle:
actual_checksum = read_checksum_from_png.read_checksum(
filehandle)
return DriverOutput(actual_text, actual_image, actual_checksum,
actual_audio)
def write_test_output(self, test_input, output, is_reftest):
if output.audio:
self._stdout.write('Content-Type: audio/wav\n')
self._stdout.write('Content-Transfer-Encoding: base64\n')
self._stdout.write(base64.b64encode(output.audio))
self._stdout.write('\n')
else:
self._stdout.write('Content-Type: text/plain\n')
# FIXME: Note that we don't ensure there is a trailing newline!
# This mirrors actual (Mac) DRT behavior but is a bug.
if output.text:
self._stdout.write(output.text)
self._stdout.write('#EOF\n')
if test_input.should_run_pixel_test and output.image_hash:
self._stdout.write('\n')
self._stdout.write('ActualHash: %s\n' % output.image_hash)
self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash)
if output.image_hash != test_input.image_hash:
self._stdout.write('Content-Type: image/png\n')
self._stdout.write('Content-Length: %s\n' % len(output.image))
self._stdout.write(output.image)
self._stdout.write('#EOF\n')
self._stdout.flush()
self._stderr.write('#EOF\n')
self._stderr.flush()
class MockDRT(object):
@classmethod
def determine_full_port_name(cls, host, options, port_name):
"""Return a fully-specified port name that can be used to construct objects."""
# Subclasses will usually override this.
return cls.port_name
def __init__(self, options, args, host, stdin, stdout, stderr):
self._options = options
self._args = args
self._host = host
self._stdout = stdout
self._stdin = stdin
self._stderr = stderr
port_name = None
if options.platform:
port_name = options.platform
self._port = PortFactory(host).get(port_name=port_name,
options=options)
self._driver = self._port.create_driver(0)
def run(self):
while True:
line = self._stdin.readline()
if not line:
break
self.run_one_test(self.parse_input(line))
return 0
def parse_input(self, line):
return _DRTInput(line)
def run_one_test(self, test_input):
port = self._port
if test_input.uri.startswith('http://') or test_input.uri.startswith(
'https://'):
test_name = self._driver.uri_to_test(test_input.uri)
else:
test_name = port.relative_test_filename(test_input.uri)
actual_text = port.expected_text(test_name)
actual_audio = port.expected_audio(test_name)
if self._options.pixel_tests and test_input.checksum:
actual_checksum = port.expected_checksum(test_name)
actual_image = port.expected_image(test_name)
if actual_audio:
self._stdout.write('Content-Type: audio/wav\n')
self._stdout.write('Content-Transfer-Encoding: base64\n')
output = base64.b64encode(actual_audio)
self._stdout.write(output)
else:
self._stdout.write('Content-Type: text/plain\n')
# FIXME: Note that we don't ensure there is a trailing newline!
# This mirrors actual (Mac) DRT behavior but is a bug.
self._stdout.write(actual_text)
self._stdout.write('#EOF\n')
if self._options.pixel_tests and test_input.checksum:
self._stdout.write('\n')
self._stdout.write('ActualHash: %s\n' % actual_checksum)
self._stdout.write('ExpectedHash: %s\n' % test_input.checksum)
if actual_checksum != test_input.checksum:
self._stdout.write('Content-Type: image/png\n')
self._stdout.write('Content-Length: %s\n' % len(actual_image))
self._stdout.write(actual_image)
self._stdout.write('#EOF\n')
self._stdout.flush()
self._stderr.flush()
class MockDRTPort(object):
port_name = 'mock'
@classmethod
def determine_full_port_name(cls, host, options, port_name):
return port_name
def __init__(self, host, port_name, **kwargs):
self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''),
**kwargs)
self.__delegate_driver_class = self.__delegate._driver_class
self.__delegate._driver_class = types.MethodType(
self._driver_class, self.__delegate)
def __getattr__(self, name):
return getattr(self.__delegate, name)
def check_build(self, needs_http, printer):
return test_run_results.OK_EXIT_STATUS
def check_sys_deps(self, needs_http):
return test_run_results.OK_EXIT_STATUS
def _driver_class(self, delegate):
return self._mocked_driver_maker
def _mocked_driver_maker(self,
port,
worker_number,
pixel_tests,
no_timeout=False):
path_to_this_file = self.host.filesystem.abspath(
__file__.replace('.pyc', '.py'))
driver = self.__delegate_driver_class()(self, worker_number,
pixel_tests, no_timeout)
driver.cmd_line = self._overriding_cmd_line(
driver.cmd_line, self.__delegate._path_to_driver(), sys.executable,
path_to_this_file, self.__delegate.name())
return driver
@staticmethod
def _overriding_cmd_line(original_cmd_line, driver_path, python_exe,
this_file, port_name):
def new_cmd_line(pixel_tests, per_test_args):
cmd_line = original_cmd_line(pixel_tests, per_test_args)
index = cmd_line.index(driver_path)
cmd_line[index:index +
1] = [python_exe, this_file, '--platform', port_name]
return cmd_line
return new_cmd_line
def start_helper(self):
pass
def start_http_server(self, additional_dirs, number_of_servers):
pass
def start_websocket_server(self):
pass
def acquire_http_lock(self):
pass
def stop_helper(self):
pass
def stop_http_server(self):
pass
def stop_websocket_server(self):
pass
def release_http_lock(self):
pass
def _make_wdiff_available(self):
self.__delegate._wdiff_available = True
def setup_environ_for_server(self):
env = self.__delegate.setup_environ_for_server()
# We need to propagate PATH down so the python code can find the checkout.
env['PATH'] = os.environ['PATH']
return env
def lookup_virtual_test_args(self, test_name):
suite = self.__delegate.lookup_virtual_suite(test_name)
return suite.args + [
'--virtual-test-suite-name', suite.name,
'--virtual-test-suite-base', suite.base
]
def lookup_virtual_reference_args(self, test_name):
suite = self.__delegate.lookup_virtual_suite(test_name)
return suite.reference_args + [
'--virtual-test-suite-name', suite.name,
'--virtual-test-suite-base', suite.base
]
def _assert_baseline_path(self, port_name, baseline_path):
port = PortFactory(MockSystemHost()).get(port_name)
self.assertEquals(port.name(), port_name)
self.assertEquals(port.baseline_path(), port._webkit_baseline_path(baseline_path))
def test_paths(port_name):
return PortFactory(host).get(port_name).tests([])
def test_paths(port_name):
return chromium_gpu._default_tests_paths(
PortFactory(MockSystemHost()).get(port_name))
class MockDRTPort(object):
"""MockPort implementation of the Port interface."""
def __init__(self, host, **kwargs):
prefix = "mock-"
if "port_name" in kwargs:
kwargs["port_name"] = kwargs["port_name"][len(prefix) :]
self._host = host
self.__delegate = PortFactory(host).get(**kwargs)
self.__real_name = prefix + self.__delegate.name()
def real_name(self):
return self.__real_name
def __getattr__(self, name):
return getattr(self.__delegate, name)
def check_build(self, needs_http):
return True
def check_sys_deps(self, needs_http):
return True
def driver_cmd_line(self):
driver = self.create_driver(0)
return driver.cmd_line()
def _path_to_driver(self):
return self._host.filesystem.abspath(__file__)
def create_driver(self, worker_number):
# We need to create a driver object as the delegate would, but
# overwrite the path to the driver binary in its command line. We do
# this by actually overwriting its cmd_line() method with a proxy
# method that splices in the mock_drt path and command line arguments
# in place of the actual path to the driver binary.
def overriding_cmd_line():
cmd = self.__original_driver_cmd_line()
index = cmd.index(self.__delegate._path_to_driver())
# FIXME: Why does this need to use sys.executable (instead of something mockable)?
cmd[index : index + 1] = [sys.executable, self._path_to_driver(), "--platform", self.name()]
return cmd
delegated_driver = self.__delegate.create_driver(worker_number)
self.__original_driver_cmd_line = delegated_driver.cmd_line
delegated_driver.cmd_line = overriding_cmd_line
return delegated_driver
def start_helper(self):
pass
def start_http_server(self):
pass
def start_websocket_server(self):
pass
def acquire_http_lock(self):
pass
def stop_helper(self):
pass
def stop_http_server(self):
pass
def stop_websocket_server(self):
pass
def release_http_lock(self):
pass
