def __init__(self, host, **kwargs): prefix = 'mock-' if 'port_name' in kwargs: kwargs['port_name'] = kwargs['port_name'][len(prefix):] self._host = host self.__delegate = PortFactory(host).get(**kwargs) self.__real_name = prefix + self.__delegate.name()
def test_basic(self): cmd = [ sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"' ] host = SystemHost() factory = PortFactory(host) port = factory.get() now = time.time() proc = server_process.ServerProcess(port, 'python', cmd) proc.write('') self.assertEquals(proc.poll(), None) self.assertFalse(proc.has_crashed()) # check that doing a read after an expired deadline returns # nothing immediately. line = proc.read_stdout_line(now - 1) self.assertEquals(line, None) line = proc.read_stdout_line(now + 1.0) self.assertEquals(line.strip(), "stdout") line = proc.read_stderr_line(now + 1.0) self.assertEquals(line.strip(), "stderr") proc.stop()
def test_basic(self): cmd = [sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"'] host = SystemHost() factory = PortFactory(host) port = factory.get() now = time.time() proc = server_process.ServerProcess(port, 'python', cmd) proc.write('') self.assertEqual(proc.poll(), None) self.assertFalse(proc.has_crashed()) # check that doing a read after an expired deadline returns # nothing immediately. line = proc.read_stdout_line(now - 1) self.assertEqual(line, None) # FIXME: This part appears to be flaky. line should always be non-None. # FIXME: https://bugs.webkit.org/show_bug.cgi?id=88280 line = proc.read_stdout_line(now + 1.0) if line: self.assertEqual(line.strip(), "stdout") line = proc.read_stderr_line(now + 1.0) if line: self.assertEqual(line.strip(), "stderr") proc.stop(0)
def test_basic(self): cmd = [ sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"' ] host = SystemHost() factory = PortFactory(host) port = factory.get() now = time.time() proc = server_process.ServerProcess(port, 'python', cmd) proc.write('') self.assertEquals(proc.poll(), None) self.assertFalse(proc.has_crashed()) # check that doing a read after an expired deadline returns # nothing immediately. line = proc.read_stdout_line(now - 1) self.assertEquals(line, None) # FIXME: This part appears to be flaky. line should always be non-None. # FIXME: https://bugs.webkit.org/show_bug.cgi?id=88280 line = proc.read_stdout_line(now + 1.0) if line: self.assertEquals(line.strip(), "stdout") line = proc.read_stderr_line(now + 1.0) if line: self.assertEquals(line.strip(), "stderr") proc.stop(0)
def test_path_to_expectations(self): port_factory = PortFactory(MockSystemHost()) for port_name in ("google-chrome-linux32", "google-chrome-linux64", "google-chrome-mac", "google-chrome-win"): self.assertTrue( port_factory.get(port_name) .path_to_test_expectations_file() .endswith("platform/chromium/TestExpectations") )
def _verify_baseline_search_path_startswith(self, port_name, expected_platform_dirs): port = PortFactory(MockSystemHost()).get(port_name=port_name) actual_platform_dirs = [ port._filesystem.basename(path) for path in port.baseline_search_path() ] self.assertEqual(expected_platform_dirs, actual_platform_dirs[0:len(expected_platform_dirs)])
def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0)
def __init__(self, log_executive=False, executive_throws_when_run=None, initialize_scm_by_default=True, web=None, scm=None, os_name=None, os_version=None): MockSystemHost.__init__(self, log_executive, executive_throws_when_run, os_name=os_name, os_version=os_version) add_unit_tests_to_mock_filesystem(self.filesystem) self.web = web or MockWeb() self._scm = scm # FIXME: we should never initialize the SCM by default, since the real # object doesn't either. This has caused at least one bug (see bug 89498). if initialize_scm_by_default: self.initialize_scm() self.buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self.builders = Builders()
def assertTest(self, test_name, pixel_tests, expected_checksum=None, drt_output=None, host=None, expected_text=None): port_name = 'test' host = host or MockSystemHost() test.add_unit_tests_to_mock_filesystem(host.filesystem) port = PortFactory(host).get(port_name) drt_input, drt_output = self.make_input_output( port, test_name, pixel_tests, expected_checksum, drt_output, drt_input=None, expected_text=expected_text) args = ['--dump-render-tree', '--platform', port_name, '-'] stdin = newstringio.StringIO(drt_input) stdout = newstringio.StringIO() stderr = newstringio.StringIO() options, args = mock_drt.parse_options(args) drt = self.make_drt(options, args, host, stdin, stdout, stderr) res = drt.run() self.assertEqual(res, 0) # We use the StringIO.buflist here instead of getvalue() because # the StringIO might be a mix of unicode/ascii and 8-bit strings. self.assertEqual(stdout.buflist, drt_output) self.assertEqual(stderr.getvalue(), '#EOF\n')
def __init__(self, log_executive=False, initialize_scm_by_default=True, web=None, scm=None, os_name=None, os_version=None, time_return_val=123): super(MockHost, self).__init__(log_executive=log_executive, os_name=os_name, os_version=os_version, time_return_val=time_return_val) add_unit_tests_to_mock_filesystem(self.filesystem) self.web = web or MockWeb() self._scm = scm # TODO(qyearsley): we should never initialize the SCM by default, since # the real object doesn't either. This has caused at least one bug # (see bug 89498). if initialize_scm_by_default: self.initialize_scm() self.buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self.builders = BuilderList(BUILDERS)
def assertTest(self, test_name, pixel_tests, expected_checksum=None, drt_output=None, host=None, expected_text=None): port_name = 'test' host = host or MockSystemHost() test.add_unit_tests_to_mock_filesystem(host.filesystem) port = PortFactory(host).get(port_name) drt_input, drt_output = self.make_input_output( port, test_name, pixel_tests, expected_checksum, drt_output, drt_input=None, expected_text=expected_text) args = ['--run-layout-test', '--platform', port_name, '-'] stdin = io.BytesIO(drt_input) stdout = io.BytesIO() stderr = io.BytesIO() options, args = mock_drt.parse_options(args) drt = self.make_drt(options, args, host, stdin, stdout, stderr) res = drt.run() self.assertEqual(res, 0) self.assertEqual(stdout.getvalue(), ''.join(drt_output)) self.assertEqual(stderr.getvalue(), '#EOF\n')
def __init__(self, host, **kwargs): prefix = "mock-" if "port_name" in kwargs: kwargs["port_name"] = kwargs["port_name"][len(prefix) :] self._host = host self.__delegate = PortFactory(host).get(**kwargs) self.__real_name = prefix + self.__delegate.name()
def _verify_expectations_overrides(self, port_name): host = MockSystemHost() chromium_port = PortFactory(host).get("chromium-mac-leopard") chromium_base = chromium_port.path_from_chromium_base() port = PortFactory(host).get(port_name=port_name, options=None) expected_chromium_overrides = '// chromium overrides\n' expected_chrome_overrides = '// chrome overrides\n' chromium_path = host.filesystem.join(chromium_base, 'webkit', 'tools', 'layout_tests', 'test_expectations.txt') chrome_path = host.filesystem.join(chromium_base, 'webkit', 'tools', 'layout_tests', 'test_expectations_chrome.txt') host.filesystem.files[chromium_path] = expected_chromium_overrides host.filesystem.files[chrome_path] = None actual_chrome_overrides = port.test_expectations_overrides() self.assertEqual(expected_chromium_overrides, actual_chrome_overrides) host.filesystem.files[chrome_path] = expected_chrome_overrides actual_chrome_overrides = port.test_expectations_overrides() self.assertEqual( actual_chrome_overrides, expected_chromium_overrides + expected_chrome_overrides)
def _verify_expectations_overrides(self, port_name): host = MockSystemHost() chromium_port = PortFactory(host).get("chromium-mac-leopard") chromium_base = chromium_port.path_from_chromium_base() port = PortFactory(host).get(port_name=port_name, options=None) expected_chromium_overrides = '// chromium overrides\n' expected_chrome_overrides = '// chrome overrides\n' chromium_path = host.filesystem.join(chromium_base, 'webkit', 'tools', 'layout_tests', 'test_expectations.txt') chrome_path = host.filesystem.join(chromium_base, 'webkit', 'tools', 'layout_tests', 'test_expectations_chrome.txt') host.filesystem.files[chromium_path] = expected_chromium_overrides host.filesystem.files[chrome_path] = None actual_chrome_overrides = port.test_expectations_overrides() self.assertEqual(expected_chromium_overrides, actual_chrome_overrides) host.filesystem.files[chrome_path] = expected_chrome_overrides actual_chrome_overrides = port.test_expectations_overrides() self.assertEqual(actual_chrome_overrides, expected_chromium_overrides + expected_chrome_overrides)
class Host(SystemHost): def __init__(self): SystemHost.__init__(self) self.web = web.Web() # FIXME: Checkout should own the scm object. self._scm = None self._checkout = None # Everything below this line is WebKit-specific and belongs on a higher-level object. self.bugs = bugzilla.Bugzilla() self.buildbot = buildbot.BuildBot() # FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world. # In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem # so for now we just pass along the whole Host object. # FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency). self.port_factory = PortFactory(self) self._engage_awesome_locale_hacks() # We call this from the Host constructor, as it's one of the # earliest calls made for all webkitpy-based programs. def _engage_awesome_locale_hacks(self): # To make life easier on our non-english users, we override # the locale environment variables inside webkitpy. # If we don't do this, programs like SVN will output localized # messages and svn.py will fail to parse them. # FIXME: We should do these overrides *only* for the subprocesses we know need them! # This hack only works in unix environments. os.environ['LANGUAGE'] = 'en' os.environ['LANG'] = 'en_US.UTF-8' os.environ['LC_MESSAGES'] = 'en_US.UTF-8' os.environ['LC_ALL'] = '' def initialize_scm(self, patch_directories=None): detector = SCMDetector(self.filesystem, self.executive) self._scm = detector.default_scm(patch_directories) self._checkout = Checkout(self.scm()) def scm(self): return self._scm def checkout(self): return self._checkout def buildbot_for_builder_name(self, name): if self.port_factory.get_from_builder_name(name).is_chromium(): return self.chromium_buildbot() return self.buildbot @memoized def watch_list(self): return WatchListLoader(self.filesystem).load()
def assert_port_works(self, port_name, input_name=None, platform=None): host = MockSystemHost() host.filesystem = FileSystem() # FIXME: This test should not use a real filesystem! # test that we got the right port mock_options = MockOptions(accelerated_2d_canvas=None, accelerated_video=None, builder_name='foo', child_processes=None) if input_name and platform: port = PortFactory(host).get(host, platform=platform, port_name=input_name, options=mock_options) else: port = PortFactory(host).get(host, port_name=port_name, options=mock_options) self.assertTrue(port._options.accelerated_2d_canvas) self.assertTrue(port._options.accelerated_video) self.assertTrue(port._options.experimental_fully_parallel) self.assertEqual(port._options.builder_name, 'foo - GPU') self.assertTrue(port.name().startswith(port_name)) # test that it has the right directories in front of the search path. paths = port.baseline_search_path() self.assertEqual(port._webkit_baseline_path(port_name), paths[0]) if port_name == 'chromium-gpu-linux': self.assertEqual(port._webkit_baseline_path('chromium-gpu-win'), paths[1]) self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[2]) else: self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[1]) # Test that we're limiting to the correct directories. # These two tests are picked mostly at random, but we make sure they # exist separately from being filtered out by the port. # Note that this is using a real filesystem. files = port.tests(None) path = 'fast/html/keygen.html' self.assertTrue(port._filesystem.exists(port.abspath_for_test(path))) self.assertFalse(path in files)
def test_pixeltest__fails(self): host = MockSystemHost() url = '#URL:file://' url = url + '%s/failures/expected/image_checksum.html' % PortFactory(host).get('test').layout_tests_dir() self.assertTest('failures/expected/image_checksum.html', pixel_tests=True, expected_checksum='image_checksum', drt_output=[url + '\n', '#MD5:image_checksum-checksum\n', 'image_checksum-txt', '\n', '#EOF\n'], host=host) self.assertEqual(host.filesystem.written_files, {'/tmp/png_result0.png': 'image_checksum\x8a-pngtEXtchecksum\x00image_checksum-checksum'})
def test_basic(self): cmd = [sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"'] host = SystemHost() factory = PortFactory(host) port = factory.get() now = time.time() proc = server_process.ServerProcess(port, 'python', cmd) proc.write('') self.assertEquals(proc.poll(), None) self.assertFalse(proc.has_crashed()) # check that doing a read after an expired deadline returns # nothing immediately. line = proc.read_stdout_line(now - 1) self.assertEquals(line, None) line = proc.read_stdout_line(now + 1.0) self.assertEquals(line.strip(), "stdout") line = proc.read_stderr_line(now + 1.0) self.assertEquals(line.strip(), "stderr") proc.stop()
def __init__(self): SystemHost.__init__(self) self.web = web.Web() self._git = None # Everything below this line is WebKit-specific and belongs on a higher-level object. self.buildbot = BuildBot() # FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world. # In order to instantiate a port correctly, we have to pass it at least an executive, user, git, and filesystem # so for now we just pass along the whole Host object. # FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency). self.port_factory = PortFactory(self) self.builders = BuilderList.load_default_builder_list(self.filesystem)
def __init__(self, log_executive=False, web=None, git=None, os_name=None, os_version=None, time_return_val=123): super(MockHost, self).__init__(log_executive=log_executive, os_name=os_name, os_version=os_version, time_return_val=time_return_val) add_unit_tests_to_mock_filesystem(self.filesystem) self._add_base_manifest_to_mock_filesystem(self.filesystem) self.web = web or MockWeb() self._git = git self.buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self.builders = BuilderList({ 'Fake Test Win10': { 'port_name': 'win-win10', 'specifiers': ['Win10', 'Release'] }, 'Fake Test Linux': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'Fake Test Linux (dbg)': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Debug'] }, 'Fake Test Mac10.12': { 'port_name': 'mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'] }, 'Fake Test Linux Try Bot': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, })
def __init__(self): self.bugs = bugzilla.Bugzilla() self.buildbot = buildbot.BuildBot() self.executive = executive.Executive() self.web = web.Web() self._irc = None self.filesystem = filesystem.FileSystem() self.workspace = workspace.Workspace(self.filesystem, self.executive) self._port = None self.user = user.User() self._scm = None self._checkout = None self.status_server = statusserver.StatusServer() # FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world. # In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem # so for now we just pass along the whole Host object. self.port_factory = PortFactory(self) self.platform = platforminfo.PlatformInfo()
def __init__(self): SystemHost.__init__(self) self.web = web.Web() # FIXME: Checkout should own the scm object. self._scm = None self._checkout = None # Everything below this line is WebKit-specific and belongs on a higher-level object. self.bugs = bugzilla.Bugzilla() self.buildbot = buildbot.BuildBot() # FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world. # In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem # so for now we just pass along the whole Host object. # FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency). self.port_factory = PortFactory(self) self._engage_awesome_locale_hacks()
def __init__( self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm, logged_before=False ): """ Args: running_port: the Port the script is running on. target_port: the Port the script uses to find port-specific configuration information like the test_expectations.txt file location and the list of test platforms. platform: the test platform to rebaseline options: the command-line options object. url_fetcher: object that can fetch objects from URLs zip_factory: optional object that can fetch zip files from URLs scm: scm object for adding new baselines logged_before: whether the previous running port logged anything. """ self._platform = platform self._options = options self._port = running_port self._filesystem = running_port._filesystem self._target_port = target_port # FIXME: This should get its PortFactory from a Host object. # Note: using running_port.executive, running_port.user since we can't get them from a host. self._rebaseline_port = PortFactory().get( platform, options, filesystem=self._filesystem, executive=running_port.executive, user=running_port.user ) self._rebaselining_tests = set() self._rebaselined_tests = [] self._logged_before = logged_before self.did_log = False # Create tests and expectations helper which is used to: # -. compile list of tests that need rebaselining. # -. update the tests in test_expectations file after rebaseline # is done. expectations_str = self._rebaseline_port.test_expectations() self._test_expectations = test_expectations.TestExpectations( self._rebaseline_port, None, expectations_str, self._rebaseline_port.test_configuration(), False ) self._url_fetcher = url_fetcher self._zip_factory = zip_factory self._scm = scm
def __init__(self, log_executive=False, executive_throws_when_run=None, initialize_scm_by_default=True): MockSystemHost.__init__(self, log_executive, executive_throws_when_run) add_unit_tests_to_mock_filesystem(self.filesystem) self.web = MockWeb() self._checkout = MockCheckout() self._scm = None # FIXME: we should never initialize the SCM by default, since the real # object doesn't either. This has caused at least one bug (see bug 89498). if initialize_scm_by_default: self.initialize_scm() self.bugs = MockBugzilla() self.buildbot = MockBuildBot() self._chromium_buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self._watch_list = MockWatchList()
def test_pixeltest__fails(self): host = MockSystemHost() url = '#URL:file://' if sys.platform == 'win32': host = MockSystemHost(os_name='win', os_version='xp') url = '#URL:file:///' url = url + '%s/failures/expected/checksum.html' % PortFactory( host).get('test').layout_tests_dir() self.assertTest('failures/expected/checksum.html', pixel_tests=True, expected_checksum='wrong-checksum', drt_output=[ url + '\n', '#MD5:checksum-checksum\n', 'checksum-txt', '\n', '#EOF\n' ], host=host) self.assertEquals( host.filesystem.written_files, { '/tmp/png_result0.png': 'checksum\x8a-pngtEXtchecksum\x00checksum-checksum' })
def __init__(self, log_executive=False, executive_throws_when_run=None): MockSystemHost.__init__(self, log_executive, executive_throws_when_run) add_unit_tests_to_mock_filesystem(self.filesystem) self.web = MockWeb() self._checkout = MockCheckout() self._scm = MockSCM(filesystem=self.filesystem, executive=self.executive) # Various pieces of code (wrongly) call filesystem.chdir(checkout_root). # Making the checkout_root exist in the mock filesystem makes that chdir not raise. self.filesystem.maybe_make_directory(self._scm.checkout_root) self.bugs = MockBugzilla() self.buildbot = MockBuildBot() self._chromium_buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self._watch_list = MockWatchList()
def __init__(self, log_executive=False, web=None, git=None, os_name=None, os_version=None, time_return_val=123): super(MockHost, self).__init__(log_executive=log_executive, os_name=os_name, os_version=os_version, time_return_val=time_return_val) add_unit_tests_to_mock_filesystem(self.filesystem) self._add_base_manifest_to_mock_filesystem(self.filesystem) self.web = web or MockWeb() self._git = git self.buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self.builders = BuilderList(BUILDERS)
class MockDRT(object): def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0) def run(self): while True: line = self._stdin.readline() if not line: return 0 driver_input = self.input_from_line(line) dirname, basename = self._port.split_test(driver_input.test_name) is_reftest = (self._port.reference_files(driver_input.test_name) or self._port.is_reference_html_file(self._port._filesystem, dirname, basename)) output = self.output_for_test(driver_input, is_reftest) self.write_test_output(driver_input, output, is_reftest) def input_from_line(self, line): vals = line.strip().split("'") uri = vals[0] checksum = None should_run_pixel_tests = False if len(vals) == 2 and vals[1] == '--pixel-test': should_run_pixel_tests = True elif len(vals) == 3 and vals[1] == '--pixel-test': should_run_pixel_tests = True checksum = vals[2] elif len(vals) != 1: raise NotImplementedError if uri.startswith('http://') or uri.startswith('https://'): test_name = self._driver.uri_to_test(uri) else: test_name = self._port.relative_test_filename(uri) return DriverInput(test_name, 0, checksum, should_run_pixel_tests) def output_for_test(self, test_input, is_reftest): port = self._port if self._options.virtual_test_suite_name: test_input.test_name = test_input.test_name.replace(self._options.virtual_test_suite_base, self._options.virtual_test_suite_name) actual_text = port.expected_text(test_input.test_name) actual_audio = port.expected_audio(test_input.test_name) actual_image = None actual_checksum = None if is_reftest: # Make up some output for reftests. actual_text = 'reference text\n' actual_checksum = 'mock-checksum' actual_image = 'blank' if test_input.test_name.endswith('-mismatch.html'): actual_text = 'not reference text\n' actual_checksum = 'not-mock-checksum' actual_image = 'not blank' elif test_input.should_run_pixel_test and test_input.image_hash: actual_checksum = port.expected_checksum(test_input.test_name) actual_image = port.expected_image(test_input.test_name) if self._options.actual_directory: actual_path = port._filesystem.join(self._options.actual_directory, test_input.test_name) root, _ = port._filesystem.splitext(actual_path) text_path = root + '-actual.txt' if port._filesystem.exists(text_path): actual_text = port._filesystem.read_binary_file(text_path) audio_path = root + '-actual.wav' if port._filesystem.exists(audio_path): actual_audio = port._filesystem.read_binary_file(audio_path) image_path = root + '-actual.png' if port._filesystem.exists(image_path): actual_image = port._filesystem.read_binary_file(image_path) with port._filesystem.open_binary_file_for_reading(image_path) as filehandle: actual_checksum = read_checksum_from_png.read_checksum(filehandle) return DriverOutput(actual_text, actual_image, actual_checksum, actual_audio) def write_test_output(self, test_input, output, is_reftest): if output.audio: self._stdout.write('Content-Type: audio/wav\n') self._stdout.write('Content-Transfer-Encoding: base64\n') self._stdout.write(base64.b64encode(output.audio)) self._stdout.write('\n') else: self._stdout.write('Content-Type: text/plain\n') # FIXME: Note that we don't ensure there is a trailing newline! # This mirrors actual (Mac) DRT behavior but is a bug. if output.text: self._stdout.write(output.text) self._stdout.write('#EOF\n') if test_input.should_run_pixel_test and output.image_hash: self._stdout.write('\n') self._stdout.write('ActualHash: %s\n' % output.image_hash) self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash) if output.image_hash != test_input.image_hash: self._stdout.write('Content-Type: image/png\n') self._stdout.write('Content-Length: %s\n' % len(output.image)) self._stdout.write(output.image) self._stdout.write('#EOF\n') self._stdout.flush() self._stderr.write('#EOF\n') self._stderr.flush()
def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__delegate_driver_class = self.__delegate._driver_class self.__delegate._driver_class = types.MethodType( self._driver_class, self.__delegate)
def test_graphics_type(self): port = PortFactory(MockSystemHost()).get('chromium-gpu-mac') self.assertEquals('gpu', port.graphics_type())
def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs)
def _verify_baseline_path(self, expected_path, port_name): port = PortFactory(MockSystemHost()).get(port_name=port_name) path = port.baseline_search_path()[0] self.assertEqual(expected_path, port._filesystem.basename(path))
class MockDRTPort(object): """MockPort implementation of the Port interface.""" def __init__(self, host, **kwargs): prefix = 'mock-' if 'port_name' in kwargs: kwargs['port_name'] = kwargs['port_name'][len(prefix):] self._host = host self.__delegate = PortFactory(host).get(**kwargs) self.__real_name = prefix + self.__delegate.name() def real_name(self): return self.__real_name def __getattr__(self, name): return getattr(self.__delegate, name) def check_build(self, needs_http): return True def check_sys_deps(self, needs_http): return True def driver_cmd_line(self): driver = self.create_driver(0) return driver.cmd_line() def _path_to_driver(self): return self._host.filesystem.abspath(__file__) def create_driver(self, worker_number): # We need to create a driver object as the delegate would, but # overwrite the path to the driver binary in its command line. We do # this by actually overwriting its cmd_line() method with a proxy # method that splices in the mock_drt path and command line arguments # in place of the actual path to the driver binary. def overriding_cmd_line(): cmd = self.__original_driver_cmd_line() index = cmd.index(self.__delegate._path_to_driver()) # FIXME: Why does this need to use sys.executable (instead of something mockable)? cmd[index:index + 1] = [ sys.executable, self._path_to_driver(), '--platform', self.name() ] return cmd delegated_driver = self.__delegate.create_driver(worker_number) self.__original_driver_cmd_line = delegated_driver.cmd_line delegated_driver.cmd_line = overriding_cmd_line return delegated_driver def start_helper(self): pass def start_http_server(self): pass def start_websocket_server(self): pass def acquire_http_lock(self): pass def stop_helper(self): pass def stop_http_server(self): pass def stop_websocket_server(self): pass def release_http_lock(self): pass
class MockDRT(object): def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0) def run(self): while True: line = self._stdin.readline() if not line: return 0 driver_input = self.input_from_line(line) dirname, basename = self._port.split_test(driver_input.test_name) is_reftest = (self._port.reference_files(driver_input.test_name) or self._port.is_reference_html_file(self._port._filesystem, dirname, basename)) output = self.output_for_test(driver_input, is_reftest) self.write_test_output(driver_input, output, is_reftest) def input_from_line(self, line): vals = line.strip().split("'") if len(vals) == 1: uri = vals[0] checksum = None else: uri = vals[0] checksum = vals[1] if uri.startswith('http://') or uri.startswith('https://'): test_name = self._driver.uri_to_test(uri) else: test_name = self._port.relative_test_filename(uri) return DriverInput(test_name, 0, checksum, self._options.pixel_tests) def output_for_test(self, test_input, is_reftest): port = self._port actual_text = port.expected_text(test_input.test_name) actual_audio = port.expected_audio(test_input.test_name) actual_image = None actual_checksum = None if is_reftest: # Make up some output for reftests. actual_text = 'reference text\n' actual_checksum = 'mock-checksum' actual_image = 'blank' if test_input.test_name.endswith('-mismatch.html'): actual_text = 'not reference text\n' actual_checksum = 'not-mock-checksum' actual_image = 'not blank' elif self._options.pixel_tests and test_input.image_hash: actual_checksum = port.expected_checksum(test_input.test_name) actual_image = port.expected_image(test_input.test_name) return DriverOutput(actual_text, actual_image, actual_checksum, actual_audio) def write_test_output(self, test_input, output, is_reftest): if output.audio: self._stdout.write('Content-Type: audio/wav\n') self._stdout.write('Content-Transfer-Encoding: base64\n') self._stdout.write(base64.b64encode(output.audio)) else: self._stdout.write('Content-Type: text/plain\n') # FIXME: Note that we don't ensure there is a trailing newline! # This mirrors actual (Mac) DRT behavior but is a bug. if output.text: self._stdout.write(output.text) self._stdout.write('#EOF\n') if self._options.pixel_tests and output.image_hash: self._stdout.write('\n') self._stdout.write('ActualHash: %s\n' % output.image_hash) self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash) if output.image_hash != test_input.image_hash: self._stdout.write('Content-Type: image/png\n') self._stdout.write('Content-Length: %s\n' % len(output.image)) self._stdout.write(output.image) self._stdout.write('#EOF\n') self._stdout.flush() self._stderr.write('#EOF\n') self._stderr.flush()
def _verify_expectations_overrides(self, port_name): host = MockSystemHost() port = PortFactory(host).get(port_name=port_name, options=None) self.assertTrue('TestExpectations' in port.expectations_files()[0]) self.assertTrue('skia_test_expectations.txt' in port.expectations_files()[1]) self.assertTrue('test_expectations_chrome.txt' in port.expectations_files()[-1])
class Rebaseliner(object): """Class to produce new baselines for a given platform.""" REVISION_REGEX = r"<a href=\"(\d+)/\">" def __init__( self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm, logged_before=False ): """ Args: running_port: the Port the script is running on. target_port: the Port the script uses to find port-specific configuration information like the test_expectations.txt file location and the list of test platforms. platform: the test platform to rebaseline options: the command-line options object. url_fetcher: object that can fetch objects from URLs zip_factory: optional object that can fetch zip files from URLs scm: scm object for adding new baselines logged_before: whether the previous running port logged anything. """ self._platform = platform self._options = options self._port = running_port self._filesystem = running_port._filesystem self._target_port = target_port # FIXME: This should get its PortFactory from a Host object. # Note: using running_port.executive, running_port.user since we can't get them from a host. self._rebaseline_port = PortFactory().get( platform, options, filesystem=self._filesystem, executive=running_port.executive, user=running_port.user ) self._rebaselining_tests = set() self._rebaselined_tests = [] self._logged_before = logged_before self.did_log = False # Create tests and expectations helper which is used to: # -. compile list of tests that need rebaselining. # -. update the tests in test_expectations file after rebaseline # is done. expectations_str = self._rebaseline_port.test_expectations() self._test_expectations = test_expectations.TestExpectations( self._rebaseline_port, None, expectations_str, self._rebaseline_port.test_configuration(), False ) self._url_fetcher = url_fetcher self._zip_factory = zip_factory self._scm = scm def run(self): """Run rebaseline process.""" log_dashed_string("Compiling rebaselining tests", self._platform, logging.DEBUG) if not self._compile_rebaselining_tests(): return False if not self._rebaselining_tests: return True self.did_log = True log_dashed_string("Downloading archive", self._platform, logging.DEBUG) archive_file = self._download_buildbot_archive() _log.debug("") if not archive_file: _log.error("No archive found.") return False log_dashed_string("Extracting and adding new baselines", self._platform, logging.DEBUG) self._extract_and_add_new_baselines(archive_file) archive_file.close() log_dashed_string("Updating rebaselined tests in file", self._platform) if len(self._rebaselining_tests) != len(self._rebaselined_tests): _log.debug("") _log.debug("NOT ALL TESTS WERE REBASELINED.") _log.debug(" Number marked for rebaselining: %d", len(self._rebaselining_tests)) _log.debug(" Number actually rebaselined: %d", len(self._rebaselined_tests)) _log.info("") return False _log.debug(" All tests needing rebaselining were successfully rebaselined.") _log.info("") return True def remove_rebaselining_expectations(self, tests, backup): """if backup is True, we backup the original test expectations file.""" new_expectations = self._test_expectations.remove_rebaselined_tests(tests) path = self._target_port.path_to_test_expectations_file() if backup: date_suffix = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) backup_file = "%s.orig.%s" % (path, date_suffix) if self._filesystem.exists(backup_file): self._filesystem.remove(backup_file) _log.debug('Saving original file to "%s"', backup_file) self._filesystem.move(path, backup_file) self._filesystem.write_text_file(path, new_expectations) # self._scm.add(path) def get_rebaselined_tests(self): return self._rebaselined_tests def _compile_rebaselining_tests(self): """Compile list of tests that need rebaselining for the platform. Returns: False if reftests are wrongly marked as 'needs rebaselining' or True """ self._rebaselining_tests = self._test_expectations.get_rebaselining_failures() if not self._rebaselining_tests: _log.info("%s: No tests to rebaseline.", self._platform) return True fs = self._target_port._filesystem for test in self._rebaselining_tests: test_abspath = self._target_port.abspath_for_test(test) if fs.exists(self._target_port.reftest_expected_filename(test_abspath)) or fs.exists( self._target_port.reftest_expected_mismatch_filename(test_abspath) ): _log.error("%s seems to be a reftest. We can not rebase for reftests.", test) self._rebaselining_tests = set() return False if not self._logged_before: _log.info("") _log.info("%s: Rebaselining %d tests:", self._platform, len(self._rebaselining_tests)) test_no = 1 for test in self._rebaselining_tests: _log.debug(" %d: %s", test_no, test) test_no += 1 return True def _get_latest_revision(self, url): """Get the latest layout test revision number from buildbot. Args: url: Url to retrieve layout test revision numbers. Returns: latest revision or None on failure. """ _log.debug('Url to retrieve revision: "%s"', url) content = self._url_fetcher.fetch(url) revisions = re.findall(self.REVISION_REGEX, content) if not revisions: _log.error('Failed to find revision, content: "%s"', content) return None revisions.sort(key=int) _log.debug(" Latest revision: %s", revisions[len(revisions) - 1]) return revisions[len(revisions) - 1] def _get_archive_dir_name(self, platform): """Get name of the layout test archive directory. Returns: Directory name or None on failure """ if platform in ARCHIVE_DIR_NAME_DICT: return ARCHIVE_DIR_NAME_DICT[platform] else: _log.error("Cannot find platform key %s in archive " "directory name dictionary", platform) return None def _get_archive_url(self): """Generate the url to download latest layout test archive. Returns: Url to download archive or None on failure """ if self._options.force_archive_url: return self._options.force_archive_url dir_name = self._get_archive_dir_name(self._platform) if not dir_name: return None _log.debug('Buildbot platform dir name: "%s"', dir_name) url_base = "%s/%s/" % (self._options.archive_url, dir_name) latest_revision = self._get_latest_revision(url_base) if latest_revision is None or latest_revision <= 0: return None archive_url = "%s%s/layout-test-results.zip" % (url_base, latest_revision) _log.info(" Using %s", archive_url) return archive_url def _download_buildbot_archive(self): """Download layout test archive file from buildbot and return a handle to it.""" url = self._get_archive_url() if url is None: return None archive_file = zipfileset.ZipFileSet(url, filesystem=self._filesystem, zip_factory=self._zip_factory) _log.debug("Archive downloaded") return archive_file def _extract_and_add_new_baselines(self, zip_file): """Extract new baselines from the zip file and add them to SVN repository. Returns: List of tests that have been rebaselined or None on failure.""" zip_namelist = zip_file.namelist() _log.debug("zip file namelist:") for name in zip_namelist: _log.debug(" " + name) _log.debug('Platform dir: "%s"', self._platform) self._rebaselined_tests = [] for test_no, test in enumerate(self._rebaselining_tests): _log.debug("Test %d: %s", test_no + 1, test) self._extract_and_add_new_baseline(test, zip_file) def _extract_and_add_new_baseline(self, test, zip_file): found = False scm_error = False test_basename = self._filesystem.splitext(test)[0] for suffix in BASELINE_SUFFIXES: archive_test_name = "layout-test-results/%s-actual%s" % (test_basename, suffix) _log.debug(' Archive test file name: "%s"', archive_test_name) if not archive_test_name in zip_file.namelist(): _log.debug(" %s file not in archive.", suffix) continue found = True _log.debug(" %s file found in archive.", suffix) temp_name = self._extract_from_zip_to_tempfile(zip_file, archive_test_name) expected_filename = "%s-expected%s" % (test_basename, suffix) expected_fullpath = self._filesystem.join(self._rebaseline_port.baseline_path(), expected_filename) expected_fullpath = self._filesystem.normpath(expected_fullpath) _log.debug(' Expected file full path: "%s"', expected_fullpath) relpath = self._filesystem.relpath(expected_fullpath, self._target_port.layout_tests_dir()) # TODO(victorw): for now, the rebaselining tool checks whether # or not THIS baseline is duplicate and should be skipped. # We could improve the tool to check all baselines in upper # and lower levels and remove all duplicated baselines. if self._is_dup_baseline(temp_name, expected_fullpath, test, suffix, self._platform): self._filesystem.remove(temp_name) if self._filesystem.exists(expected_fullpath): _log.info(" Removing %s" % relpath) self._delete_baseline(expected_fullpath) _log.debug(" %s is a duplicate" % relpath) # FIXME: We consider a duplicate baseline a success in the normal case. # FIXME: This may not be what you want sometimes; should this be # FIXME: controllable? self._rebaselined_tests.append(test) continue if suffix == ".checksum" and self._png_has_same_checksum(temp_name, test, expected_fullpath): self._filesystem.remove(temp_name) # If an old checksum exists, delete it. self._delete_baseline(expected_fullpath) continue self._filesystem.maybe_make_directory(self._filesystem.dirname(expected_fullpath)) self._filesystem.move(temp_name, expected_fullpath) path_from_base = self._filesystem.relpath(expected_fullpath) if self._scm.exists(path_from_base): _log.info(" Updating %s" % relpath) else: _log.info(" Adding %s" % relpath) if self._scm.add(expected_fullpath, return_exit_code=True): # FIXME: print detailed diagnose messages scm_error = True elif suffix != ".checksum": self._create_html_baseline_files(expected_fullpath) if not found: _log.warn("No results in archive for %s" % test) elif scm_error: _log.warn("Failed to add baselines to your repository.") else: _log.debug(" Rebaseline succeeded.") self._rebaselined_tests.append(test) def _extract_from_zip_to_tempfile(self, zip_file, filename): """Extracts |filename| from |zip_file|, a ZipFileSet. Returns the full path name to the extracted file.""" data = zip_file.read(filename) suffix = self._filesystem.splitext(filename)[1] tempfile, temp_name = self._filesystem.open_binary_tempfile(suffix) tempfile.write(data) tempfile.close() return temp_name def _png_has_same_checksum(self, checksum_path, test, checksum_expected_fullpath): """Returns True if the fallback png for |checksum_expected_fullpath| contains the same checksum.""" fs = self._filesystem png_fullpath = self._first_fallback_png_for_test(test) if not fs.exists(png_fullpath): _log.error(" Checksum without png file found! Expected %s to exist." % png_fullpath) return False with fs.open_binary_file_for_reading(png_fullpath) as filehandle: checksum_in_png = read_checksum_from_png.read_checksum(filehandle) checksum_in_text_file = fs.read_text_file(checksum_path) if checksum_in_png and checksum_in_png != checksum_in_text_file: _log.error( " checksum in %s and %s don't match! Continuing" " to copy but please investigate." % (checksum_expected_fullpath, png_fullpath) ) return checksum_in_text_file == checksum_in_png def _first_fallback_png_for_test(self, test): all_baselines = self._rebaseline_port.expected_baselines(test, ".png", True) return self._filesystem.join(all_baselines[0][0], all_baselines[0][1]) def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix, platform): """Check whether a baseline is duplicate and can fallback to same baseline for another platform. For example, if a test has same baseline on linux and windows, then we only store windows baseline and linux baseline will fallback to the windows version. Args: new_baseline: temp filename containing the new baseline results baseline_path: baseline expectation file name. test: test name. suffix: file suffix of the expected results, including dot; e.g. '.txt' or '.png'. platform: baseline platform 'mac', 'win' or 'linux'. Returns: True if the baseline is unnecessary. False otherwise. """ all_baselines = self._rebaseline_port.expected_baselines(test, suffix, True) for fallback_dir, fallback_file in all_baselines: if not fallback_dir or not fallback_file: continue fallback_fullpath = self._filesystem.normpath(self._filesystem.join(fallback_dir, fallback_file)) if fallback_fullpath.lower() == baseline_path.lower(): continue fallback_dir_relpath = self._filesystem.relpath(fallback_dir, self._target_port.layout_tests_dir()) if fallback_dir_relpath == "": fallback_dir_relpath = "<generic>" new_output = self._filesystem.read_binary_file(new_baseline) fallback_output = self._filesystem.read_binary_file(fallback_fullpath) is_image = baseline_path.lower().endswith(".png") if not self._diff_baselines(new_output, fallback_output, is_image): _log.info(" Skipping %s (matches %s)", test, fallback_dir_relpath) return True return False return False def _diff_baselines(self, output1, output2, is_image): """Check whether two baselines are different. Args: output1, output2: contents of the baselines to compare. Returns: True if two files are different or have different extensions. False otherwise. """ if is_image: return self._port.diff_image(output1, output2)[0] return self._port.compare_text(output1, output2) def _delete_baseline(self, filename): """Remove the file from repository and delete it from disk. Args: filename: full path of the file to delete. """ if not filename or not self._filesystem.isfile(filename): return self._scm.delete(filename) def _create_html_baseline_files(self, baseline_fullpath): """Create baseline files (old, new and diff) in html directory. The files are used to compare the rebaselining results. Args: baseline_fullpath: full path of the expected baseline file. """ baseline_relpath = self._filesystem.relpath(baseline_fullpath) _log.debug(' Html: create baselines for "%s"', baseline_relpath) if not baseline_fullpath or not self._filesystem.exists(baseline_fullpath): _log.debug(' Html: Does not exist: "%s"', baseline_fullpath) return if not self._scm.exists(baseline_relpath): _log.debug(' Html: Does not exist in scm: "%s"', baseline_relpath) return # Copy the new baseline to html directory for result comparison. baseline_filename = self._filesystem.basename(baseline_fullpath) new_file = get_result_file_fullpath( self._filesystem, self._options.html_directory, baseline_filename, self._platform, "new" ) self._filesystem.copyfile(baseline_fullpath, new_file) _log.debug(' Html: copied new baseline file from "%s" to "%s".', baseline_fullpath, new_file) # Get the old baseline from the repository and save to the html directory. try: output = self._scm.show_head(baseline_relpath) except ScriptError, e: _log.warning(e) output = "" if (not output) or (output.upper().rstrip().endswith("NO SUCH FILE OR DIRECTORY")): _log.warning(' No base file: "%s"', baseline_fullpath) return base_file = get_result_file_fullpath( self._filesystem, self._options.html_directory, baseline_filename, self._platform, "old" ) if base_file.upper().endswith(".PNG"): self._filesystem.write_binary_file(base_file, output) else: self._filesystem.write_text_file(base_file, output) _log.debug(' Html: created old baseline file: "%s".', base_file) # Get the diff between old and new baselines and save to the html dir. diff_file = get_result_file_fullpath( self._filesystem, self._options.html_directory, baseline_filename, self._platform, "diff" ) has_diff = False if baseline_filename.upper().endswith(".TXT"): output = self._scm.diff_for_file(baseline_relpath, log=_log) if output: self._filesystem.write_text_file(diff_file, output) has_diff = True elif baseline_filename.upper().endswith(".PNG"): old_file = get_result_file_fullpath( self._filesystem, self._options.html_directory, baseline_filename, self._platform, "old" ) new_file = get_result_file_fullpath( self._filesystem, self._options.html_directory, baseline_filename, self._platform, "new" ) _log.debug(' Html: diffing "%s" and "%s"', old_file, new_file) old_output = self._filesystem.read_binary_file(old_file) new_output = self._filesystem.read_binary_file(new_file) image_diff = self._port.diff_image(old_output, new_output)[0] self._filesystem.write_binary_file(diff_file, image_diff) if has_diff: _log.debug(' Html: created baseline diff file: "%s".', diff_file)
class MockDRT(object): @classmethod def determine_full_port_name(cls, host, options, port_name): """Return a fully-specified port name that can be used to construct objects.""" # Subclasses will usually override this. return cls.port_name def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0) def run(self): while True: line = self._stdin.readline() if not line: break self.run_one_test(self.parse_input(line)) return 0 def parse_input(self, line): return _DRTInput(line) def run_one_test(self, test_input): port = self._port if test_input.uri.startswith('http://') or test_input.uri.startswith('https://'): test_name = self._driver.uri_to_test(test_input.uri) else: test_name = port.relative_test_filename(test_input.uri) actual_text = port.expected_text(test_name) actual_audio = port.expected_audio(test_name) if self._options.pixel_tests and test_input.checksum: actual_checksum = port.expected_checksum(test_name) actual_image = port.expected_image(test_name) if actual_audio: self._stdout.write('Content-Type: audio/wav\n') self._stdout.write('Content-Transfer-Encoding: base64\n') output = base64.b64encode(actual_audio) self._stdout.write(output) else: self._stdout.write('Content-Type: text/plain\n') # FIXME: Note that we don't ensure there is a trailing newline! # This mirrors actual (Mac) DRT behavior but is a bug. self._stdout.write(actual_text) self._stdout.write('#EOF\n') if self._options.pixel_tests and test_input.checksum: self._stdout.write('\n') self._stdout.write('ActualHash: %s\n' % actual_checksum) self._stdout.write('ExpectedHash: %s\n' % test_input.checksum) if actual_checksum != test_input.checksum: self._stdout.write('Content-Type: image/png\n') self._stdout.write('Content-Length: %s\n' % len(actual_image)) self._stdout.write(actual_image) self._stdout.write('#EOF\n') self._stdout.flush() self._stderr.flush()
class MockDRTPort(object): """MockPort implementation of the Port interface.""" port_name = 'mock' @classmethod def determine_full_port_name(cls, host, options, port_name): """Return a fully-specified port name that can be used to construct objects.""" # Subclasses will usually override this. return port_name def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__real_name = port_name self._host = host def real_name(self): return self.__real_name def __getattr__(self, name): return getattr(self.__delegate, name) def check_build(self, needs_http): return True def check_sys_deps(self, needs_http): return True def driver_cmd_line(self): driver = self.create_driver(0) return driver.cmd_line() def _path_to_driver(self): return self._host.filesystem.abspath(__file__) def create_driver(self, worker_number): # We need to create a driver object as the delegate would, but # overwrite the path to the driver binary in its command line. We do # this by actually overwriting its cmd_line() method with a proxy # method that splices in the mock_drt path and command line arguments # in place of the actual path to the driver binary. def overriding_cmd_line(): cmd = self.__original_driver_cmd_line() index = cmd.index(self.__delegate._path_to_driver()) # FIXME: Why does this need to use sys.executable (instead of something mockable)? cmd[index:index + 1] = [sys.executable, self._path_to_driver(), '--platform', self.name()] return cmd delegated_driver = self.__delegate.create_driver(worker_number) self.__original_driver_cmd_line = delegated_driver.cmd_line delegated_driver.cmd_line = overriding_cmd_line return delegated_driver def start_helper(self): pass def start_http_server(self): pass def start_websocket_server(self): pass def acquire_http_lock(self): pass def stop_helper(self): pass def stop_http_server(self): pass def stop_websocket_server(self): pass def release_http_lock(self): pass
def _verify_baseline_search_path_startswith(self, port_name, expected_platform_dirs): port = PortFactory(MockSystemHost()).get(port_name=port_name) actual_platform_dirs = [port._filesystem.basename(path) for path in port.baseline_search_path()] self.assertEqual(expected_platform_dirs, actual_platform_dirs[0:len(expected_platform_dirs)])
def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__real_name = port_name self._host = host
class MockDRTPort(object): port_name = 'mock' @classmethod def determine_full_port_name(cls, host, options, port_name): return port_name def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__delegate_driver_class = self.__delegate._driver_class self.__delegate._driver_class = types.MethodType(self._driver_class, self.__delegate) def __getattr__(self, name): return getattr(self.__delegate, name) def check_build(self, needs_http, printer): return True def check_sys_deps(self, needs_http): return True def _driver_class(self, delegate): return self._mocked_driver_maker def _mocked_driver_maker(self, port, worker_number, pixel_tests, no_timeout=False): path_to_this_file = self.host.filesystem.abspath(__file__.replace('.pyc', '.py')) driver = self.__delegate_driver_class()(self, worker_number, pixel_tests, no_timeout) driver.cmd_line = self._overriding_cmd_line(driver.cmd_line, self.__delegate._path_to_driver(), sys.executable, path_to_this_file, self.__delegate.name()) return driver @staticmethod def _overriding_cmd_line(original_cmd_line, driver_path, python_exe, this_file, port_name): def new_cmd_line(pixel_tests, per_test_args): cmd_line = original_cmd_line(pixel_tests, per_test_args) index = cmd_line.index(driver_path) cmd_line[index:index + 1] = [python_exe, this_file, '--platform', port_name] return cmd_line return new_cmd_line def start_helper(self): pass def start_http_server(self, number_of_servers): pass def start_websocket_server(self): pass def acquire_http_lock(self): pass def stop_helper(self): pass def stop_http_server(self): pass def stop_websocket_server(self): pass def release_http_lock(self): pass def _make_wdiff_available(self): self.__delegate._wdiff_available = True def setup_environ_for_server(self, server_name): env = self.__delegate.setup_environ_for_server() # We need to propagate PATH down so the python code can find the checkout. env['PATH'] = os.environ['PATH'] return env def lookup_virtual_test_args(self, test_name): suite = self.__delegate.lookup_virtual_suite(test_name) return suite.args + ['--virtual-test-suite-name', suite.name, '--virtual-test-suite-base', suite.base]
def assert_port_works(self, port_name, input_name=None, platform=None): host = MockSystemHost() host.filesystem = FileSystem( ) # FIXME: This test should not use a real filesystem! # test that we got the right port mock_options = MockOptions(accelerated_2d_canvas=None, accelerated_video=None, builder_name='foo', child_processes=None) if input_name and platform: port = PortFactory(host).get(host, platform=platform, port_name=input_name, options=mock_options) else: port = PortFactory(host).get(host, port_name=port_name, options=mock_options) self.assertTrue(port._options.accelerated_2d_canvas) self.assertTrue(port._options.accelerated_video) self.assertTrue(port._options.experimental_fully_parallel) self.assertEqual(port._options.builder_name, 'foo - GPU') self.assertTrue(port.name().startswith(port_name)) # test that it has the right directories in front of the search path. paths = port.baseline_search_path() self.assertEqual(port._webkit_baseline_path(port_name), paths[0]) if port_name == 'chromium-gpu-linux': self.assertEqual(port._webkit_baseline_path('chromium-gpu-win'), paths[1]) self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[2]) else: self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[1]) # Test that we're limiting to the correct directories. # These two tests are picked mostly at random, but we make sure they # exist separately from being filtered out by the port. # Note that this is using a real filesystem. files = port.tests(None) path = 'fast/html/keygen.html' self.assertTrue(port._filesystem.exists(port.abspath_for_test(path))) self.assertFalse(path in files)
def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__delegate_driver_class = self.__delegate._driver_class self.__delegate._driver_class = types.MethodType(self._driver_class, self.__delegate)
class MockDRT(object): def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0) def run(self): while True: line = self._stdin.readline() if not line: break self.run_one_test(self.parse_input(line)) return 0 def parse_input(self, line): return _DRTInput(line) def run_one_test(self, test_input): port = self._port if test_input.uri.startswith("http://") or test_input.uri.startswith("https://"): test_name = self._driver.uri_to_test(test_input.uri) else: test_name = port.relative_test_filename(test_input.uri) actual_text = port.expected_text(test_name) actual_audio = port.expected_audio(test_name) if self._options.pixel_tests and test_input.checksum: actual_checksum = port.expected_checksum(test_name) actual_image = port.expected_image(test_name) if actual_audio: self._stdout.write("Content-Type: audio/wav\n") self._stdout.write("Content-Transfer-Encoding: base64\n") output = base64.b64encode(actual_audio) self._stdout.write(output) else: self._stdout.write("Content-Type: text/plain\n") # FIXME: Note that we don't ensure there is a trailing newline! # This mirrors actual (Mac) DRT behavior but is a bug. self._stdout.write(actual_text) self._stdout.write("#EOF\n") if self._options.pixel_tests and test_input.checksum: self._stdout.write("\n") self._stdout.write("ActualHash: %s\n" % actual_checksum) self._stdout.write("ExpectedHash: %s\n" % test_input.checksum) if actual_checksum != test_input.checksum: self._stdout.write("Content-Type: image/png\n") self._stdout.write("Content-Length: %s\n" % len(actual_image)) self._stdout.write(actual_image) self._stdout.write("#EOF\n") self._stdout.flush() self._stderr.flush()
class MockDRTPort(object): """MockPort implementation of the Port interface.""" port_name = 'mock' @classmethod def determine_full_port_name(cls, host, options, port_name): """Return a fully-specified port name that can be used to construct objects.""" # Subclasses will usually override this. return port_name def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__real_name = port_name self._host = host def real_name(self): return self.__real_name def __getattr__(self, name): return getattr(self.__delegate, name) def check_build(self, needs_http): return True def check_sys_deps(self, needs_http): return True def driver_cmd_line(self): driver = self.create_driver(0) return driver.cmd_line() def _path_to_driver(self): return self._host.filesystem.abspath(__file__) def create_driver(self, worker_number): # We need to create a driver object as the delegate would, but # overwrite the path to the driver binary in its command line. We do # this by actually overwriting its cmd_line() method with a proxy # method that splices in the mock_drt path and command line arguments # in place of the actual path to the driver binary. def overriding_cmd_line(): cmd = self.__original_driver_cmd_line() index = cmd.index(self.__delegate._path_to_driver()) # FIXME: Why does this need to use sys.executable (instead of something mockable)? cmd[index:index + 1] = [ sys.executable, self._path_to_driver(), '--platform', self.name() ] return cmd delegated_driver = self.__delegate.create_driver(worker_number) self.__original_driver_cmd_line = delegated_driver.cmd_line delegated_driver.cmd_line = overriding_cmd_line return delegated_driver def start_helper(self): pass def start_http_server(self): pass def start_websocket_server(self): pass def acquire_http_lock(self): pass def stop_helper(self): pass def stop_http_server(self): pass def stop_websocket_server(self): pass def release_http_lock(self): pass
class MockDRT(object): def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0) def run(self): self._stdout.write("#READY\n") self._stdout.flush() while True: line = self._stdin.readline() if not line: return 0 driver_input = self.input_from_line(line) dirname, basename = self._port.split_test(driver_input.test_name) is_reftest = (self._port.reference_files(driver_input.test_name) or self._port.is_reference_html_file( self._port._filesystem, dirname, basename)) output = self.output_for_test(driver_input, is_reftest) self.write_test_output(driver_input, output, is_reftest) def input_from_line(self, line): vals = line.strip().split("'") uri = vals[0] checksum = None should_run_pixel_tests = False if len(vals) == 2 and vals[1] == '--pixel-test': should_run_pixel_tests = True elif len(vals) == 3 and vals[1] == '--pixel-test': should_run_pixel_tests = True checksum = vals[2] elif len(vals) != 1: raise NotImplementedError if uri.startswith('http://') or uri.startswith('https://'): test_name = self._driver.uri_to_test(uri) else: test_name = self._port.relative_test_filename(uri) return DriverInput(test_name, 0, checksum, should_run_pixel_tests, args=[]) def output_for_test(self, test_input, is_reftest): port = self._port if self._options.virtual_test_suite_name: test_input.test_name = test_input.test_name.replace( self._options.virtual_test_suite_base, self._options.virtual_test_suite_name) actual_text = port.expected_text(test_input.test_name) actual_audio = port.expected_audio(test_input.test_name) actual_image = None actual_checksum = None if is_reftest: # Make up some output for reftests. actual_text = 'reference text\n' actual_checksum = 'mock-checksum' actual_image = 'blank' if test_input.test_name.endswith('-mismatch.html'): actual_text = 'not reference text\n' actual_checksum = 'not-mock-checksum' actual_image = 'not blank' elif test_input.should_run_pixel_test and test_input.image_hash: actual_checksum = port.expected_checksum(test_input.test_name) actual_image = port.expected_image(test_input.test_name) if self._options.actual_directory: actual_path = port._filesystem.join(self._options.actual_directory, test_input.test_name) root, _ = port._filesystem.splitext(actual_path) text_path = root + '-actual.txt' if port._filesystem.exists(text_path): actual_text = port._filesystem.read_binary_file(text_path) audio_path = root + '-actual.wav' if port._filesystem.exists(audio_path): actual_audio = port._filesystem.read_binary_file(audio_path) image_path = root + '-actual.png' if port._filesystem.exists(image_path): actual_image = port._filesystem.read_binary_file(image_path) with port._filesystem.open_binary_file_for_reading( image_path) as filehandle: actual_checksum = read_checksum_from_png.read_checksum( filehandle) return DriverOutput(actual_text, actual_image, actual_checksum, actual_audio) def write_test_output(self, test_input, output, is_reftest): if output.audio: self._stdout.write('Content-Type: audio/wav\n') self._stdout.write('Content-Transfer-Encoding: base64\n') self._stdout.write(base64.b64encode(output.audio)) self._stdout.write('\n') else: self._stdout.write('Content-Type: text/plain\n') # FIXME: Note that we don't ensure there is a trailing newline! # This mirrors actual (Mac) DRT behavior but is a bug. if output.text: self._stdout.write(output.text) self._stdout.write('#EOF\n') if test_input.should_run_pixel_test and output.image_hash: self._stdout.write('\n') self._stdout.write('ActualHash: %s\n' % output.image_hash) self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash) if output.image_hash != test_input.image_hash: self._stdout.write('Content-Type: image/png\n') self._stdout.write('Content-Length: %s\n' % len(output.image)) self._stdout.write(output.image) self._stdout.write('#EOF\n') self._stdout.flush() self._stderr.write('#EOF\n') self._stderr.flush()
class MockDRT(object): @classmethod def determine_full_port_name(cls, host, options, port_name): """Return a fully-specified port name that can be used to construct objects.""" # Subclasses will usually override this. return cls.port_name def __init__(self, options, args, host, stdin, stdout, stderr): self._options = options self._args = args self._host = host self._stdout = stdout self._stdin = stdin self._stderr = stderr port_name = None if options.platform: port_name = options.platform self._port = PortFactory(host).get(port_name=port_name, options=options) self._driver = self._port.create_driver(0) def run(self): while True: line = self._stdin.readline() if not line: break self.run_one_test(self.parse_input(line)) return 0 def parse_input(self, line): return _DRTInput(line) def run_one_test(self, test_input): port = self._port if test_input.uri.startswith('http://') or test_input.uri.startswith( 'https://'): test_name = self._driver.uri_to_test(test_input.uri) else: test_name = port.relative_test_filename(test_input.uri) actual_text = port.expected_text(test_name) actual_audio = port.expected_audio(test_name) if self._options.pixel_tests and test_input.checksum: actual_checksum = port.expected_checksum(test_name) actual_image = port.expected_image(test_name) if actual_audio: self._stdout.write('Content-Type: audio/wav\n') self._stdout.write('Content-Transfer-Encoding: base64\n') output = base64.b64encode(actual_audio) self._stdout.write(output) else: self._stdout.write('Content-Type: text/plain\n') # FIXME: Note that we don't ensure there is a trailing newline! # This mirrors actual (Mac) DRT behavior but is a bug. self._stdout.write(actual_text) self._stdout.write('#EOF\n') if self._options.pixel_tests and test_input.checksum: self._stdout.write('\n') self._stdout.write('ActualHash: %s\n' % actual_checksum) self._stdout.write('ExpectedHash: %s\n' % test_input.checksum) if actual_checksum != test_input.checksum: self._stdout.write('Content-Type: image/png\n') self._stdout.write('Content-Length: %s\n' % len(actual_image)) self._stdout.write(actual_image) self._stdout.write('#EOF\n') self._stdout.flush() self._stderr.flush()
class MockDRTPort(object): port_name = 'mock' @classmethod def determine_full_port_name(cls, host, options, port_name): return port_name def __init__(self, host, port_name, **kwargs): self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''), **kwargs) self.__delegate_driver_class = self.__delegate._driver_class self.__delegate._driver_class = types.MethodType( self._driver_class, self.__delegate) def __getattr__(self, name): return getattr(self.__delegate, name) def check_build(self, needs_http, printer): return test_run_results.OK_EXIT_STATUS def check_sys_deps(self, needs_http): return test_run_results.OK_EXIT_STATUS def _driver_class(self, delegate): return self._mocked_driver_maker def _mocked_driver_maker(self, port, worker_number, pixel_tests, no_timeout=False): path_to_this_file = self.host.filesystem.abspath( __file__.replace('.pyc', '.py')) driver = self.__delegate_driver_class()(self, worker_number, pixel_tests, no_timeout) driver.cmd_line = self._overriding_cmd_line( driver.cmd_line, self.__delegate._path_to_driver(), sys.executable, path_to_this_file, self.__delegate.name()) return driver @staticmethod def _overriding_cmd_line(original_cmd_line, driver_path, python_exe, this_file, port_name): def new_cmd_line(pixel_tests, per_test_args): cmd_line = original_cmd_line(pixel_tests, per_test_args) index = cmd_line.index(driver_path) cmd_line[index:index + 1] = [python_exe, this_file, '--platform', port_name] return cmd_line return new_cmd_line def start_helper(self): pass def start_http_server(self, additional_dirs, number_of_servers): pass def start_websocket_server(self): pass def acquire_http_lock(self): pass def stop_helper(self): pass def stop_http_server(self): pass def stop_websocket_server(self): pass def release_http_lock(self): pass def _make_wdiff_available(self): self.__delegate._wdiff_available = True def setup_environ_for_server(self): env = self.__delegate.setup_environ_for_server() # We need to propagate PATH down so the python code can find the checkout. env['PATH'] = os.environ['PATH'] return env def lookup_virtual_test_args(self, test_name): suite = self.__delegate.lookup_virtual_suite(test_name) return suite.args + [ '--virtual-test-suite-name', suite.name, '--virtual-test-suite-base', suite.base ] def lookup_virtual_reference_args(self, test_name): suite = self.__delegate.lookup_virtual_suite(test_name) return suite.reference_args + [ '--virtual-test-suite-name', suite.name, '--virtual-test-suite-base', suite.base ]
def _assert_baseline_path(self, port_name, baseline_path): port = PortFactory(MockSystemHost()).get(port_name) self.assertEquals(port.name(), port_name) self.assertEquals(port.baseline_path(), port._webkit_baseline_path(baseline_path))
def test_paths(port_name): return PortFactory(host).get(port_name).tests([])
def test_paths(port_name): return chromium_gpu._default_tests_paths( PortFactory(MockSystemHost()).get(port_name))
class MockDRTPort(object): """MockPort implementation of the Port interface.""" def __init__(self, host, **kwargs): prefix = "mock-" if "port_name" in kwargs: kwargs["port_name"] = kwargs["port_name"][len(prefix) :] self._host = host self.__delegate = PortFactory(host).get(**kwargs) self.__real_name = prefix + self.__delegate.name() def real_name(self): return self.__real_name def __getattr__(self, name): return getattr(self.__delegate, name) def check_build(self, needs_http): return True def check_sys_deps(self, needs_http): return True def driver_cmd_line(self): driver = self.create_driver(0) return driver.cmd_line() def _path_to_driver(self): return self._host.filesystem.abspath(__file__) def create_driver(self, worker_number): # We need to create a driver object as the delegate would, but # overwrite the path to the driver binary in its command line. We do # this by actually overwriting its cmd_line() method with a proxy # method that splices in the mock_drt path and command line arguments # in place of the actual path to the driver binary. def overriding_cmd_line(): cmd = self.__original_driver_cmd_line() index = cmd.index(self.__delegate._path_to_driver()) # FIXME: Why does this need to use sys.executable (instead of something mockable)? cmd[index : index + 1] = [sys.executable, self._path_to_driver(), "--platform", self.name()] return cmd delegated_driver = self.__delegate.create_driver(worker_number) self.__original_driver_cmd_line = delegated_driver.cmd_line delegated_driver.cmd_line = overriding_cmd_line return delegated_driver def start_helper(self): pass def start_http_server(self): pass def start_websocket_server(self): pass def acquire_http_lock(self): pass def stop_helper(self): pass def stop_http_server(self): pass def stop_websocket_server(self): pass def release_http_lock(self): pass