def run_test(self,
tests,
expected_stdout,
platform='test-win-win7',
**kwargs):
options_defaults = {
'all': False,
'csv': False,
'full': False,
'platform': platform,
'include_keyword': [],
'exclude_keyword': [],
'paths': False,
}
options_defaults.update(kwargs)
options = optparse.Values(dict(**options_defaults))
tool = MockBlinkTool()
tool.port_factory.all_port_names = lambda: [
'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11',
'test-mac-mac10.10', 'test-win-win10', 'test-win-win7'
]
command = PrintExpectations()
oc = OutputCapture()
try:
oc.capture_output()
command.execute(options, tests, tool)
finally:
stdout, _, _ = oc.restore_output()
self.assertMultiLineEqual(stdout, expected_stdout)
class OutputCaptureTest(unittest.TestCase):
def setUp(self):
self.output = OutputCapture()
def log_all_levels(self):
_log.info('INFO')
_log.warning('WARN')
_log.error('ERROR')
_log.critical('CRITICAL')
def assert_logged(self, expected_logs):
actual_stdout, actual_stderr, actual_logs = self.output.restore_output(
)
self.assertEqual('', actual_stdout)
self.assertEqual('', actual_stderr)
self.assertMultiLineEqual(expected_logs, actual_logs)
def test_initial_log_level(self):
self.output.capture_output()
self.log_all_levels()
self.assert_logged('INFO\nWARN\nERROR\nCRITICAL\n')
def test_set_log_level(self):
self.output.set_log_level(logging.ERROR)
self.output.capture_output()
self.log_all_levels()
self.output.set_log_level(logging.WARN)
self.log_all_levels()
self.assert_logged('ERROR\nCRITICAL\nWARN\nERROR\nCRITICAL\n')
def test_start_cmd(self):
# Fails on win - see https://bugs.webkit.org/show_bug.cgi?id=84726
if sys.platform == 'win32':
return
def fake_pid(_):
host.filesystem.write_text_file('/tmp/WebKit/httpd.pid', '42')
return True
host = MockHost()
host.executive = MockExecutive(should_log=True)
test_port = test.TestPort(host)
host.filesystem.write_text_file(test_port.path_to_apache_config_file(),
'')
server = ApacheHTTP(test_port,
'/mock/output_dir',
additional_dirs=[],
number_of_servers=4)
server._check_that_all_ports_are_available = lambda: True
server._is_server_running_on_all_ports = lambda: True
server._wait_for_action = fake_pid
oc = OutputCapture()
try:
oc.capture_output()
server.start()
server.stop()
finally:
_, _, logs = oc.restore_output()
self.assertIn('StartServers 4', logs)
self.assertIn('MinSpareServers 4', logs)
self.assertIn('MaxSpareServers 4', logs)
def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
oc = OutputCapture()
try:
oc.capture_output()
self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
finally:
_, _, logs = oc.restore_output()
self.assertNotEquals(logs, '')
def test_help_command(self):
oc = OutputCapture()
oc.capture_output()
tool = BlinkTool('path')
tool.main(['tool', 'help'])
out, err, logs = oc.restore_output()
self.assertTrue(out.startswith('Usage: '))
self.assertEqual('', err)
self.assertEqual('', logs)
def test_skipped_entry_dont_exist(self):
port = MockHost().port_factory.get(
'test-win-win7',
options=optparse.Values({'ignore_tests': ['foo/bar/baz.html']}))
expectations_dict = OrderedDict()
expectations_dict['expectations'] = ''
port.expectations_dict = lambda: expectations_dict
capture = OutputCapture()
capture.capture_output()
TestExpectations(port)
_, _, logs = capture.restore_output()
self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
def test_help_argument(self):
oc = OutputCapture()
oc.capture_output()
tool = BlinkTool('path')
try:
tool.main(['tool', '--help'])
except SystemExit:
pass # optparse calls sys.exit after showing help.
finally:
out, err, logs = oc.restore_output()
self.assertTrue(out.startswith('Usage: '))
self.assertEqual('', err)
self.assertEqual('', logs)
def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
self.tool.executive = MockExecutive()
port = self.tool.port_factory.get('test-win-win7')
baseline_relative_path = 'platform/test-win-win10/failures/expected/image-expected.txt'
baseline_local_absolute_path = port.host.filesystem.join(
port.web_tests_dir(), baseline_relative_path)
self._write(baseline_local_absolute_path, 'original win10 result')
actual_result_url = (
'https://test-results.appspot.com/data/layout_results/MOCK_Win10/'
+ 'results/layout-test-results/failures/expected/image-actual.txt')
self.tool.web.urls[actual_result_url] = 'new win10 result'
oc = OutputCapture()
try:
options = optparse.Values({
'optimize': True,
'builder': 'MOCK Win10',
'port_name': None,
'suffixes': 'txt',
'verbose': True,
'test': 'failures/expected/image.html',
'results_directory': None,
'build_number': None,
'step_name': None,
})
oc.capture_output()
self.command.execute(options, [], self.tool)
finally:
out, _, _ = oc.restore_output()
self.assertItemsEqual(self.tool.web.urls_fetched, [actual_result_url])
self.assertMultiLineEqual(self._read(baseline_local_absolute_path),
'new win10 result')
self.assertFalse(
self.tool.filesystem.exists(
self.tool.filesystem.join(
port.web_tests_dir(),
'platform/test-win-win7/failures/expected/image-expected.txt'
)))
self.assertMultiLineEqual(
out,
'{"remove-lines": [{"test": "failures/expected/image.html", "port_name": "test-win-win10"}]}\n'
)
def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
self.tool.executive = MockExecutive()
port = self.tool.port_factory.get('test-win-win7')
self._write(
port.host.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win10/failures/expected/image-expected.txt'
), 'original win10 result')
oc = OutputCapture()
try:
options = optparse.Values({
'optimize': True,
'builder': 'MOCK Win10',
'port_name': None,
'suffixes': 'txt',
'verbose': True,
'test': 'failures/expected/image.html',
'results_directory': None,
'build_number': None,
'step_name': None,
})
oc.capture_output()
self.command.execute(options, [], self.tool)
finally:
out, _, _ = oc.restore_output()
self.assertMultiLineEqual(
self._read(
self.tool.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win10/failures/expected/image-expected.txt'
)), 'MOCK Web result, convert 404 to None=True')
self.assertFalse(
self.tool.filesystem.exists(
self.tool.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win7/failures/expected/image-expected.txt'
)))
self.assertMultiLineEqual(
out,
'{"remove-lines": [{"test": "failures/expected/image.html", "port_name": "test-win-win10"}]}\n'
)
class PrintBaselinesTest(unittest.TestCase):
def setUp(self):
self.oc = None
self.tool = MockBlinkTool()
self.test_port = self.tool.port_factory.get('test-win-win7')
self.tool.port_factory.get = lambda port_name=None: self.test_port
self.tool.port_factory.all_port_names = lambda: [
'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11',
'test-mac-mac10.10', 'test-win-win10', 'test-win-win7'
]
def tearDown(self):
if self.oc:
self.restore_output()
def capture_output(self):
self.oc = OutputCapture()
self.oc.capture_output()
def restore_output(self):
stdout, stderr, logs = self.oc.restore_output()
self.oc = None
return (stdout, stderr, logs)
def test_basic(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'include_virtual_tests': False,
'csv': False,
'platform': None
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, ('// For test-win-win7\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'))
def test_multiple(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'include_virtual_tests': False,
'csv': False,
'platform': 'test-win-*'
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, ('// For test-win-win10\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'
'\n'
'// For test-win-win7\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'))
def test_csv(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'platform': '*win7',
'csv': True,
'include_virtual_tests': False
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, (
'test-win-win7,passes/text.html,None,png,passes/text-expected.png,None\n'
'test-win-win7,passes/text.html,None,txt,passes/text-expected.txt,None\n'
))