def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): self.tool.executive = MockExecutive() port = self.tool.port_factory.get('test-win-win7') self._write( port.host.filesystem.join( port.layout_tests_dir(), 'platform/test-win-win10/failures/expected/image-expected.txt'), 'original win10 result') oc = OutputCapture() try: options = optparse.Values({ 'optimize': True, 'builder': 'MOCK Win10', 'suffixes': 'txt', 'verbose': True, 'test': 'failures/expected/image.html', 'results_directory': None, 'build_number': None }) oc.capture_output() self.command.execute(options, [], self.tool) finally: out, _, _ = oc.restore_output() self.assertMultiLineEqual( self._read(self.tool.filesystem.join( port.layout_tests_dir(), 'platform/test-win-win10/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True') self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'))) self.assertMultiLineEqual( out, '{"remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win10"}]}\n')
def test_rebaseline_reftest(self): self._write('userscripts/another-test.html', 'test data') self._write('userscripts/another-test-expected.html', 'generic result') OutputCapture().assert_outputs( self, self.command._rebaseline_test_and_update_expectations, args=[self.options(suffixes='png')], expected_logs='Cannot rebaseline image result for reftest: userscripts/another-test.html\n') self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {'remove-lines': []})
def test_convert_attributes_if_needed(self): """Tests convert_attributes_if_needed() using a reference file that has some relative src paths.""" test_html = """<html> <head> <script src="../../some-script.js"></script> <style src="../../../some-style.css"></style> </head> <body> <img src="../../../../some-image.jpg"> </body> </html> """ test_reference_support_info = { 'reference_relpath': '../', 'files': ['../../some-script.js', '../../../some-style.css', '../../../../some-image.jpg'], 'elements': ['script', 'style', 'img'] } converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, test_reference_support_info) oc = OutputCapture() oc.capture_output() try: converter.feed(test_html) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_reference_relative_paths(converted, test_reference_support_info)
def run_test(self, tests, expected_stdout, platform='test-win-win7', **kwargs): options_defaults = { 'all': False, 'csv': False, 'full': False, 'platform': platform, 'include_keyword': [], 'exclude_keyword': [], 'paths': False, } options_defaults.update(kwargs) options = optparse.Values(dict(**options_defaults)) tool = MockBlinkTool() tool.port_factory.all_port_names = lambda: [ 'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11', 'test-mac-mac10.10', 'test-win-win10', 'test-win-win7' ] command = PrintExpectations() oc = OutputCapture() try: oc.capture_output() command.execute(options, tests, tool) finally: stdout, _, _ = oc.restore_output() self.assertMultiLineEqual(stdout, expected_stdout)
def assert_execute_outputs(self, command, args=None, expected_stdout='', expected_stderr='', expected_exception=None, expected_logs=None, options=optparse.Values(), tool=MockBlinkTool()): args = args or [] options.blocks = None options.cc = 'MOCK cc' options.component = 'MOCK component' options.confirm = True options.email = 'MOCK email' options.git_commit = 'MOCK git commit' options.obsolete_patches = True options.open_bug = True options.port = 'MOCK port' options.update_changelogs = False options.quiet = True options.reviewer = 'MOCK reviewer' OutputCapture().assert_outputs(self, command.execute, [options, args, tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr, expected_exception=expected_exception, expected_logs=expected_logs)
def test_analyze_test_reftest_match_and_mismatch(self): test_html = """<head> <link rel="match" href="green-box-ref.xht" /> <link rel="match" href="blue-box-ref.xht" /> <link rel="mismatch" href="orange-box-notref.xht" /> </head> """ oc = OutputCapture() oc.capture_output() try: test_path = '/some/madeup/path/' parser = TestParser(test_path + 'somefile.html', MockHost()) test_info = parser.analyze_test(test_contents=test_html) finally: _, _, logs = oc.restore_output() self.assertNotEqual(test_info, None, 'did not find a test') self.assertTrue('test' in test_info.keys(), 'did not find a test file') self.assertTrue('reference' in test_info.keys(), 'did not find a reference file') self.assertTrue(test_info['reference'].startswith(test_path), 'reference path is not correct') self.assertFalse('refsupport' in test_info.keys(), 'there should be no refsupport files for this test') self.assertFalse('jstest' in test_info.keys(), 'test should not have been analyzed as a jstest') self.assertEqual( logs, 'Multiple references are not supported. Importing the first ref defined in somefile.html\n' )
def test_parse_output_with_failing_line(self): output = DriverOutput(""" Running 20 times Ignoring warm-up run (1115) some-unrecognizable-line Time: values 1080, 1120, 1095, 1101, 1104 ms avg 1100 ms median 1101 ms stdev 14.50862 ms min 1080 ms max 1120 ms """, image=None, image_hash=None, audio=None) output_capture = OutputCapture() output_capture.capture_output() try: test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') test.run_single = lambda driver, path, time_out_ms: output self.assertFalse(test._run_with_driver(None, None)) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'ERROR: some-unrecognizable-line\n')
def test_parse_output_with_subtests(self): output = DriverOutput(""" Running 20 times some test: [1, 2, 3, 4, 5] other test = else: [6, 7, 8, 9, 10] Ignoring warm-up run (1115) Time: values 1080, 1120, 1095, 1101, 1104 ms avg 1100 ms median 1101 ms stdev 14.50862 ms min 1080 ms max 1120 ms """, image=None, image_hash=None, audio=None) output_capture = OutputCapture() output_capture.capture_output() try: test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') self._assert_results_are_correct(test, output) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, '')
def test_start_cmd(self): # Fails on win - see https://bugs.webkit.org/show_bug.cgi?id=84726 if sys.platform in ('cygwin', 'win32'): return def fake_pid(_): host.filesystem.write_text_file('/tmp/WebKit/httpd.pid', '42') return True host = MockHost() host.executive = MockExecutive(should_log=True) test_port = test.TestPort(host) host.filesystem.write_text_file(test_port.path_to_apache_config_file(), '') server = ApacheHTTP(test_port, "/mock/output_dir", additional_dirs=[], number_of_servers=4) server._check_that_all_ports_are_available = lambda: True server._is_server_running_on_all_ports = lambda: True server._wait_for_action = fake_pid oc = OutputCapture() try: oc.capture_output() server.start() server.stop() finally: _, _, logs = oc.restore_output() self.assertIn("StartServers 4", logs) self.assertIn("MinSpareServers 4", logs) self.assertIn("MaxSpareServers 4", logs)
def test_convert_for_webkit_nothing_to_convert(self): """Tests convert_for_webkit() using a basic test that has nothing to convert.""" test_html = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html xmlns="http://www.w3.org/1999/xhtml"> <head> <title>CSS Test: DESCRIPTION OF TEST</title> <link rel="author" title="NAME_OF_AUTHOR" href="mailto:EMAIL OR http://CONTACT_PAGE"/> <link rel="help" href="RELEVANT_SPEC_SECTION"/> <meta name="assert" content="TEST ASSERTION"/> <style type="text/css"><![CDATA[ CSS FOR TEST ]]></style> </head> <body> CONTENT OF TEST </body> </html> """ converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None) oc = OutputCapture() oc.capture_output() try: converter.feed(test_html) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_no_conversion_happened(converted, test_html)
def test_convert_for_webkit_properties_only(self): """Tests convert_for_webkit() using a test that has 2 prefixed properties: 1 in a style block + 1 inline style.""" test_html = """<html> <head> <link href="/resources/testharness.css" rel="stylesheet" type="text/css"> <script src="/resources/testharness.js"></script> <style type="text/css"> #block1 { @test0@: propvalue; } </style> </head> <body> <div id="elem1" style="@test1@: propvalue;"></div> </body> </html> """ fake_dir_path = self.fake_dir_path('harnessandprops') converter = _W3CTestConverter(fake_dir_path, DUMMY_FILENAME, None) test_content = self.generate_test_content(converter.prefixed_properties, 1, test_html) oc = OutputCapture() oc.capture_output() try: converter.feed(test_content[1]) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_prefixed_properties(converted, test_content[0])
def test_check_httpd_success(self): port = self.make_port(executive=MockExecutive()) port.path_to_apache = lambda: '/usr/sbin/httpd' capture = OutputCapture() capture.capture_output() self.assertTrue(port.check_httpd()) _, _, logs = capture.restore_output() self.assertEqual('', logs)
def test_httpd_returns_error_code(self): port = self.make_port(executive=MockExecutive(exit_code=1)) port.path_to_apache = lambda: '/usr/sbin/httpd' capture = OutputCapture() capture.capture_output() self.assertFalse(port.check_httpd()) _, _, logs = capture.restore_output() self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
def test_help_command(self): oc = OutputCapture() oc.capture_output() tool = WebKitPatch('path') tool.main(['tool', 'help']) out, err, logs = oc.restore_output() self.assertTrue(out.startswith('Usage: ')) self.assertEqual('', err) self.assertEqual('', logs)
def test_pretty_patch_os_error(self): port = self.make_port(executive=MockExecutive(exception=OSError)) oc = OutputCapture() oc.capture_output() self.assertEqual(port.pretty_patch_text("patch.txt"), port._pretty_patch_error_html) # This tests repeated calls to make sure we cache the result. self.assertEqual(port.pretty_patch_text("patch.txt"), port._pretty_patch_error_html) oc.restore_output()
def test_rebaseline_expectations_noop(self): self._zero_out_test_expectations() oc = OutputCapture() try: oc.capture_output() self.command.execute(self.options(), [], self.tool) finally: _, _, logs = oc.restore_output() self.assertEqual(self.tool.filesystem.written_files, {}) self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n')
def test_run_test_set_for_parser_tests(self): runner, _ = self.create_runner() tests = self._tests_for_runner(runner, ['Bindings/event-target-wrapper.html', 'Parser/some-parser.html']) output = OutputCapture() output.capture_output() try: unexpected_result_count = runner._run_tests_set(tests) finally: _, _, log = output.restore_output() self.assertEqual(unexpected_result_count, 0) self.assertEqual(self._normalize_output(log), EventTargetWrapperTestData.output + SomeParserTestData.output)
def test_run_test_set(self): runner, _ = self.create_runner() tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html', 'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html']) output = OutputCapture() output.capture_output() try: unexpected_result_count = runner._run_tests_set(tests) finally: _, _, log = output.restore_output() self.assertEqual(unexpected_result_count, len(tests) - 1) self.assertTrue('\nRESULT group_name: test_name= 42 ms\n' in log)
def test_help_argument(self): oc = OutputCapture() oc.capture_output() tool = WebKitPatch('path') try: tool.main(['tool', '--help']) except SystemExit: pass # optparse calls sys.exit after showing help. finally: out, err, logs = oc.restore_output() self.assertTrue(out.startswith('Usage: ')) self.assertEqual('', err) self.assertEqual('', logs)
def test_create_zip(self): workspace = Workspace(None, MockExecutive(should_log=True)) expected_logs = "MOCK run_command: ['zip', '-9', '-r', '/zip/path', '.'], cwd=/source/path\n" class MockZipFile(object): def __init__(self, path): self.filename = path archive = OutputCapture().assert_outputs( self, workspace.create_zip, ["/zip/path", "/source/path", MockZipFile], expected_logs=expected_logs) self.assertEqual(archive.filename, "/zip/path")
def test_required_arguments(self): class TrivialCommandWithRequiredAndOptionalArgs(TrivialCommand): argument_names = "ARG1 ARG2 [ARG3]" two_required_arguments = TrivialCommandWithRequiredAndOptionalArgs() expected_logs = ( "2 arguments required, 1 argument provided. Provided: 'foo' Required: ARG1 ARG2\n" "See 'dummy-tool help trivial' for usage.\n") exit_code = OutputCapture().assert_outputs( self, two_required_arguments.check_arguments_and_execute, [None, ["foo"], DummyTool()], expected_logs=expected_logs) self.assertEqual(exit_code, 1)
def _test_run_with_json_output(self, runner, filesystem, upload_succeeds=False, results_shown=True, expected_exit_code=0, repeat=1, compare_logs=True): filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') filesystem.write_text_file( runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') uploaded = [False] def mock_upload_json(hostname, json_path, host_path=None): # FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition. self.assertIn(hostname, ['some.host']) self.assertIn(json_path, ['/mock-checkout/output.json']) self.assertIn(host_path, [None, '/api/report']) uploaded[0] = upload_succeeds return upload_succeeds runner._upload_json = mock_upload_json runner._timestamp = 123456789 runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37, 460000) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(runner.run(), expected_exit_code) finally: _, _, logs = output_capture.restore_output() if not expected_exit_code and compare_logs: expected_logs = '' for i in xrange(repeat): runs = ' (Run %d of %d)' % (i + 1, repeat) if repeat > 1 else '' expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output if results_shown: expected_logs += 'MOCK: user.open_url: file://...\n' self.assertEqual(self._normalize_output(logs), expected_logs) self.assertEqual(uploaded[0], upload_succeeds) return logs
def test_run_memory_test(self): runner, port = self.create_runner_and_setup_results_template() runner._timestamp = 123456789 port.host.filesystem.write_text_file(runner._base_path + '/Parser/memory-test.html', 'some content') output = OutputCapture() output.capture_output() try: unexpected_result_count = runner.run() finally: _, _, log = output.restore_output() self.assertEqual(unexpected_result_count, 0) self.assertEqual(self._normalize_output(log), MemoryTestData.output + '\nMOCK: user.open_url: file://...\n') parser_tests = self._load_output_json(runner)[0]['tests']['Parser']['tests'] self.assertEqual(parser_tests['memory-test']['metrics']['Time'], MemoryTestData.results) self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results) self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results)
def test_optimize_all_suffixes_by_default(self): test_port = self.tool.port_factory.get('test') self._write_test_file(test_port, 'another/test.html', "Dummy test contents") self._write_test_file( test_port, 'platform/test-mac-mac10.10/another/test-expected.txt', "result A") self._write_test_file( test_port, 'platform/test-mac-mac10.10/another/test-expected.png', "result A png") self._write_test_file(test_port, 'another/test-expected.txt', "result A") self._write_test_file(test_port, 'another/test-expected.png', "result A png") try: oc = OutputCapture() oc.capture_output() self.command.execute( optparse.Values({ 'suffixes': 'txt,wav,png', 'no_modify_scm': True, 'platform': 'test-mac-mac10.10' }), ['another/test.html'], self.tool) finally: oc.restore_output() self.assertFalse( self.tool.filesystem.exists( self.tool.filesystem.join( test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt'))) self.assertFalse( self.tool.filesystem.exists( self.tool.filesystem.join( test_port.layout_tests_dir(), 'platform/mac/another/test-expected.png'))) self.assertTrue( self.tool.filesystem.exists( self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt'))) self.assertTrue( self.tool.filesystem.exists( self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.png')))
def run_prompt_test(inputs, expected_result, can_choose_multiple=False): def mock_raw_input(message): return inputs.pop(0) output_capture = OutputCapture() actual_result = output_capture.assert_outputs( self, User.prompt_with_list, args=['title', ['foo', 'bar']], kwargs={ 'can_choose_multiple': can_choose_multiple, 'raw_input': mock_raw_input }, expected_stdout='title\n 1. foo\n 2. bar\n') self.assertEqual(actual_result, expected_result) self.assertEqual(len(inputs), 0)
def test_create_zip_exception(self): workspace = Workspace( None, MockExecutive(should_log=True, should_throw=True)) expected_logs = """MOCK run_command: ['zip', '-9', '-r', '/zip/path', '.'], cwd=/source/path Workspace.create_zip failed in /source/path: MOCK ScriptError output: MOCK output of child process """ class MockZipFile(object): def __init__(self, path): self.filename = path archive = OutputCapture().assert_outputs( self, workspace.create_zip, ["/zip/path", "/source/path", MockZipFile], expected_logs=expected_logs) self.assertIsNone(archive)
def setUp(self): self.output = OutputCapture()
def capture_output(self): self.oc = OutputCapture() self.oc.capture_output()
def test_upload_json(self): runner, port = self.create_runner() port.host.filesystem.files['/mock-checkout/some.json'] = 'some content' class MockFileUploader: called = [] upload_single_text_file_throws = False upload_single_text_file_return_value = None @classmethod def reset(cls): cls.called = [] cls.upload_single_text_file_throws = False cls.upload_single_text_file_return_value = None def __init__(mock, url, timeout): self.assertEqual(url, 'https://some.host/some/path') self.assertTrue(isinstance(timeout, int) and timeout) mock.called.append('FileUploader') def upload_single_text_file(mock, filesystem, content_type, filename): self.assertEqual(filesystem, port.host.filesystem) self.assertEqual(content_type, 'application/json') self.assertEqual(filename, 'some.json') mock.called.append('upload_single_text_file') if mock.upload_single_text_file_throws: raise Exception return mock.upload_single_text_file_return_value MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( 'OK') self.assertTrue( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) MockFileUploader.reset() MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( 'Some error') output = OutputCapture() output.capture_output() self.assertFalse( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) _, _, logs = output.restore_output() self.assertEqual( logs, 'Uploaded JSON to https://some.host/some/path but got a bad response:\nSome error\n' ) # Throwing an exception upload_single_text_file shouldn't blow up _upload_json MockFileUploader.reset() MockFileUploader.upload_single_text_file_throws = True self.assertFalse( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) MockFileUploader.reset() MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( '{"status": "OK"}') self.assertTrue( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) MockFileUploader.reset() MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( '{"status": "SomethingHasFailed", "failureStored": false}') output = OutputCapture() output.capture_output() self.assertFalse( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) _, _, logs = output.restore_output() serialized_json = json.dumps( { 'status': 'SomethingHasFailed', 'failureStored': False }, indent=4) self.assertEqual( logs, 'Uploaded JSON to https://some.host/some/path but got an error:\n%s\n' % serialized_json)
def test_convert_prefixed_properties(self): """Tests convert_prefixed_properties() file that has 20 properties requiring the -webkit- prefix. The properties are: 10 in one style block + 5 in another style block + 5 inline styles, including one with multiple prefixed properties. 2 when prefixed properties appear in comments without ending ';'. The properties in the test content are in all sorts of wack formatting. """ test_html = """<html> <style type="text/css"><![CDATA[ .block1 { width: 300px; height: 300px } .block2 { @test0@: propvalue; } .block3{@test1@: propvalue;} .block4 { @test2@:propvalue; } .block5{ @test3@ :propvalue; } #block6 { @test4@ : propvalue; } #block7 { @test5@: propvalue; } #block8 { @test6@: propvalue; } #block9:pseudo { @test7@: propvalue; @test8@: propvalue propvalue propvalue; } ]]></style> </head> <body> <div id="elem1" style="@test9@: propvalue;"></div> <div id="elem2" style="propname: propvalue; @test10@ : propvalue; propname:propvalue;"></div> <div id="elem2" style="@test11@: propvalue; @test12@ : propvalue; @test13@ :propvalue;"></div> <div id="elem3" style="@test14@:propvalue"></div> </body> <style type="text/css"><![CDATA[ .block10{ @test15@: propvalue; } .block11{ @test16@: propvalue; } .block12{ @test17@: propvalue; } #block13:pseudo { @test18@: propvalue; @test19@: propvalue; } #missing-semicolon-in-comments { /* @test20@: propvalue */ @test21@: propvalue; } ]]></style> </html> """ converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None) test_content = self.generate_test_content(converter.prefixed_properties, 22, test_html) oc = OutputCapture() oc.capture_output() try: converter.feed(test_content[1]) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_prefixed_properties(converted, test_content[0])