def test_help_command(self): oc = OutputCapture() oc.capture_output() tool = WebKitPatch('path') tool.main(['tool', 'help']) out, err, logs = oc.restore_output() self.assertTrue(out.startswith('Usage: ')) self.assertEqual('', err) self.assertEqual('', logs)
def test_run_test_set_for_parser_tests(self): runner, _ = self.create_runner() tests = self._tests_for_runner(runner, ['Bindings/event-target-wrapper.html', 'Parser/some-parser.html']) output = OutputCapture() output.capture_output() try: unexpected_result_count = runner._run_tests_set(tests) finally: _, _, log = output.restore_output() self.assertEqual(unexpected_result_count, 0) self.assertEqual(self._normalize_output(log), EventTargetWrapperTestData.output + SomeParserTestData.output)
def test_rebaseline_expectations_noop(self): self._zero_out_test_expectations() oc = OutputCapture() try: oc.capture_output() self.command.execute(self.options(), [], self.tool) finally: _, _, logs = oc.restore_output() self.assertEqual(self.tool.filesystem.written_files, {}) self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n')
def test_run_test_set(self): runner, _ = self.create_runner() tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html', 'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html']) output = OutputCapture() output.capture_output() try: unexpected_result_count = runner._run_tests_set(tests) finally: _, _, log = output.restore_output() self.assertEqual(unexpected_result_count, len(tests) - 1) self.assertTrue('\nRESULT group_name: test_name= 42 ms\n' in log)
def test_help_argument(self): oc = OutputCapture() oc.capture_output() tool = WebKitPatch('path') try: tool.main(['tool', '--help']) except SystemExit: pass # optparse calls sys.exit after showing help. finally: out, err, logs = oc.restore_output() self.assertTrue(out.startswith('Usage: ')) self.assertEqual('', err) self.assertEqual('', logs)
def test_convert_attributes_if_needed(self): """Tests convert_attributes_if_needed() using a reference file that has some relative src paths.""" test_html = """<html> <head> <script src="../../some-script.js"></script> <style src="../../../some-style.css"></style> </head> <body> <img src="../../../../some-image.jpg"> </body> </html> """ test_reference_support_info = { 'reference_relpath': '../', 'files': [ '../../some-script.js', '../../../some-style.css', '../../../../some-image.jpg' ], 'elements': ['script', 'style', 'img'] } converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, test_reference_support_info) oc = OutputCapture() oc.capture_output() try: converter.feed(test_html) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_reference_relative_paths(converted, test_reference_support_info)
def _test_run_with_json_output(self, runner, filesystem, upload_succeeds=False, results_shown=True, expected_exit_code=0, repeat=1, compare_logs=True): filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') filesystem.write_text_file( runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') uploaded = [False] def mock_upload_json(hostname, json_path, host_path=None): # FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition. self.assertIn(hostname, ['some.host']) self.assertIn(json_path, ['/mock-checkout/output.json']) self.assertIn(host_path, [None, '/api/report']) uploaded[0] = upload_succeeds return upload_succeeds runner._upload_json = mock_upload_json runner._timestamp = 123456789 runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37, 460000) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(runner.run(), expected_exit_code) finally: _, _, logs = output_capture.restore_output() if not expected_exit_code and compare_logs: expected_logs = '' for i in xrange(repeat): runs = ' (Run %d of %d)' % (i + 1, repeat) if repeat > 1 else '' expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output if results_shown: expected_logs += 'MOCK: user.open_url: file://...\n' self.assertEqual(self._normalize_output(logs), expected_logs) self.assertEqual(uploaded[0], upload_succeeds) return logs
def test_run_memory_test(self): runner, port = self.create_runner_and_setup_results_template() runner._timestamp = 123456789 port.host.filesystem.write_text_file(runner._base_path + '/Parser/memory-test.html', 'some content') output = OutputCapture() output.capture_output() try: unexpected_result_count = runner.run() finally: _, _, log = output.restore_output() self.assertEqual(unexpected_result_count, 0) self.assertEqual(self._normalize_output(log), MemoryTestData.output + '\nMOCK: user.open_url: file://...\n') parser_tests = self._load_output_json(runner)[0]['tests']['Parser']['tests'] self.assertEqual(parser_tests['memory-test']['metrics']['Time'], MemoryTestData.results) self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results) self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results)
def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): self.tool.executive = MockExecutive() port = self.tool.port_factory.get('test-win-win7') self._write( port.host.filesystem.join( port.layout_tests_dir(), 'platform/test-win-win10/failures/expected/image-expected.txt' ), 'original win10 result') oc = OutputCapture() try: options = optparse.Values({ 'optimize': True, 'builder': 'MOCK Win10', 'port_name': None, 'suffixes': 'txt', 'verbose': True, 'test': 'failures/expected/image.html', 'results_directory': None, 'build_number': None }) oc.capture_output() self.command.execute(options, [], self.tool) finally: out, _, _ = oc.restore_output() self.assertMultiLineEqual( self._read( self.tool.filesystem.join( port.layout_tests_dir(), 'platform/test-win-win10/failures/expected/image-expected.txt' )), 'MOCK Web result, convert 404 to None=True') self.assertFalse( self.tool.filesystem.exists( self.tool.filesystem.join( port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt' ))) self.assertMultiLineEqual( out, '{"remove-lines": [{"test": "failures/expected/image.html", "port_name": "test-win-win10"}]}\n' )
def test_parse_output(self): output = DriverOutput(""" Running 20 times Ignoring warm-up run (1115) Time: values 1080, 1120, 1095, 1101, 1104 ms avg 1100 ms median 1101 ms stdev 14.50862 ms min 1080 ms max 1120 ms """, image=None, image_hash=None, audio=None) output_capture = OutputCapture() output_capture.capture_output() try: test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') self._assert_results_are_correct(test, output) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, '')
class PrintBaselinesTest(unittest.TestCase): def setUp(self): self.oc = None self.tool = MockBlinkTool() self.test_port = self.tool.port_factory.get('test-win-win7') self.tool.port_factory.get = lambda port_name=None: self.test_port self.tool.port_factory.all_port_names = lambda: [ 'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11', 'test-mac-mac10.10', 'test-win-win10', 'test-win-win7' ] def tearDown(self): if self.oc: self.restore_output() def capture_output(self): self.oc = OutputCapture() self.oc.capture_output() def restore_output(self): stdout, stderr, logs = self.oc.restore_output() self.oc = None return (stdout, stderr, logs) def test_basic(self): command = PrintBaselines() self.capture_output() options = optparse.Values({ 'all': False, 'include_virtual_tests': False, 'csv': False, 'platform': None }) command.execute(options, ['passes/text.html'], self.tool) stdout, _, _ = self.restore_output() self.assertMultiLineEqual(stdout, ('// For test-win-win7\n' 'passes/text-expected.png\n' 'passes/text-expected.txt\n')) def test_multiple(self): command = PrintBaselines() self.capture_output() options = optparse.Values({ 'all': False, 'include_virtual_tests': False, 'csv': False, 'platform': 'test-win-*' }) command.execute(options, ['passes/text.html'], self.tool) stdout, _, _ = self.restore_output() self.assertMultiLineEqual(stdout, ('// For test-win-win10\n' 'passes/text-expected.png\n' 'passes/text-expected.txt\n' '\n' '// For test-win-win7\n' 'passes/text-expected.png\n' 'passes/text-expected.txt\n')) def test_csv(self): command = PrintBaselines() self.capture_output() options = optparse.Values({ 'all': False, 'platform': '*win7', 'csv': True, 'include_virtual_tests': False }) command.execute(options, ['passes/text.html'], self.tool) stdout, _, _ = self.restore_output() self.assertMultiLineEqual(stdout, ( 'test-win-win7,passes/text.html,None,png,passes/text-expected.png,None\n' 'test-win-win7,passes/text.html,None,txt,passes/text-expected.txt,None\n' ))
def test_upload_json(self): runner, port = self.create_runner() port.host.filesystem.files['/mock-checkout/some.json'] = 'some content' class MockFileUploader: called = [] upload_single_text_file_throws = False upload_single_text_file_return_value = None @classmethod def reset(cls): cls.called = [] cls.upload_single_text_file_throws = False cls.upload_single_text_file_return_value = None def __init__(mock, url, timeout): self.assertEqual(url, 'https://some.host/some/path') self.assertTrue(isinstance(timeout, int) and timeout) mock.called.append('FileUploader') def upload_single_text_file(mock, filesystem, content_type, filename): self.assertEqual(filesystem, port.host.filesystem) self.assertEqual(content_type, 'application/json') self.assertEqual(filename, 'some.json') mock.called.append('upload_single_text_file') if mock.upload_single_text_file_throws: raise Exception return mock.upload_single_text_file_return_value MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( 'OK') self.assertTrue( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) MockFileUploader.reset() MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( 'Some error') output = OutputCapture() output.capture_output() self.assertFalse( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) _, _, logs = output.restore_output() self.assertEqual( logs, 'Uploaded JSON to https://some.host/some/path but got a bad response:\nSome error\n' ) # Throwing an exception upload_single_text_file shouldn't blow up _upload_json MockFileUploader.reset() MockFileUploader.upload_single_text_file_throws = True self.assertFalse( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) MockFileUploader.reset() MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( '{"status": "OK"}') self.assertTrue( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) MockFileUploader.reset() MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO( '{"status": "SomethingHasFailed", "failureStored": false}') output = OutputCapture() output.capture_output() self.assertFalse( runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) _, _, logs = output.restore_output() serialized_json = json.dumps( { 'status': 'SomethingHasFailed', 'failureStored': False }, indent=4) self.assertEqual( logs, 'Uploaded JSON to https://some.host/some/path but got an error:\n%s\n' % serialized_json)
def test_convert_prefixed_properties(self): """Tests convert_prefixed_properties() file that has 20 properties requiring the -webkit- prefix. The properties are: 10 in one style block + 5 in another style block + 5 inline styles, including one with multiple prefixed properties. 2 when prefixed properties appear in comments without ending ';'. The properties in the test content are in all sorts of wack formatting. """ test_html = """<html> <style type="text/css"><![CDATA[ .block1 { width: 300px; height: 300px } .block2 { @test0@: propvalue; } .block3{@test1@: propvalue;} .block4 { @test2@:propvalue; } .block5{ @test3@ :propvalue; } #block6 { @test4@ : propvalue; } #block7 { @test5@: propvalue; } #block8 { @test6@: propvalue; } #block9:pseudo { @test7@: propvalue; @test8@: propvalue propvalue propvalue; } ]]></style> </head> <body> <div id="elem1" style="@test9@: propvalue;"></div> <div id="elem2" style="propname: propvalue; @test10@ : propvalue; propname:propvalue;"></div> <div id="elem2" style="@test11@: propvalue; @test12@ : propvalue; @test13@ :propvalue;"></div> <div id="elem3" style="@test14@:propvalue"></div> </body> <style type="text/css"><![CDATA[ .block10{ @test15@: propvalue; } .block11{ @test16@: propvalue; } .block12{ @test17@: propvalue; } #block13:pseudo { @test18@: propvalue; @test19@: propvalue; } #missing-semicolon-in-comments { /* @test20@: propvalue */ @test21@: propvalue; } ]]></style> </html> """ converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None) test_content = self.generate_test_content(converter.prefixed_properties, 22, test_html) oc = OutputCapture() oc.capture_output() try: converter.feed(test_content[1]) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_prefixed_properties(converted, test_content[0])