def test_run_test_pause_before_testing(self): class TestDriverWithStartCount(MainTest.TestDriver): start_count = 0 def start(self): TestDriverWithStartCount.start_count += 1 runner, port = self.create_runner( args=["--pause-before-testing"], driver_class=TestDriverWithStartCount) tests = self._tests_for_runner(runner, ['inspector/pass.html']) output = OutputCapture() output.capture_output() try: unexpected_result_count = runner._run_tests_set(tests, port) self.assertEqual(TestDriverWithStartCount.start_count, 1) finally: stdout, stderr, log = output.restore_output() self.assertEqual(stderr, "Ready to run test?\n") self.assertEqual( self.normalizeFinishedTime(log), "Running inspector/pass.html (1 of 1)\nRESULT group_name: test_name= 42 ms\nFinished: 0.1 s\n\n" )
def test_convert_test_harness_paths(self): """ Tests convert_testharness_paths() with a test that uses all three testharness files """ test_html = """<head> <link href="/resources/testharness.css" rel="stylesheet" type="text/css"> <script src="/resources/testharness.js"></script> <script src="/resources/testharnessreport.js"></script> </head> """ fake_dir_path = self.fake_dir_path('testharnesspaths') converter = _W3CTestConverter(fake_dir_path, DUMMY_FILENAME, None) oc = OutputCapture() oc.capture_output() try: converter.feed(test_html) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_test_harness_paths(converter, converted[2], fake_dir_path, 2, 1)
def test_convert_for_webkit_properties_only(self): """ Tests convert_for_webkit() using a test that has 2 prefixed properties: 1 in a style block + 1 inline style """ test_html = """<html> <head> <link href="/resources/testharness.css" rel="stylesheet" type="text/css"> <script src="/resources/testharness.js"></script> <style type="text/css"> #block1 { @test0@: propvalue; } </style> </head> <body> <div id="elem1" style="@test1@: propvalue;"></div> </body> </html> """ fake_dir_path = self.fake_dir_path('harnessandprops') converter = _W3CTestConverter(fake_dir_path, DUMMY_FILENAME) test_content = self.generate_test_content( converter.prefixed_properties, 1, test_html) oc = OutputCapture() oc.capture_output() try: converter.feed(test_content[1]) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_test_harness_paths(converter, converted[1], fake_dir_path, 1, 1) self.verify_prefixed_properties(converted, test_content[0])
def test_webkit_run_unit_tests(self): tool = MockTool(log_executive=True) tool._deprecated_port.run_python_unittests_command = lambda: None tool._deprecated_port.run_perl_unittests_command = lambda: None step = RunTests(tool, MockOptions(test=True, non_interactive=True, quiet=False)) if sys.platform != "cygwin": expected_logs = """Running bindings generation tests MOCK run_and_throw_if_fail: ['mock-run-bindings-tests'], cwd=/mock-checkout Running WebKit unit tests MOCK run_and_throw_if_fail: ['mock-run-webkit-unit-tests'], cwd=/mock-checkout Running run-webkit-tests MOCK run_and_throw_if_fail: ['mock-run-webkit-tests', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures=30', '--quiet', '--skip-failing-tests'], cwd=/mock-checkout """ else: expected_logs = """Running bindings generation tests MOCK run_and_throw_if_fail: ['mock-run-bindings-tests'], cwd=/mock-checkout Running WebKit unit tests MOCK run_and_throw_if_fail: ['mock-run-webkit-unit-tests'], cwd=/mock-checkout Running run-webkit-tests MOCK run_and_throw_if_fail: ['mock-run-webkit-tests', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures=30', '--no-build'], cwd=/mock-checkout """ OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
def test_convert_for_webkit_harness_and_properties(self): """ Tests convert_for_webkit() using a basic JS test that uses testharness.js and testharness.css and has 4 prefixed properties: 3 in a style block + 1 inline style """ test_html = """<html> <head> <link href="/resources/testharness.css" rel="stylesheet" type="text/css"> <script src="/resources/testharness.js"></script> <style type="text/css"> #block1 { @test0@: propvalue; } #block2 { @test1@: propvalue; } #block3 { @test2@: propvalue; } </style> </head> <body> <div id="elem1" style="@test3@: propvalue;"></div> </body> </html> """ converter = W3CTestConverter() fake_dir_path = self.fake_dir_path(converter, 'harnessandprops') oc = OutputCapture() oc.capture_output() try: test_content = self.generate_test_content( converter.prefixed_properties, 2, test_html) converted = converter.convert_html(fake_dir_path, test_content[1]) finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_test_harness_paths(converter, converted[1], fake_dir_path, 1, 1) self.verify_prefixed_properties(converted, test_content[0])
def test_analyze_test_reftest_multiple_matches(self): test_html = """<head> <link rel="match" href="green-box-ref.xht" /> <link rel="match" href="blue-box-ref.xht" /> <link rel="match" href="orange-box-ref.xht" /> </head> """ oc = OutputCapture() oc.capture_output() try: test_path = '/some/madeup/path/' parser = TestParser(test_path + 'somefile.html', MockHost()) test_info = parser.analyze_test(test_contents=test_html) finally: _, _, logs = oc.restore_output() self.assertNotEqual(test_info, None, 'did not find a test') self.assertTrue('test' in test_info.keys(), 'did not find a test file') self.assertTrue('reference' in test_info.keys(), 'did not find a reference file') self.assertTrue(test_info['reference'].startswith(test_path), 'reference path is not correct') self.assertFalse('refsupport' in test_info.keys(), 'there should be no refsupport files for this test') self.assertFalse('jstest' in test_info.keys(), 'test should not have been analyzed as a jstest') self.assertEqual(logs, 'Multiple references are not supported. Importing the first ref defined in somefile.html\n')
def _test_run_with_json_output(self, runner, filesystem, upload_succeeds=False, results_shown=True, expected_exit_code=0, repeat=1, compare_logs=True): filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') filesystem.write_text_file(runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') uploaded = [False] def mock_upload_json(hostname, json_path, host_path=None): # FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition. self.assertIn(hostname, ['some.host']) self.assertIn(json_path, ['/mock-checkout/output.json']) self.assertIn(host_path, [None, '/api/report']) uploaded[0] = upload_succeeds return upload_succeeds runner._upload_json = mock_upload_json runner._timestamp = 123456789 runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37, 460000) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(runner.run(), expected_exit_code) finally: stdout, stderr, logs = output_capture.restore_output() if not expected_exit_code and compare_logs: expected_logs = '' for i in xrange(repeat): runs = ' (Run %d of %d)' % (i + 1, repeat) if repeat > 1 else '' expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output if results_shown: expected_logs += 'MOCK: user.open_url: file://...\n' self.assertEqual(self._normalize_output(logs), expected_logs) self.assertEqual(uploaded[0], upload_succeeds) return logs
def test_run_memory_test(self): runner, port = self.create_runner_and_setup_results_template() runner._timestamp = 123456789 port.host.filesystem.write_text_file( runner._base_path + '/Parser/memory-test.html', 'some content') output = OutputCapture() output.capture_output() try: unexpected_result_count = runner.run() finally: stdout, stderr, log = output.restore_output() self.assertEqual(unexpected_result_count, 0) self.assertEqual( self._normalize_output(log), MemoryTestData.output + '\nMOCK: user.open_url: file://...\n') parser_tests = self._load_output_json( runner)[0]['tests']['Parser']['tests'] self.assertEqual(parser_tests['memory-test']['metrics']['Time'], MemoryTestData.results) self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results) self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results)
def test_prepare_calls_run_single(self): output_capture = OutputCapture() output_capture.capture_output() called = [False] def run_single(driver, url, time_out_ms, record): self.assertTrue(record) self.assertEqual(url, 'http://some-test/') called[0] = True return False test, port = self._setup_test() test.run_single = run_single try: self.assertEqual(test.prepare(time_out_ms=100), False) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertTrue(called[0]) self.assertEqual(test._archive_path, '/path/some-dir/some-test.wpr') self.assertEqual(test._url, 'http://some-test/') self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, "Preparing replay for some-test.replay\nFailed to prepare a replay for some-test.replay\n")
def test_rebaseline(self): self.command._builders_to_pull_from = lambda: [ MockBuilder('MOCK builder') ] self.command._tests_to_update = lambda builder: [ 'mock/path/to/test.html' ] self._zero_out_test_expectations() old_exact_matches = builders._exact_matches oc = OutputCapture() try: builders._exact_matches = { "MOCK builder": { "port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"]) }, } oc.capture_output() self.command.execute( MockOptions(optimize=False, builders=None, suffixes="txt,png", verbose=True, move_overwritten_baselines=False), [], self.tool) finally: oc.restore_output() builders._exact_matches = old_exact_matches calls = filter(lambda x: x[0] != 'perl', self.tool.executive.calls) self.assertEqual(calls, [[[ 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html', '--verbose' ]]])
def test_double_flaky_test_failure(self): commit_queue = FailingTestCommitQueue([ None, None, None, None, ScriptError("MOCK test failure"), ScriptError("MOCK test failure again"), ], [ "foo.html", "bar.html", "foo.html", ]) # The (subtle) point of this test is that report_flaky_tests does not appear # in the expected_stderr for this run. # Note also that there is no attempt to run the tests w/o the patch. expected_stderr = """run_webkit_patch: ['clean'] command_passed: success_message='Cleaned working directory' patch='197' run_webkit_patch: ['update'] command_passed: success_message='Updated working directory' patch='197' run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] command_passed: success_message='Applied patch' patch='197' run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] command_passed: success_message='Built patch' patch='197' run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' archive_last_layout_test_results: patch='197' run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' """ tool = MockTool(log_executive=True) patch = tool.bugs.fetch_attachment(197) task = CommitQueueTask(commit_queue, patch) success = OutputCapture().assert_outputs( self, task.run, expected_stderr=expected_stderr) self.assertEqual(success, False)
def test_convert_prefixed_properties(self): """ Tests convert_prefixed_properties() file that has 20 properties requiring the -webkit- prefix: 10 in one style block + 5 in another style block + 5 inline styles, including one with multiple prefixed properties. The properties in the test content are in all sorts of wack formatting. """ test_html = """<html> <style type="text/css"><![CDATA[ .block1 { width: 300px; height: 300px } .block2 { @test0@: propvalue; } .block3{@test1@: propvalue;} .block4 { @test2@:propvalue; } .block5{ @test3@ :propvalue; } #block6 { @test4@ : propvalue; } #block7 { @test5@: propvalue; } #block8 { @test6@: propvalue; } #block9:pseudo { @test7@: propvalue; @test8@: propvalue propvalue propvalue;; } ]]></style> </head> <body> <div id="elem1" style="@test9@: propvalue;"></div> <div id="elem2" style="propname: propvalue; @test10@ : propvalue; propname:propvalue;"></div> <div id="elem2" style="@test11@: propvalue; @test12@ : propvalue; @test13@ :propvalue;"></div> <div id="elem3" style="@test14@:propvalue"></div> </body> <style type="text/css"><![CDATA[ .block10{ @test15@: propvalue; } .block11{ @test16@: propvalue; } .block12{ @test17@: propvalue; } #block13:pseudo { @test18@: propvalue; @test19@: propvalue; } ]]></style> </html> """ converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None) test_content = self.generate_test_content( converter.prefixed_properties, 20, test_html) oc = OutputCapture() oc.capture_output() try: converter.feed(test_content[1]) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) self.verify_prefixed_properties(converted, test_content[0])
def test_rollout_no_reason(self): expected_logs = "MOCK: irc.post: mock_nick: Usage: rollout SVN_REVISION [SVN_REVISIONS] REASON\n" OutputCapture().assert_outputs(self, run, args=["rollout 21654"], expected_logs=expected_logs)
def _assert_step_output_with_bug(self, step, bug_id, expected_stderr, options=None): state = {'bug_id': bug_id} OutputCapture().assert_outputs(self, self._run_step, [step, MockTool(), options, state], expected_stderr=expected_stderr)
def test_update_step(self): tool = MockTool() options = self._step_options() options.update = True expected_stderr = "Updating working directory\n" OutputCapture().assert_outputs(self, self._run_step, [steps.Update, tool, options], expected_stderr=expected_stderr)
def test_show_results_html_file(self): port = self.make_port() port._executive = MockExecutive(should_log=True) expected_stderr = "MOCK: user.open_url: test.html\n" OutputCapture().assert_outputs(self, port.show_results_html_file, ["test.html"], expected_stderr=expected_stderr)
def test_guess_reviewer_from_bug(self): capture = OutputCapture() step = UpdateChangeLogsWithReviewer(MockTool(), MockOptions()) expected_stderr = "0 reviewed patches on bug 75, cannot infer reviewer.\n" capture.assert_outputs(self, step._guess_reviewer_from_bug, [75], expected_stderr=expected_stderr)
def test_lgr(self): expected_stderr = "MOCK: irc.post: mock_nick: http://trac.webkit.org/changeset/9479\n" OutputCapture().assert_outputs(self, run, args=["last-green-revision"], expected_stderr=expected_stderr)
def test_multi_rollout_no_reason(self): expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION [SVN_REVISIONS] REASON\n" OutputCapture().assert_outputs(self, run, args=["rollout 21654 21655 r21656"], expected_stderr=expected_stderr)
def test_add_cc_to_bug(self): bugzilla = Bugzilla() bugzilla.browser = MockBrowser() bugzilla.authenticate = lambda: None expected_stderr = "Adding ['*****@*****.**'] to the CC list for bug 42\n" OutputCapture().assert_outputs(self, bugzilla.add_cc_to_bug, [42, ["*****@*****.**"]], expected_stderr=expected_stderr)
def test_rollout_bananas(self): expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION [SVN_REVISIONS] REASON\n" OutputCapture().assert_outputs(self, run, args=["rollout bananas"], expected_stderr=expected_stderr)
def _post_to_path(self, path, body=None, expected_stderr=None, expected_stdout=None, server=None): handler = TestGardeningHTTPRequestHandler(server or MockServer()) handler.path = path handler.body = body OutputCapture().assert_outputs(self, handler.do_POST, expected_stderr=expected_stderr, expected_stdout=expected_stdout)
def test_help(self): expected_stderr = "MOCK: irc.post: mock_nick: Available commands: whois, hi, last-green-revision, rollout, restart, help\n" OutputCapture().assert_outputs(self, run, args=["help"], expected_stderr=expected_stderr)
def test_show_results_html_file(self): port = self.make_port() # Delay setting a should_log executive to avoid logging from MacPort.__init__. port._executive = MockExecutive(should_log=True) expected_logs = "MOCK popen: ['Tools/Scripts/run-safari', '--release', '--no-saved-state', '-NSOpen', 'test.html'], cwd=/mock-checkout\n" OutputCapture().assert_outputs(self, port.show_results_html_file, ["test.html"], expected_logs=expected_logs)
def test_disabled(self): step = SuggestReviewers(MockTool(), MockOptions(suggest_reviewers=False)) OutputCapture().assert_outputs(self, step.run, [{}])
def test_convert_style_multiple_url(self): """ Tests convert_attributes_if_needed() using a reference file that has several relative URL paths in the style """ test_html = """<html> <head> <style type="text/css"> .redSquare { position: absolute; left:50px; width: 100px; height: 100px; background-image:url(../support/yyy.png); } .greenSquare { position: absolute; left:50px; width: 100px; height: 100px; background-image:url(../support/yy.png); } .yellowSquare { position: absolute; left:50px; width: 100px; height: 100px; background-image:url(../../another/directory/x.png); } .container { position: absolute; } </style> </head> <body> </body> </html> """ test_reference_support_info = { 'reference_relpath': '../', 'files': [ '../support/yyy.png', '../support/yy.png', '../../another/directory/x.png' ] } converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, test_reference_support_info) oc = OutputCapture() oc.capture_output() try: converter.feed(test_html) converter.close() converted = converter.output() finally: oc.restore_output() self.verify_conversion_happened(converted) for path in test_reference_support_info['files']: expected_path = re.sub( test_reference_support_info['reference_relpath'], '', path, 1) expected_url = 'background-image:url(' + expected_path + ');' self.assertTrue( expected_url in converted[2], 'relative path ' + path + ' was not converted correcty')
def begin_work_queue(self): output_capture = OutputCapture() output_capture.capture_output() CommitQueue.begin_work_queue(self) output_capture.restore_output()
def _assert_tool_main_outputs(self, tool, main_args, expected_stdout, expected_stderr = "", expected_exit_code=0): exit_code = OutputCapture().assert_outputs(self, tool.main, [main_args], expected_stdout=expected_stdout, expected_stderr=expected_stderr) self.assertEqual(exit_code, expected_exit_code)
def capture_output(self): self.oc = OutputCapture() self.oc.capture_output()
def test_required_arguments(self): two_required_arguments = TrivialCommand(argument_names="ARG1 ARG2 [ARG3]") expected_missing_args_error = "2 arguments required, 1 argument provided. Provided: 'foo' Required: ARG1 ARG2\nSee 'trivial-tool help trivial' for usage.\n" exit_code = OutputCapture().assert_outputs(self, two_required_arguments.check_arguments_and_execute, [None, ["foo"], TrivialTool()], expected_stderr=expected_missing_args_error) self.assertEqual(exit_code, 1)