def test_was_interrupted(self): self.assertTrue( LayoutTestResults.results_from_string( 'ADD_RESULTS({"tests":{},"interrupted":true});'). run_was_interrupted()) self.assertFalse( LayoutTestResults.results_from_string( 'ADD_RESULTS({"tests":{},"interrupted":false});'). run_was_interrupted())
def test_was_interrupted(self): self.assertTrue( LayoutTestResults.results_from_string('ADD_RESULTS({"tests":{},"interrupted":true});').run_was_interrupted() ) self.assertFalse( LayoutTestResults.results_from_string( 'ADD_RESULTS({"tests":{},"interrupted":false});' ).run_was_interrupted() )
def _create_layout_test_results(self): results_path = self._host.filesystem.join(self._results_directory, "full_results.json") results_html = self._read_file_contents(results_path) if not results_html: return None return LayoutTestResults.results_from_string(results_html)
def builder_data(): old_builder_data() # have prototype-chocolate only fail on "MOCK Leopard". self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "PASS", "actual": "PASS TEXT", "is_unexpected": true }, "prototype-chocolate.html": { "expected": "FAIL", "actual": "PASS" }, "prototype-strawberry.html": { "expected": "PASS", "actual": "IMAGE PASS", "is_unexpected": true } } } } });""") return self.command._builder_data
def fetch_layout_test_results(self, results_url): # FIXME: This should cache that the result was a 404 and stop hitting the network. results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "full_results.json")) if not results_file: results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "results.html")) # results_from_string accepts either ORWT html or NRWT json. return LayoutTestResults.results_from_string(results_file)
def fetch_layout_test_results(self, results_url): # FIXME: This should cache that the result was a 404 and stop hitting the network. results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "full_results.json")) if not results_file: results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "results.html")) # results_from_string accepts either ORWT html or NRWT json. return LayoutTestResults.results_from_string(results_file)
def fetch_layout_test_results(self, results_url): """Returns a LayoutTestResults object for results fetched from a given URL.""" results_file = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file(results_url, "failing_results.json")) revision = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file(results_url, "LAST_CHANGE")) if not revision: results_file = None return LayoutTestResults.results_from_string(results_file, revision)
def test_unexpected_mismatch_results(self): results = LayoutTestResults.results_from_string(self.example_full_results_json) self.assertEqual( [r.test_name() for r in results.unexpected_mismatch_results()], [ 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-taco.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html' ])
def fetch_layout_test_results(self, results_url): """Returns a LayoutTestResults object for results fetched from a given URL.""" results_file = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file(results_url, "failing_results.json")) revision = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file(results_url, "LAST_CHANGE")) if not revision: results_file = None return LayoutTestResults.results_from_string(results_file, revision)
def test_actual_results(self): results = LayoutTestResults.results_from_string( self.example_full_results_json) self.assertEqual( results.actual_results("fast/dom/prototype-banana.html"), "PASS") self.assertEqual( results.actual_results("fast/dom/prototype-taco.html"), "PASS TEXT") self.assertEqual(results.actual_results("nonexistant.html"), "")
def fetch_layout_test_results(self, results_url): # FIXME: This should cache that the result was a 404 and stop hitting the network. results_file = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file_from_results(results_url, "failing_results.json")) revision = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file_from_results(results_url, "LAST_CHANGE")) if not revision: results_file = None return LayoutTestResults.results_from_string(results_file, revision)
def test_unexpected_mismatch_results(self): results = LayoutTestResults.results_from_string( self.example_full_results_json) self.assertEqual([ r.test_name() for r in results.unexpected_mismatch_results() ], [ 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-taco.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html' ])
def test_didnt_run_as_expected_results(self): results = LayoutTestResults.results_from_string(self.example_full_results_json) self.assertEqual( [r.test_name() for r in results.didnt_run_as_expected_results()], [ 'fast/dom/prototype-banana.html', 'fast/dom/prototype-crashy.html', 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-newtest.html', 'fast/dom/prototype-strawberry.html', 'fast/dom/prototype-taco.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html', ])
def _tests_to_update(self, attachment, bot_type=None): _log.info("Processing attachment " + str(attachment.id())) zip_file = self.unzip(attachment.contents()) results = LayoutTestResults.results_from_string( zip_file.read("full_results.json")) results_to_update = [ result.test_name for result in results.failing_test_results() if result.type == test_expectations.TEXT ] return { result: zip_file.read( TestResultWriter.actual_filename(result, self.filesystem)) for result in results_to_update }
def _setup_mock_builder_data(self): data = LayoutTestResults.results_from_string("""ADD_RESULTS({ "tests": { "userscripts": { "first-test.html": { "expected": "PASS", "actual": "IMAGE+TEXT" }, "second-test.html": { "expected": "FAIL", "actual": "IMAGE+TEXT" } } } });""") for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']: self.command._builder_data[builder] = data
def _tests_to_update(self, bot_name): _log.info("{} archive: {}".format( bot_name, self.ews_results[bot_name]['layout-tests-archive-url'])) _log.info("{} status: {}".format( bot_name, self.ews_results[bot_name]['layout-tests-results-string'])) layout_tests_archive_request = requests.get( self.ews_results[bot_name]['layout-tests-archive-url']) layout_tests_archive_content = layout_tests_archive_request.content zip_file = self.unzip(layout_tests_archive_content) results = LayoutTestResults.results_from_string( zip_file.read("full_results.json")) results_to_update = [ result.test_name for result in results.failing_test_results() if result.type in [test_expectations.TEXT, test_expectations.MISSING] ] return { result: zip_file.read( TestResultWriter.actual_filename(result, self.filesystem)) for result in results_to_update }
def _create_layout_test_results(self): results_path = self._tool.port().layout_tests_results_path() results_html = self._read_file_contents(results_path) if not results_html: return None return LayoutTestResults.results_from_string(results_html)
def fetch_layout_test_results(self, _): return LayoutTestResults.results_from_string( layouttestresults_unittest.LayoutTestResultsTest. example_full_results_json)
def layout_test_results(self): if not self._layout_test_results: # FIXME: This should cache that the result was a 404 and stop hitting the network. self._layout_test_results = LayoutTestResults.results_from_string(self._fetch_results_html()) return self._layout_test_results
def test_results_from_string(self): self.assertEqual(LayoutTestResults.results_from_string(None), None) self.assertEqual(LayoutTestResults.results_from_string(""), None) results = LayoutTestResults.results_from_string(self._example_results_html) self.assertEqual(len(results.failing_tests()), 0)
def fetch_results(self, build): return self._canned_results.get( build, LayoutTestResults.results_from_string(LayoutTestResultsTest.example_full_results_json))
def fetch_layout_test_results(self, _): return LayoutTestResults.results_from_string(LayoutTestResultsTest.example_full_results_json)
def test_results_from_string(self): self.assertEqual(LayoutTestResults.results_from_string(None), None) self.assertEqual(LayoutTestResults.results_from_string(""), None) results = LayoutTestResults.results_from_string( ORWTResultsHTMLParserTest._example_results_html) self.assertEqual(len(results.failing_tests()), 0)
def latest_layout_test_results(self): return LayoutTestResults.results_from_string(layouttestresults_unittest.LayoutTestResultsTest.example_full_results_json)
def test_tests_matching_failure_types(self): results = LayoutTestResults.results_from_string(ORWTResultsHTMLParserTest._example_results_html_with_failing_tests) failing_tests = results.tests_matching_failure_types([test_failures.FailureTextMismatch]) self.assertEqual(len(results.failing_tests()), 2)
def test_chromium_revision(self): self.assertEqual(LayoutTestResults.results_from_string(self.example_full_results_json).chromium_revision(), 1234)
def test_missing_baseline(self): results = LayoutTestResults.results_from_string(self.example_full_results_json) missing_results = results.missing_results() self.assertEqual(len(missing_results), 1) self.assertEqual(missing_results[0].test_name(), "fast/dom/prototype-newtest.html")
def builder_data(): self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "PASS", "actual": "PASS TEXT", "is_unexpected": true } } } } });""") return self.command._builder_data
def fetch_results(self, build): return self._canned_results.get( build, LayoutTestResults.results_from_string( LayoutTestResultsTest.example_full_results_json))
def test_tests_matching_failure_types(self): results = LayoutTestResults.results_from_string( ORWTResultsHTMLParserTest._example_results_html_with_failing_tests) failing_tests = results.tests_matching_failure_types( [test_failures.FailureTextMismatch]) self.assertEqual(len(results.failing_tests()), 1)
def test_results_from_string(self): self.assertIsNone(LayoutTestResults.results_from_string(None)) self.assertIsNone(LayoutTestResults.results_from_string(""))
def _create_layout_test_results(self): results_path = self._host.filesystem.join(self._results_directory, "full_results.json") results_html = self._read_file_contents(results_path) if not results_html: return None return LayoutTestResults.results_from_string(results_html)
def test_actual_results(self): results = LayoutTestResults.results_from_string(self.example_full_results_json) self.assertEqual(results.result_for_test("fast/dom/prototype-banana.html").actual_results(), "PASS") self.assertEqual(results.result_for_test("fast/dom/prototype-taco.html").actual_results(), "PASS TEXT") self.assertFalse(results.result_for_test("nonexistant.html"))
def layout_test_results(self): results_path = self._tool.port().layout_tests_results_path() results_html = self._read_file_contents(results_path) if not results_html: return None return LayoutTestResults.results_from_string(results_html)
def fetch_layout_test_results(self, results_url): # FIXME: This should cache that the result was a 404 and stop hitting the network. results_file = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file_from_results(results_url, "failing_results.json")) return LayoutTestResults.results_from_string(results_file)
def test_results_from_string(self): self.assertEqual(LayoutTestResults.results_from_string(None), None) self.assertEqual(LayoutTestResults.results_from_string(""), None)
def layout_test_results(self): if not self._layout_test_results: # FIXME: This should cache that the result was a 404 and stop hitting the network. self._layout_test_results = LayoutTestResults.results_from_string( self._fetch_results_html()) return self._layout_test_results
def builder_data(): self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ "tests": { "userscripts": { "another-test.html": { "expected": "PASS", "actual": "PASS TEXT" }, "images.svg": { "expected": "FAIL", "actual": "IMAGE+TEXT" } } } });""") return self.command._builder_data