def test_was_interrupted(self):
self.assertTrue(
LayoutTestResults.results_from_string(
'ADD_RESULTS({"tests":{},"interrupted":true});').
run_was_interrupted())
self.assertFalse(
LayoutTestResults.results_from_string(
'ADD_RESULTS({"tests":{},"interrupted":false});').
run_was_interrupted())
def test_was_interrupted(self):
self.assertTrue(
LayoutTestResults.results_from_string('ADD_RESULTS({"tests":{},"interrupted":true});').run_was_interrupted()
)
self.assertFalse(
LayoutTestResults.results_from_string(
'ADD_RESULTS({"tests":{},"interrupted":false});'
).run_was_interrupted()
)
def _create_layout_test_results(self):
results_path = self._host.filesystem.join(self._results_directory,
"full_results.json")
results_html = self._read_file_contents(results_path)
if not results_html:
return None
return LayoutTestResults.results_from_string(results_html)
def builder_data():
old_builder_data()
# have prototype-chocolate only fail on "MOCK Leopard".
self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "PASS",
"actual": "PASS TEXT",
"is_unexpected": true
},
"prototype-chocolate.html": {
"expected": "FAIL",
"actual": "PASS"
},
"prototype-strawberry.html": {
"expected": "PASS",
"actual": "IMAGE PASS",
"is_unexpected": true
}
}
}
}
});""")
return self.command._builder_data
def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "full_results.json"))
if not results_file:
results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "results.html"))
# results_from_string accepts either ORWT html or NRWT json.
return LayoutTestResults.results_from_string(results_file)
def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "full_results.json"))
if not results_file:
results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "results.html"))
# results_from_string accepts either ORWT html or NRWT json.
return LayoutTestResults.results_from_string(results_file)
def fetch_layout_test_results(self, results_url):
"""Returns a LayoutTestResults object for results fetched from a given URL."""
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(results_url, "failing_results.json"))
revision = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(results_url, "LAST_CHANGE"))
if not revision:
results_file = None
return LayoutTestResults.results_from_string(results_file, revision)
def test_unexpected_mismatch_results(self):
results = LayoutTestResults.results_from_string(self.example_full_results_json)
self.assertEqual(
[r.test_name() for r in results.unexpected_mismatch_results()],
[
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-taco.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html'
])
def fetch_layout_test_results(self, results_url):
"""Returns a LayoutTestResults object for results fetched from a given URL."""
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(results_url, "failing_results.json"))
revision = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(results_url, "LAST_CHANGE"))
if not revision:
results_file = None
return LayoutTestResults.results_from_string(results_file, revision)
def test_actual_results(self):
results = LayoutTestResults.results_from_string(
self.example_full_results_json)
self.assertEqual(
results.actual_results("fast/dom/prototype-banana.html"), "PASS")
self.assertEqual(
results.actual_results("fast/dom/prototype-taco.html"),
"PASS TEXT")
self.assertEqual(results.actual_results("nonexistant.html"), "")
def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file_from_results(results_url,
"failing_results.json"))
revision = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file_from_results(results_url, "LAST_CHANGE"))
if not revision:
results_file = None
return LayoutTestResults.results_from_string(results_file, revision)
def test_unexpected_mismatch_results(self):
results = LayoutTestResults.results_from_string(
self.example_full_results_json)
self.assertEqual([
r.test_name() for r in results.unexpected_mismatch_results()
], [
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-taco.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html'
])
def test_didnt_run_as_expected_results(self):
results = LayoutTestResults.results_from_string(self.example_full_results_json)
self.assertEqual(
[r.test_name() for r in results.didnt_run_as_expected_results()],
[
'fast/dom/prototype-banana.html',
'fast/dom/prototype-crashy.html',
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'fast/dom/prototype-strawberry.html',
'fast/dom/prototype-taco.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
])
def _tests_to_update(self, attachment, bot_type=None):
_log.info("Processing attachment " + str(attachment.id()))
zip_file = self.unzip(attachment.contents())
results = LayoutTestResults.results_from_string(
zip_file.read("full_results.json"))
results_to_update = [
result.test_name for result in results.failing_test_results()
if result.type == test_expectations.TEXT
]
return {
result: zip_file.read(
TestResultWriter.actual_filename(result, self.filesystem))
for result in results_to_update
}
def _setup_mock_builder_data(self):
data = LayoutTestResults.results_from_string("""ADD_RESULTS({
"tests": {
"userscripts": {
"first-test.html": {
"expected": "PASS",
"actual": "IMAGE+TEXT"
},
"second-test.html": {
"expected": "FAIL",
"actual": "IMAGE+TEXT"
}
}
}
});""")
for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']:
self.command._builder_data[builder] = data
def _tests_to_update(self, bot_name):
_log.info("{} archive: {}".format(
bot_name, self.ews_results[bot_name]['layout-tests-archive-url']))
_log.info("{} status: {}".format(
bot_name,
self.ews_results[bot_name]['layout-tests-results-string']))
layout_tests_archive_request = requests.get(
self.ews_results[bot_name]['layout-tests-archive-url'])
layout_tests_archive_content = layout_tests_archive_request.content
zip_file = self.unzip(layout_tests_archive_content)
results = LayoutTestResults.results_from_string(
zip_file.read("full_results.json"))
results_to_update = [
result.test_name for result in results.failing_test_results() if
result.type in [test_expectations.TEXT, test_expectations.MISSING]
]
return {
result: zip_file.read(
TestResultWriter.actual_filename(result, self.filesystem))
for result in results_to_update
}
def _create_layout_test_results(self):
results_path = self._tool.port().layout_tests_results_path()
results_html = self._read_file_contents(results_path)
if not results_html:
return None
return LayoutTestResults.results_from_string(results_html)
def fetch_layout_test_results(self, _):
return LayoutTestResults.results_from_string(
layouttestresults_unittest.LayoutTestResultsTest.
example_full_results_json)
def layout_test_results(self):
if not self._layout_test_results:
# FIXME: This should cache that the result was a 404 and stop hitting the network.
self._layout_test_results = LayoutTestResults.results_from_string(self._fetch_results_html())
return self._layout_test_results
def test_results_from_string(self):
self.assertEqual(LayoutTestResults.results_from_string(None), None)
self.assertEqual(LayoutTestResults.results_from_string(""), None)
results = LayoutTestResults.results_from_string(self._example_results_html)
self.assertEqual(len(results.failing_tests()), 0)
def fetch_results(self, build):
return self._canned_results.get(
build,
LayoutTestResults.results_from_string(LayoutTestResultsTest.example_full_results_json))
def fetch_layout_test_results(self, _):
return LayoutTestResults.results_from_string(LayoutTestResultsTest.example_full_results_json)
def test_results_from_string(self):
self.assertEqual(LayoutTestResults.results_from_string(None), None)
self.assertEqual(LayoutTestResults.results_from_string(""), None)
results = LayoutTestResults.results_from_string(
ORWTResultsHTMLParserTest._example_results_html)
self.assertEqual(len(results.failing_tests()), 0)
def latest_layout_test_results(self):
return LayoutTestResults.results_from_string(layouttestresults_unittest.LayoutTestResultsTest.example_full_results_json)
def test_tests_matching_failure_types(self):
results = LayoutTestResults.results_from_string(ORWTResultsHTMLParserTest._example_results_html_with_failing_tests)
failing_tests = results.tests_matching_failure_types([test_failures.FailureTextMismatch])
self.assertEqual(len(results.failing_tests()), 2)
def test_chromium_revision(self):
self.assertEqual(LayoutTestResults.results_from_string(self.example_full_results_json).chromium_revision(), 1234)
def test_missing_baseline(self):
results = LayoutTestResults.results_from_string(self.example_full_results_json)
missing_results = results.missing_results()
self.assertEqual(len(missing_results), 1)
self.assertEqual(missing_results[0].test_name(), "fast/dom/prototype-newtest.html")
def builder_data():
self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "PASS",
"actual": "PASS TEXT",
"is_unexpected": true
}
}
}
}
});""")
return self.command._builder_data
def fetch_results(self, build):
return self._canned_results.get(
build,
LayoutTestResults.results_from_string(
LayoutTestResultsTest.example_full_results_json))
def test_tests_matching_failure_types(self):
results = LayoutTestResults.results_from_string(
ORWTResultsHTMLParserTest._example_results_html_with_failing_tests)
failing_tests = results.tests_matching_failure_types(
[test_failures.FailureTextMismatch])
self.assertEqual(len(results.failing_tests()), 1)
def test_results_from_string(self):
self.assertIsNone(LayoutTestResults.results_from_string(None))
self.assertIsNone(LayoutTestResults.results_from_string(""))
def _create_layout_test_results(self):
results_path = self._host.filesystem.join(self._results_directory, "full_results.json")
results_html = self._read_file_contents(results_path)
if not results_html:
return None
return LayoutTestResults.results_from_string(results_html)
def test_actual_results(self):
results = LayoutTestResults.results_from_string(self.example_full_results_json)
self.assertEqual(results.result_for_test("fast/dom/prototype-banana.html").actual_results(), "PASS")
self.assertEqual(results.result_for_test("fast/dom/prototype-taco.html").actual_results(), "PASS TEXT")
self.assertFalse(results.result_for_test("nonexistant.html"))
def layout_test_results(self):
results_path = self._tool.port().layout_tests_results_path()
results_html = self._read_file_contents(results_path)
if not results_html:
return None
return LayoutTestResults.results_from_string(results_html)
def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file_from_results(results_url, "failing_results.json"))
return LayoutTestResults.results_from_string(results_file)
def test_results_from_string(self):
self.assertEqual(LayoutTestResults.results_from_string(None), None)
self.assertEqual(LayoutTestResults.results_from_string(""), None)
def layout_test_results(self):
if not self._layout_test_results:
# FIXME: This should cache that the result was a 404 and stop hitting the network.
self._layout_test_results = LayoutTestResults.results_from_string(
self._fetch_results_html())
return self._layout_test_results
def builder_data():
self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
"tests": {
"userscripts": {
"another-test.html": {
"expected": "PASS",
"actual": "PASS TEXT"
},
"images.svg": {
"expected": "FAIL",
"actual": "IMAGE+TEXT"
}
}
}
});""")
return self.command._builder_data
