def setUp(self):
self.host = MockHost()
self.host.port_factory = MockPortFactory(self.host)
self.port = self.host.port_factory.get()
# Create a testing manifest containing any test files that we
# might interact with.
self.host.filesystem.write_text_file(
self.port.web_tests_dir() + '/external/' + BASE_MANIFEST_NAME,
json.dumps({
'items': {
'reftest': {
'reftest.html': [
'c3f2fb6f436da59d43aeda0a7e8a018084557033',
[None, [['reftest-ref.html', '==']], {}],
]
},
'testharness': {
'test.html': [
'd933fd981d4a33ba82fb2b000234859bdda1494e',
[None, {}]
],
'crash.html': [
'd933fd981d4a33ba82fb2b000234859bdda1494e',
[None, {}]
],
'variant.html': [
'b8db5972284d1ac6bbda0da81621d9bca5d04ee7',
['variant.html?foo=bar/abc', {}],
['variant.html?foo=baz', {}],
],
'dir': {
'multiglob.https.any.js': [
'd6498c3e388e0c637830fa080cca78b0ab0e5305',
['dir/multiglob.https.any.window.html', {}],
['dir/multiglob.https.any.worker.html', {}],
],
},
},
},
}))
self.host.filesystem.write_text_file(
os.path.join(WEB_TESTS_DIR, "fast", "harness", "results.html"),
"results-viewer-body")
self.wpt_adapter = BaseWptScriptAdapter(self.host)
self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME
self.wpt_adapter.sink = MockResultSink()
class BaseWptScriptAdapterTest(unittest.TestCase):
def setUp(self):
self.host = MockHost()
self.host.port_factory = MockPortFactory(self.host)
self.port = self.host.port_factory.get()
# Create a testing manifest containing any test files that we
# might interact with.
self.host.filesystem.write_text_file(
self.port.web_tests_dir() + '/external/' + BASE_MANIFEST_NAME,
json.dumps({
'items': {
'reftest': {
'reftest.html': [
'c3f2fb6f436da59d43aeda0a7e8a018084557033',
[None, [['reftest-ref.html', '==']], {}],
]
},
'testharness': {
'test.html': [
'd933fd981d4a33ba82fb2b000234859bdda1494e',
[None, {}]
],
'crash.html': [
'd933fd981d4a33ba82fb2b000234859bdda1494e',
[None, {}]
],
'variant.html': [
'b8db5972284d1ac6bbda0da81621d9bca5d04ee7',
['variant.html?foo=bar/abc', {}],
['variant.html?foo=baz', {}],
],
'dir': {
'multiglob.https.any.js': [
'd6498c3e388e0c637830fa080cca78b0ab0e5305',
['dir/multiglob.https.any.window.html', {}],
['dir/multiglob.https.any.worker.html', {}],
],
},
},
},
}))
self.host.filesystem.write_text_file(
os.path.join(WEB_TESTS_DIR, "fast", "harness", "results.html"),
"results-viewer-body")
self.wpt_adapter = BaseWptScriptAdapter(self.host)
self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME
self.wpt_adapter.sink = MockResultSink()
def _create_json_output(self, json_dict):
"""Writing some json output for processing."""
self.host.filesystem.write_text_file(OUTPUT_JSON_FILENAME,
json.dumps(json_dict))
def _load_json_output(self, filename=OUTPUT_JSON_FILENAME):
"""Loads the json output after post-processing."""
return json.loads(self.host.filesystem.read_text_file(filename))
def test_result_sink_for_test_expected_result(self):
json_dict = {
'tests': {
'fail': {
'test.html?variant1': {
'expected': 'PASS FAIL',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
},
'path_delimiter': os.path.sep,
}
test_abs_path = os.path.join(WEB_TESTS_DIR, 'external', 'wpt', 'fail',
'test.html')
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
baseline_artifacts = {
'wpt_actual_status': ['OK'],
'actual_text':
[(os.path.join('layout-test-results', 'external', 'wpt', 'fail',
'test_variant1-actual.txt'))]
}
self.assertEquals(self.wpt_adapter.sink.sink_requests, [{
'test':
os.path.join('external', 'wpt', 'fail', 'test.html?variant1'),
'test_path':
test_abs_path,
'result': {
'actual': 'FAIL',
'expected': set(['PASS', 'FAIL']),
'unexpected': False,
'artifacts': baseline_artifacts
}
}])
def test_result_sink_for_test_variant(self):
json_dict = {
'tests': {
'fail': {
'test.html?variant1': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
},
'path_delimiter': os.path.sep,
}
test_abs_path = os.path.join(WEB_TESTS_DIR, 'external', 'wpt', 'fail',
'test.html')
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
baseline_artifacts = {
'wpt_actual_status': ['OK'],
'actual_text':
[(os.path.join('layout-test-results', 'external', 'wpt', 'fail',
'test_variant1-actual.txt'))]
}
self.assertEquals(self.wpt_adapter.sink.sink_requests, [{
'test':
os.path.join('external', 'wpt', 'fail', 'test.html?variant1'),
'test_path':
test_abs_path,
'result': {
'actual': 'FAIL',
'expected': set(['PASS']),
'unexpected': True,
'artifacts': baseline_artifacts
}
}])
def test_result_sink_artifacts(self):
json_dict = {
'tests': {
'fail': {
'test.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
test_abs_path = os.path.join(WEB_TESTS_DIR, 'external', 'wpt', 'fail',
'test.html')
baseline_artifacts = {
'wpt_actual_status': ['OK'],
'actual_text': [
os.path.join('layout-test-results', 'external', 'wpt', 'fail',
'test-actual.txt')
]
}
self.assertEquals(
self.wpt_adapter.sink.sink_requests,
[{
'test': os.path.join('external', 'wpt', 'fail', 'test.html'),
'test_path': test_abs_path,
'result': {
'actual': 'FAIL',
'expected': set(['PASS']),
'unexpected': True,
'artifacts': baseline_artifacts
}
}])
def test_write_jsons(self):
# Ensure that various JSONs are written to the correct location.
json_dict = {
'tests': {
'pass': {
'test.html': {
'expected': 'PASS',
'actual': 'PASS',
'artifacts': {
'wpt_actual_status': ['OK'],
},
},
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'artifacts': {
'wpt_actual_status': ['OK'],
},
'is_unexpected': True,
},
'fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['ERROR'],
},
'is_unexpected': True,
'is_regression': True,
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
# The correctness of the output JSON is verified by other tests.
written_files = self.wpt_adapter.fs.written_files
self.assertEqual(
written_files[OUTPUT_JSON_FILENAME],
written_files[os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
'full_results.json')])
# Verify JSONP
full_results_jsonp = written_files[os.path.join(
LAYOUT_TEST_RESULTS_SUBDIR, 'full_results_jsonp.js')]
match = re.match(r'ADD_FULL_RESULTS\((.*)\);$', full_results_jsonp)
self.assertIsNotNone(match)
self.assertEqual(
match.group(1),
written_files[OUTPUT_JSON_FILENAME].decode(encoding='utf-8'))
failing_results_jsonp = written_files[os.path.join(
LAYOUT_TEST_RESULTS_SUBDIR, 'failing_results.json')]
match = re.match(r'ADD_RESULTS\((.*)\);$', failing_results_jsonp)
self.assertIsNotNone(match)
failing_results = json.loads(match.group(1))
# Verify filtering of failing_results.json
self.assertIn('fail.html', failing_results['tests']['external']['wpt'])
# We shouldn't have unexpected passes or empty dirs after filtering.
self.assertNotIn('unexpected_pass.html',
failing_results['tests']['external']['wpt'])
self.assertNotIn('pass', failing_results['tests']['external']['wpt'])
def test_write_text_outputs(self):
# Ensure that text outputs are written to the correct location.
# We only generate an actual.txt if our actual wasn't PASS, so in this
# case we shouldn't write anything.
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'PASS',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
actual_path = os.path.join(artifact_subdir, "test-actual.txt")
diff_path = os.path.join(artifact_subdir, "test-diff.txt")
pretty_diff_path = os.path.join(artifact_subdir,
"test-pretty-diff.html")
self.assertNotIn(actual_path, written_files)
self.assertNotIn(diff_path, written_files)
self.assertNotIn(pretty_diff_path, written_files)
# Now we change the outcome to be a FAIL, which should generate an
# actual.txt
json_dict['tests']['test.html']['actual'] = 'FAIL'
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join(artifact_subdir, "test-actual.txt")
self.assertEqual("test.html actual text",
written_files[actual_path].decode(encoding='utf-8'))
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
updated_json = self._load_json_output()
self.assertFalse("wpt_actual_metadata" in updated_json["tests"]
["external"]["wpt"]["test.html"]["artifacts"])
self.assertEqual([actual_path], updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"]["actual_text"])
self.assertIn(actual_path, written_files)
self.assertNotIn(diff_path, written_files)
self.assertNotIn(pretty_diff_path, written_files)
def test_write_log_artifact(self):
# Ensure that crash log artifacts are written to the correct location.
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['ERROR'],
'wpt_log': ['test.html exceptions'],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
stderr_path = os.path.join(artifact_subdir, "test-stderr.txt")
self.assertEqual("test.html exceptions",
written_files[stderr_path].decode(encoding='utf-8'))
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
updated_json = self._load_json_output()
self.assertFalse("wpt_log" in updated_json["tests"]["external"]["wpt"]
["test.html"]["artifacts"])
self.assertEqual([stderr_path], updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"]["stderr"])
self.assertTrue(updated_json["tests"]["external"]["wpt"]["test.html"]
["has_stderr"])
def test_write_crashlog_artifact(self):
# Ensure that crash log artifacts are written to the correct location.
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'CRASH',
'artifacts': {
'wpt_actual_status': ['CRASH'],
'wpt_crash_log': ['test.html crashed!'],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
crash_log_path = os.path.join(artifact_subdir, "test-crash-log.txt")
self.assertEqual(
"test.html crashed!",
written_files[crash_log_path].decode(encoding='utf-8'))
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
updated_json = self._load_json_output()
self.assertFalse("wpt_crash_log" in updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"])
self.assertEqual([crash_log_path], updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"]["crash_log"])
def test_write_screenshot_artifacts(self):
# Ensure that screenshots are written to the correct filenames and
# their bytes are base64 decoded. The image diff should also be created.
json_dict = {
'tests': {
'reftest.html': {
'expected': 'PASS',
'actual': 'PASS',
'artifacts': {
'wpt_actual_status': ['PASS'],
'screenshots': [
'reftest.html:abcd',
'reftest-ref.html:bcde',
],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
actual_image_path = os.path.join(artifact_subdir, "reftest-actual.png")
self.assertEqual(base64.b64decode('abcd'),
written_files[actual_image_path])
expected_image_path = os.path.join(artifact_subdir,
"reftest-expected.png")
self.assertEqual(base64.b64decode('bcde'),
written_files[expected_image_path])
diff_image_path = os.path.join(artifact_subdir, "reftest-diff.png")
# Make sure the diff image was written but don't check the contents,
# assume that diff_image does the right thing.
self.assertIsNotNone(written_files[diff_image_path])
# Ensure the artifacts in the json were replaced with the location of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse("screenshots" in updated_json["tests"]["external"]
["wpt"]["reftest.html"]["artifacts"])
self.assertEqual([actual_image_path], updated_json["tests"]["external"]
["wpt"]["reftest.html"]["artifacts"]["actual_image"])
self.assertEqual([expected_image_path],
updated_json["tests"]["external"]["wpt"]
["reftest.html"]["artifacts"]["expected_image"])
self.assertEqual([diff_image_path], updated_json["tests"]["external"]
["wpt"]["reftest.html"]["artifacts"]["image_diff"])
def test_copy_expected_output(self):
# Check that an -expected.txt file is created from a checked-in metadata
# ini file if it exists for a test
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
# Also create a checked-in metadata file for this test
self.host.filesystem.write_text_file(
os.path.join(EXTERNAL_WPT_TESTS_DIR, "test.html.ini"),
"test.html checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
actual_path = os.path.join(artifact_subdir, "test-actual.txt")
self.assertEqual("test.html actual text",
written_files[actual_path].decode(encoding='utf-8'))
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join(artifact_subdir, "test-expected.txt")
self.assertEqual("test.html checked-in metadata",
written_files[expected_path].decode(encoding='utf-8'))
# Ensure the artifacts in the json were replaced with the locations of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse("wpt_actual_metadata" in updated_json["tests"]
["external"]["wpt"]["test.html"]["artifacts"])
self.assertEqual([actual_path], updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"]["actual_text"])
self.assertEqual([expected_path], updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"]["expected_text"])
# Ensure that a diff was also generated. There should be both additions
# and deletions for this test since we have expected output. We don't
# validate the entire diff files to avoid checking line numbers/markup.
diff_path = os.path.join(artifact_subdir, "test-diff.txt")
self.assertIn("-test.html checked-in metadata",
written_files[diff_path].decode(encoding='utf-8'))
self.assertIn("+test.html actual text",
written_files[diff_path].decode(encoding='utf-8'))
self.assertEqual([diff_path], updated_json["tests"]["external"]["wpt"]
["test.html"]["artifacts"]["text_diff"])
pretty_diff_path = os.path.join(artifact_subdir,
"test-pretty-diff.html")
self.assertIn("test.html checked-in metadata",
written_files[pretty_diff_path].decode(encoding='utf-8'))
self.assertIn("test.html actual text",
written_files[pretty_diff_path].decode(encoding='utf-8'))
self.assertEqual([pretty_diff_path], updated_json["tests"]["external"]
["wpt"]["test.html"]["artifacts"]["pretty_text_diff"])
def test_expected_output_for_variant(self):
# Check that an -expected.txt file is created from a checked-in metadata
# ini file for a variant test. Variants are a little different because
# we have to use the manifest to map a test name to the test file, and
# determine the associated metadata from the test file.
# Check that an -expected.txt file is created from a checked-in metadata
# ini file if it exists for a test
json_dict = {
'tests': {
'variant.html?foo=bar/abc': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['variant bar/abc actual text'],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
# Also create a checked-in metadata file for this test. This filename
# matches the test *file* name, not the test name (which includes the
# variant).
self.host.filesystem.write_text_file(
os.path.join(EXTERNAL_WPT_TESTS_DIR, "variant.html.ini"),
"variant.html checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
actual_path = os.path.join(artifact_subdir,
"variant_foo=bar_abc-actual.txt")
self.assertEqual("variant bar/abc actual text",
written_files[actual_path].decode(encoding='utf-8'))
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join(artifact_subdir,
"variant_foo=bar_abc-expected.txt")
self.assertEqual("variant.html checked-in metadata",
written_files[expected_path].decode(encoding='utf-8'))
# Ensure the artifacts in the json were replaced with the locations of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse(
"wpt_actual_metadata" in updated_json["tests"]["external"]["wpt"]
["variant.html?foo=bar/abc"]["artifacts"])
self.assertEqual(
[actual_path], updated_json["tests"]["external"]["wpt"]
["variant.html?foo=bar/abc"]["artifacts"]["actual_text"])
self.assertEqual(
[expected_path], updated_json["tests"]["external"]["wpt"]
["variant.html?foo=bar/abc"]["artifacts"]["expected_text"])
def test_expected_output_for_multiglob(self):
# Check that an -expected.txt file is created from a checked-in metadata
# ini file for a multiglobal test. Multi-globals are a little different
# because we have to use the manifest to map a test name to the test
# file, and determine the associated metadata from the test file.
#
# Also note that the "dir" is both a directory and a part of the test
# name, so the path delimiter remains a / (ie: dir/multiglob) even on
# Windows.
json_dict = {
'tests': {
'dir/multiglob.https.any.worker.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata':
['dir/multiglob worker actual text'],
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
# Also create a checked-in metadata file for this test. This filename
# matches the test *file* name, not the test name (which includes test
# scope).
self.host.filesystem.write_text_file(
os.path.join(EXTERNAL_WPT_TESTS_DIR,
"dir/multiglob.https.any.js.ini"),
"dir/multiglob checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "external",
"wpt")
actual_path = os.path.join(
artifact_subdir, "dir/multiglob.https.any.worker-actual.txt")
self.assertEqual("dir/multiglob worker actual text",
written_files[actual_path].decode(encoding='utf-8'))
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join(
artifact_subdir, "dir/multiglob.https.any.worker-expected.txt")
self.assertEqual("dir/multiglob checked-in metadata",
written_files[expected_path].decode(encoding='utf-8'))
# Ensure the artifacts in the json were replaced with the locations of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse(
"wpt_actual_metadata" in updated_json["tests"]["external"]["wpt"]
["dir/multiglob.https.any.worker.html"]["artifacts"])
self.assertEqual([actual_path], updated_json["tests"]["external"]
["wpt"]["dir/multiglob.https.any.worker.html"]
["artifacts"]["actual_text"])
self.assertEqual([expected_path], updated_json["tests"]["external"]
["wpt"]["dir/multiglob.https.any.worker.html"]
["artifacts"]["expected_text"])
class BaseWptScriptAdapterTest(unittest.TestCase):
def setUp(self):
self.host = MockHost()
self.host.port_factory = MockPortFactory(self.host)
self.port = self.host.port_factory.get()
# Create a testing manifest containing any test files that we
# might interact with.
self.host.filesystem.write_text_file(
self.port.web_tests_dir() + '/external/' + BASE_MANIFEST_NAME,
json.dumps({
'items': {
'reftest': {
'reftest.html': [
'c3f2fb6f436da59d43aeda0a7e8a018084557033',
[None, [['reftest-ref.html', '==']], {}],
]
},
'testharness': {
'test.html': [
'd933fd981d4a33ba82fb2b000234859bdda1494e',
[None, {}]
],
'crash.html': [
'd933fd981d4a33ba82fb2b000234859bdda1494e',
[None, {}]
],
'variant.html': [
'b8db5972284d1ac6bbda0da81621d9bca5d04ee7',
['variant.html?foo=bar/abc', {}],
['variant.html?foo=baz', {}],
],
'dir': {
'multiglob.https.any.js': [
'd6498c3e388e0c637830fa080cca78b0ab0e5305',
['dir/multiglob.https.any.window.html', {}],
['dir/multiglob.https.any.worker.html', {}],
],
},
},
},
}))
self.host.filesystem.write_text_file(
os.path.join(WEB_TESTS_DIR, "fast", "harness", "results.html"),
"results-viewer-body")
self.wpt_adapter = BaseWptScriptAdapter(self.host)
self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME
def _create_json_output(self, json_dict):
"""Writing some json output for processing."""
self.host.filesystem.write_text_file(OUTPUT_JSON_FILENAME,
json.dumps(json_dict))
def _load_json_output(self):
"""Loads the json output after post-processing."""
return json.loads(
self.host.filesystem.read_text_file(OUTPUT_JSON_FILENAME))
def test_write_log_artifact(self):
# Ensure that log artifacts are written to the correct location.
# We only generate an actual.txt if our actual wasn't PASS, so in this
# case we shouldn't write anything.
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'PASS',
'artifacts': {
'wpt_actual_status': ['OK'],
'log': ['test.html actual text'],
},
},
},
'path_delimiter': '/',
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results", "test-actual.txt")
diff_path = os.path.join("layout-test-results", "test-diff.txt")
pretty_diff_path = os.path.join("layout-test-results",
"test-pretty-diff.html")
assert actual_path not in written_files
assert diff_path not in written_files
assert pretty_diff_path not in written_files
# Now we change the outcome to be a FAIL, which should generate an
# actual.txt
json_dict['tests']['test.html']['actual'] = 'FAIL'
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results", "test-actual.txt")
self.assertEqual("test.html actual text", written_files[actual_path])
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
updated_json = self._load_json_output()
self.assertFalse(
"log" in updated_json["tests"]["test.html"]["artifacts"])
self.assertEqual(
[actual_path],
updated_json["tests"]["test.html"]["artifacts"]["actual_text"])
# Ensure that a diff was also generated. Since there's no expected
# output, the actual text is all new. We don't validate the entire diff
# files to avoid checking line numbers/markup.
diff_path = os.path.join("layout-test-results", "test-diff.txt")
self.assertIn("+test.html actual text", written_files[diff_path])
self.assertEqual(
[diff_path],
updated_json["tests"]["test.html"]["artifacts"]["text_diff"])
pretty_diff_path = os.path.join("layout-test-results",
"test-pretty-diff.html")
self.assertIn("test.html actual text", written_files[pretty_diff_path])
self.assertEqual([
pretty_diff_path
], updated_json["tests"]["test.html"]["artifacts"]["pretty_text_diff"])
def test_write_crashlog_artifact(self):
# Ensure that crash log artifacts are written to the correct location.
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'CRASH',
'artifacts': {
'wpt_actual_status': ['CRASH'],
'wpt_crash_log': ['test.html crashed!'],
},
},
},
'path_delimiter': '/',
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
crash_log_path = os.path.join("layout-test-results",
"test-crash-log.txt")
self.assertEqual("test.html crashed!", written_files[crash_log_path])
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
updated_json = self._load_json_output()
self.assertFalse(
"wpt_crash_log" in updated_json["tests"]["test.html"]["artifacts"])
self.assertEqual(
[crash_log_path],
updated_json["tests"]["test.html"]["artifacts"]["crash_log"])
def test_write_screenshot_artifacts(self):
# Ensure that screenshots are written to the correct filenames and
# their bytes are base64 decoded.
json_dict = {
'tests': {
'reftest.html': {
'expected': 'PASS',
'actual': 'PASS',
'artifacts': {
'wpt_actual_status': ['PASS'],
'screenshots': [
'reftest.html:abcd',
'reftest-ref.html:bcde',
],
},
},
},
'path_delimiter': '/',
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_image_path = os.path.join("layout-test-results",
"reftest-actual.png")
self.assertEqual(base64.b64decode('abcd'),
written_files[actual_image_path])
expected_image_path = os.path.join("layout-test-results",
"reftest-expected.png")
self.assertEqual(base64.b64decode('bcde'),
written_files[expected_image_path])
# Ensure the artifacts in the json were replaced with the location of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse("screenshots" in updated_json["tests"]["reftest.html"]
["artifacts"])
self.assertEqual(
[actual_image_path],
updated_json["tests"]["reftest.html"]["artifacts"]["actual_image"])
self.assertEqual([expected_image_path], updated_json["tests"]
["reftest.html"]["artifacts"]["expected_image"])
def test_copy_expected_output(self):
# Check that an -expected.txt file is created from a checked-in metadata
# ini file if it exists for a test
json_dict = {
'tests': {
'test.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'log': ['test.html actual text'],
},
},
},
'path_delimiter': '/',
}
self._create_json_output(json_dict)
# Also create a checked-in metadata file for this test
self.host.filesystem.write_text_file(
os.path.join(EXTERNAL_WPT_TESTS_DIR, "test.html.ini"),
"test.html checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results", "test-actual.txt")
self.assertEqual("test.html actual text", written_files[actual_path])
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join("layout-test-results",
"test-expected.txt")
self.assertEqual("test.html checked-in metadata",
written_files[expected_path])
# Ensure the artifacts in the json were replaced with the locations of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse(
"log" in updated_json["tests"]["test.html"]["artifacts"])
self.assertEqual(
[actual_path],
updated_json["tests"]["test.html"]["artifacts"]["actual_text"])
self.assertEqual(
[expected_path],
updated_json["tests"]["test.html"]["artifacts"]["expected_text"])
# Ensure that a diff was also generated. There should be both additions
# and deletions for this test since we have expected output. We don't
# validate the entire diff files to avoid checking line numbers/markup.
diff_path = os.path.join("layout-test-results", "test-diff.txt")
self.assertIn("-test.html checked-in metadata",
written_files[diff_path])
self.assertIn("+test.html actual text", written_files[diff_path])
self.assertEqual(
[diff_path],
updated_json["tests"]["test.html"]["artifacts"]["text_diff"])
pretty_diff_path = os.path.join("layout-test-results",
"test-pretty-diff.html")
self.assertIn("test.html checked-in metadata",
written_files[pretty_diff_path])
self.assertIn("test.html actual text", written_files[pretty_diff_path])
self.assertEqual([
pretty_diff_path
], updated_json["tests"]["test.html"]["artifacts"]["pretty_text_diff"])
def test_expected_output_for_variant(self):
# Check that an -expected.txt file is created from a checked-in metadata
# ini file for a variant test. Variants are a little different because
# we have to use the manifest to map a test name to the test file, and
# determine the associated metadata from the test file.
# Check that an -expected.txt file is created from a checked-in metadata
# ini file if it exists for a test
json_dict = {
'tests': {
'variant.html?foo=bar/abc': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'log': ['variant bar/abc actual text'],
},
},
},
'path_delimiter': '/',
}
self._create_json_output(json_dict)
# Also create a checked-in metadata file for this test. This filename
# matches the test *file* name, not the test name (which includes the
# variant).
self.host.filesystem.write_text_file(
os.path.join(EXTERNAL_WPT_TESTS_DIR, "variant.html.ini"),
"variant.html checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results",
"variant_foo=bar_abc-actual.txt")
self.assertEqual("variant bar/abc actual text",
written_files[actual_path])
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join("layout-test-results",
"variant_foo=bar_abc-expected.txt")
self.assertEqual("variant.html checked-in metadata",
written_files[expected_path])
# Ensure the artifacts in the json were replaced with the locations of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse("log" in updated_json["tests"]
["variant.html?foo=bar/abc"]["artifacts"])
self.assertEqual([actual_path],
updated_json["tests"]["variant.html?foo=bar/abc"]
["artifacts"]["actual_text"])
self.assertEqual([expected_path],
updated_json["tests"]["variant.html?foo=bar/abc"]
["artifacts"]["expected_text"])
def test_expected_output_for_multiglob(self):
# Check that an -expected.txt file is created from a checked-in metadata
# ini file for a multiglobal test. Multi-globals are a little different
# because we have to use the manifest to map a test name to the test
# file, and determine the associated metadata from the test file.
#
# Also note that the "dir" is both a directory and a part of the test
# name, so the path delimiter remains a / (ie: dir/multiglob) even on
# Windows.
json_dict = {
'tests': {
'dir/multiglob.https.any.worker.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'log': ['dir/multiglob worker actual text'],
},
},
},
'path_delimiter': '/',
}
self._create_json_output(json_dict)
# Also create a checked-in metadata file for this test. This filename
# matches the test *file* name, not the test name (which includes test
# scope).
self.host.filesystem.write_text_file(
os.path.join(EXTERNAL_WPT_TESTS_DIR,
"dir/multiglob.https.any.js.ini"),
"dir/multiglob checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join(
"layout-test-results", "dir/multiglob.https.any.worker-actual.txt")
self.assertEqual("dir/multiglob worker actual text",
written_files[actual_path])
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join(
"layout-test-results",
"dir/multiglob.https.any.worker-expected.txt")
self.assertEqual("dir/multiglob checked-in metadata",
written_files[expected_path])
# Ensure the artifacts in the json were replaced with the locations of
# the newly-created files.
updated_json = self._load_json_output()
self.assertFalse("log" in updated_json["tests"]
["dir/multiglob.https.any.worker.html"]["artifacts"])
self.assertEqual(
[actual_path],
updated_json["tests"]["dir/multiglob.https.any.worker.html"]
["artifacts"]["actual_text"])
self.assertEqual(
[expected_path],
updated_json["tests"]["dir/multiglob.https.any.worker.html"]
["artifacts"]["expected_text"])