def test_is_testharness_output_passing_with_console_messages(self):
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' CONSOLE ERROR: BLAH \n'
' Harness: the test ran to completion.'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' CONSOLE WARNING: BLAH \n'
' Harness: the test ran to completion.'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'CONSOLE LOG: error.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'Harness: the test ran to completion.\n'
'\n'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'CONSOLE ERROR: error.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'Harness: the test ran to completion.\n'
'\n'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'CONSOLE WARNING: error.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'Harness: the test ran to completion.\n'
'\n'))
def test_is_testharness_output_passing_with_console_messages(self):
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' CONSOLE ERROR: BLAH \n'
' Harness: the test ran to completion.'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' CONSOLE WARNING: BLAH \n'
'PASS: some passing method\n'
' Harness: the test ran to completion.'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'CONSOLE LOG: error.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'Harness: the test ran to completion.\n'
'\n'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'CONSOLE ERROR: error.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'Harness: the test ran to completion.\n'
'\n'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'CONSOLE WARNING: error.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'Harness: the test ran to completion.\n'
'\n'))
def test_is_testharness_output_passing_with_timeout_or_notrun(self):
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' TIMEOUT: bah \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' NOTRUN: bah \n'
' Harness: the test ran to completion.'))
def test_is_testharness_output_passing_with_timeout_or_notrun(self):
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' TIMEOUT: bah \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' NOTRUN: bah \n'
' Harness: the test ran to completion.'))
def test_is_testharness_output_passing_no_pass(self):
# If there are no PASS lines, then the test is not considered to pass.
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' Foo bar \n'
' Harness: the test ran to completion.'))
def test_is_testharness_output_passing_unexpected_content(self):
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' Foo bar \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'RANDOM TEXT.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
'.Harness: the test ran to completion.\n'
'\n'))
def test_is_testharness_output_passing_with_pass_and_random_text(self):
self.assertTrue(testharness_results.is_testharness_output_passing(
'RANDOM TEXT.\n'
'This is a testharness.js-based test.\n'
'PASS: things are fine.\n'
' Harness: the test ran to completion.\n'
'\n'))
def test_is_testharness_output_passing_basic_examples(self):
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' PASS: foo bar \n'
' Harness: the test ran to completion.'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' PASS: foo bar FAIL \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' PASS: foo bar \n'
'FAIL \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' FAIL: bah \n'
' Harness: the test ran to completion.'))
def test_is_testharness_output_passing_basic_examples(self):
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' PASS: foo bar \n'
' Harness: the test ran to completion.'))
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' PASS: foo bar FAIL \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' PASS: foo bar \n'
'FAIL \n'
' Harness: the test ran to completion.'))
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' FAIL: bah \n'
' Harness: the test ran to completion.'))
def _compare_testharness_test(self, driver_output, expected_driver_output):
if expected_driver_output.image or expected_driver_output.audio or expected_driver_output.text:
return False, []
if driver_output.image or driver_output.audio or self._is_render_tree(driver_output.text):
return False, []
text = driver_output.text or ''
if not testharness_results.is_testharness_output(text):
return False, []
if not testharness_results.is_testharness_output_passing(text):
return True, [test_failures.FailureTestHarnessAssertion()]
return True, []
def test_is_testharness_output_passing(self):
test_data = [
{'content': 'This is a testharness.js-based test.\n Harness: the test ran to completion.', 'result': True},
{'content': 'This is a testharness.js-based test.\n \n Harness: the test ran to completion.', 'result': False},
{'content': 'This is a testharness.js-based test.\n PASS: foo bar \n Harness: the test ran to completion.', 'result': True},
{'content': 'This is a testharness.js-based test.\n PASS: foo bar FAIL \n Harness: the test ran to completion.', 'result': True},
{'content': 'This is a testharness.js-based test.\n PASS: foo bar \nFAIL \n Harness: the test ran to completion.', 'result': False},
{'content': 'This is a testharness.js-based test.\n CONSOLE ERROR: BLAH \n Harness: the test ran to completion.', 'result': True},
{'content': 'This is a testharness.js-based test.\n Foo bar \n Harness: the test ran to completion.', 'result': False},
{'content': 'This is a testharness.js-based test.\n FAIL: bah \n Harness: the test ran to completion.', 'result': False},
{'content': 'This is a testharness.js-based test.\n TIMEOUT: bah \n Harness: the test ran to completion.', 'result': False},
{'content': 'This is a testharness.js-based test.\n NOTRUN: bah \n Harness: the test ran to completion.', 'result': False},
{'content': 'CONSOLE LOG: error.\nThis is a testharness.js-based test.\nPASS: things are fine.\nHarness: the test ran to completion.\n\n', 'result': True},
{'content': 'CONSOLE ERROR: error.\nThis is a testharness.js-based test.\nPASS: things are fine.\nHarness: the test ran to completion.\n\n', 'result': True},
{'content': 'RANDOM TEXT.\nThis is a testharness.js-based test.\nPASS: things are fine.\n.Harness: the test ran to completion.\n\n', 'result': False},
]
for data in test_data:
self.assertEqual(data['result'], testharness_results.is_testharness_output_passing(data['content']))
def test_is_testharness_output_passing(self):
test_data = [
{
'content':
'This is a testharness.js-based test.\n Harness: the test ran to completion.',
'result': True
},
{
'content':
'This is a testharness.js-based test.\n \n Harness: the test ran to completion.',
'result': False
},
{
'content':
'This is a testharness.js-based test.\n PASS: foo bar \n Harness: the test ran to completion.',
'result': True
},
{
'content':
'This is a testharness.js-based test.\n PASS: foo bar FAIL \n Harness: the test ran to completion.',
'result': True
},
{
'content':
'This is a testharness.js-based test.\n PASS: foo bar \nFAIL \n Harness: the test ran to completion.',
'result': False
},
{
'content':
'This is a testharness.js-based test.\n CONSOLE ERROR: BLAH \n Harness: the test ran to completion.',
'result': True
},
{
'content':
'This is a testharness.js-based test.\n CONSOLE WARNING: BLAH \n Harness: the test ran to completion.',
'result': True
},
{
'content':
'This is a testharness.js-based test.\n Foo bar \n Harness: the test ran to completion.',
'result': False
},
{
'content':
'This is a testharness.js-based test.\n FAIL: bah \n Harness: the test ran to completion.',
'result': False
},
{
'content':
'This is a testharness.js-based test.\n TIMEOUT: bah \n Harness: the test ran to completion.',
'result': False
},
{
'content':
'This is a testharness.js-based test.\n NOTRUN: bah \n Harness: the test ran to completion.',
'result': False
},
{
'content':
('CONSOLE LOG: error.\nThis is a testharness.js-based test.\n'
'PASS: things are fine.\nHarness: the test ran to completion.\n\n'
),
'result':
True
},
{
'content':
('CONSOLE ERROR: error.\nThis is a testharness.js-based test.\n'
'PASS: things are fine.\nHarness: the test ran to completion.\n\n'
),
'result':
True
},
{
'content':
('CONSOLE WARNING: error.\nThis is a testharness.js-based test.\n'
'PASS: things are fine.\nHarness: the test ran to completion.\n\n'
),
'result':
True
},
{
'content':
('RANDOM TEXT.\nThis is a testharness.js-based test.\n'
'PASS: things are fine.\n.Harness: the test ran to completion.\n\n'
),
'result':
False
},
]
for data in test_data:
self.assertEqual(
data['result'],
testharness_results.is_testharness_output_passing(
data['content']))
def test_is_testharness_output_passing_empty_content(self):
self.assertFalse(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' Harness: the test ran to completion.'))
def test_is_testharness_output_passing_empty_content(self):
self.assertTrue(testharness_results.is_testharness_output_passing(
'This is a testharness.js-based test.\n'
' Harness: the test ran to completion.'))
