def test_finds_not_ok(self):
"""The parser extracts a not ok line."""
parser = Parser()
line = parser.parse_line('not ok - This is a failing test line.')
self.assertEqual('test', line.category)
self.assertFalse(line.ok)
self.assertTrue(line.number is None)
def test_finds_ok(self):
"""The parser extracts an ok line."""
parser = Parser()
line = parser.parse_line("ok - This is a passing test line.")
self.assertEqual("test", line.category)
self.assertTrue(line.ok)
self.assertTrue(line.number is None)
def test_diagnostic_line(self):
"""The parser extracts a diagnostic line."""
text = '# An example diagnostic line'
parser = Parser()
line = parser.parse_line(text)
self.assertEqual('diagnostic', line.category)
self.assertEqual(text, line.text)
def test_finds_not_ok(self):
"""The parser extracts a not ok line."""
parser = Parser()
line = parser.parse_line("not ok - This is a failing test line.")
self.assertEqual("test", line.category)
self.assertFalse(line.ok)
self.assertTrue(line.number is None)
self.assertEqual("", line.directive.text)
def test_parse_empty_file(self):
temp = tempfile.NamedTemporaryFile(delete=False)
temp.close()
parser = Parser()
lines = []
for line in parser.parse_file(temp.name):
lines.append(line)
self.assertEqual(0, len(lines))
def test_parses_stdin(self):
parser = Parser()
lines = []
for line in parser.parse_stdin():
lines.append(line)
self.assertEqual(3, len(lines))
self.assertEqual('plan', lines[0].category)
self.assertEqual('test', lines[1].category)
self.assertTrue(lines[1].ok)
self.assertEqual('test', lines[2].category)
self.assertFalse(lines[2].ok)
def test_finds_directive(self):
"""The parser extracts a directive"""
parser = Parser()
test_line = 'not ok - This line fails # TODO not implemented'
line = parser.parse_line(test_line)
directive = line.directive
self.assertEqual('test', line.category)
self.assertEqual('TODO not implemented', directive.text)
self.assertFalse(directive.skip)
self.assertTrue(directive.todo)
self.assertEqual('not implemented', directive.reason)
def parse(self, tap_file, progress_cb=None):
self.backend.configure()
test_execution_id = None
trace_back = []
for line in Parser().parse_file(tap_file):
if isinstance(line, Result):
# before parsing the 'next' result line add
# traceback as comment to the previous TE
if test_execution_id and trace_back:
self.backend.add_comment(test_execution_id,
"\n" + "\n".join(trace_back))
trace_back = []
elif isinstance(line, Diagnostic):
trace_back.append(line.text[2:])
continue
else:
continue
test_case, _ = self.backend.test_case_get_or_create(
line.description)
test_case_id = test_case['id']
self.backend.add_test_case_to_plan(test_case_id,
self.backend.plan_id)
comment = 'Result recorded via Kiwi TCMS tap-plugin'
if line.ok:
status_id = self.backend.get_status_id('PASSED')
else:
status_id = self.backend.get_status_id('FAILED')
if line.skip:
status_id = self.backend.get_status_id('WAIVED')
comment = line.directive.text
if line.todo:
status_id = self.backend.get_status_id('PAUSED')
comment = line.directive.text
for execution in self.backend.add_test_case_to_run(
test_case_id,
self.backend.run_id,
):
self.backend.update_test_execution(execution["id"], status_id,
comment)
if progress_cb:
progress_cb()
self.backend.finish_test_run()
def test_parses_text(self):
sample = inspect.cleandoc(u"""1..2
ok 1 A passing test
not ok 2 A failing test""")
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
self.assertEqual(3, len(lines))
self.assertEqual('plan', lines[0].category)
self.assertEqual('test', lines[1].category)
self.assertTrue(lines[1].ok)
self.assertEqual('test', lines[2].category)
self.assertFalse(lines[2].ok)
def test_malformed_yaml(self):
self.maxDiff = None
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
---
test: sample yaml
\tfail: tabs are not allowed!
...
not ok 2 A failing test"""
)
yaml_err = inspect.cleandoc(
u"""
WARNING: Optional imports not found, TAP 13 output will be
ignored. To parse yaml, see requirements in docs:
https://tappy.readthedocs.io/en/latest/consumers.html#tap-version-13"""
)
parser = Parser()
lines = []
with captured_output() as (parse_out, _):
for line in parser.parse_text(sample):
lines.append(line)
try:
import yaml # noqa
from more_itertools import peekable # noqa
self.assertEqual(4, len(lines))
self.assertEqual(13, lines[0].version)
with captured_output() as (out, _):
self.assertIsNone(lines[2].yaml_block)
self.assertEqual(
"Error parsing yaml block. Check formatting.", out.getvalue().strip()
)
self.assertEqual("test", lines[3].category)
self.assertIsNone(lines[3].yaml_block)
except ImportError:
self.assertEqual(8, len(lines))
self.assertEqual(13, lines[0].version)
for l in list(range(3, 7)):
self.assertEqual("unknown", lines[l].category)
self.assertEqual("test", lines[7].category)
self.assertEqual(yaml_err, parse_out.getvalue().strip())
def test_parses_yaml_more_complex(self):
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
---
message: test
severity: fail
data:
got:
- foo
expect:
- bar
output: |-
a multiline string
must be handled properly
even with | pipes
| here > and: there"""
)
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
if have_yaml:
converted_yaml = yaml.safe_load(
u'''
message: test
severity: fail
data:
got:
- foo
expect:
- bar
output: "a multiline string\\nmust be handled properly\\neven with | pipes\\n| here > and: there"''' # noqa
)
self.assertEqual(3, len(lines))
self.assertEqual(13, lines[0].version)
self.assertEqual(converted_yaml, lines[2].yaml_block)
else:
self.assertEqual(16, len(lines))
self.assertEqual(13, lines[0].version)
for line_index in list(range(3, 11)):
self.assertEqual("unknown", lines[line_index].category)
def test_parses_yaml_no_start(self):
sample = inspect.cleandoc(u"""TAP version 13
1..2
ok 1 A passing test
test: sample yaml
...
not ok 2 A failing test""")
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
self.assertEqual(6, len(lines))
self.assertEqual(13, lines[0].version)
self.assertIsNone(lines[2].yaml_block)
for l in list(range(3, 5)):
self.assertEqual('unknown', lines[l].category)
self.assertEqual('test', lines[5].category)
def test_parses_yaml_more_complex(self):
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
---
message: test
severity: fail
data:
got:
- foo
expect:
- bar"""
)
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
try:
import yaml
from more_itertools import peekable # noqa
converted_yaml = yaml.load(
u"""
message: test
severity: fail
data:
got:
- foo
expect:
- bar"""
)
self.assertEqual(3, len(lines))
self.assertEqual(13, lines[0].version)
self.assertEqual(converted_yaml, lines[2].yaml_block)
except ImportError:
self.assertEqual(11, len(lines))
self.assertEqual(13, lines[0].version)
for l in list(range(3, 11)):
self.assertEqual("unknown", lines[l].category)
def test_parses_file(self):
sample = inspect.cleandoc("""1..2
ok 1 A passing test
not ok 2 A failing test""")
temp = tempfile.NamedTemporaryFile(delete=False)
temp.write(sample.encode('utf-8'))
temp.close()
parser = Parser()
lines = []
for line in parser.parse_file(temp.name):
lines.append(line)
self.assertEqual(3, len(lines))
self.assertEqual('plan', lines[0].category)
self.assertEqual('test', lines[1].category)
self.assertTrue(lines[1].ok)
self.assertIsNone(lines[1].yaml_block)
self.assertEqual('test', lines[2].category)
self.assertFalse(lines[2].ok)
def test_parses_stdin(self, mock_sys):
mock_sys.stdin.__iter__.return_value = iter([
'1..2\n',
'ok 1 A passing test\n',
'not ok 2 A failing test\n',
])
mock_sys.stdin.__enter__.return_value = None
mock_sys.stdin.__exit__.return_value = None
parser = Parser()
lines = []
for line in parser.parse_stdin():
lines.append(line)
self.assertEqual(3, len(lines))
self.assertEqual('plan', lines[0].category)
self.assertEqual('test', lines[1].category)
self.assertTrue(lines[1].ok)
self.assertEqual('test', lines[2].category)
self.assertFalse(lines[2].ok)
def get_first_error(text):
try:
parser = Parser()
tap_lines = parser.parse_text(text)
# TODO: refactor
lines = text.split("\n")
output = []
seeking = False
for ix, tap_line in enumerate(tap_lines):
if seeking and (tap_line.category == "test"
or "Backtrace:" in lines[ix]):
break
if not seeking and tap_line.category == "test" and not tap_line.ok:
seeking = True
if seeking:
output.append(lines[ix])
if message := "\n".join(output):
return message
else:
def test_parses_yaml_no_association(self):
sample = inspect.cleandoc(u"""TAP version 13
1..2
ok 1 A passing test
# Diagnostic line
---
test: sample yaml
...
not ok 2 A failing test""")
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
self.assertEqual(8, len(lines))
self.assertEqual(13, lines[0].version)
self.assertIsNone(lines[2].yaml_block)
self.assertEqual('diagnostic', lines[3].category)
for l in list(range(4, 7)):
self.assertEqual('unknown', lines[l].category)
self.assertEqual('test', lines[7].category)
def test_errors_on_old_version(self):
"""The TAP spec dictates that anything less than 13 is an error."""
parser = Parser()
with self.assertRaises(ValueError):
parser.parse_line('TAP version 12')
def __init__(self):
self._parser = Parser()
def test_parses_yaml_more_complex(self):
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
---
message: test
severity: fail
data:
got:
- foo
expect:
- bar
output: |-
a multiline string
must be handled properly
even with | pipes
| here > and: there"""
)
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
if have_yaml:
converted_yaml = yaml.safe_load(
u'''
message: test
severity: fail
data:
got:
- foo
expect:
- bar
output: |-
a multiline string
must be handled properly
even with | pipes
| here > and: there"""
)
self.assertEqual(3, len(lines))
self.assertEqual(13, lines[0].version)
self.assertEqual(converted_yaml, lines[2].yaml_block)
else:
self.assertEqual(16, len(lines))
self.assertEqual(13, lines[0].version)
for l in list(range(3, 11)):
self.assertEqual("unknown", lines[l].category)
def test_parses_yaml_no_association(self):
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
# Diagnostic line
---
test: sample yaml
...
not ok 2 A failing test"""
)
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
self.assertEqual(8, len(lines))
self.assertEqual(13, lines[0].version)
self.assertIsNone(lines[2].yaml_block)
self.assertEqual("diagnostic", lines[3].category)
for l in list(range(4, 7)):
self.assertEqual("unknown", lines[l].category)
self.assertEqual("test", lines[7].category)
def test_parses_yaml_no_start(self):
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
test: sample yaml
...
not ok 2 A failing test"""
)
parser = Parser()
lines = []
for line in parser.parse_text(sample):
lines.append(line)
self.assertEqual(6, len(lines))
self.assertEqual(13, lines[0].version)
self.assertIsNone(lines[2].yaml_block)
for l in list(range(3, 5)):
self.assertEqual("unknown", lines[l].category)
self.assertEqual("test", lines[5].category)
def test_malformed_yaml(self):
self.maxDiff = None
sample = inspect.cleandoc(
u"""TAP version 13
1..2
ok 1 A passing test
---
test: sample yaml
\tfail: tabs are not allowed!
...
not ok 2 A failing test"""
)
yaml_err = inspect.cleandoc(
u"""
WARNING: Optional imports not found, TAP 13 output will be
ignored. To parse yaml, see requirements in docs:
https://tappy.readthedocs.io/en/latest/consumers.html#tap-version-13"""
)
parser = Parser()
lines = []
with captured_output() as (parse_out, _):
for line in parser.parse_text(sample):
lines.append(line)
if have_yaml:
self.assertEqual(4, len(lines))
self.assertEqual(13, lines[0].version)
with captured_output() as (out, _):
self.assertIsNone(lines[2].yaml_block)
self.assertEqual(
"Error parsing yaml block. Check formatting.", out.getvalue().strip()
)
self.assertEqual("test", lines[3].category)
self.assertIsNone(lines[3].yaml_block)
else:
self.assertEqual(8, len(lines))
self.assertEqual(13, lines[0].version)
for l in list(range(3, 7)):
self.assertEqual("unknown", lines[l].category)
self.assertEqual("test", lines[7].category)
self.assertEqual(yaml_err, parse_out.getvalue().strip())
def get_score(text):
parser = Parser()
lines = text.split("\n")
gen = parser.parse_text(text)
return sum(test.ok for ix, test in enumerate(gen)
if test.category == "test" and "WARNING" not in lines[ix])
import sys
from tap.parser import Parser
from tap.line import Result, Unknown, Diagnostic
out = ["### TAP Output for " + sys.argv[2]]
p = Parser()
in_error = False
for line in p.parse_file(sys.argv[1]):
if isinstance(line, Result):
if in_error:
out.append("")
out.append("</pre></code></details>")
out.append("")
out.append("----")
out.append("")
in_error = False
if not line.ok and not line.todo:
in_error = True
out.append("FAILURE Test #%d: ``%s``" %
(line.number, line.description))
out.append("")
out.append("<details><summary>Show log</summary><code><pre>")
elif isinstance(line, Diagnostic) and in_error:
out.append(line.text)
from tap.parser import Parser
output = """
TAP version 13
1..3
DsMakeSpn: HTTP/LAB1-W2K8R2-GW.lab1.awake.local
ok 1 - TestDsMakeSpn # 0.004000
ok 2 - TestDsCrackNames # 0.000000
ok 3 - TestDsCrackNames # SKIP
[100%] Built target kos-qemu-image-TestDsParse-sim
"""
lines = []
parser = Parser()
for line in parser.parse_text(output):
if line.category == "test":
d = line.directive
print(line.ok, line.description, line.skip, d.text)
if line.category == "plan":
print(line.expected_tests)
