def test_adds_not_ok(self):
tracker = Tracker()
tracker.add_not_ok('FakeTestCase', 'a description')
line = tracker._test_cases['FakeTestCase'][0]
self.assertEqual(line.status, 'not ok')
self.assertEqual(line.description, 'a description')
self.assertEqual(line.directive, '')
def test_adds_skip(self):
tracker = Tracker()
tracker.add_skip('FakeTestCase', 'a description', 'a reason')
line = tracker._test_cases['FakeTestCase'][0]
self.assertEqual(line.status, 'ok')
self.assertEqual(line.description, 'a description')
self.assertEqual(line.directive, '# SKIP a reason')
def test_adds_skip(self):
tracker = Tracker()
tracker.add_skip('FakeTestCase', 'a description', 'a reason')
line = tracker._test_cases['FakeTestCase'][0]
self.assertTrue(line.ok)
self.assertEqual(line.description, 'a description')
self.assertEqual(line.directive.text, 'SKIP a reason')
def test_adds_skip(self):
tracker = Tracker()
tracker.add_skip("FakeTestCase", "a description", "a reason")
line = tracker._test_cases["FakeTestCase"][0]
self.assertTrue(line.ok)
self.assertEqual(line.description, "a description")
self.assertEqual(line.directive.text, "SKIP a reason")
def test_streaming_writes_plan(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.combined_line_number = 42
tracker.generate_tap_reports()
self.assertEqual(stream.getvalue(), '1..42\n')
def test_write_plan_first_combined(self):
outdir = tempfile.mkdtemp()
tracker = Tracker(streaming=False, outdir=outdir, combined=True)
tracker.set_plan(123)
tracker.generate_tap_reports()
with open(os.path.join(outdir, "testresults.tap"), "r") as f:
lines = f.readlines()
self.assertEqual(lines[0], "1..123\n")
def test_streaming_does_not_write_files(self):
outdir = tempfile.mkdtemp()
stream = StringIO()
tracker = Tracker(outdir=outdir, streaming=True, stream=stream)
tracker.add_ok("FakeTestCase", "YESSS!")
tracker.generate_tap_reports()
self.assertFalse(os.path.exists(os.path.join(outdir, "FakeTestCase.tap")))
def test_does_not_write_header(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream, header=False)
tracker.add_skip('FakeTestCase', 'YESSS!', 'a reason')
expected = inspect.cleandoc(
"""ok 1 - YESSS! # SKIP a reason""")
self.assertEqual(stream.getvalue().strip(), expected)
def test_generates_tap_reports_in_existing_outdir(self):
outdir = tempfile.mkdtemp()
tracker = Tracker(outdir=outdir)
tracker.add_ok('FakeTestCase', 'I should be in the specified dir.')
tracker.generate_tap_reports()
tap_file = os.path.join(outdir, 'FakeTestCase.tap')
self.assertTrue(os.path.exists(tap_file))
def test_generates_tap_reports_in_new_outdir(self):
tempdir = tempfile.mkdtemp()
outdir = os.path.join(tempdir, "non", "existent", "path")
tracker = Tracker(outdir=outdir)
tracker.add_ok("FakeTestCase", "I should be in the specified dir.")
tracker.generate_tap_reports()
tap_file = os.path.join(outdir, "FakeTestCase.tap")
self.assertTrue(os.path.exists(tap_file))
def test_add_skip_writes_to_stream_while_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_skip('FakeTestCase', 'YESSS!', 'a reason')
expected = inspect.cleandoc(
"""# TAP results for FakeTestCase
ok 1 - YESSS! # SKIP a reason
""")
self.assertEqual(stream.getvalue().strip(), expected)
def test_individual_report_has_no_plan_when_combined(self):
outdir = tempfile.mkdtemp()
tracker = Tracker(outdir=outdir, combined=True)
tracker.add_ok('FakeTestCase', 'Look ma, no plan!')
out_file = StringIO()
tracker.generate_tap_report(
'FakeTestCase', tracker._test_cases['FakeTestCase'], out_file)
report = out_file.getvalue()
self.assertTrue('Look ma' in report)
self.assertFalse('1..' in report)
def test_add_skip_writes_to_stream_while_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_skip('FakeTestCase', 'YESSS!', 'a reason')
expected = inspect.cleandoc(
"""{header}
ok 1 - YESSS! # SKIP a reason
""".format(
header=self._make_header('FakeTestCase')))
self.assertEqual(stream.getvalue().strip(), expected)
def test_add_not_ok_writes_to_stream_while_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_not_ok("FakeTestCase", "YESSS!")
expected = inspect.cleandoc(
"""{header}
not ok 1 YESSS!
""".format(
header=self._make_header("FakeTestCase")
)
)
self.assertEqual(stream.getvalue().strip(), expected)
def test_add_ok_writes_to_stream_while_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_ok('FakeTestCase', 'YESSS!')
tracker.add_ok('AnotherTestCase', 'Sure.')
expected = inspect.cleandoc(
"""# TAP results for FakeTestCase
ok 1 - YESSS!
# TAP results for AnotherTestCase
ok 2 - Sure.
""")
self.assertEqual(stream.getvalue().strip(), expected)
def test_add_ok_writes_to_stream_while_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_ok('FakeTestCase', 'YESSS!')
tracker.add_ok('AnotherTestCase', 'Sure.')
expected = inspect.cleandoc(
"""{header_1}
ok 1 - YESSS!
{header_2}
ok 2 - Sure.
""".format(
header_1=self._make_header('FakeTestCase'),
header_2=self._make_header('AnotherTestCase')))
self.assertEqual(stream.getvalue().strip(), expected)
def test_streaming_writes_tap_version_13(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_skip("FakeTestCase", "YESSS!", "a reason")
expected = inspect.cleandoc(
"""
TAP version 13
{header}
ok 1 YESSS! # SKIP a reason
""".format(
header=self._make_header("FakeTestCase")
)
)
self.assertEqual(stream.getvalue().strip(), expected)
def configure(self, options, conf):
super(TAP, self).configure(options, conf)
if self.enabled:
self.tracker = Tracker(
outdir=options.tap_outdir, combined=options.tap_combined,
streaming=options.tap_stream)
self._format = options.tap_format
def test_combined_results_in_one_file_tap_version_12(self):
outdir = tempfile.mkdtemp()
tracker = Tracker(outdir=outdir, combined=True)
tracker.add_ok("FakeTestCase", "YESSS!")
tracker.add_ok("DifferentFakeTestCase", "GOAAL!")
tracker.generate_tap_reports()
self.assertFalse(os.path.exists(os.path.join(outdir, "FakeTestCase.tap")))
self.assertFalse(
os.path.exists(os.path.join(outdir, "DifferentFakeTestCase.tap"))
)
with open(os.path.join(outdir, "testresults.tap"), "r") as f:
report = f.read()
expected = inspect.cleandoc(
"""{header_1}
ok 1 YESSS!
{header_2}
ok 2 GOAAL!
1..2
""".format(
header_1=self._make_header("FakeTestCase"),
header_2=self._make_header("DifferentFakeTestCase"),
)
)
self.assertEqual(report.strip(), expected)
def test_add_ok_writes_to_stream_while_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.add_ok("FakeTestCase", "YESSS!")
tracker.add_ok("AnotherTestCase", "Sure.")
expected = inspect.cleandoc(
"""{header_1}
ok 1 YESSS!
{header_2}
ok 2 Sure.
""".format(
header_1=self._make_header("FakeTestCase"),
header_2=self._make_header("AnotherTestCase"),
)
)
self.assertEqual(stream.getvalue().strip(), expected)
class TAP(Plugin):
"""This plugin provides test results in the Test Anything Protocol format.
"""
name = 'tap'
def options(self, parser, env=os.environ):
super(TAP, self).options(parser, env=env)
parser.add_option(
'--tap-outdir',
help='An optional output directory to write TAP files to. If the'
' directory does not exist, it will be created.')
def configure(self, options, conf):
super(TAP, self).configure(options, conf)
if self.enabled:
self.tracker = Tracker(outdir=options.tap_outdir)
def finalize(self, results):
self.tracker.generate_tap_reports()
def addError(self, test, err):
err_cls, reason, _ = err
if err_cls != SkipTest:
self.tracker.add_not_ok(
self._cls_name(test), self._description(test))
else:
self.tracker.add_skip(
self._cls_name(test), self._description(test), reason)
def addFailure(self, test, err):
self.tracker.add_not_ok(self._cls_name(test), self._description(test))
def addSuccess(self, test):
self.tracker.add_ok(self._cls_name(test), self._description(test))
def _cls_name(self, test):
# nose masks the true test case name so the real class name is found
# under the test attribute.
return test.test.__class__.__name__
def _description(self, test):
return test.shortDescription() or str(test)
def test_combined_results_in_one_file(self):
outdir = tempfile.mkdtemp()
tracker = Tracker(outdir=outdir, combined=True)
tracker.add_ok('FakeTestCase', 'YESSS!')
tracker.add_ok('DifferentFakeTestCase', 'GOAAL!')
tracker.generate_tap_reports()
self.assertFalse(
os.path.exists(os.path.join(outdir, 'FakeTestCase.tap')))
self.assertFalse(
os.path.exists(os.path.join(outdir, 'DifferentFakeTestCase.tap')))
with open(os.path.join(outdir, 'testresults.tap'), 'r') as f:
report = f.read()
expected = inspect.cleandoc(
"""# TAP results for FakeTestCase
ok 1 - YESSS!
# TAP results for DifferentFakeTestCase
ok 2 - GOAAL!
1..2
""")
self.assertEqual(report.strip(), expected)
def test_adds_not_ok_with_diagnostics(self):
tracker = Tracker()
tracker.add_not_ok(
'FakeTestCase', 'a description', diagnostics='# more info\n')
line = tracker._test_cases['FakeTestCase'][0]
self.assertEqual('# more info\n', line.diagnostics)
def configure(self, options, conf):
super(TAP, self).configure(options, conf)
if self.enabled:
self.tracker = Tracker(outdir=options.tap_outdir)
def test_adds_ok(self):
tracker = Tracker()
tracker.add_ok("FakeTestCase", "a description")
line = tracker._test_cases["FakeTestCase"][0]
self.assertTrue(line.ok)
self.assertEqual(line.description, "a description")
def test_write_plan_first_streaming(self):
stream = StringIO()
tracker = Tracker(streaming=True, stream=stream)
tracker.set_plan(123)
tracker.generate_tap_reports()
self.assertEqual(stream.getvalue(), "1..123\n")
def test_header_set_by_init(self):
tracker = Tracker(header=False)
self.assertFalse(tracker.header)
class TAPTestResult(TextTestResult):
# This attribute will store the user's desired output directory.
OUTDIR = None
def __init__(self, stream, descriptions, verbosity):
super(TAPTestResult, self).__init__(stream, descriptions, verbosity)
self.tracker = Tracker(outdir=self.OUTDIR)
def stopTestRun(self):
"""Once the test run is complete, generate each of the TAP files."""
super(TAPTestResult, self).stopTestRun()
self.tracker.generate_tap_reports()
def addError(self, test, err):
super(TAPTestResult, self).addError(test, err)
self.tracker.add_not_ok(self._cls_name(test), self._description(test))
def addFailure(self, test, err):
super(TAPTestResult, self).addFailure(test, err)
self.tracker.add_not_ok(self._cls_name(test), self._description(test))
def addSuccess(self, test):
super(TAPTestResult, self).addSuccess(test)
self.tracker.add_ok(self._cls_name(test), self._description(test))
def addSkip(self, test, reason):
super(TAPTestResult, self).addSkip(test, reason)
self.tracker.add_skip(
self._cls_name(test), self._description(test), reason)
def addExpectedFailure(self, test, err):
super(TAPTestResult, self).addExpectedFailure(test, err)
self.tracker.add_not_ok(self._cls_name(test), self._description(test),
'(expected failure)')
def addUnexpectedSuccess(self, test):
super(TAPTestResult, self).addUnexpectedSuccess(test)
self.tracker.add_ok(self._cls_name(test), self._description(test),
'(unexpected success)')
def _cls_name(self, test):
return test.__class__.__name__
def _description(self, test):
return test.shortDescription() or str(test)
def test_sanitizes_tap_file_path(self):
tracker = Tracker()
file_path = tracker._get_tap_file_path('an awful \\ testcase / name\n')
self.assertEqual('an-awful---testcase---name-.tap', file_path)
def test_get_default_tap_file_path(self):
tracker = Tracker()
file_path = tracker._get_tap_file_path("foo")
self.assertEqual("foo.tap", file_path)
def test_sanitizes_tap_file_path(self):
tracker = Tracker()
file_path = tracker._get_tap_file_path("an awful \\ testcase / name\n")
self.assertEqual("an-awful---testcase---name-.tap", file_path)
def setUp(self):
"""The pytest plugin uses module scope so a fresh tracker
must be installed each time."""
# When running this suite with pytest, save and restore the tracker.
self._tracker = plugin.tracker
plugin.tracker = Tracker()
def test_adds_not_ok_with_diagnostics(self):
tracker = Tracker()
tracker.add_not_ok("FakeTestCase", "a description", diagnostics="# more info\n")
line = tracker._test_cases["FakeTestCase"][0]
self.assertEqual("# more info\n", line.diagnostics)
def test_results_not_combined_by_default(self):
tracker = Tracker()
self.assertFalse(tracker.combined)
def test_get_default_tap_file_path(self):
tracker = Tracker()
file_path = tracker._get_tap_file_path('foo')
self.assertEqual('foo.tap', file_path)
def test_adds_not_ok(self):
tracker = Tracker()
tracker.add_not_ok('FakeTestCase', 'a description')
line = tracker._test_cases['FakeTestCase'][0]
self.assertFalse(line.ok)
self.assertEqual(line.description, 'a description')
def test_adds_not_ok(self):
tracker = Tracker()
tracker.add_not_ok('FakeTestCase', 'a description')
line = tracker._test_cases['FakeTestCase'][0]
self.assertFalse(line.ok)
self.assertEqual(line.description, 'a description')
def __init__(self, stream, descriptions, verbosity):
super(TAPTestResult, self).__init__(stream, descriptions, verbosity)
self.tracker = Tracker(outdir=self.OUTDIR)
def test_header_displayed_by_default(self):
tracker = Tracker()
self.assertTrue(tracker.header)
def test_tracker_has_stream(self):
tracker = Tracker()
self.assertTrue(tracker.stream is None)
def test_has_test_cases(self):
tracker = Tracker()
self.assertEqual(tracker._test_cases, {})
def test_get_default_tap_file_path(self):
tracker = Tracker()
file_path = tracker._get_tap_file_path('foo')
self.assertEqual('foo.tap', file_path)
def test_tracks_class(self):
tracker = Tracker()
tracker._track("FakeTestClass")
self.assertEqual(tracker._test_cases.get("FakeTestClass"), [])
def test_adds_not_ok_with_diagnostics(self):
tracker = Tracker()
tracker.add_not_ok(
'FakeTestCase', 'a description', diagnostics='# more info\n')
line = tracker._test_cases['FakeTestCase'][0]
self.assertEqual('# more info\n', line.diagnostics)
def test_write_plan_immediate_streaming(self):
stream = StringIO()
Tracker(streaming=True, stream=stream, plan=123)
self.assertEqual(stream.getvalue(), "1..123\n")
def test_tracks_class(self):
tracker = Tracker()
tracker._track('FakeTestClass')
self.assertEqual(tracker._test_cases.get('FakeTestClass'), [])
def test_write_plan_first_not_combined(self):
outdir = tempfile.mkdtemp()
tracker = Tracker(streaming=False, outdir=outdir, combined=False)
with self.assertRaises(ValueError):
tracker.set_plan(123)
def _make_one(cls):
# Yep, the stream is not being closed.
stream = open(os.devnull, "w")
result = TAPTestResult(stream, False, 0)
result.tracker = Tracker()
return result
from cStringIO import StringIO
except ImportError: # pragma: no cover
from io import StringIO
import sys
from py.io import TerminalWriter
import pytest
import six
from tap.formatter import format_as_diagnostics
from tap.tracker import Tracker
from pytest_tap.i18n import _
# Because of how pytest hooks work, there is not much choice
# except to use module level state. Ugh.
tracker = Tracker()
ENABLED = False
def pytest_addoption(parser):
"""Include all the command line options."""
group = parser.getgroup("terminal reporting", "reporting", after="general")
group.addoption(
"--tap-stream",
default=False,
action="store_true",
help=_("Stream TAP output instead of the default test runner output."),
)
group.addoption(
"--tap-files",
default=False,
def test_tracker_does_not_stream_by_default(self):
tracker = Tracker()
self.assertFalse(tracker.streaming)
class TAP(Plugin):
"""This plugin provides test results in the Test Anything Protocol format.
"""
name = 'tap'
def options(self, parser, env=os.environ):
super(TAP, self).options(parser, env=env)
parser.add_option(
'--tap-stream',
default=False,
action='store_true',
help=_('Stream TAP output instead of the default test runner'
' output.'))
parser.add_option(
'--tap-outdir',
metavar='PATH',
help=_('An optional output directory to write TAP files to. '
'If the directory does not exist, it will be created.'))
parser.add_option(
'--tap-combined',
default=False,
action='store_true',
help=_('Store all TAP test results into a combined output file.'))
parser.add_option(
'--tap-format',
default='',
metavar='FORMAT',
help=_('An optional format string for the TAP output.'
' The format options are:'
' {short_description} for the short description, and'
' {method_name} for the test method name.'))
def configure(self, options, conf):
super(TAP, self).configure(options, conf)
if self.enabled:
self.tracker = Tracker(outdir=options.tap_outdir,
combined=options.tap_combined,
streaming=options.tap_stream)
self._format = options.tap_format
def finalize(self, results):
self.tracker.generate_tap_reports()
def setOutputStream(self, stream):
# When streaming is on, hijack the stream and return a dummy to send
# standard nose output to oblivion.
if self.tracker.streaming:
self.tracker.stream = stream
return DummyStream()
return stream
def addError(self, test, err):
err_cls, reason, _ = err
if err_cls != SkipTest:
self.tracker.add_not_ok(self._cls_name(test),
self._description(test))
else:
self.tracker.add_skip(self._cls_name(test),
self._description(test), reason)
def addFailure(self, test, err):
self.tracker.add_not_ok(self._cls_name(test), self._description(test))
def addSuccess(self, test):
self.tracker.add_ok(self._cls_name(test), self._description(test))
def _cls_name(self, test):
if isinstance(test, ContextSuite):
# In the class setup and teardown, test is a ContextSuite
# instead of a test case. Grab the name from the context.
return test.context.__name__
else:
# nose masks the true test case name so the real class name
# is found under the test attribute.
return test.test.__class__.__name__
def _description(self, test):
if self._format:
try:
return self._format.format(
method_name=str(test),
short_description=test.shortDescription() or '')
except KeyError:
sys.exit(
_('Bad format string: {format}\n'
'Replacement options are: {{short_description}} and '
'{{method_name}}').format(format=self._format))
return test.shortDescription() or str(test)