def test_tags(self):
output = self.run_command(['-s', '--with-tag', 'a'],
_b("tags: a\n"
"test: foo\n"
"success: foo\n"
"tags: -a\n"
"test: bar\n"
"success: bar\n"
"test: baz\n"
"tags: a\n"
"success: baz\n"))
events = self.to_events(output)
foo = subunit.RemotedTestCase('foo')
baz = subunit.RemotedTestCase('baz')
self.assertEqual([
('tags', set(['a']), set()),
('startTest', foo),
('addSuccess', foo),
('stopTest', foo),
('tags', set(), set(['a'])),
('startTest', baz),
('tags', set(['a']), set()),
('addSuccess', baz),
('stopTest', baz),
], events)
def test_add_success(self):
protocol = MockTestProtocolServerClient()
protocol.addSuccess(subunit.RemotedTestCase("test old mcdonald"))
self.assertEqual(protocol.start_calls, [])
self.assertEqual(protocol.end_calls, [])
self.assertEqual(protocol.error_calls, [])
self.assertEqual(protocol.failure_calls, [])
self.assertEqual(protocol.success_calls,
[subunit.RemotedTestCase("test old mcdonald")])
def test_add_failure(self):
protocol = MockTestProtocolServerClient()
protocol.addFailure(subunit.RemotedTestCase("old mcdonald"),
subunit.RemoteError("omg it works"))
self.assertEqual(protocol.start_calls, [])
self.assertEqual(protocol.end_calls, [])
self.assertEqual(protocol.error_calls, [])
self.assertEqual(protocol.failure_calls,
[(subunit.RemotedTestCase("old mcdonald"),
subunit.RemoteError("omg it works"))])
self.assertEqual(protocol.success_calls, [])
def test_run(self):
runner = MockTestProtocolServerClient()
test = self.SampleExecTestCase("test_sample_method")
test.run(runner)
mcdonald = subunit.RemotedTestCase("old mcdonald")
bing = subunit.RemotedTestCase("bing crosby")
an_error = subunit.RemotedTestCase("an error")
self.assertEqual(runner.error_calls,
[(an_error, subunit.RemoteError())])
self.assertEqual(
runner.failure_calls,
[(bing, subunit.RemoteError("foo.c:53:ERROR invalid state\n"))])
self.assertEqual(runner.start_calls, [mcdonald, bing, an_error])
self.assertEqual(runner.end_calls, [mcdonald, bing, an_error])
def outSideTestaddSkip(self, offset, line):
"""A 'skip:' directive has been read."""
test_name = line[offset:-1].decode('utf8')
self.parser._current_test = subunit.RemotedTestCase(test_name)
self.parser.current_test_description = test_name
self.parser._state = self.parser._reading_skip_details
self.parser._reading_skip_details.set_simple()
self.parser.subunitLineReceived(line)
def test_only_one_time_sent(self):
# If we receive a single time event followed by a non-time event, we
# send exactly one time event.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase('foo'))
self.assertEquals([('time', a_time)], result._events[:-1])
def _add_prefix(self, test):
prefix = ""
suffix = ""
if self.prefix is not None:
prefix = self.prefix
if self.suffix is not None:
suffix = self.suffix
return subunit.RemotedTestCase(prefix + test.id() + suffix)
def test_duplicate_times_not_sent(self):
# Many time events with the exact same time are collapsed into one
# time event.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
for i in range(5):
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase('foo'))
self.assertEquals([('time', a_time)], result._events[:-1])
def test_story(self):
client = unittest.TestResult()
out = StringIO()
protocol = subunit.TestProtocolServer(client, forward_stream=out)
pipe = StringIO("test old mcdonald\n"
"success old mcdonald\n")
protocol.readFrom(pipe)
mcdonald = subunit.RemotedTestCase("old mcdonald")
self.assertEqual(client.testsRun, 1)
self.assertEqual(pipe.getvalue(), out.getvalue())
def test_lost_connected_after_failure(self):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("failure old mcdonald\n")
self.protocol.lostConnection()
test = subunit.RemotedTestCase("old mcdonald")
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls, [])
self.assertEqual(self.client.failure_calls,
[(self.test, subunit.RemoteError())])
self.assertEqual(self.client.success_calls, [])
def test_time_collapsed_to_first_and_last(self):
# If there are many consecutive time events, only the first and last
# are sent through.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
times = [self.make_time() for i in range(5)]
for a_time in times:
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase('foo'))
self.assertEquals([('time', times[0]), ('time', times[-1])],
result._events[:-1])
def test_skip_preserved(self):
subunit_stream = _b('\n'.join(["test: foo", "skip: foo", ""]))
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
self.run_tests(result_filter, subunit_stream)
foo = subunit.RemotedTestCase('foo')
self.assertEquals([
('startTest', foo),
('addSkip', foo, {}),
('stopTest', foo),
], result._events)
def test_story(self):
client = unittest.TestResult()
protocol = subunit.TestProtocolServer(client)
pipe = StringIO("test old mcdonald\n"
"success old mcdonald\n"
"test bing crosby\n"
"failure bing crosby [\n"
"foo.c:53:ERROR invalid state\n"
"]\n"
"test an error\n"
"error an error\n")
protocol.readFrom(pipe)
mcdonald = subunit.RemotedTestCase("old mcdonald")
bing = subunit.RemotedTestCase("bing crosby")
an_error = subunit.RemotedTestCase("an error")
self.assertEqual(client.errors, [(an_error, 'RemoteException: \n\n')])
self.assertEqual(
client.failures,
[(bing, "RemoteException: foo.c:53:ERROR invalid state\n\n")])
self.assertEqual(client.testsRun, 3)
def test_tags_collapsed_inside_of_tests_different_ordering(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase('foo')
tag_collapser.startTest(test)
tag_collapser.tags(set(), set(['a']))
tag_collapser.tags(set(['a', 'b']), set())
tag_collapser.tags(set(['c']), set())
tag_collapser.stopTest(test)
self.assertEquals([('startTest', test),
('tags', set(['a', 'b', 'c']), set()),
('stopTest', test)], result._events)
def test_no_times_inserted(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
foo = subunit.RemotedTestCase('foo')
tag_collapser.startTest(foo)
tag_collapser.addSuccess(foo)
tag_collapser.stopTest(foo)
self.assertEquals([('time', a_time), ('startTest', foo),
('addSuccess', foo), ('stopTest', foo)],
result._events)
def test_simple(self):
test = subunit.RemotedTestCase("A test description")
self.assertRaises(NotImplementedError, test.setUp)
self.assertRaises(NotImplementedError, test.tearDown)
self.assertEqual("A test description", test.shortDescription())
self.assertEqual("A test description", test.id())
self.assertEqual("A test description (subunit.RemotedTestCase)",
"%s" % test)
self.assertEqual(
"<subunit.RemotedTestCase description="
"'A test description'>", "%r" % test)
result = unittest.TestResult()
test.run(result)
self.assertEqual([(test, _remote_exception_str + ": "
"Cannot run RemotedTestCases.\n\n")], result.errors)
self.assertEqual(1, result.testsRun)
another_test = subunit.RemotedTestCase("A test description")
self.assertEqual(test, another_test)
different_test = subunit.RemotedTestCase("ofo")
self.assertNotEqual(test, different_test)
self.assertNotEqual(another_test, different_test)
def test_run(self):
result = ExtendedTestResult()
test = self.SampleExecTestCase("test_sample_method")
test.run(result)
mcdonald = subunit.RemotedTestCase("old mcdonald")
bing = subunit.RemotedTestCase("bing crosby")
bing_details = {}
bing_details['traceback'] = Content(ContentType("text", "x-traceback",
{'charset': 'utf8'}), lambda:["foo.c:53:ERROR invalid state\n"])
an_error = subunit.RemotedTestCase("an error")
error_details = {}
self.assertEqual([
('startTest', mcdonald),
('addSuccess', mcdonald),
('stopTest', mcdonald),
('startTest', bing),
('addFailure', bing, bing_details),
('stopTest', bing),
('startTest', an_error),
('addError', an_error, error_details),
('stopTest', an_error),
], result._events)
def test_story(self):
client = unittest.TestResult()
protocol = subunit.TestProtocolServer(client)
pipe = BytesIO(
_b("test old mcdonald\n"
"success old mcdonald\n"
"test bing crosby\n"
"failure bing crosby [\n"
"foo.c:53:ERROR invalid state\n"
"]\n"
"test an error\n"
"error an error\n"))
protocol.readFrom(pipe)
bing = subunit.RemotedTestCase("bing crosby")
an_error = subunit.RemotedTestCase("an error")
self.assertEqual(client.errors,
[(an_error, _remote_exception_str + '\n')])
self.assertEqual(
client.failures,
[(bing, _remote_exception_str + ": Text attachment: traceback\n"
"------------\nfoo.c:53:ERROR invalid state\n"
"------------\n\n")])
self.assertEqual(client.testsRun, 3)
def test_tags_sent_before_result(self):
# Because addSuccess and friends tend to send subunit output
# immediately, and because 'tags:' before a result line means
# something different to 'tags:' after a result line, we need to be
# sure that tags are emitted before 'addSuccess' (or whatever).
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase('foo')
tag_collapser.startTest(test)
tag_collapser.tags(set(['a']), set())
tag_collapser.addSuccess(test)
tag_collapser.stopTest(test)
self.assertEquals([('startTest', test), ('tags', set(['a']), set()),
('addSuccess', test), ('stopTest', test)],
result._events)
def test_tags_forwarded_after_tests(self):
test = subunit.RemotedTestCase('foo')
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.startTestRun()
tag_collapser.startTest(test)
tag_collapser.addSuccess(test)
tag_collapser.stopTest(test)
tag_collapser.tags(set(['a']), set(['b']))
tag_collapser.stopTestRun()
self.assertEqual([
('startTestRun', ),
('startTest', test),
('addSuccess', test),
('stopTest', test),
('tags', set(['a']), set(['b'])),
('stopTestRun', ),
], result._events)
def test_time_ordering_preserved(self):
# Passing a subunit stream through TestResultFilter preserves the
# relative ordering of 'time' directives and any other subunit
# directives that are still included.
date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC)
date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC)
date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC)
subunit_stream = _b('\n'.join([
"time: %s", "test: foo", "time: %s", "error: foo", "time: %s", ""
]) % (date_a, date_b, date_c))
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
self.run_tests(result_filter, subunit_stream)
foo = subunit.RemotedTestCase('foo')
self.assertEquals([('time', date_a), ('startTest', foo),
('time', date_b), ('addError', foo, {}),
('stopTest', foo), ('time', date_c)],
result._events)
def test_tags_tracked_correctly(self):
tag_filter = make_tag_filter(['a'], [])
result = ExtendedTestResult()
result_filter = TestResultFilter(result,
filter_success=False,
filter_predicate=tag_filter)
input_stream = _b("test: foo\n"
"tags: a\n"
"successful: foo\n"
"test: bar\n"
"successful: bar\n")
self.run_tests(result_filter, input_stream)
foo = subunit.RemotedTestCase('foo')
self.assertEquals([
('startTest', foo),
('tags', set(['a']), set()),
('addSuccess', foo),
('stopTest', foo),
], result._events)
def test_time_passes_through_filtered_tests(self):
# Passing a subunit stream through TestResultFilter preserves 'time'
# directives even if a specific test is filtered out.
date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC)
date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC)
date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC)
subunit_stream = _b('\n'.join([
"time: %s", "test: foo", "time: %s", "success: foo", "time: %s", ""
]) % (date_a, date_b, date_c))
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
result_filter.startTestRun()
self.run_tests(result_filter, subunit_stream)
result_filter.stopTestRun()
foo = subunit.RemotedTestCase('foo')
self.maxDiff = None
self.assertEqual([
('startTestRun', ),
('time', date_a),
('time', date_c),
('stopTestRun', ),
], result._events)
def setUp(self):
self.client = Python26TestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.test = subunit.RemotedTestCase("old mcdonald")
def test_default(self):
output = self.run_command([], _b("test: foo\n" "skip: foo\n"))
events = self.to_events(output)
foo = subunit.RemotedTestCase('foo')
self.assertEqual([('startTest', foo), ('addSkip', foo, {}),
('stopTest', foo)], events)
def setUp(self):
self.stdout = BytesIO()
self.test = subunit.RemotedTestCase("old mcdonald")
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client, self.stdout)
def test_start_testing_colon(self):
self.protocol.lineReceived(_b("testing: old mcdonald\n"))
self.assertEqual(
self.client._events,
[('startTest', subunit.RemotedTestCase("old mcdonald"))])
def parse_results(msg_ops, statistics, fh):
exitcode = 0
open_tests = {}
while fh:
l = fh.readline()
if l == "":
break
parts = l.split(None, 1)
if not len(parts) == 2 or not l.startswith(parts[0]):
msg_ops.output_msg(l)
continue
command = parts[0].rstrip(":")
arg = parts[1]
if command in ("test", "testing"):
msg_ops.control_msg(l)
name = arg.rstrip()
test = subunit.RemotedTestCase(name)
if name in open_tests:
msg_ops.addError(open_tests.pop(name),
subunit.RemoteError(u"Test already running"))
msg_ops.startTest(test)
open_tests[name] = test
elif command == "time":
msg_ops.control_msg(l)
try:
dt = subunit.iso8601.parse_date(arg.rstrip("\n"))
except TypeError, e:
print "Unable to parse time line: %s" % arg.rstrip("\n")
else:
msg_ops.time(dt)
elif command in VALID_RESULTS:
msg_ops.control_msg(l)
result = command
grp = re.match("(.*?)( \[)?([ \t]*)( multipart)?\n", arg)
(testname, hasreason) = (grp.group(1), grp.group(2))
if hasreason:
reason = ""
# reason may be specified in next lines
terminated = False
while fh:
l = fh.readline()
if l == "":
break
msg_ops.control_msg(l)
if l == "]\n":
terminated = True
break
else:
reason += l
remote_error = subunit.RemoteError(reason.decode("utf-8"))
if not terminated:
statistics['TESTS_ERROR'] += 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"reason (%s) interrupted" %
result))
return 1
else:
reason = None
remote_error = subunit.RemoteError(u"No reason specified")
if result in ("success", "successful"):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_EXPECTED_OK'] += 1
msg_ops.addSuccess(test)
elif result in ("xfail", "knownfail"):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_EXPECTED_FAIL'] += 1
msg_ops.addExpectedFailure(test, remote_error)
elif result in ("uxsuccess", ):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_UNEXPECTED_OK'] += 1
msg_ops.addUnexpectedSuccess(test, remote_error)
exitcode = 1
elif result in ("failure", "fail"):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_UNEXPECTED_FAIL'] += 1
exitcode = 1
msg_ops.addFailure(test, remote_error)
elif result == "skip":
statistics['TESTS_SKIP'] += 1
# Allow tests to be skipped without prior announcement of test
try:
test = open_tests.pop(testname)
except KeyError:
test = subunit.RemotedTestCase(testname)
msg_ops.addSkip(test, reason)
elif result == "error":
statistics['TESTS_ERROR'] += 1
exitcode = 1
try:
test = open_tests.pop(testname)
except KeyError:
test = subunit.RemotedTestCase(testname)
msg_ops.addError(test, remote_error)
elif result == "skip-testsuite":
msg_ops.skip_testsuite(testname)
elif result == "testsuite-success":
msg_ops.end_testsuite(testname, "success", reason)
elif result == "testsuite-failure":
msg_ops.end_testsuite(testname, "failure", reason)
exitcode = 1
elif result == "testsuite-xfail":
msg_ops.end_testsuite(testname, "xfail", reason)
elif result == "testsuite-error":
msg_ops.end_testsuite(testname, "error", reason)
exitcode = 1
else:
raise AssertionError("Recognized but unhandled result %r" %
result)
msg_ops.start_testsuite(arg.strip())
elif command == "progress":
arg = arg.strip()
if arg == "pop":
msg_ops.progress(None, subunit.PROGRESS_POP)
elif arg == "push":
msg_ops.progress(None, subunit.PROGRESS_PUSH)
elif arg[0] in '+-':
msg_ops.progress(int(arg), subunit.PROGRESS_CUR)
else:
msg_ops.progress(int(arg), subunit.PROGRESS_SET)
else:
msg_ops.output_msg(l)
while open_tests:
test = subunit.RemotedTestCase(open_tests.popitem()[1])
msg_ops.addError(
test, subunit.RemoteError(u"was started but never finished!"))
statistics['TESTS_ERROR'] += 1
exitcode = 1
return exitcode
class SubunitOps(subunit.TestProtocolClient, TestsuiteEnabledTestResult):
# The following are Samba extensions:
def start_testsuite(self, name):
self._stream.write("testsuite: %s\n" % name)
def skip_testsuite(self, name, reason=None):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(_b("test mcdonalds farm\n"))
self.test = subunit.RemotedTestCase("mcdonalds farm")
