def check_success_or_xfail(self, as_success, error_message=None):
if as_success:
self.assertEqual([
('startTest', self.test),
('addSuccess', self.test),
('stopTest', self.test),
], self.client._events)
else:
details = {}
if error_message is not None:
details['traceback'] = Content(
ContentType("text", "x-traceback", {'charset': 'utf8'}),
lambda: [_b(error_message)])
if isinstance(self.client, ExtendedTestResult):
value = details
else:
if error_message is not None:
value = subunit.RemoteError(
_u("Text attachment: traceback\n"
"------------\n") + _u(error_message) +
_u("------------\n"))
else:
value = subunit.RemoteError()
self.assertEqual([
('startTest', self.test),
('addExpectedFailure', self.test, value),
('stopTest', self.test),
], self.client._events)
def test_eq(self):
error = subunit.RemoteError(_u("Something went wrong"))
another_error = subunit.RemoteError(_u("Something went wrong"))
different_error = subunit.RemoteError(_u("boo!"))
self.assertEqual(error, another_error)
self.assertNotEqual(error, different_error)
self.assertNotEqual(different_error, another_error)
def test_add_failure(self):
protocol = MockTestProtocolServerClient()
protocol.addFailure(subunit.RemotedTestCase("old mcdonald"),
subunit.RemoteError("omg it works"))
self.assertEqual(protocol.start_calls, [])
self.assertEqual(protocol.end_calls, [])
self.assertEqual(protocol.error_calls, [])
self.assertEqual(protocol.failure_calls,
[(subunit.RemotedTestCase("old mcdonald"),
subunit.RemoteError("omg it works"))])
self.assertEqual(protocol.success_calls, [])
def test_run(self):
runner = MockTestProtocolServerClient()
test = self.SampleExecTestCase("test_sample_method")
test.run(runner)
mcdonald = subunit.RemotedTestCase("old mcdonald")
bing = subunit.RemotedTestCase("bing crosby")
an_error = subunit.RemotedTestCase("an error")
self.assertEqual(runner.error_calls,
[(an_error, subunit.RemoteError())])
self.assertEqual(
runner.failure_calls,
[(bing, subunit.RemoteError("foo.c:53:ERROR invalid state\n"))])
self.assertEqual(runner.start_calls, [mcdonald, bing, an_error])
self.assertEqual(runner.end_calls, [mcdonald, bing, an_error])
def test_add_error(self):
"""Test stopTest on a TestProtocolClient."""
self.protocol.addError(self.test, subunit.RemoteError("phwoar"))
self.assertEqual(
self.io.getvalue(), 'error: %s [\n'
"RemoteException: phwoar\n"
"]\n" % self.test.id())
def test_add_failure(self):
"""Test addFailure on a TestProtocolClient."""
self.protocol.addFailure(self.test, subunit.RemoteError(_u("boo qux")))
self.assertEqual(
self.io.getvalue(),
_b(('failure: %s [\n' + _remote_exception_str + ': boo qux\n]\n') %
self.test.id()))
def simple_failure_keyword(self, keyword):
self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls, [])
self.assertEqual(self.client.failure_calls,
[(self.test, subunit.RemoteError())])
def test_keywords_during_failure(self):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("failure: old mcdonald [\n")
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("failure a\n")
self.protocol.lineReceived("failure: a\n")
self.protocol.lineReceived("error a\n")
self.protocol.lineReceived("error: a\n")
self.protocol.lineReceived("success a\n")
self.protocol.lineReceived("success: a\n")
self.protocol.lineReceived("successful a\n")
self.protocol.lineReceived("successful: a\n")
self.protocol.lineReceived(" ]\n")
self.protocol.lineReceived("]\n")
self.assertEqual(self.stdout.getvalue(), "")
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.failure_calls,
[(self.test,
subunit.RemoteError("test old mcdonald\n"
"failure a\n"
"failure: a\n"
"error a\n"
"error: a\n"
"success a\n"
"success: a\n"
"successful a\n"
"successful: a\n"
"]\n"))])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls, [])
self.assertEqual(self.client.success_calls, [])
def test_failure_empty_message(self):
self.protocol.lineReceived("failure mcdonalds farm [\n")
self.protocol.lineReceived("]\n")
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls, [])
self.assertEqual(self.client.failure_calls,
[(self.test, subunit.RemoteError())])
def test_add_error(self):
"""Test stopTest on a TestProtocolClient."""
self.protocol.addError(self.test,
subunit.RemoteError(_u("phwoar crikey")))
self.assertEqual(
self.io.getvalue(),
_b(('error: %s [\n' + _remote_exception_str + ": phwoar crikey\n"
"]\n") % self.test.id()))
def test_add_expected_failure(self):
"""Test addExpectedFailure on a TestProtocolClient."""
self.protocol.addExpectedFailure(
self.test, subunit.RemoteError(_u("phwoar crikey")))
self.assertEqual(
self.io.getvalue(),
_b(('xfail: %s [\n' + _remote_exception_str + ": phwoar crikey\n"
"]\n") % self.test.id()))
def failure_quoted_bracket(self, keyword):
self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
self.protocol.lineReceived(" ]\n")
self.protocol.lineReceived("]\n")
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls, [])
self.assertEqual(self.client.failure_calls,
[(self.test, subunit.RemoteError("]\n"))])
def test_lost_connected_after_error(self):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("error old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual([
('startTest', self.test),
('addError', self.test, subunit.RemoteError(u"")),
('stopTest', self.test),
], self.client._events)
def setUp(self):
self.io = StringIO()
self.protocol = subunit.TestProtocolClient(self.io)
self.test = TestTestProtocolClient("test_start_test")
self.sample_details = {'something':Content(
ContentType('text', 'plain'), lambda:['serialised\nform'])}
self.sample_tb_details = dict(self.sample_details)
self.sample_tb_details['traceback'] = TracebackContent(
subunit.RemoteError(u"boo qux"), self.test)
def test_lost_connected_after_failure(self):
self.protocol.lineReceived(_b("test old mcdonald\n"))
self.protocol.lineReceived(_b("failure old mcdonald\n"))
self.protocol.lostConnection()
self.assertEqual([
('startTest', self.test),
('addFailure', self.test, subunit.RemoteError(_u(""))),
('stopTest', self.test),
], self.client._events)
def test_keywords_after_failure(self):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("failure old mcdonald\n")
self.keywords_before_test()
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls, [])
self.assertEqual(self.client.failure_calls,
[(self.test, subunit.RemoteError())])
self.assertEqual(self.client.success_calls, [])
def test_lost_connection_after_start(self):
self.protocol.lineReceived(_b("test old mcdonald\n"))
self.protocol.lostConnection()
failure = subunit.RemoteError(
_u("lost connection during test 'old mcdonald'"))
self.assertEqual([
('startTest', self.test),
('addError', self.test, failure),
('stopTest', self.test),
], self.client._events)
def test_lost_connected_after_error(self):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("error old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.failure_calls, [])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls,
[(self.test, subunit.RemoteError(""))])
self.assertEqual(self.client.success_calls, [])
def do_connection_lost(self, outcome, opening):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("%s old mcdonald %s" % (outcome, opening))
self.protocol.lostConnection()
failure = subunit.RemoteError(
u"lost connection during %s report of test 'old mcdonald'" %
outcome)
self.assertEqual([
('startTest', self.test),
('addError', self.test, failure),
('stopTest', self.test),
], self.client._events)
def addFailure(self, test, details=None):
test = self._add_prefix(test)
xfail_reason = find_in_list(self.expected_failures, test.id())
if xfail_reason is not None:
self.xfail_added += 1
self.total_xfail += 1
if details is not None:
details = subunit.RemoteError(
unicode(details[1]) + xfail_reason.decode("utf-8"))
else:
details = subunit.RemoteError(xfail_reason.decode("utf-8"))
self._ops.addExpectedFailure(test, details)
else:
self.fail_added += 1
self.total_fail += 1
self._ops.addFailure(test, details)
if self.output:
self._ops.output_msg(self.output)
if self.fail_immediately:
raise ImmediateFail()
self.output = None
def test_lost_connection_during_failure(self):
self.protocol.lineReceived("test old mcdonald\n")
self.protocol.lineReceived("failure old mcdonald [\n")
self.protocol.lostConnection()
self.assertEqual(self.client.start_calls, [self.test])
self.assertEqual(self.client.end_calls, [self.test])
self.assertEqual(self.client.error_calls,
[(self.test,
subunit.RemoteError("lost connection during "
"failure report"
" of test 'old mcdonald'"))])
self.assertEqual(self.client.failure_calls, [])
self.assertEqual(self.client.success_calls, [])
def test_addError(self):
self.result.addError(self, subunit.RemoteError())
def test_addExpectedFailure(self):
self.result.addExpectedFailure(self, subunit.RemoteError())
def parse_results(msg_ops, statistics, fh):
exitcode = 0
open_tests = {}
while fh:
l = fh.readline()
if l == "":
break
parts = l.split(None, 1)
if not len(parts) == 2 or not l.startswith(parts[0]):
msg_ops.output_msg(l)
continue
command = parts[0].rstrip(":")
arg = parts[1]
if command in ("test", "testing"):
msg_ops.control_msg(l)
name = arg.rstrip()
test = subunit.RemotedTestCase(name)
if name in open_tests:
msg_ops.addError(open_tests.pop(name),
subunit.RemoteError(u"Test already running"))
msg_ops.startTest(test)
open_tests[name] = test
elif command == "time":
msg_ops.control_msg(l)
try:
dt = subunit.iso8601.parse_date(arg.rstrip("\n"))
except TypeError, e:
print "Unable to parse time line: %s" % arg.rstrip("\n")
else:
msg_ops.time(dt)
elif command in VALID_RESULTS:
msg_ops.control_msg(l)
result = command
grp = re.match("(.*?)( \[)?([ \t]*)( multipart)?\n", arg)
(testname, hasreason) = (grp.group(1), grp.group(2))
if hasreason:
reason = ""
# reason may be specified in next lines
terminated = False
while fh:
l = fh.readline()
if l == "":
break
msg_ops.control_msg(l)
if l == "]\n":
terminated = True
break
else:
reason += l
remote_error = subunit.RemoteError(reason.decode("utf-8"))
if not terminated:
statistics['TESTS_ERROR'] += 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"reason (%s) interrupted" %
result))
return 1
else:
reason = None
remote_error = subunit.RemoteError(u"No reason specified")
if result in ("success", "successful"):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_EXPECTED_OK'] += 1
msg_ops.addSuccess(test)
elif result in ("xfail", "knownfail"):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_EXPECTED_FAIL'] += 1
msg_ops.addExpectedFailure(test, remote_error)
elif result in ("uxsuccess", ):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_UNEXPECTED_OK'] += 1
msg_ops.addUnexpectedSuccess(test, remote_error)
exitcode = 1
elif result in ("failure", "fail"):
try:
test = open_tests.pop(testname)
except KeyError:
statistics['TESTS_ERROR'] += 1
exitcode = 1
msg_ops.addError(
subunit.RemotedTestCase(testname),
subunit.RemoteError(u"Test was never started"))
else:
statistics['TESTS_UNEXPECTED_FAIL'] += 1
exitcode = 1
msg_ops.addFailure(test, remote_error)
elif result == "skip":
statistics['TESTS_SKIP'] += 1
# Allow tests to be skipped without prior announcement of test
try:
test = open_tests.pop(testname)
except KeyError:
test = subunit.RemotedTestCase(testname)
msg_ops.addSkip(test, reason)
elif result == "error":
statistics['TESTS_ERROR'] += 1
exitcode = 1
try:
test = open_tests.pop(testname)
except KeyError:
test = subunit.RemotedTestCase(testname)
msg_ops.addError(test, remote_error)
elif result == "skip-testsuite":
msg_ops.skip_testsuite(testname)
elif result == "testsuite-success":
msg_ops.end_testsuite(testname, "success", reason)
elif result == "testsuite-failure":
msg_ops.end_testsuite(testname, "failure", reason)
exitcode = 1
elif result == "testsuite-xfail":
msg_ops.end_testsuite(testname, "xfail", reason)
elif result == "testsuite-error":
msg_ops.end_testsuite(testname, "error", reason)
exitcode = 1
else:
raise AssertionError("Recognized but unhandled result %r" %
result)
arg = arg.strip()
if arg == "pop":
msg_ops.progress(None, subunit.PROGRESS_POP)
elif arg == "push":
msg_ops.progress(None, subunit.PROGRESS_PUSH)
elif arg[0] in '+-':
msg_ops.progress(int(arg), subunit.PROGRESS_CUR)
else:
msg_ops.progress(int(arg), subunit.PROGRESS_SET)
else:
msg_ops.output_msg(l)
while open_tests:
test = subunit.RemotedTestCase(open_tests.popitem()[1])
msg_ops.addError(
test, subunit.RemoteError(u"was started but never finished!"))
statistics['TESTS_ERROR'] += 1
exitcode = 1
return exitcode
class SubunitOps(subunit.TestProtocolClient, TestsuiteEnabledTestResult):
# The following are Samba extensions:
def start_testsuite(self, name):
self._stream.write("testsuite: %s\n" % name)
def skip_testsuite(self, name, reason=None):
if reason:
self._stream.write("skip-testsuite: %s [\n%s\n]\n" %
def test_empty_constructor(self):
self.assertEqual(subunit.RemoteError(), subunit.RemoteError(_u("")))
def test_add_failure(self):
"""Test addFailure on a TestProtocolClient."""
self.protocol.addFailure(self.test, subunit.RemoteError("boo"))
self.assertEqual(
self.io.getvalue(),
'failure: %s [\nRemoteException: boo\n]\n' % self.test.id())
