def test_test_complete(self):
assert_equal(self.reporter.result_queue.qsize(), 0)
test_case = DummyTestCase()
fake_test_result = TestResult(test_case.run)
self.reporter.test_complete(fake_test_result.to_dict())
assert_equal(self.reporter.result_queue.qsize(), 0)
def test_test_complete(self):
assert_equal(self.reporter.result_queue.qsize(), 0)
test_case = DummyTestCase()
fake_test_result = TestResult(test_case.run)
self.reporter.test_complete(fake_test_result.to_dict())
assert_equal(self.reporter.result_queue.qsize(), 0)
def test_previous_run(self):
"""Insert a test result with two previous runs, and make sure it works properly."""
conn = self.reporter.conn
test_case = DummyTestCase()
results = [TestResult(test_case.test_pass) for _ in range(3)]
previous_run = None
for result in results:
if previous_run:
result.start(previous_run=previous_run.to_dict())
else:
result.start()
result.end_in_success()
previous_run = result
self.reporter.test_complete(results[-1].to_dict())
assert self.reporter.report() # Make sure all results are inserted.
test_results = self._get_test_results(conn)
assert_equal(len(test_results), 3)
for result in test_results:
assert_equal(result['method_name'], 'test_pass')
def test_previous_run(self):
"""Insert a test result with two previous runs, and make sure it works properly."""
conn = self.reporter.conn
test_case = DummyTestCase()
results = [TestResult(test_case.test_pass) for _ in xrange(3)]
previous_run = None
for result in results:
if previous_run:
result.start(previous_run=previous_run.to_dict())
else:
result.start()
result.end_in_success()
previous_run = result
self.reporter.test_complete(results[-1].to_dict())
assert self.reporter.report() # Make sure all results are inserted.
test_results = list(
conn.execute(
SA.select(columns=TestResults.columns + Tests.columns,
from_obj=TestResults.join(
Tests, TestResults.c.test == Tests.c.id))))
assert_equal(len(test_results), 3)
for result in test_results:
assert_equal(result['method_name'], 'test_pass')
def test_traceback_size_limit(self):
"""Insert a failure with a long exception and make sure it gets truncated."""
conn = self.reporter.conn
test_case = DummyTestCase()
result = TestResult(test_case.test_fail)
result.start()
result.end_in_failure(
(type(AssertionError), AssertionError('A' * 200), None))
with patch.object(self.reporter.options, 'sql_traceback_size', 50):
with patch.object(
result,
'format_exception_info') as mock_format_exception_info:
mock_format_exception_info.return_value = "AssertionError: %s\n%s\n" % (
'A' * 200, 'A' * 200)
self.reporter.test_complete(result.to_dict())
assert self.reporter.report()
failure = conn.execute(self.reporter.Failures.select()).fetchone()
assert_equal(len(failure.traceback), 50)
assert_equal(len(failure.error), 50)
assert_in('Exception truncated.', failure.traceback)
assert_in('Exception truncated.', failure.error)
def run_fixture(self, fixture, function_to_call, enter_callback=None, exit_callback=None):
result = TestResult(fixture)
try:
result.start()
if enter_callback:
enter_callback(result)
if result.record(function_to_call):
result.end_in_success()
else:
return result.exception_infos
except (KeyboardInterrupt, SystemExit):
result.end_in_interruption(sys.exc_info())
raise
finally:
if exit_callback:
exit_callback(result)
class TestResultTestCase(TestCase):
@setup
def setup_test_result(self):
self.test_method = mock.MagicMock(__name__='test_name')
self.test_result = TestResult(self.test_method)
def _set_exc_info(self, exc_type):
value, tb = mock.Mock(), mock.Mock(tb_next=None)
tb.tb_frame.f_globals.has_key.return_value = False
self.test_result.exception_info = exc_type, value, tb
return value, tb
@mock.patch('testify.test_result.traceback.format_exception',
autospec=True)
def test_format_exception_info_assertion(self, mock_format_exception):
value, tb = self._set_exc_info(AssertionError)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_called_with(AssertionError, value, tb, 1)
assert_equal(formatted, mock_format_exception.return_value)
@mock.patch('testify.test_result.traceback.format_exception',
autospec=True)
def test_format_exception_info_error(self, mock_format_exception):
value, tb = self._set_exc_info(ValueError)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_called_with(ValueError, value, tb)
assert_equal(formatted, mock_format_exception.return_value)
@mock.patch('testify.test_result.fancy_tb_formatter')
def test_format_exception_info_assertion_pretty(self, mock_format):
value, tb = self._set_exc_info(AssertionError)
formatted = self.test_result.format_exception_info(pretty=True)
mock_format.assert_called_with(AssertionError, value, tb, 1)
assert_equal(formatted, mock_format.return_value)
@mock.patch('testify.test_result.fancy_tb_formatter')
def test_format_exception_info_error_pretty(self, mock_format):
value, tb = self._set_exc_info(ValueError)
formatted = self.test_result.format_exception_info(pretty=True)
mock_format.assert_called_with(ValueError, value, tb)
assert_equal(formatted, mock_format.return_value)
class TestResultTestCase(TestCase):
@setup
def setup_test_result(self):
self.test_method = mock.MagicMock(__name__='test_name')
self.test_result = TestResult(self.test_method)
def _set_exc_info(self, exc_type):
value, tb = mock.Mock(), mock.Mock(tb_next=None)
tb.tb_frame.f_globals.has_key.return_value = False
self.test_result.exception_info = exc_type, value, tb
return value, tb
@mock.patch('testify.test_result.traceback.format_exception', autospec=True)
def test_format_exception_info_assertion(self, mock_format_exception):
value, tb = self._set_exc_info(AssertionError)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_called_with(AssertionError, value, tb, 1)
assert_equal(formatted, mock_format_exception.return_value)
@mock.patch('testify.test_result.traceback.format_exception', autospec=True)
def test_format_exception_info_error(self, mock_format_exception):
value, tb = self._set_exc_info(ValueError)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_called_with(ValueError, value, tb)
assert_equal(formatted, mock_format_exception.return_value)
@mock.patch('testify.test_result.fancy_tb_formatter')
def test_format_exception_info_assertion_pretty(self, mock_format):
value, tb = self._set_exc_info(AssertionError)
formatted = self.test_result.format_exception_info(pretty=True)
mock_format.assert_called_with(AssertionError, value, tb, 1)
assert_equal(formatted, mock_format.return_value)
@mock.patch('testify.test_result.fancy_tb_formatter')
def test_format_exception_info_error_pretty(self, mock_format):
value, tb = self._set_exc_info(ValueError)
formatted = self.test_result.format_exception_info(pretty=True)
mock_format.assert_called_with(ValueError, value, tb)
assert_equal(formatted, mock_format.return_value)
def test_happy_path(self):
conn = self.reporter.conn
test_case = DummyTestCase()
results = [
TestResult(test_case.test_pass),
TestResult(test_case.test_fail),
]
chunk = []
for result in results:
result.start()
result.end_in_success()
chunk.append(result.to_dict())
# In production, Someone Else takes care of manipulating the reporter's
# result_queue. We'll just mock the method we care about to avoid
# violating the Law of Demeter.
with patch.object(self.reporter.result_queue, 'task_done') as mock_task_done:
self.reporter._report_results_by_chunk(conn, chunk)
assert_equal(len(results), mock_task_done.call_count)
test_results = self._get_test_results(conn)
assert_equal(len(results), len(test_results))
def run_fixture(self, fixture, function_to_call, enter_callback=None, exit_callback=None):
result = TestResult(fixture)
try:
result.start()
if enter_callback:
enter_callback(result)
if result.record(function_to_call):
result.end_in_success()
else:
return result.exception_infos
except (KeyboardInterrupt, SystemExit):
result.end_in_interruption(sys.exc_info())
raise
finally:
if exit_callback:
exit_callback(result)
def run_fixture(self, fixture, function_to_call, enter_callback=None, exit_callback=None):
result = TestResult(fixture)
try:
result.start()
if enter_callback:
enter_callback(result)
if result.record(function_to_call):
result.end_in_success()
else:
return result.exception_infos
finally:
if exit_callback:
exit_callback(result)
def run_fixture(self,
fixture,
function_to_call,
enter_callback=None,
exit_callback=None):
result = TestResult(fixture)
try:
result.start()
if enter_callback:
enter_callback(result)
if result.record(function_to_call):
result.end_in_success()
else:
return result.exception_infos
finally:
if exit_callback:
exit_callback(result)
def test_json_reporter_reports(self):
self.set_options()
with mock_conf_files():
self.reporter = TestCaseJSONReporter(self.options)
test_case = TestCase()
fake_test_result = TestResult(test_case.run)
with mock.patch.object(datetime, 'datetime',
**{'now.return_value': start_time}):
fake_test_result.start()
with mock.patch.object(datetime, 'datetime',
**{'now.return_value': end_time}):
fake_test_result._complete()
self.reporter.test_case_complete(fake_test_result.to_dict())
assert_equal(
json.loads(self.reporter.log_file.getvalue()),
json.loads(output_str),
)
def test_json_reporter_reports(self):
self.set_options()
with mock_conf_files():
self.reporter = TestCaseJSONReporter(self.options)
test_case = TestCase()
fake_test_result = TestResult(test_case.run)
with mock.patch.object(
datetime, 'datetime', **{'now.return_value': start_time}
):
fake_test_result.start()
with mock.patch.object(datetime, 'datetime', **{'now.return_value': end_time}):
fake_test_result._complete()
self.reporter.test_case_complete(fake_test_result.to_dict())
assert_equal(
json.loads(self.reporter.log_file.getvalue()),
json.loads(output_str),
)
def test_frame_stripping(self, mock_format_exception):
"""On assertion error, testify strips head and tail frame which originate from testify."""
test_result = TestResult(lambda:'wat', runner_id='foo!')
test_result.start()
root_tb = tb = mock.Mock()
testify_frames = [True, True, False, True, False, True, True]
for testify_frame in testify_frames:
tb.tb_next = mock.Mock()
tb = tb.tb_next
tb.configure_mock(**{'tb_frame.f_globals.has_key.return_value': testify_frame})
tb.tb_next = None
tb = root_tb.tb_next
test_result.end_in_failure((AssertionError, 'wat', tb))
formatted = test_result.format_exception_info()
assert_equal(formatted, 'Traceback: AssertionError\n')
# It should format three frames of the stack, starting with the third frame.
mock_format_exception.assert_called_with(AssertionError, 'wat', tb.tb_next.tb_next, 3)
def test_traceback_size_limit(self):
"""Insert a failure with a long exception and make sure it gets truncated."""
conn = self.reporter.conn
test_case = DummyTestCase()
result = TestResult(test_case.test_fail)
result.start()
result.end_in_failure((type(AssertionError), AssertionError('A' * 200), None))
with patch.object(self.reporter.options, 'sql_traceback_size', 50):
with patch.object(result, 'format_exception_info') as mock_format_exception_info:
mock_format_exception_info.return_value = ["AssertionError: %s" % ('A' * 200), 'A' * 200]
self.reporter.test_complete(result.to_dict())
assert self.reporter.report()
failure = conn.execute(Failures.select()).fetchone()
assert_equal(len(failure.traceback), 50)
assert_equal(len(failure.error), 50)
assert_in('Exception truncated.', failure.traceback)
assert_in('Exception truncated.', failure.error)
def setup_test_result(self):
self.test_method = mock.MagicMock(__name__='test_name')
self.test_result = TestResult(self.test_method)
def setup_test_result(self):
self.test_method = mock.MagicMock(__name__='test_name')
self.test_result = TestResult(self.test_method)
class TestResultTestCase(TestCase):
@setup
def setup_test_result(self):
self.test_method = mock.MagicMock(__name__='test_name')
self.test_result = TestResult(self.test_method)
def _append_exc_info(self, exc_type):
value, tb = mock.Mock(), mock.Mock(tb_next=None)
tb.tb_frame.f_globals.has_key.return_value = False
self.test_result.exception_infos.append((exc_type, value, tb))
return value, tb
@mock.patch('traceback.format_exception', wraps=fake_format_exception)
def test_format_exception_info_assertion(self, mock_format_exception):
value, tb = self._append_exc_info(AssertionError)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_called_with(AssertionError, value, tb, 1)
assert_equal(formatted, 'Traceback: AssertionError\n')
@mock.patch('traceback.format_exception', wraps=fake_format_exception)
def test_format_exception_info_error(self, mock_format_exception):
value, tb = self._append_exc_info(ValueError)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_called_with(ValueError, value, tb, None)
assert_equal(formatted, 'Traceback: ValueError\n')
@mock.patch('testify.test_result.fancy_tb_formatter')
def test_format_exception_info_assertion_pretty(self, mock_format):
value, tb = self._append_exc_info(AssertionError)
formatted = self.test_result.format_exception_info(pretty=True)
mock_format.assert_called_with(AssertionError, value, tb, 1)
assert_equal(formatted, mock_format.return_value)
@mock.patch('testify.test_result.fancy_tb_formatter')
def test_format_exception_info_error_pretty(self, mock_format):
value, tb = self._append_exc_info(ValueError)
formatted = self.test_result.format_exception_info(pretty=True)
mock_format.assert_called_with(ValueError, value, tb)
assert_equal(formatted, mock_format.return_value)
@mock.patch('traceback.format_exception', wraps=fake_format_exception)
def test_format_exception_info_multiple(self, mock_format_exception):
class Error1(Exception): pass
class Error2(Exception): pass
value1, tb1 = self._append_exc_info(Error1)
value2, tb2 = self._append_exc_info(Error2)
formatted = self.test_result.format_exception_info()
mock_format_exception.assert_has_calls([
mock.call(Error1, value1, tb1, None),
mock.call(Error2, value2, tb2, None),
])
assert_equal(
formatted,
''.join((
'There were multiple errors in this test:\n',
'Traceback: Error1\n',
'Traceback: Error2\n',
))
)
def mock_test_result(self):
test_method = mock.Mock(__name__='test_name')
with mock.patch('testify.TestCase.test_result', new_callable=mock.PropertyMock) as test_result:
test_result.return_value = TestResult(test_method)
yield
