def setUp(self):
super(TestFlakyPlugin, self).setUp()
test_mod = 'box.test.flaky.flaky_plugin'
self._mock_test_result = MagicMock()
self._mock_stream = MagicMock(spec=StringIO)
with patch(test_mod + '.TextTestResult') as flaky_result:
with patch(test_mod + '.StringIO') as string_io:
string_io.return_value = self._mock_stream
flaky_result.return_value = self._mock_test_result
self._flaky_plugin = FlakyPlugin()
self._mock_test = MagicMock(name='flaky_plugin_test')
self._mock_test_case = MagicMock(name='flaky_plugin_test_case',
spec=TestCase)
self._mock_test_case.address = MagicMock()
self._mock_test_case.test = self._mock_test
self._mock_test_module_name = 'test_module'
self._mock_test_class_name = 'TestClass'
self._mock_test_method_name = 'test_method'
self._mock_test_names = '{}:{}.{}'.format(self._mock_test_module_name,
self._mock_test_class_name,
self._mock_test_method_name)
self._mock_exception = Exception('Error in {}'.format(
self._mock_test_method_name))
self._mock_stack_trace = ''
self._mock_exception_type = Exception
self._mock_error = (self._mock_exception_type, self._mock_exception,
self._mock_stack_trace)
def setUp(self):
super(TestFlakyPlugin, self).setUp()
test_mod = 'box.test.flaky.flaky_plugin'
self._mock_test_result = MagicMock()
self._mock_stream = MagicMock(spec=StringIO)
with patch(test_mod + '.TextTestResult') as flaky_result:
with patch(test_mod + '.StringIO') as string_io:
string_io.return_value = self._mock_stream
flaky_result.return_value = self._mock_test_result
self._flaky_plugin = FlakyPlugin()
self._mock_test = MagicMock(name='flaky_plugin_test')
self._mock_test_case = MagicMock(
name='flaky_plugin_test_case',
spec=TestCase
)
self._mock_test_case.address = MagicMock()
self._mock_test_case.test = self._mock_test
self._mock_test_module_name = 'test_module'
self._mock_test_class_name = 'TestClass'
self._mock_test_method_name = 'test_method'
self._mock_test_names = '{}:{}.{}'.format(
self._mock_test_module_name,
self._mock_test_class_name,
self._mock_test_method_name
)
self._mock_exception = Exception('Error in {}'.format(
self._mock_test_method_name)
)
self._mock_stack_trace = ''
self._mock_exception_type = Exception
self._mock_error = (
self._mock_exception_type,
self._mock_exception,
self._mock_stack_trace
)
class TestFlakyPlugin(TestCase):
def setUp(self):
super(TestFlakyPlugin, self).setUp()
test_mod = 'box.test.flaky.flaky_plugin'
self._mock_test_result = MagicMock()
self._mock_stream = MagicMock(spec=StringIO)
with patch(test_mod + '.TextTestResult') as flaky_result:
with patch(test_mod + '.StringIO') as string_io:
string_io.return_value = self._mock_stream
flaky_result.return_value = self._mock_test_result
self._flaky_plugin = FlakyPlugin()
self._mock_test = MagicMock(name='flaky_plugin_test')
self._mock_test_case = MagicMock(
name='flaky_plugin_test_case',
spec=TestCase
)
self._mock_test_case.address = MagicMock()
self._mock_test_case.test = self._mock_test
self._mock_test_module_name = 'test_module'
self._mock_test_class_name = 'TestClass'
self._mock_test_method_name = 'test_method'
self._mock_test_names = '{}:{}.{}'.format(
self._mock_test_module_name,
self._mock_test_class_name,
self._mock_test_method_name
)
self._mock_exception = Exception('Error in {}'.format(
self._mock_test_method_name)
)
self._mock_stack_trace = ''
self._mock_exception_type = Exception
self._mock_error = (
self._mock_exception_type,
self._mock_exception,
self._mock_stack_trace
)
def _expect_call_test_address(self):
self._mock_test_case.address.return_value = (
None,
None,
self._mock_test_names
)
def _expect_test_flaky(self, is_test_method, max_runs, min_passes):
self._expect_call_test_address()
if is_test_method:
for flaky_attr in FlakyNames():
setattr(self._mock_test, flaky_attr, None)
mock_test_method = getattr(
self._mock_test,
self._mock_test_method_name
)
flaky(max_runs, min_passes)(mock_test_method)
else:
flaky(max_runs, min_passes)(self._mock_test)
mock_test_method = getattr(
self._mock_test,
self._mock_test_method_name
)
for flaky_attr in FlakyNames():
setattr(mock_test_method, flaky_attr, None)
def _expect_test_not_flaky(self):
self._expect_call_test_address()
for test_object in (
self._mock_test,
getattr(self._mock_test, self._mock_test_method_name)
):
for flaky_attr in FlakyNames():
setattr(test_object, flaky_attr, None)
def _assert_test_ignored(self):
self._mock_test_case.address.assert_called_with()
self.assertEqual(self._mock_test_case.mock_calls, [call.address()])
self.assertEqual(self._mock_test.mock_calls, [])
def _get_flaky_attributes(self, is_test_method):
if is_test_method:
mock_test_method = getattr(
self._mock_test,
self._mock_test_method_name
)
test_object = mock_test_method
else:
test_object = self._mock_test
actual_flaky_attributes = {
attr: getattr(
test_object,
attr
) for attr in FlakyNames()
}
return actual_flaky_attributes
def _set_flaky_attribute(self, is_test_method, attr, value):
if is_test_method:
mock_test_method = getattr(
self._mock_test,
self._mock_test_method_name
)
test_object = mock_test_method
else:
test_object = self._mock_test
setattr(test_object, attr, value)
def _assert_flaky_attributes_contains(
self,
expected_flaky_attributes,
):
actual_flaky_attributes = self._get_flaky_attributes(True)
self.assertDictContainsSubset(
expected_flaky_attributes,
actual_flaky_attributes,
'Unexpected flaky attributes, {} vs {}'.format(
expected_flaky_attributes,
actual_flaky_attributes
)
)
def test_flaky_plugin_report(self):
flaky_report = 'Flaky tests passed; others failed. ' \
'No more tests; that ship has sailed.'
self._test_flaky_plugin_report(flaky_report)
def test_flaky_plugin_handles_success_for_test_method(self):
self._test_flaky_plugin_handles_success()
def test_flaky_plugin_handles_success_for_test_instance(self):
self._test_flaky_plugin_handles_success(is_test_method=False)
def test_flaky_plugin_handles_success_for_needs_rerun(self):
self._test_flaky_plugin_handles_success(min_passes=2)
def test_flaky_plugin_ignores_success_for_non_flaky_test(self):
self._expect_test_not_flaky()
self._flaky_plugin.addSuccess(self._mock_test_case)
self._assert_test_ignored()
def test_flaky_plugin_ignores_error_for_non_flaky_test(self):
self._expect_test_not_flaky()
self._flaky_plugin.handleError(self._mock_test_case, None)
self._assert_test_ignored()
def test_flaky_plugin_ignores_failure_for_non_flaky_test(self):
self._expect_test_not_flaky()
self._flaky_plugin.handleFailure(self._mock_test_case, None)
self._assert_test_ignored()
def test_flaky_plugin_handles_error_for_test_method(self):
self._test_flaky_plugin_handles_failure_or_error()
def test_flaky_plugin_handles_error_for_test_instance(self):
self._test_flaky_plugin_handles_failure_or_error(is_test_method=False)
def test_flaky_plugin_handles_failure_for_test_method(self):
self._test_flaky_plugin_handles_failure_or_error(is_failure=True)
def test_flaky_plugin_handles_failure_for_test_instance(self):
self._test_flaky_plugin_handles_failure_or_error(
is_failure=True,
is_test_method=False
)
def test_flaky_plugin_handles_failure_for_no_more_retries(self):
self._test_flaky_plugin_handles_failure_or_error(
is_failure=True,
max_runs=1
)
def test_flaky_plugin_handles_additional_errors(self):
self._test_flaky_plugin_handles_failure_or_error(
current_errors=[self._mock_error]
)
def _test_flaky_plugin_handles_failure_or_error(
self,
current_errors=None,
current_passes=0,
current_runs=0,
is_failure=False,
is_test_method=True,
max_runs=2,
min_passes=1,
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
if current_errors is None:
current_errors = [self._mock_error]
else:
current_errors.append(self._mock_error)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_ERRORS,
current_errors
)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_PASSES,
current_passes
)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_RUNS,
current_runs
)
too_few_passes = current_passes < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_failure = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
if is_failure:
actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
self._mock_test_case,
self._mock_error
)
else:
actual_plugin_handles_failure = self._flaky_plugin.handleError(
self._mock_test_case,
self._mock_error
)
self.assertEqual(
expected_plugin_handles_failure,
actual_plugin_handles_failure,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_failure else '',
'' if actual_plugin_handles_failure else ' not'
)
)
self._assert_flaky_attributes_contains(
{
FlakyNames.CURRENT_RUNS: current_runs + 1,
FlakyNames.CURRENT_ERRORS: current_errors
}
)
expected_test_case_calls = [call.address(), call.address()]
if expected_plugin_handles_failure:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
' failed ({} runs remaining out of {}).'.format(
max_runs - current_runs - 1, max_runs
),
'\n\t',
unicode(self._mock_error[0]),
'\n\t',
unicode(self._mock_error[1].message),
'\n\t',
unicode(self._mock_error[2]),
'\n'
])]
else:
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
' failed; it passed {} out of the required {} times.'.format(
current_passes,
min_passes
),
'\n\t',
unicode(self._mock_error[0]),
'\n\t',
unicode(self._mock_error[1].message),
'\n\t',
unicode(self._mock_error[2]),
'\n'
])]
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls: {} vs {}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls
)
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_handles_success(
self,
current_passes=0,
current_runs=0,
is_test_method=True,
max_runs=2,
min_passes=1
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_PASSES,
current_passes
)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_RUNS,
current_runs
)
too_few_passes = current_passes + 1 < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_success = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
actual_plugin_handles_success = self._flaky_plugin.addSuccess(
self._mock_test_case
)
self.assertEqual(
expected_plugin_handles_success,
actual_plugin_handles_success,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_success else '',
'' if actual_plugin_handles_success else ' not'
)
)
self._assert_flaky_attributes_contains(
{
FlakyNames.CURRENT_PASSES: current_passes + 1,
FlakyNames.CURRENT_RUNS: current_runs + 1,
}
)
expected_test_case_calls = [call.address(), call.address()]
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
" passed {} out of the required {} times. ".format(
current_passes + 1, min_passes,
),
])]
if expected_plugin_handles_success:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls.append(
call.write(
'Running test again until it passes {} times.\n'.format(
min_passes,
),
),
)
else:
expected_stream_calls.append(call.write('Success!\n'))
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls = {} vs {}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls,
),
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_report(self, expected_stream_value):
mock_stream = MagicMock()
self._mock_stream.getvalue.return_value = expected_stream_value
self._flaky_plugin.report(mock_stream)
self.assertEqual(
mock_stream.mock_calls,
[
call.write('===Flaky Test Report===\n\n'),
call.write(expected_stream_value),
call.write('\n===End Flaky Test Report===\n'),
],
)
class TestFlakyPlugin(TestCase):
def setUp(self):
super(TestFlakyPlugin, self).setUp()
test_mod = 'box.test.flaky.flaky_plugin'
self._mock_test_result = MagicMock()
self._mock_stream = MagicMock(spec=StringIO)
with patch(test_mod + '.TextTestResult') as flaky_result:
with patch(test_mod + '.StringIO') as string_io:
string_io.return_value = self._mock_stream
flaky_result.return_value = self._mock_test_result
self._flaky_plugin = FlakyPlugin()
self._mock_test = MagicMock(name='flaky_plugin_test')
self._mock_test_case = MagicMock(name='flaky_plugin_test_case',
spec=TestCase)
self._mock_test_case.address = MagicMock()
self._mock_test_case.test = self._mock_test
self._mock_test_module_name = 'test_module'
self._mock_test_class_name = 'TestClass'
self._mock_test_method_name = 'test_method'
self._mock_test_names = '{}:{}.{}'.format(self._mock_test_module_name,
self._mock_test_class_name,
self._mock_test_method_name)
self._mock_exception = Exception('Error in {}'.format(
self._mock_test_method_name))
self._mock_stack_trace = ''
self._mock_exception_type = Exception
self._mock_error = (self._mock_exception_type, self._mock_exception,
self._mock_stack_trace)
def _expect_call_test_address(self):
self._mock_test_case.address.return_value = (None, None,
self._mock_test_names)
def _expect_test_flaky(self, is_test_method, max_runs, min_passes):
self._expect_call_test_address()
if is_test_method:
for flaky_attr in FlakyNames():
setattr(self._mock_test, flaky_attr, None)
mock_test_method = getattr(self._mock_test,
self._mock_test_method_name)
flaky(max_runs, min_passes)(mock_test_method)
else:
flaky(max_runs, min_passes)(self._mock_test)
mock_test_method = getattr(self._mock_test,
self._mock_test_method_name)
for flaky_attr in FlakyNames():
setattr(mock_test_method, flaky_attr, None)
def _expect_test_not_flaky(self):
self._expect_call_test_address()
for test_object in (self._mock_test,
getattr(self._mock_test,
self._mock_test_method_name)):
for flaky_attr in FlakyNames():
setattr(test_object, flaky_attr, None)
def _assert_test_ignored(self):
self._mock_test_case.address.assert_called_with()
self.assertEqual(self._mock_test_case.mock_calls, [call.address()])
self.assertEqual(self._mock_test.mock_calls, [])
def _get_flaky_attributes(self, is_test_method):
if is_test_method:
mock_test_method = getattr(self._mock_test,
self._mock_test_method_name)
test_object = mock_test_method
else:
test_object = self._mock_test
actual_flaky_attributes = {
attr: getattr(test_object, attr)
for attr in FlakyNames()
}
return actual_flaky_attributes
def _set_flaky_attribute(self, is_test_method, attr, value):
if is_test_method:
mock_test_method = getattr(self._mock_test,
self._mock_test_method_name)
test_object = mock_test_method
else:
test_object = self._mock_test
setattr(test_object, attr, value)
def _assert_flaky_attributes_contains(
self,
expected_flaky_attributes,
):
actual_flaky_attributes = self._get_flaky_attributes(True)
self.assertDictContainsSubset(
expected_flaky_attributes, actual_flaky_attributes,
'Unexpected flaky attributes, {} vs {}'.format(
expected_flaky_attributes, actual_flaky_attributes))
def test_flaky_plugin_report(self):
flaky_report = 'Flaky tests passed; others failed. ' \
'No more tests; that ship has sailed.'
self._test_flaky_plugin_report(flaky_report)
def test_flaky_plugin_handles_success_for_test_method(self):
self._test_flaky_plugin_handles_success()
def test_flaky_plugin_handles_success_for_test_instance(self):
self._test_flaky_plugin_handles_success(is_test_method=False)
def test_flaky_plugin_handles_success_for_needs_rerun(self):
self._test_flaky_plugin_handles_success(min_passes=2)
def test_flaky_plugin_ignores_success_for_non_flaky_test(self):
self._expect_test_not_flaky()
self._flaky_plugin.addSuccess(self._mock_test_case)
self._assert_test_ignored()
def test_flaky_plugin_ignores_error_for_non_flaky_test(self):
self._expect_test_not_flaky()
self._flaky_plugin.handleError(self._mock_test_case, None)
self._assert_test_ignored()
def test_flaky_plugin_ignores_failure_for_non_flaky_test(self):
self._expect_test_not_flaky()
self._flaky_plugin.handleFailure(self._mock_test_case, None)
self._assert_test_ignored()
def test_flaky_plugin_handles_error_for_test_method(self):
self._test_flaky_plugin_handles_failure_or_error()
def test_flaky_plugin_handles_error_for_test_instance(self):
self._test_flaky_plugin_handles_failure_or_error(is_test_method=False)
def test_flaky_plugin_handles_failure_for_test_method(self):
self._test_flaky_plugin_handles_failure_or_error(is_failure=True)
def test_flaky_plugin_handles_failure_for_test_instance(self):
self._test_flaky_plugin_handles_failure_or_error(is_failure=True,
is_test_method=False)
def test_flaky_plugin_handles_failure_for_no_more_retries(self):
self._test_flaky_plugin_handles_failure_or_error(is_failure=True,
max_runs=1)
def test_flaky_plugin_handles_additional_errors(self):
self._test_flaky_plugin_handles_failure_or_error(
current_errors=[self._mock_error])
def _test_flaky_plugin_handles_failure_or_error(
self,
current_errors=None,
current_passes=0,
current_runs=0,
is_failure=False,
is_test_method=True,
max_runs=2,
min_passes=1,
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
if current_errors is None:
current_errors = [self._mock_error]
else:
current_errors.append(self._mock_error)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_ERRORS,
current_errors)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_PASSES,
current_passes)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_RUNS,
current_runs)
too_few_passes = current_passes < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_failure = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
if is_failure:
actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
self._mock_test_case, self._mock_error)
else:
actual_plugin_handles_failure = self._flaky_plugin.handleError(
self._mock_test_case, self._mock_error)
self.assertEqual(
expected_plugin_handles_failure, actual_plugin_handles_failure,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_failure else '',
'' if actual_plugin_handles_failure else ' not'))
self._assert_flaky_attributes_contains({
FlakyNames.CURRENT_RUNS:
current_runs + 1,
FlakyNames.CURRENT_ERRORS:
current_errors
})
expected_test_case_calls = [call.address(), call.address()]
if expected_plugin_handles_failure:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
' failed ({} runs remaining out of {}).'.format(
max_runs - current_runs - 1, max_runs), '\n\t',
unicode(self._mock_error[0]), '\n\t',
unicode(self._mock_error[1].message), '\n\t',
unicode(self._mock_error[2]), '\n'
])
]
else:
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
' failed; it passed {} out of the required {} times.'.
format(current_passes, min_passes), '\n\t',
unicode(self._mock_error[0]), '\n\t',
unicode(self._mock_error[1].message), '\n\t',
unicode(self._mock_error[2]), '\n'
])
]
self.assertEqual(
self._mock_test_case.mock_calls, expected_test_case_calls,
'Unexpected TestCase calls: {} vs {}'.format(
self._mock_test_case.mock_calls, expected_test_case_calls))
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_handles_success(self,
current_passes=0,
current_runs=0,
is_test_method=True,
max_runs=2,
min_passes=1):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_PASSES,
current_passes)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_RUNS,
current_runs)
too_few_passes = current_passes + 1 < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_success = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
actual_plugin_handles_success = self._flaky_plugin.addSuccess(
self._mock_test_case)
self.assertEqual(
expected_plugin_handles_success, actual_plugin_handles_success,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_success else '',
'' if actual_plugin_handles_success else ' not'))
self._assert_flaky_attributes_contains({
FlakyNames.CURRENT_PASSES:
current_passes + 1,
FlakyNames.CURRENT_RUNS:
current_runs + 1,
})
expected_test_case_calls = [call.address(), call.address()]
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
" passed {} out of the required {} times. ".format(
current_passes + 1,
min_passes,
),
])
]
if expected_plugin_handles_success:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls.append(
call.write(
'Running test again until it passes {} times.\n'.format(
min_passes, ), ), )
else:
expected_stream_calls.append(call.write('Success!\n'))
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls = {} vs {}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls,
),
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_report(self, expected_stream_value):
mock_stream = MagicMock()
self._mock_stream.getvalue.return_value = expected_stream_value
self._flaky_plugin.report(mock_stream)
self.assertEqual(
mock_stream.mock_calls,
[
call.write('===Flaky Test Report===\n\n'),
call.write(expected_stream_value),
call.write('\n===End Flaky Test Report===\n'),
],
)
