def _test_flaky_plugin_handles_success(self,
current_passes=0,
current_runs=0,
is_test_method=True,
max_runs=2,
min_passes=1):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_PASSES,
current_passes)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_RUNS,
current_runs)
too_few_passes = current_passes + 1 < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_success = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
actual_plugin_handles_success = self._flaky_plugin.addSuccess(
self._mock_test_case)
self.assertEqual(
expected_plugin_handles_success, actual_plugin_handles_success,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_success else '',
'' if actual_plugin_handles_success else ' not'))
self._assert_flaky_attributes_contains({
FlakyNames.CURRENT_PASSES:
current_passes + 1,
FlakyNames.CURRENT_RUNS:
current_runs + 1,
})
expected_test_case_calls = [call.address(), call.address()]
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
" passed {} out of the required {} times. ".format(
current_passes + 1,
min_passes,
),
])
]
if expected_plugin_handles_success:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls.append(
call.write(
'Running test again until it passes {} times.\n'.format(
min_passes, ), ), )
else:
expected_stream_calls.append(call.write('Success!\n'))
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls = {} vs {}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls,
),
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _assert_test_ignored(self):
self._mock_test_case.address.assert_called_with()
self.assertEqual(self._mock_test_case.mock_calls, [call.address()])
self.assertEqual(self._mock_test.mock_calls, [])
def _test_flaky_plugin_handles_success(
self,
current_passes=0,
current_runs=0,
is_test_method=True,
max_runs=2,
min_passes=1
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_PASSES,
current_passes
)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_RUNS,
current_runs
)
too_few_passes = current_passes + 1 < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_success = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
actual_plugin_handles_success = self._flaky_plugin.addSuccess(
self._mock_test_case
)
self.assertEqual(
expected_plugin_handles_success,
actual_plugin_handles_success,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_success else '',
'' if actual_plugin_handles_success else ' not'
)
)
self._assert_flaky_attributes_contains(
{
FlakyNames.CURRENT_PASSES: current_passes + 1,
FlakyNames.CURRENT_RUNS: current_runs + 1,
}
)
expected_test_case_calls = [call.address(), call.address()]
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
" passed {} out of the required {} times. ".format(
current_passes + 1, min_passes,
),
])]
if expected_plugin_handles_success:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls.append(
call.write(
'Running test again until it passes {} times.\n'.format(
min_passes,
),
),
)
else:
expected_stream_calls.append(call.write('Success!\n'))
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls = {} vs {}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls,
),
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_handles_failure_or_error(
self,
current_errors=None,
current_passes=0,
current_runs=0,
is_failure=False,
is_test_method=True,
max_runs=2,
min_passes=1,
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
if current_errors is None:
current_errors = [self._mock_error]
else:
current_errors.append(self._mock_error)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_ERRORS,
current_errors
)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_PASSES,
current_passes
)
self._set_flaky_attribute(
is_test_method,
FlakyNames.CURRENT_RUNS,
current_runs
)
too_few_passes = current_passes < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_failure = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
if is_failure:
actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
self._mock_test_case,
self._mock_error
)
else:
actual_plugin_handles_failure = self._flaky_plugin.handleError(
self._mock_test_case,
self._mock_error
)
self.assertEqual(
expected_plugin_handles_failure,
actual_plugin_handles_failure,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_failure else '',
'' if actual_plugin_handles_failure else ' not'
)
)
self._assert_flaky_attributes_contains(
{
FlakyNames.CURRENT_RUNS: current_runs + 1,
FlakyNames.CURRENT_ERRORS: current_errors
}
)
expected_test_case_calls = [call.address(), call.address()]
if expected_plugin_handles_failure:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
' failed ({} runs remaining out of {}).'.format(
max_runs - current_runs - 1, max_runs
),
'\n\t',
unicode(self._mock_error[0]),
'\n\t',
unicode(self._mock_error[1].message),
'\n\t',
unicode(self._mock_error[2]),
'\n'
])]
else:
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
' failed; it passed {} out of the required {} times.'.format(
current_passes,
min_passes
),
'\n\t',
unicode(self._mock_error[0]),
'\n\t',
unicode(self._mock_error[1].message),
'\n\t',
unicode(self._mock_error[2]),
'\n'
])]
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls: {} vs {}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls
)
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _assert_test_ignored(self):
self._mock_test_case.address.assert_called_with()
self.assertEqual(self._mock_test_case.mock_calls, [call.address()])
self.assertEqual(self._mock_test.mock_calls, [])
def _test_flaky_plugin_handles_failure_or_error(
self,
current_errors=None,
current_passes=0,
current_runs=0,
is_failure=False,
is_test_method=True,
max_runs=2,
min_passes=1,
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
if current_errors is None:
current_errors = [self._mock_error]
else:
current_errors.append(self._mock_error)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_ERRORS,
current_errors)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_PASSES,
current_passes)
self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_RUNS,
current_runs)
too_few_passes = current_passes < min_passes
retries_remaining = current_runs + 1 < max_runs
expected_plugin_handles_failure = too_few_passes and retries_remaining
self._flaky_plugin.prepareTestCase(self._mock_test_case)
if is_failure:
actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
self._mock_test_case, self._mock_error)
else:
actual_plugin_handles_failure = self._flaky_plugin.handleError(
self._mock_test_case, self._mock_error)
self.assertEqual(
expected_plugin_handles_failure, actual_plugin_handles_failure,
'Expected plugin{} to handle the test run, but it did{}.'.format(
' to' if expected_plugin_handles_failure else '',
'' if actual_plugin_handles_failure else ' not'))
self._assert_flaky_attributes_contains({
FlakyNames.CURRENT_RUNS:
current_runs + 1,
FlakyNames.CURRENT_ERRORS:
current_errors
})
expected_test_case_calls = [call.address(), call.address()]
if expected_plugin_handles_failure:
expected_test_case_calls.append(call.run(self._mock_test_result))
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
' failed ({} runs remaining out of {}).'.format(
max_runs - current_runs - 1, max_runs), '\n\t',
unicode(self._mock_error[0]), '\n\t',
unicode(self._mock_error[1].message), '\n\t',
unicode(self._mock_error[2]), '\n'
])
]
else:
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
' failed; it passed {} out of the required {} times.'.
format(current_passes, min_passes), '\n\t',
unicode(self._mock_error[0]), '\n\t',
unicode(self._mock_error[1].message), '\n\t',
unicode(self._mock_error[2]), '\n'
])
]
self.assertEqual(
self._mock_test_case.mock_calls, expected_test_case_calls,
'Unexpected TestCase calls: {} vs {}'.format(
self._mock_test_case.mock_calls, expected_test_case_calls))
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_handles_failure_or_error(
self,
current_errors=None,
current_passes=0,
current_runs=0,
is_failure=False,
is_test_method=True,
max_runs=2,
min_passes=1,
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
if current_errors is None:
current_errors = [self._mock_error]
else:
current_errors.append(self._mock_error)
self._set_flaky_attribute(
FlakyNames.CURRENT_ERRORS,
current_errors,
)
self._set_flaky_attribute(
FlakyNames.CURRENT_PASSES,
current_passes,
)
self._set_flaky_attribute(
FlakyNames.CURRENT_RUNS,
current_runs,
)
retries_remaining = current_runs + 1 < max_runs
too_few_passes = current_passes < min_passes
expected_plugin_handles_failure = too_few_passes and retries_remaining
did_plugin_retry_test = max_runs > 1
self._flaky_plugin.prepareTestCase(self._mock_test_case)
if is_failure:
actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
self._mock_test_case,
self._mock_error,
)
else:
actual_plugin_handles_failure = self._flaky_plugin.handleError(
self._mock_test_case,
self._mock_error,
)
self.assertEqual(
expected_plugin_handles_failure,
actual_plugin_handles_failure,
'Expected plugin{0} to handle the test run, but it did{1}.'.format(
' to' if expected_plugin_handles_failure else '',
'' if actual_plugin_handles_failure else ' not'
),
)
self._assert_flaky_attributes_contains(
{
FlakyNames.CURRENT_RUNS: current_runs + 1,
FlakyNames.CURRENT_ERRORS: tuple(current_errors),
},
)
expected_test_case_calls = [call.address(), call.address()]
expected_result_calls = []
if expected_plugin_handles_failure:
expected_test_case_calls.append(('__hash__',))
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
' failed ({0} runs remaining out of {1}).'.format(
max_runs - current_runs - 1, max_runs
),
'\n\t',
unicode_type(self._mock_error[0]),
'\n\t',
unicode_type(self._mock_error[1]),
'\n\t',
unicode_type(self._mock_error[2]),
'\n',
])]
else:
if did_plugin_retry_test:
if is_failure:
expected_result_calls.append(
call.addFailure(
self._mock_test_case,
self._mock_error,
),
)
else:
expected_result_calls.append(
call.addError(self._mock_test_case, self._mock_error),
)
expected_stream_calls = [call.writelines([
self._mock_test_method_name,
' failed; it passed {0} out of the required {1} times.'.format(
current_passes,
min_passes
),
'\n\t',
unicode_type(self._mock_error[0]),
'\n\t',
unicode_type(self._mock_error[1]),
'\n\t',
unicode_type(self._mock_error[2]),
'\n'
])]
self.assertEqual(
self._mock_nose_result.mock_calls,
expected_result_calls,
)
self.assertEqual(
self._mock_test_case.mock_calls,
expected_test_case_calls,
'Unexpected TestCase calls: {0} vs {1}'.format(
self._mock_test_case.mock_calls,
expected_test_case_calls
)
)
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
def _test_flaky_plugin_handles_failure_or_error(
self,
current_errors=None,
current_passes=0,
current_runs=0,
is_failure=False,
is_test_method=True,
max_runs=2,
min_passes=1,
):
self._expect_test_flaky(is_test_method, max_runs, min_passes)
if current_errors is None:
current_errors = [self._mock_error]
else:
current_errors.append(self._mock_error)
self._set_flaky_attribute(
FlakyNames.CURRENT_ERRORS,
current_errors,
)
self._set_flaky_attribute(
FlakyNames.CURRENT_PASSES,
current_passes,
)
self._set_flaky_attribute(
FlakyNames.CURRENT_RUNS,
current_runs,
)
retries_remaining = current_runs + 1 < max_runs
too_few_passes = current_passes < min_passes
expected_plugin_handles_failure = too_few_passes and retries_remaining
did_plugin_retry_test = max_runs > 1
self._flaky_plugin.prepareTestCase(self._mock_test_case)
if is_failure:
actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
self._mock_test_case,
self._mock_error,
)
else:
actual_plugin_handles_failure = self._flaky_plugin.handleError(
self._mock_test_case,
self._mock_error,
)
self.assertEqual(
expected_plugin_handles_failure,
actual_plugin_handles_failure,
'Expected plugin{0} to handle the test run, but it did{1}.'.format(
' to' if expected_plugin_handles_failure else '',
'' if actual_plugin_handles_failure else ' not'),
)
self._assert_flaky_attributes_contains(
{
FlakyNames.CURRENT_RUNS: current_runs + 1,
FlakyNames.CURRENT_ERRORS: tuple(current_errors),
}, )
expected_test_case_calls = [call.address(), call.address()]
expected_result_calls = []
if expected_plugin_handles_failure:
expected_test_case_calls.append(('__hash__', ))
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
' failed ({0} runs remaining out of {1}).'.format(
max_runs - current_runs - 1, max_runs),
'\n\t',
unicode_type(self._mock_error[0]),
'\n\t',
unicode_type(self._mock_error[1]),
'\n\t',
unicode_type(self._mock_error[2]),
'\n',
])
]
else:
if did_plugin_retry_test:
if is_failure:
expected_result_calls.append(
call.addFailure(
self._mock_test_case,
self._mock_error,
), )
else:
expected_result_calls.append(
call.addError(self._mock_test_case,
self._mock_error), )
expected_stream_calls = [
call.writelines([
self._mock_test_method_name,
' failed; it passed {0} out of the required {1} times.'.
format(current_passes, min_passes), '\n\t',
unicode_type(self._mock_error[0]), '\n\t',
unicode_type(self._mock_error[1]), '\n\t',
unicode_type(self._mock_error[2]), '\n'
])
]
self.assertEqual(
self._mock_nose_result.mock_calls,
expected_result_calls,
)
self.assertEqual(
self._mock_test_case.mock_calls, expected_test_case_calls,
'Unexpected TestCase calls: {0} vs {1}'.format(
self._mock_test_case.mock_calls, expected_test_case_calls))
self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)