예제 #1
0
    def _test_flaky_plugin_handles_success(self,
                                           current_passes=0,
                                           current_runs=0,
                                           is_test_method=True,
                                           max_runs=2,
                                           min_passes=1):
        self._expect_test_flaky(is_test_method, max_runs, min_passes)
        self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_PASSES,
                                  current_passes)
        self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_RUNS,
                                  current_runs)

        too_few_passes = current_passes + 1 < min_passes
        retries_remaining = current_runs + 1 < max_runs
        expected_plugin_handles_success = too_few_passes and retries_remaining

        self._flaky_plugin.prepareTestCase(self._mock_test_case)
        actual_plugin_handles_success = self._flaky_plugin.addSuccess(
            self._mock_test_case)

        self.assertEqual(
            expected_plugin_handles_success, actual_plugin_handles_success,
            'Expected plugin{} to handle the test run, but it did{}.'.format(
                ' to' if expected_plugin_handles_success else '',
                '' if actual_plugin_handles_success else ' not'))
        self._assert_flaky_attributes_contains({
            FlakyNames.CURRENT_PASSES:
            current_passes + 1,
            FlakyNames.CURRENT_RUNS:
            current_runs + 1,
        })
        expected_test_case_calls = [call.address(), call.address()]
        expected_stream_calls = [
            call.writelines([
                self._mock_test_method_name,
                " passed {} out of the required {} times. ".format(
                    current_passes + 1,
                    min_passes,
                ),
            ])
        ]
        if expected_plugin_handles_success:
            expected_test_case_calls.append(call.run(self._mock_test_result))
            expected_stream_calls.append(
                call.write(
                    'Running test again until it passes {} times.\n'.format(
                        min_passes, ), ), )
        else:
            expected_stream_calls.append(call.write('Success!\n'))
        self.assertEqual(
            self._mock_test_case.mock_calls,
            expected_test_case_calls,
            'Unexpected TestCase calls = {} vs {}'.format(
                self._mock_test_case.mock_calls,
                expected_test_case_calls,
            ),
        )
        self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
예제 #2
0
 def _assert_test_ignored(self):
     self._mock_test_case.address.assert_called_with()
     self.assertEqual(self._mock_test_case.mock_calls, [call.address()])
     self.assertEqual(self._mock_test.mock_calls, [])
예제 #3
0
    def _test_flaky_plugin_handles_success(
        self,
        current_passes=0,
        current_runs=0,
        is_test_method=True,
        max_runs=2,
        min_passes=1
    ):
        self._expect_test_flaky(is_test_method, max_runs, min_passes)
        self._set_flaky_attribute(
            is_test_method,
            FlakyNames.CURRENT_PASSES,
            current_passes
        )
        self._set_flaky_attribute(
            is_test_method,
            FlakyNames.CURRENT_RUNS,
            current_runs
        )

        too_few_passes = current_passes + 1 < min_passes
        retries_remaining = current_runs + 1 < max_runs
        expected_plugin_handles_success = too_few_passes and retries_remaining

        self._flaky_plugin.prepareTestCase(self._mock_test_case)
        actual_plugin_handles_success = self._flaky_plugin.addSuccess(
            self._mock_test_case
        )

        self.assertEqual(
            expected_plugin_handles_success,
            actual_plugin_handles_success,
            'Expected plugin{} to handle the test run, but it did{}.'.format(
                ' to' if expected_plugin_handles_success else '',
                '' if actual_plugin_handles_success else ' not'
            )
        )
        self._assert_flaky_attributes_contains(
            {
                FlakyNames.CURRENT_PASSES: current_passes + 1,
                FlakyNames.CURRENT_RUNS: current_runs + 1,
            }
        )
        expected_test_case_calls = [call.address(), call.address()]
        expected_stream_calls = [call.writelines([
            self._mock_test_method_name,
            " passed {} out of the required {} times. ".format(
                current_passes + 1, min_passes,
            ),
        ])]
        if expected_plugin_handles_success:
            expected_test_case_calls.append(call.run(self._mock_test_result))
            expected_stream_calls.append(
                call.write(
                    'Running test again until it passes {} times.\n'.format(
                        min_passes,
                    ),
                ),
            )
        else:
            expected_stream_calls.append(call.write('Success!\n'))
        self.assertEqual(
            self._mock_test_case.mock_calls,
            expected_test_case_calls,
            'Unexpected TestCase calls = {} vs {}'.format(
                self._mock_test_case.mock_calls,
                expected_test_case_calls,
            ),
        )
        self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
예제 #4
0
    def _test_flaky_plugin_handles_failure_or_error(
        self,
        current_errors=None,
        current_passes=0,
        current_runs=0,
        is_failure=False,
        is_test_method=True,
        max_runs=2,
        min_passes=1,
    ):
        self._expect_test_flaky(is_test_method, max_runs, min_passes)
        if current_errors is None:
            current_errors = [self._mock_error]
        else:
            current_errors.append(self._mock_error)
        self._set_flaky_attribute(
            is_test_method,
            FlakyNames.CURRENT_ERRORS,
            current_errors
        )
        self._set_flaky_attribute(
            is_test_method,
            FlakyNames.CURRENT_PASSES,
            current_passes
        )
        self._set_flaky_attribute(
            is_test_method,
            FlakyNames.CURRENT_RUNS,
            current_runs
        )

        too_few_passes = current_passes < min_passes
        retries_remaining = current_runs + 1 < max_runs
        expected_plugin_handles_failure = too_few_passes and retries_remaining

        self._flaky_plugin.prepareTestCase(self._mock_test_case)
        if is_failure:
            actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
                self._mock_test_case,
                self._mock_error
            )
        else:
            actual_plugin_handles_failure = self._flaky_plugin.handleError(
                self._mock_test_case,
                self._mock_error
            )

        self.assertEqual(
            expected_plugin_handles_failure,
            actual_plugin_handles_failure,
            'Expected plugin{} to handle the test run, but it did{}.'.format(
                ' to' if expected_plugin_handles_failure else '',
                '' if actual_plugin_handles_failure else ' not'
            )
        )
        self._assert_flaky_attributes_contains(
            {
                FlakyNames.CURRENT_RUNS: current_runs + 1,
                FlakyNames.CURRENT_ERRORS: current_errors
            }
        )
        expected_test_case_calls = [call.address(), call.address()]
        if expected_plugin_handles_failure:
            expected_test_case_calls.append(call.run(self._mock_test_result))
            expected_stream_calls = [call.writelines([
                self._mock_test_method_name,
                ' failed ({} runs remaining out of {}).'.format(
                    max_runs - current_runs - 1, max_runs
                ),
                '\n\t',
                unicode(self._mock_error[0]),
                '\n\t',
                unicode(self._mock_error[1].message),
                '\n\t',
                unicode(self._mock_error[2]),
                '\n'
            ])]
        else:
            expected_stream_calls = [call.writelines([
                self._mock_test_method_name,
                ' failed; it passed {} out of the required {} times.'.format(
                    current_passes,
                    min_passes
                ),
                '\n\t',
                unicode(self._mock_error[0]),
                '\n\t',
                unicode(self._mock_error[1].message),
                '\n\t',
                unicode(self._mock_error[2]),
                '\n'
            ])]
        self.assertEqual(
            self._mock_test_case.mock_calls,
            expected_test_case_calls,
            'Unexpected TestCase calls: {} vs {}'.format(
                self._mock_test_case.mock_calls,
                expected_test_case_calls
            )
        )
        self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
예제 #5
0
 def _assert_test_ignored(self):
     self._mock_test_case.address.assert_called_with()
     self.assertEqual(self._mock_test_case.mock_calls, [call.address()])
     self.assertEqual(self._mock_test.mock_calls, [])
예제 #6
0
    def _test_flaky_plugin_handles_failure_or_error(
        self,
        current_errors=None,
        current_passes=0,
        current_runs=0,
        is_failure=False,
        is_test_method=True,
        max_runs=2,
        min_passes=1,
    ):
        self._expect_test_flaky(is_test_method, max_runs, min_passes)
        if current_errors is None:
            current_errors = [self._mock_error]
        else:
            current_errors.append(self._mock_error)
        self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_ERRORS,
                                  current_errors)
        self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_PASSES,
                                  current_passes)
        self._set_flaky_attribute(is_test_method, FlakyNames.CURRENT_RUNS,
                                  current_runs)

        too_few_passes = current_passes < min_passes
        retries_remaining = current_runs + 1 < max_runs
        expected_plugin_handles_failure = too_few_passes and retries_remaining

        self._flaky_plugin.prepareTestCase(self._mock_test_case)
        if is_failure:
            actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
                self._mock_test_case, self._mock_error)
        else:
            actual_plugin_handles_failure = self._flaky_plugin.handleError(
                self._mock_test_case, self._mock_error)

        self.assertEqual(
            expected_plugin_handles_failure, actual_plugin_handles_failure,
            'Expected plugin{} to handle the test run, but it did{}.'.format(
                ' to' if expected_plugin_handles_failure else '',
                '' if actual_plugin_handles_failure else ' not'))
        self._assert_flaky_attributes_contains({
            FlakyNames.CURRENT_RUNS:
            current_runs + 1,
            FlakyNames.CURRENT_ERRORS:
            current_errors
        })
        expected_test_case_calls = [call.address(), call.address()]
        if expected_plugin_handles_failure:
            expected_test_case_calls.append(call.run(self._mock_test_result))
            expected_stream_calls = [
                call.writelines([
                    self._mock_test_method_name,
                    ' failed ({} runs remaining out of {}).'.format(
                        max_runs - current_runs - 1, max_runs), '\n\t',
                    unicode(self._mock_error[0]), '\n\t',
                    unicode(self._mock_error[1].message), '\n\t',
                    unicode(self._mock_error[2]), '\n'
                ])
            ]
        else:
            expected_stream_calls = [
                call.writelines([
                    self._mock_test_method_name,
                    ' failed; it passed {} out of the required {} times.'.
                    format(current_passes, min_passes), '\n\t',
                    unicode(self._mock_error[0]), '\n\t',
                    unicode(self._mock_error[1].message), '\n\t',
                    unicode(self._mock_error[2]), '\n'
                ])
            ]
        self.assertEqual(
            self._mock_test_case.mock_calls, expected_test_case_calls,
            'Unexpected TestCase calls: {} vs {}'.format(
                self._mock_test_case.mock_calls, expected_test_case_calls))
        self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
예제 #7
0
    def _test_flaky_plugin_handles_failure_or_error(
        self,
        current_errors=None,
        current_passes=0,
        current_runs=0,
        is_failure=False,
        is_test_method=True,
        max_runs=2,
        min_passes=1,
    ):
        self._expect_test_flaky(is_test_method, max_runs, min_passes)
        if current_errors is None:
            current_errors = [self._mock_error]
        else:
            current_errors.append(self._mock_error)
        self._set_flaky_attribute(
            FlakyNames.CURRENT_ERRORS,
            current_errors,
        )
        self._set_flaky_attribute(
            FlakyNames.CURRENT_PASSES,
            current_passes,
        )
        self._set_flaky_attribute(
            FlakyNames.CURRENT_RUNS,
            current_runs,
        )

        retries_remaining = current_runs + 1 < max_runs
        too_few_passes = current_passes < min_passes
        expected_plugin_handles_failure = too_few_passes and retries_remaining
        did_plugin_retry_test = max_runs > 1

        self._flaky_plugin.prepareTestCase(self._mock_test_case)
        if is_failure:
            actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
                self._mock_test_case,
                self._mock_error,
            )
        else:
            actual_plugin_handles_failure = self._flaky_plugin.handleError(
                self._mock_test_case,
                self._mock_error,
            )

        self.assertEqual(
            expected_plugin_handles_failure,
            actual_plugin_handles_failure,
            'Expected plugin{0} to handle the test run, but it did{1}.'.format(
                ' to' if expected_plugin_handles_failure else '',
                '' if actual_plugin_handles_failure else ' not'
            ),
        )
        self._assert_flaky_attributes_contains(
            {
                FlakyNames.CURRENT_RUNS: current_runs + 1,
                FlakyNames.CURRENT_ERRORS: tuple(current_errors),
            },
        )
        expected_test_case_calls = [call.address(), call.address()]
        expected_result_calls = []
        if expected_plugin_handles_failure:
            expected_test_case_calls.append(('__hash__',))
            expected_stream_calls = [call.writelines([
                self._mock_test_method_name,
                ' failed ({0} runs remaining out of {1}).'.format(
                    max_runs - current_runs - 1, max_runs
                ),
                '\n\t',
                unicode_type(self._mock_error[0]),
                '\n\t',
                unicode_type(self._mock_error[1]),
                '\n\t',
                unicode_type(self._mock_error[2]),
                '\n',
            ])]
        else:
            if did_plugin_retry_test:
                if is_failure:
                    expected_result_calls.append(
                        call.addFailure(
                            self._mock_test_case,
                            self._mock_error,
                        ),
                    )
                else:
                    expected_result_calls.append(
                        call.addError(self._mock_test_case, self._mock_error),
                    )
            expected_stream_calls = [call.writelines([
                self._mock_test_method_name,
                ' failed; it passed {0} out of the required {1} times.'.format(
                    current_passes,
                    min_passes
                ),
                '\n\t',
                unicode_type(self._mock_error[0]),
                '\n\t',
                unicode_type(self._mock_error[1]),
                '\n\t',
                unicode_type(self._mock_error[2]),
                '\n'
            ])]
        self.assertEqual(
            self._mock_nose_result.mock_calls,
            expected_result_calls,
        )
        self.assertEqual(
            self._mock_test_case.mock_calls,
            expected_test_case_calls,
            'Unexpected TestCase calls: {0} vs {1}'.format(
                self._mock_test_case.mock_calls,
                expected_test_case_calls
            )
        )
        self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)
예제 #8
0
    def _test_flaky_plugin_handles_failure_or_error(
        self,
        current_errors=None,
        current_passes=0,
        current_runs=0,
        is_failure=False,
        is_test_method=True,
        max_runs=2,
        min_passes=1,
    ):
        self._expect_test_flaky(is_test_method, max_runs, min_passes)
        if current_errors is None:
            current_errors = [self._mock_error]
        else:
            current_errors.append(self._mock_error)
        self._set_flaky_attribute(
            FlakyNames.CURRENT_ERRORS,
            current_errors,
        )
        self._set_flaky_attribute(
            FlakyNames.CURRENT_PASSES,
            current_passes,
        )
        self._set_flaky_attribute(
            FlakyNames.CURRENT_RUNS,
            current_runs,
        )

        retries_remaining = current_runs + 1 < max_runs
        too_few_passes = current_passes < min_passes
        expected_plugin_handles_failure = too_few_passes and retries_remaining
        did_plugin_retry_test = max_runs > 1

        self._flaky_plugin.prepareTestCase(self._mock_test_case)
        if is_failure:
            actual_plugin_handles_failure = self._flaky_plugin.handleFailure(
                self._mock_test_case,
                self._mock_error,
            )
        else:
            actual_plugin_handles_failure = self._flaky_plugin.handleError(
                self._mock_test_case,
                self._mock_error,
            )

        self.assertEqual(
            expected_plugin_handles_failure,
            actual_plugin_handles_failure,
            'Expected plugin{0} to handle the test run, but it did{1}.'.format(
                ' to' if expected_plugin_handles_failure else '',
                '' if actual_plugin_handles_failure else ' not'),
        )
        self._assert_flaky_attributes_contains(
            {
                FlakyNames.CURRENT_RUNS: current_runs + 1,
                FlakyNames.CURRENT_ERRORS: tuple(current_errors),
            }, )
        expected_test_case_calls = [call.address(), call.address()]
        expected_result_calls = []
        if expected_plugin_handles_failure:
            expected_test_case_calls.append(('__hash__', ))
            expected_stream_calls = [
                call.writelines([
                    self._mock_test_method_name,
                    ' failed ({0} runs remaining out of {1}).'.format(
                        max_runs - current_runs - 1, max_runs),
                    '\n\t',
                    unicode_type(self._mock_error[0]),
                    '\n\t',
                    unicode_type(self._mock_error[1]),
                    '\n\t',
                    unicode_type(self._mock_error[2]),
                    '\n',
                ])
            ]
        else:
            if did_plugin_retry_test:
                if is_failure:
                    expected_result_calls.append(
                        call.addFailure(
                            self._mock_test_case,
                            self._mock_error,
                        ), )
                else:
                    expected_result_calls.append(
                        call.addError(self._mock_test_case,
                                      self._mock_error), )
            expected_stream_calls = [
                call.writelines([
                    self._mock_test_method_name,
                    ' failed; it passed {0} out of the required {1} times.'.
                    format(current_passes, min_passes), '\n\t',
                    unicode_type(self._mock_error[0]), '\n\t',
                    unicode_type(self._mock_error[1]), '\n\t',
                    unicode_type(self._mock_error[2]), '\n'
                ])
            ]
        self.assertEqual(
            self._mock_nose_result.mock_calls,
            expected_result_calls,
        )
        self.assertEqual(
            self._mock_test_case.mock_calls, expected_test_case_calls,
            'Unexpected TestCase calls: {0} vs {1}'.format(
                self._mock_test_case.mock_calls, expected_test_case_calls))
        self.assertEqual(self._mock_stream.mock_calls, expected_stream_calls)