Esempio n. 1
0
    def _make_test_flaky(cls, test, max_runs=None, min_passes=None, rerun_filter=None):
        """
        Make a given test flaky.

        :param test:
            The test in question.
        :type test:
            :class:`nose.case.Test` or :class:`Function`
        :param max_runs:
            The value of the FlakyNames.MAX_RUNS attribute to use.
        :type max_runs:
            `int`
        :param min_passes:
            The value of the FlakyNames.MIN_PASSES attribute to use.
        :type min_passes:
            `int`
        :param rerun_filter:
            Filter function to decide whether a test should be rerun if it fails.
            Function signature is as follows:
                (err, name, test, plugin) -> should_rerun
            - err (`tuple` of `class`, :class:`Exception`, `traceback`):
                Information about the test failure (from sys.exc_info())
            - name (`unicode`):
                The test name
            - test (:class:`nose.case.Test` or :class:`Function`):
                The test that has raised an error
            - plugin (:class:`FlakyNosePlugin` or :class:`FlakyPytestPlugin`):
                The flaky plugin. Has a :prop:`stream` that can be written to in
                order to add to the Flaky Report.
        :type rerun_filter:
            `callable`
        """
        attrib_dict = defaults.default_flaky_attributes(max_runs, min_passes, rerun_filter)
        for attr, value in attrib_dict.items():
            cls._set_flaky_attribute(test, attr, value)
Esempio n. 2
0
def flaky(max_runs=None, min_passes=None, rerun_filter=None):
    """
    Decorator used to mark a test as "flaky". When used in conjuction with
    the flaky nosetests plugin, will cause the decorated test to be retried
    until min_passes successes are achieved out of up to max_runs test runs.

    :param max_runs:
        The maximum number of times the decorated test will be run.
    :type max_runs:
        `int`
    :param min_passes:
        The minimum number of times the test must pass to be a success.
    :type min_passes:
        `int`
    :param rerun_filter:
        Filter function to decide whether a test should be rerun if it fails.
        Function signature is as follows:
            (err, name, test, plugin) -> should_rerun
        - err (`tuple` of `class`, :class:`Exception`, `traceback`):
            Information about the test failure (from sys.exc_info())
        - name (`unicode`):
            The test name
        - test (:class:`nose.case.Test` or :class:`Function`):
            The test that has raised an error
        - plugin (:class:`FlakyNosePlugin` or :class:`FlakyPytestPlugin`):
            The flaky plugin. Has a :prop:`stream` that can be written to in
            order to add to the Flaky Report.
    :type rerun_filter:
        `callable`
    :return:
        A wrapper function that includes attributes describing the flaky test.
    :rtype:
        `callable`
    """
    # In case @flaky is applied to a function or class without arguments
    # (and without parentheses), max_runs will refer to the wrapped object.
    # In this case, the default value can be used.
    wrapped = None
    if hasattr(max_runs, '__call__'):
        wrapped, max_runs = max_runs, None
    if max_runs is None:
        max_runs = 2
    if min_passes is None:
        min_passes = 1
    if min_passes <= 0:
        raise ValueError('min_passes must be positive')
    if max_runs < min_passes:
        raise ValueError('min_passes cannot be greater than max_runs!')

    attrib = default_flaky_attributes(max_runs, min_passes, rerun_filter)

    def wrapper(wrapped_object):
        for name, value in attrib.items():
            setattr(wrapped_object, name, value)
        return wrapped_object

    return wrapper(wrapped) if wrapped is not None else wrapper
Esempio n. 3
0
 def test_flaky_plugin_handles_bare_test(self):
     self._mock_test_names = self._mock_test_method_name
     self._mock_test.test = Mock()
     self._expect_call_test_address()
     attrib = defaults.default_flaky_attributes(2, 1)
     for name, value in attrib.items():
         setattr(self._mock_test.test, name, value)
     delattr(self._mock_test, self._mock_test_method_name)
     self._flaky_plugin.prepareTestCase(self._mock_test_case)
     self.assertTrue(self._flaky_plugin.handleError(self._mock_test_case, self._mock_error))
     self.assertFalse(self._flaky_plugin.handleError(self._mock_test_case, self._mock_error))
Esempio n. 4
0
 def _make_test_flaky(cls, test, max_runs, min_passes):
     """
     Make a given test flaky.
     :param test:
         The test in question.
     :type test:
         :class:`nose.case.Test` or :class:`Function`
     :param max_runs:
         The value of the FlakyNames.MAX_RUNS attribute to use.
     :type max_runs:
         `int`
     :param min_passes:
         The value of the FlakyNames.MIN_PASSES attribute to use.
     :type min_passes:
         `int`
     """
     attrib_dict = defaults.default_flaky_attributes(max_runs, min_passes)
     for attr, value in attrib_dict.items():
         cls._set_flaky_attribute(test, attr, value)
Esempio n. 5
0
def flaky(max_runs=None, min_passes=None):
    """
    Decorator used to mark a test as "flaky". When used in conjuction with
    the flaky nosetests plugin, will cause the decorated test to be retried
    until min_passes successes are achieved out of up to max_runs test runs.
    :param max_runs:
        The maximum number of times the decorated test will be run.
    :type max_runs:
        `int`
    :param min_passes:
        The minimum number of times the test must pass to be a success.
    :type min_passes:
        `int`
    :return:
        A wrapper function that includes attributes describing the flaky test.
    :rtype:
        `callable`
    """
    if max_runs is None:
        max_runs = 2
    if min_passes is None:
        min_passes = 1
    if min_passes <= 0:
        raise ValueError('min_passes must be positive')
    # In case @flaky is applied to a function or class without arguments
    # (and without parentheses), max_runs will refer to the wrapped object.
    # In this case, the default value can be used.
    wrapped = None
    if hasattr(max_runs, '__call__'):
        wrapped = max_runs
        max_runs = 2
    if max_runs < min_passes:
        raise ValueError('min_passes cannot be greater than max_runs!')

    attrib = defaults.default_flaky_attributes(max_runs, min_passes)

    def wrapper(wrapped_object):
        for name, value in attrib.items():
            setattr(wrapped_object, name, value)
        return wrapped_object

    return wrapper(wrapped) if wrapped is not None else wrapper
Esempio n. 6
0
    def _make_test_flaky(cls, test, max_runs, min_passes):
        """
        Make a given test flaky.

        :param test:
            The test in question.
        :type test:
            :class:`nose.case.Test` or :class:`Function`
        :param max_runs:
            The value of the FlakyNames.MAX_RUNS attribute to use.
        :type max_runs:
            `int`
        :param min_passes:
            The value of the FlakyNames.MIN_PASSES attribute to use.
        :type min_passes:
            `int`
        """
        attrib_dict = defaults.default_flaky_attributes(max_runs, min_passes)
        for attr, value in attrib_dict.items():
            cls._set_flaky_attribute(test, attr, value)
Esempio n. 7
0
 def test_flaky_plugin_handles_bare_test(self):
     self._mock_test_names = self._mock_test_method_name
     self._mock_test.test = Mock()
     self._expect_call_test_address()
     attrib = defaults.default_flaky_attributes(2, 1)
     for name, value in attrib.items():
         setattr(
             self._mock_test.test,
             name,
             value,
         )
     delattr(self._mock_test, self._mock_test_method_name)
     self.assertTrue(self._flaky_plugin.handleError(
         self._mock_test_case,
         self._mock_error,
     ))
     self.assertFalse(self._flaky_plugin.handleError(
         self._mock_test_case,
         self._mock_error,
     ))