Exemplo n.º 1
0
 def test_try_till_success(self):
     a1 = Mock()
     a1.f = Mock(__name__='f', side_effect=[IOError, IOError, None])
     a = RetryWrapper(a1,
                      retry_if=lambda x: True,
                      backoff=0.0001,
                      max_tries=0)
     a.f()
     self.assertEqual(a1.f.call_count, 3)
Exemplo n.º 2
0
 def test_failure_raises_if_all_tries_fail(self):
     a1 = Mock()
     a1.f = Mock(__name__='f', side_effect=[IOError, IOError])
     a = RetryWrapper(a1,
                      retry_if=lambda x: True,
                      backoff=0.0001,
                      max_tries=2)
     with self.assertRaises(IOError):
         a.f()
     self.assertEqual(a1.f.call_count, 2)
Exemplo n.º 3
0
    def test_failure(self):
        a1 = Mock()
        a1.f = Mock(__name__='f', side_effect=[IOError, 1])
        a = RetryWrapper(a1,
                         retry_if=lambda x: True,
                         backoff=0.0001,
                         max_tries=2)

        self.assertEqual(a.f(), 1)
        self.assertEqual(a1.f.call_count, 2)
Exemplo n.º 4
0
    def test_success(self):
        a1 = Mock()
        a1.f = Mock(__name__='f', side_effect=None)
        a = RetryWrapper(a1,
                         retry_if=lambda x: True,
                         backoff=0.0001,
                         max_tries=2)

        a.f()
        a1.f.assert_called_once_with()
Exemplo n.º 5
0
 def test_try_till_success(self):
     a1 = Mock()
     a1.f = Mock(__name__='f', side_effect=[IOError, IOError, None])
     a = RetryWrapper(
         a1,
         retry_if=lambda x: True,
         backoff=0.0001,
         max_tries=0
     )
     a.f()
     self.assertEqual(a1.f.call_count, 3)
Exemplo n.º 6
0
 def test_failure_raises_if_all_tries_fail(self):
     a1 = Mock()
     a1.f = Mock(__name__='f', side_effect=[IOError, IOError])
     a = RetryWrapper(
         a1,
         retry_if=lambda x: True,
         backoff=0.0001,
         max_tries=2
     )
     with self.assertRaises(IOError):
         a.f()
     self.assertEqual(a1.f.call_count, 2)
Exemplo n.º 7
0
    def test_failure(self):
        a1 = Mock()
        a1.f = Mock(__name__='f', side_effect=[IOError, 1])
        a = RetryWrapper(
            a1,
            retry_if=lambda x: True,
            backoff=0.0001,
            max_tries=2
        )

        self.assertEqual(a.f(), 1)
        self.assertEqual(a1.f.call_count, 2)
Exemplo n.º 8
0
    def test_success(self):
        a1 = Mock()
        a1.f = Mock(__name__='f', side_effect=None)
        a = RetryWrapper(
            a1,
            retry_if=lambda x: True,
            backoff=0.0001,
            max_tries=2
        )

        a.f()
        a1.f.assert_called_once_with()
Exemplo n.º 9
0
def _wrap_aws_client(raw_client, min_backoff=None):
    """Wrap a given boto3 Client object so that it can retry when
    throttled."""
    return RetryWrapper(raw_client,
                        retry_if=_is_retriable_client_error,
                        backoff=max(_AWS_BACKOFF, min_backoff or 0),
                        multiplier=_AWS_BACKOFF_MULTIPLIER,
                        max_tries=_AWS_MAX_TRIES)
Exemplo n.º 10
0
Arquivo: s3.py Projeto: yuanda/mrjob
def wrap_aws_conn(raw_conn):
    """Wrap a given boto Connection object so that it can retry when
    throttled."""
    def retry_if(ex):
        """Retry if we get a server error indicating throttling. Also
        handle spurious 505s that are thought to be part of a load
        balancer issue inside AWS."""
        return ((isinstance(ex, boto.exception.BotoServerError) and
                 ('Throttling' in ex.body or 'RequestExpired' in ex.body
                  or ex.status == 505))
                or (isinstance(ex, socket.error)
                    and ex.args in ((104, 'Connection reset by peer'),
                                    (110, 'Connection timed out'))))

    return RetryWrapper(raw_conn,
                        retry_if=retry_if,
                        backoff=EMR_BACKOFF,
                        multiplier=EMR_BACKOFF_MULTIPLIER,
                        max_tries=EMR_MAX_TRIES)