Ejemplo n.º 1
0
    def ensure_connection(self, errback=None, max_retries=None,
            interval_start=2, interval_step=2, interval_max=30):
        """Ensure we have a connection to the server.

        If not retry establishing the connection with the settings
        specified.

        :keyword errback: Optional callback called each time the connection
          can't be established. Arguments provided are the exception
          raised and the interval that will be slept ``(exc, interval)``.

        :keyword max_retries: Maximum number of times to retry.
          If this limit is exceeded the connection error will be re-raised.

        :keyword interval_start: The number of seconds we start sleeping for.
        :keyword interval_step: How many seconds added to the interval
          for each retry.
        :keyword interval_max: Maximum number of seconds to sleep between
          each retry.

        """
        retry_over_time(self.connect, self.connection_errors, (), {},
                        errback, max_retries,
                        interval_start, interval_step, interval_max)
        return self
Ejemplo n.º 2
0
    def ensure_connection(self, errback=None, max_retries=None,
            interval_start=2, interval_step=2, interval_max=30):
        """Ensure we have a connection to the server.

        If not retry establishing the connection with the settings
        specified.

        :keyword errback: Optional callback called each time the connection
          can't be established. Arguments provided are the exception
          raised and the interval that will be slept ``(exc, interval)``.

        :keyword max_retries: Maximum number of times to retry.
          If this limit is exceeded the connection error will be re-raised.

        :keyword interval_start: The number of seconds we start sleeping for.
        :keyword interval_step: How many seconds added to the interval
          for each retry.
        :keyword interval_max: Maximum number of seconds to sleep between
          each retry.

        """
        retry_over_time(self.connect, self.connection_errors, (), {},
                        errback, max_retries,
                        interval_start, interval_step, interval_max)
        return self
Ejemplo n.º 3
0
 def start(self):
     _start = functools.partial(self.blueprint.start, self)
     try:
         retry_over_time(_start, amqp_exceptions.NotFound, errback=self.on_error)
     except WorkerTerminate:
         self.terminate()
     except Exception as exc:
         logger.critical('Unrecoverable error: %r', exc, exc_info=True)
         self.stop(exitcode=EX_FAILURE)
     except SystemExit as exc:
         self.stop(exitcode=exc.code)
     except KeyboardInterrupt:
         self.stop(exitcode=EX_FAILURE)
Ejemplo n.º 4
0
 def test_retry_once(self):
     with self.assertRaises(self.Predicate):
         utils.retry_over_time(
             self.myfun, self.Predicate,
             max_retries=1, errback=self.errback, interval_max=14,
         )
     self.assertEqual(self.index, 1)
     # no errback
     with self.assertRaises(self.Predicate):
         utils.retry_over_time(
             self.myfun, self.Predicate,
             max_retries=1, errback=None, interval_max=14,
         )
Ejemplo n.º 5
0
    def test_retry_always(self):
        Predicate = self.Predicate

        class Fun(object):

            def __init__(self):
                self.calls = 0

            def __call__(self, *args, **kwargs):
                try:
                    if self.calls >= 10:
                        return 42
                    raise Predicate()
                finally:
                    self.calls += 1
        fun = Fun()

        self.assertEqual(
            utils.retry_over_time(
                fun, self.Predicate,
                max_retries=0, errback=None, interval_max=14,
            ),
            42,
        )
        self.assertEqual(fun.calls, 11)
Ejemplo n.º 6
0
 def test_simple(self):
     prev_count, utils.count = utils.count, Mock()
     try:
         utils.count.return_value = range(1)
         x = utils.retry_over_time(self.myfun, self.Predicate,
                 errback=None, interval_max=14)
         self.assertIsNone(x)
         utils.count.return_value = range(10)
         cb = Mock()
         x = utils.retry_over_time(self.myfun, self.Predicate,
                 errback=self.errback, callback=cb, interval_max=14)
         self.assertEqual(x, 42)
         self.assertEqual(self.index, 9)
         cb.assert_called_with()
     finally:
         utils.count = prev_count
Ejemplo n.º 7
0
 def ensure(self, fun, args, **policy):
     retry_policy = dict(self.retry_policy, **policy)
     max_retries = retry_policy.get('max_retries')
     return retry_over_time(
         fun, self.connection_errors, args, {},
         partial(self.on_connection_error, max_retries),
         **retry_policy)
Ejemplo n.º 8
0
 def test_simple(self):
     prev_count, utils.count = utils.count, Mock()
     try:
         utils.count.return_value = range(1)
         x = utils.retry_over_time(self.myfun, self.Predicate,
                 errback=None, interval_max=14)
         self.assertIsNone(x)
         utils.count.return_value = range(10)
         cb = Mock()
         x = utils.retry_over_time(self.myfun, self.Predicate,
                 errback=self.errback, callback=cb, interval_max=14)
         self.assertEqual(x, 42)
         self.assertEqual(self.index, 9)
         cb.assert_called_with()
     finally:
         utils.count = prev_count
Ejemplo n.º 9
0
 def test_simple(self):
     x = utils.retry_over_time(self.myfun,
                               self.Predicate,
                               errback=self.errback,
                               interval_max=14)
     self.assertEqual(x, 42)
     self.assertEqual(self.index, 9)
Ejemplo n.º 10
0
    def test_simple(self):
        index = [0]

        class Predicate(Exception):
            pass

        def myfun():
            sleepvals = {0: None,
                         1: 2.0,
                         2: 4.0,
                         3: 6.0,
                         4: 8.0,
                         5: 10.0,
                         6: 12.0,
                         7: 14.0,
                         8: 16.0,
                         9: 16.0}
            self.assertEqual(_tried_to_sleep[0], sleepvals[index[0]])
            if index[0] < 9:
                raise Predicate()
            return 42

        def errback(exc, interval):
            index[0] += 1

        x = utils.retry_over_time(myfun, Predicate, errback=errback,
                                                    interval_max=14)
        self.assertEqual(x, 42)
        _tried_to_sleep[0] = None
        index[0] = 0
        self.assertRaises(Predicate,
                          utils.retry_over_time, myfun, Predicate,
                          max_retries=1, errback=errback, interval_max=14)
Ejemplo n.º 11
0
    def test_simple(self):
        index = [0]

        class Predicate(Exception):
            pass

        def myfun():
            sleepvals = {0: None,
                         1: 2.0,
                         2: 4.0,
                         3: 6.0,
                         4: 8.0,
                         5: 10.0,
                         6: 12.0,
                         7: 14.0,
                         8: 16.0,
                         9: 16.0}
            self.assertEqual(_tried_to_sleep[0], sleepvals[index[0]])
            if index[0] < 9:
                raise Predicate()
            return 42

        def errback(exc, interval):
            index[0] += 1

        x = utils.retry_over_time(myfun, Predicate, errback=errback,
                                                    interval_max=14)
        self.assertEqual(x, 42)
        _tried_to_sleep[0] = None
        index[0] = 0
        self.assertRaises(Predicate,
                          utils.retry_over_time, myfun, Predicate,
                          max_retries=1, errback=errback, interval_max=14)
Ejemplo n.º 12
0
    def test_retry_always(self):
        Predicate = self.Predicate

        class Fun(object):
            def __init__(self):
                self.calls = 0

            def __call__(self, *args, **kwargs):
                try:
                    if self.calls >= 10:
                        return 42
                    raise Predicate()
                finally:
                    self.calls += 1

        fun = Fun()

        self.assertEqual(
            utils.retry_over_time(
                fun,
                self.Predicate,
                max_retries=0,
                errback=None,
                interval_max=14,
            ),
            42,
        )
        self.assertEqual(fun.calls, 11)
Ejemplo n.º 13
0
 def ensure(self, fun, args, **policy):
     retry_policy = dict(self.retry_policy, **policy)
     max_retries = retry_policy.get('max_retries')
     return retry_over_time(
         fun, self.connection_errors, args, {},
         partial(self.on_connection_error, max_retries),
         **retry_policy
     )
Ejemplo n.º 14
0
 def test_retry_once(self):
     with self.assertRaises(self.Predicate):
         utils.retry_over_time(
             self.myfun,
             self.Predicate,
             max_retries=1,
             errback=self.errback,
             interval_max=14,
         )
     self.assertEqual(self.index, 1)
     # no errback
     with self.assertRaises(self.Predicate):
         utils.retry_over_time(
             self.myfun,
             self.Predicate,
             max_retries=1,
             errback=None,
             interval_max=14,
         )
Ejemplo n.º 15
0
def asynloop(obj, connection, consumer, blueprint, hub, qos,
             heartbeat, clock, hbrate=2.0):
    """Non-blocking event loop."""
    RUN = bootsteps.RUN
    update_qos = qos.update
    errors = connection.connection_errors

    on_task_received = obj.create_task_handler()

    _enable_amqheartbeats(hub.timer, connection, rate=hbrate)

    consumer.on_message = on_task_received
    obj.controller.register_with_event_loop(hub)
    obj.register_with_event_loop(hub)

    # [FAM-348]
    def on_error(exc, intervals, _):
        consumer.cancel()

        consume_error = "Task queue is not available: %s\nTrying to consume again %s..."
        interval = next(intervals)
        error(consume_error, exc, humanize_seconds(interval, 'in', ' '))
        return interval

    retry_over_time(consumer.consume, amqp.exceptions.NotFound, errback=on_error)

    obj.on_ready()

    # did_start_ok will verify that pool processes were able to start,
    # but this will only work the first time we start, as
    # maxtasksperchild will mess up metrics.
    if not obj.restart_count and not obj.pool.did_start_ok():
        raise WorkerLostError('Could not start worker processes')

    # consumer.consume() may have prefetched up to our
    # limit - drain an event so we're in a clean state
    # prior to starting our event loop.
    if connection.transport.driver_type == 'amqp':
        hub.call_soon(_quick_drain, connection)

    # FIXME: Use loop.run_forever
    # Tried and works, but no time to test properly before release.
    hub.propagate_errors = errors
    loop = hub.create_loop()

    try:
        while blueprint.state == RUN and obj.connection:
            # shutdown if signal handlers told us to.
            should_stop, should_terminate = (
                state.should_stop, state.should_terminate,
            )
            # False == EX_OK, so must use is not False
            if should_stop is not None and should_stop is not False:
                raise WorkerShutdown(should_stop)
            elif should_terminate is not None and should_stop is not False:
                raise WorkerTerminate(should_terminate)

            # We only update QoS when there's no more messages to read.
            # This groups together qos calls, and makes sure that remote
            # control commands will be prioritized over task messages.
            if qos.prev != qos.value:
                update_qos()

            try:
                next(loop)
            except StopIteration:
                loop = hub.create_loop()
    finally:
        try:
            hub.reset()
        except Exception as exc:  # pylint: disable=broad-except
            logger.exception(
                'Error cleaning up after event loop: %r', exc)
Ejemplo n.º 16
0
 def retry_over_time(self, *args, **kwargs):
     return retry_over_time(*args, **kwargs)
Ejemplo n.º 17
0
 def retry_over_time(self, *args, **kwargs):
     return retry_over_time(*args, **kwargs)
Ejemplo n.º 18
0
 def test_simple(self):
     x = utils.retry_over_time(self.myfun, self.Predicate,
             errback=self.errback, interval_max=14)
     self.assertEqual(x, 42)
     self.assertEqual(self.index, 9)