def ensure_connection(self, errback=None, max_retries=None, interval_start=2, interval_step=2, interval_max=30): """Ensure we have a connection to the server. If not retry establishing the connection with the settings specified. :keyword errback: Optional callback called each time the connection can't be established. Arguments provided are the exception raised and the interval that will be slept ``(exc, interval)``. :keyword max_retries: Maximum number of times to retry. If this limit is exceeded the connection error will be re-raised. :keyword interval_start: The number of seconds we start sleeping for. :keyword interval_step: How many seconds added to the interval for each retry. :keyword interval_max: Maximum number of seconds to sleep between each retry. """ retry_over_time(self.connect, self.connection_errors, (), {}, errback, max_retries, interval_start, interval_step, interval_max) return self
def start(self): _start = functools.partial(self.blueprint.start, self) try: retry_over_time(_start, amqp_exceptions.NotFound, errback=self.on_error) except WorkerTerminate: self.terminate() except Exception as exc: logger.critical('Unrecoverable error: %r', exc, exc_info=True) self.stop(exitcode=EX_FAILURE) except SystemExit as exc: self.stop(exitcode=exc.code) except KeyboardInterrupt: self.stop(exitcode=EX_FAILURE)
def test_retry_once(self): with self.assertRaises(self.Predicate): utils.retry_over_time( self.myfun, self.Predicate, max_retries=1, errback=self.errback, interval_max=14, ) self.assertEqual(self.index, 1) # no errback with self.assertRaises(self.Predicate): utils.retry_over_time( self.myfun, self.Predicate, max_retries=1, errback=None, interval_max=14, )
def test_retry_always(self): Predicate = self.Predicate class Fun(object): def __init__(self): self.calls = 0 def __call__(self, *args, **kwargs): try: if self.calls >= 10: return 42 raise Predicate() finally: self.calls += 1 fun = Fun() self.assertEqual( utils.retry_over_time( fun, self.Predicate, max_retries=0, errback=None, interval_max=14, ), 42, ) self.assertEqual(fun.calls, 11)
def test_simple(self): prev_count, utils.count = utils.count, Mock() try: utils.count.return_value = range(1) x = utils.retry_over_time(self.myfun, self.Predicate, errback=None, interval_max=14) self.assertIsNone(x) utils.count.return_value = range(10) cb = Mock() x = utils.retry_over_time(self.myfun, self.Predicate, errback=self.errback, callback=cb, interval_max=14) self.assertEqual(x, 42) self.assertEqual(self.index, 9) cb.assert_called_with() finally: utils.count = prev_count
def ensure(self, fun, args, **policy): retry_policy = dict(self.retry_policy, **policy) max_retries = retry_policy.get('max_retries') return retry_over_time( fun, self.connection_errors, args, {}, partial(self.on_connection_error, max_retries), **retry_policy)
def test_simple(self): x = utils.retry_over_time(self.myfun, self.Predicate, errback=self.errback, interval_max=14) self.assertEqual(x, 42) self.assertEqual(self.index, 9)
def test_simple(self): index = [0] class Predicate(Exception): pass def myfun(): sleepvals = {0: None, 1: 2.0, 2: 4.0, 3: 6.0, 4: 8.0, 5: 10.0, 6: 12.0, 7: 14.0, 8: 16.0, 9: 16.0} self.assertEqual(_tried_to_sleep[0], sleepvals[index[0]]) if index[0] < 9: raise Predicate() return 42 def errback(exc, interval): index[0] += 1 x = utils.retry_over_time(myfun, Predicate, errback=errback, interval_max=14) self.assertEqual(x, 42) _tried_to_sleep[0] = None index[0] = 0 self.assertRaises(Predicate, utils.retry_over_time, myfun, Predicate, max_retries=1, errback=errback, interval_max=14)
def ensure(self, fun, args, **policy): retry_policy = dict(self.retry_policy, **policy) max_retries = retry_policy.get('max_retries') return retry_over_time( fun, self.connection_errors, args, {}, partial(self.on_connection_error, max_retries), **retry_policy )
def asynloop(obj, connection, consumer, blueprint, hub, qos, heartbeat, clock, hbrate=2.0): """Non-blocking event loop.""" RUN = bootsteps.RUN update_qos = qos.update errors = connection.connection_errors on_task_received = obj.create_task_handler() _enable_amqheartbeats(hub.timer, connection, rate=hbrate) consumer.on_message = on_task_received obj.controller.register_with_event_loop(hub) obj.register_with_event_loop(hub) # [FAM-348] def on_error(exc, intervals, _): consumer.cancel() consume_error = "Task queue is not available: %s\nTrying to consume again %s..." interval = next(intervals) error(consume_error, exc, humanize_seconds(interval, 'in', ' ')) return interval retry_over_time(consumer.consume, amqp.exceptions.NotFound, errback=on_error) obj.on_ready() # did_start_ok will verify that pool processes were able to start, # but this will only work the first time we start, as # maxtasksperchild will mess up metrics. if not obj.restart_count and not obj.pool.did_start_ok(): raise WorkerLostError('Could not start worker processes') # consumer.consume() may have prefetched up to our # limit - drain an event so we're in a clean state # prior to starting our event loop. if connection.transport.driver_type == 'amqp': hub.call_soon(_quick_drain, connection) # FIXME: Use loop.run_forever # Tried and works, but no time to test properly before release. hub.propagate_errors = errors loop = hub.create_loop() try: while blueprint.state == RUN and obj.connection: # shutdown if signal handlers told us to. should_stop, should_terminate = ( state.should_stop, state.should_terminate, ) # False == EX_OK, so must use is not False if should_stop is not None and should_stop is not False: raise WorkerShutdown(should_stop) elif should_terminate is not None and should_stop is not False: raise WorkerTerminate(should_terminate) # We only update QoS when there's no more messages to read. # This groups together qos calls, and makes sure that remote # control commands will be prioritized over task messages. if qos.prev != qos.value: update_qos() try: next(loop) except StopIteration: loop = hub.create_loop() finally: try: hub.reset() except Exception as exc: # pylint: disable=broad-except logger.exception( 'Error cleaning up after event loop: %r', exc)
def retry_over_time(self, *args, **kwargs): return retry_over_time(*args, **kwargs)