def retry_delay(f): """Errback, verifes an op is retryable, and delays the next retry. """ # Check that operation is retryable. if not check_retryable(client, max_time, f.value): return f # Give the connection a chance to auto-heal d = sleep(get_delay(session_timeout)) d.addCallback(retry_inner) return d
def retry(client, func, *args, **kw): """Constructs a retry wrapper around a function that retries invocations. If the function execution results in an exception due to a transient connection error, the retry wrapper will reinvoke the operation after a suitable delay (fractional value of the session timeout). :param client: A ZookeeperClient instance. :param func: A callable python object that interacts with zookeeper, the callable must utilize the same zookeeper connection as passed in the `client` param. The function must return a single value (either a deferred or result value). """ retry_started = [time.time()] retry_error = False while 1: try: value = yield func(*args, **kw) except Exception, e: # For clients which aren't connected (session timeout == None) # we raise the errors to the callers. session_timeout = client.session_timeout or 0 # The longest we keep retrying is 1.2 * session timeout max_time = (session_timeout / 1000.0) * 1.2 + retry_started[0] if not check_retryable(client, max_time, e): # Check if its a persistent client error, and if so use the cb # if present to try and reconnect for client errors. if (check_error(e) and time.time() > max_time and callable(client.cb_retry_error) and not retry_error): log.debug("Retry error %r on %s @ %s", e, func.__name__, _args(args)) retry_error = True yield client.cb_retry_error(e) retry_started[0] = time.time() continue raise # Give the connection a chance to auto-heal. yield sleep(get_delay(session_timeout)) log.debug("Retry on %s @ %s", func.__name__, _args(args)) continue returnValue(value)