Esempio n. 1
0
def make_api_request(http,
                     http_request,
                     retries=7,
                     max_retry_wait=60,
                     redirections=5,
                     check_response_func=_check_response,
                     wo_retry_func=_make_api_request_no_retry):
    """Send an HTTP request via the given http, performing error/retry handling.

    :type http: :class:`httplib2.Http`
    :param http: an instance which impelements the `Http` API.

    :type http_request: :class:`Request`
    :param http_request: the request to send.

    :type retries: integer
    :param retries: Number of retries to attempt on retryable
                    responses (such as 429 or 5XX).

    :type max_retry_wait: integer
    :param max_retry_wait: Maximum number of seconds to wait when retrying.

    :type redirections: integer
    :param redirections: Number of redirects to follow.

    :type check_response_func: function taking (response, content, url).
    :param check_response_func: Function to validate the HTTP response.

    :type wo_retry_func: function taking
                         (http, request, redirections, check_response_func)
    :param wo_retry_func: Function to make HTTP request without retries.

    :rtype: :class:`Response`
    :returns: an object representing the server's response

    :raises: :exc:`gcloud.streaming.exceptions.RequestError` if no response
             could be parsed.
    """
    retry = 0
    while True:
        try:
            return wo_retry_func(http,
                                 http_request,
                                 redirections=redirections,
                                 check_response_func=check_response_func)
        except _RETRYABLE_EXCEPTIONS as exc:
            retry += 1
            if retry >= retries:
                raise
            retry_after = getattr(exc, 'retry_after', None)
            if retry_after is None:
                retry_after = calculate_wait_for_retry(retry, max_retry_wait)

            _reset_http_connections(http)
            logging.debug('Retrying request to url %s after exception %s',
                          http_request.url,
                          type(exc).__name__)
            time.sleep(retry_after)
Esempio n. 2
0
def make_api_request(http, http_request,
                     retries=7,
                     max_retry_wait=60,
                     redirections=5,
                     check_response_func=_check_response,
                     wo_retry_func=_make_api_request_no_retry):
    """Send an HTTP request via the given http, performing error/retry handling.

    :type http: :class:`httplib2.Http`
    :param http: an instance which impelements the `Http` API.

    :type http_request: :class:`Request`
    :param http_request: the request to send.

    :type retries: integer
    :param retries: Number of retries to attempt on retryable
                    responses (such as 429 or 5XX).

    :type max_retry_wait: integer
    :param max_retry_wait: Maximum number of seconds to wait when retrying.

    :type redirections: integer
    :param redirections: Number of redirects to follow.

    :type check_response_func: function taking (response, content, url).
    :param check_response_func: Function to validate the HTTP response.

    :type wo_retry_func: function taking
                         (http, request, redirections, check_response_func)
    :param wo_retry_func: Function to make HTTP request without retries.

    :rtype: :class:`Response`
    :returns: an object representing the server's response

    :raises: :exc:`gcloud.streaming.exceptions.RequestError` if no response
             could be parsed.
    """
    retry = 0
    while True:
        try:
            return wo_retry_func(
                http, http_request, redirections=redirections,
                check_response_func=check_response_func)
        except _RETRYABLE_EXCEPTIONS as exc:
            retry += 1
            if retry >= retries:
                raise
            retry_after = getattr(exc, 'retry_after', None)
            if retry_after is None:
                retry_after = calculate_wait_for_retry(retry, max_retry_wait)

            _reset_http_connections(http)
            logging.debug('Retrying request to url %s after exception %s',
                          http_request.url, type(exc).__name__)
            time.sleep(retry_after)
Esempio n. 3
0
def handle_http_exceptions(retry_args):
    """Exception handler for http failures.

    Catches known failures and rebuild the underlying HTTP connections.

    :type retry_args: :class:`_ExceptionRetryArgs`
    :param retry_args: the exception information to be evaluated.
    """
    # If the server indicates how long to wait, use that value.  Otherwise,
    # calculate the wait time on our own.
    retry_after = None

    # Transport failures
    if isinstance(retry_args.exc, (http_client.BadStatusLine,
                                   http_client.IncompleteRead,
                                   http_client.ResponseNotReady)):
        logging.debug('Caught HTTP error %s, retrying: %s',
                      type(retry_args.exc).__name__, retry_args.exc)
    elif isinstance(retry_args.exc, socket.gaierror):
        logging.debug(
            'Caught socket address error, retrying: %s', retry_args.exc)
    elif isinstance(retry_args.exc, socket.timeout):
        logging.debug(
            'Caught socket timeout error, retrying: %s', retry_args.exc)
    elif isinstance(retry_args.exc, socket.error):
        logging.debug('Caught socket error, retrying: %s', retry_args.exc)
    elif isinstance(retry_args.exc, httplib2.ServerNotFoundError):
        logging.debug(
            'Caught server not found error, retrying: %s', retry_args.exc)
    elif isinstance(retry_args.exc, ValueError):
        # oauth2client tries to JSON-decode the response, which can result
        # in a ValueError if the response was invalid. Until that is fixed in
        # oauth2client, need to handle it here.
        logging.debug('Response content was invalid (%s), retrying',
                      retry_args.exc)
    elif isinstance(retry_args.exc, RequestError):
        logging.debug('Request returned no response, retrying')
    # API-level failures
    elif isinstance(retry_args.exc, BadStatusCodeError):
        logging.debug('Response returned status %s, retrying',
                      retry_args.exc.status_code)
    elif isinstance(retry_args.exc, RetryAfterError):
        logging.debug('Response returned a retry-after header, retrying')
        retry_after = retry_args.exc.retry_after
    else:
        raise
    _rebuild_http_connections(retry_args.http)
    logging.debug('Retrying request to url %s after exception %s',
                  retry_args.http_request.url, retry_args.exc)
    time.sleep(
        retry_after or calculate_wait_for_retry(
            retry_args.num_retries, max_wait=retry_args.max_retry_wait))
Esempio n. 4
0
 def _callFUT(self, *args, **kw):
     from gcloud.streaming.util import calculate_wait_for_retry
     return calculate_wait_for_retry(*args, **kw)
Esempio n. 5
0
 def _callFUT(self, *args, **kw):
     from gcloud.streaming.util import calculate_wait_for_retry
     return calculate_wait_for_retry(*args, **kw)