def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
if (self.max_request_size and kwargs.get('body')
and self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError(
"Request size %i bytes exceeds published limit of %i bytes" %
(len(kwargs['body']), self.max_request_size))
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
headers['X-External-Client'] = '1'
headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
if not headers.get('X-Request-Id'):
headers['X-Request-Id'] = self._request_id()
retryable = method in ['DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
retry_count = self._retry_count if retryable else 0
if (not retryable and
time.time() - self._last_request_time > self._max_keepalive_idle):
# High probability of failure due to connection atrophy. Make
# sure this request [re]opens a new connection by closing and
# forgetting all cached connections first.
for conn in self.connections.values():
conn.close()
self.connections.clear()
delay = self._retry_delay_initial
for _ in range(retry_count):
self._last_request_time = time.time()
try:
return self.orig_http_request(uri,
method,
headers=headers,
**kwargs)
except http.client.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay,
exc_info=True)
except socket.error:
# This is the one case where httplib2 doesn't close the
# underlying connection first. Close all open
# connections, expecting this object only has the one
# connection to the API server. This is safe because
# httplib2 reopens connections when needed.
_logger.debug("Retrying API request in %d s after socket error",
delay,
exc_info=True)
for conn in self.connections.values():
conn.close()
except httplib2.SSLHandshakeError as e:
# Intercept and re-raise with a better error message.
raise httplib2.SSLHandshakeError(
"Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority."
% (uri, e))
time.sleep(delay)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
return self.orig_http_request(uri, method, headers=headers, **kwargs)
def _intercept_http_request(self, uri, **kwargs):
from httplib import BadStatusLine
if (self.max_request_size and kwargs.get('body')
and self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError(
"Request size %i bytes exceeds published limit of %i bytes" %
(len(kwargs['body']), self.max_request_size))
if 'headers' not in kwargs:
kwargs['headers'] = {}
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
kwargs['headers']['X-External-Client'] = '1'
kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
try:
return self.orig_http_request(uri, **kwargs)
except BadStatusLine:
# This is how httplib tells us that it tried to reuse an
# existing connection but it was already closed by the
# server. In that case, yes, we would like to retry.
# Unfortunately, we are not absolutely certain that the
# previous call did not succeed, so this is slightly
# risky.
return self.orig_http_request(uri, **kwargs)
def _intercept_http_request(self, uri, **kwargs):
if (self.max_request_size and kwargs.get('body')
and self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError(
"Request size %i bytes exceeds published limit of %i bytes" %
(len(kwargs['body']), self.max_request_size))
if 'headers' not in kwargs:
kwargs['headers'] = {}
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
kwargs['headers']['X-External-Client'] = '1'
kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
retryable = kwargs.get(
'method', 'GET') in ['DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
retry_count = self._retry_count if retryable else 0
if (not retryable and
time.time() - self._last_request_time > self._max_keepalive_idle):
# High probability of failure due to connection atrophy. Make
# sure this request [re]opens a new connection by closing and
# forgetting all cached connections first.
for conn in self.connections.itervalues():
conn.close()
self.connections.clear()
delay = self._retry_delay_initial
for _ in range(retry_count):
self._last_request_time = time.time()
try:
return self.orig_http_request(uri, **kwargs)
except httplib.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay,
exc_info=True)
except socket.error:
# This is the one case where httplib2 doesn't close the
# underlying connection first. Close all open
# connections, expecting this object only has the one
# connection to the API server. This is safe because
# httplib2 reopens connections when needed.
_logger.debug("Retrying API request in %d s after socket error",
delay,
exc_info=True)
for conn in self.connections.itervalues():
conn.close()
time.sleep(delay)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
return self.orig_http_request(uri, **kwargs)