def test_retry_with_backoff(self):
con = Connection()
con.connection = Mock()
connect_method = "libcloud.common.base.Connection.request"
with patch(connect_method) as mock_connect:
mock_connect.__name__ = "mock_connect"
with self.assertRaises(socket.gaierror):
mock_connect.side_effect = socket.gaierror("")
retry_request = retry(timeout=2, retry_delay=0.1, backoff=1)
retry_request(con.request)(action="/")
self.assertGreater(mock_connect.call_count, 1, "Retry logic failed")
def test_retry_with_backoff(self):
con = Connection()
con.connection = Mock()
connect_method = 'libcloud.common.base.Connection.request'
with patch(connect_method) as mock_connect:
mock_connect.__name__ = 'mock_connect'
with self.assertRaises(socket.gaierror):
mock_connect.side_effect = socket.gaierror('')
retry_request = retry(timeout=0.2, retry_delay=0.1, backoff=1)
retry_request(con.request)(action='/')
self.assertGreater(mock_connect.call_count, 1,
'Retry logic failed')
def test_retry_with_timeout(self):
con = Connection()
con.connection = Mock()
connect_method = 'libcloud.common.base.Connection.request'
with patch(connect_method) as mock_connect:
mock_connect.__name__ = 'mock_connect'
with self.assertRaises(socket.gaierror):
mock_connect.side_effect = socket.gaierror('')
retry_request = retry(timeout=2, retry_delay=.1,
backoff=1)
retry_request(con.request)(action='/')
self.assertGreater(mock_connect.call_count, 1,
'Retry logic failed')
def request(self,
action,
params=None,
data=None,
headers=None,
method='GET',
raw=False,
stream=False):
"""
Request a given `action`.
Basically a wrapper around the connection
object's `request` that does some helpful pre-processing.
:type action: ``str``
:param action: A path. This can include arguments. If included,
any extra parameters are appended to the existing ones.
:type params: ``dict``
:param params: Optional mapping of additional parameters to send. If
None, leave as an empty ``dict``.
:type data: ``unicode``
:param data: A body of data to send with the request.
:type headers: ``dict``
:param headers: Extra headers to add to the request
None, leave as an empty ``dict``.
:type method: ``str``
:param method: An HTTP method such as "GET" or "POST".
:type raw: ``bool``
:param raw: True to perform a "raw" request aka only send the headers
and use the rawResponseCls class. This is used with
storage API when uploading a file.
:type stream: ``bool``
:param stream: True to return an iterator in Response.iter_content
and allow streaming of the response data
(for downloading large files)
:return: An :class:`Response` instance.
:rtype: :class:`Response` instance
"""
if params is None:
params = {}
else:
params = copy.copy(params)
if headers is None:
headers = {}
else:
headers = copy.copy(headers)
retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
False) or RETRY_FAILED_HTTP_REQUESTS
action = self.morph_action_hook(action)
self.action = action
self.method = method
self.data = data
# Extend default parameters
params = self.add_default_params(params)
# Add cache busting parameters (if enabled)
if self.cache_busting and method == 'GET':
params = self._add_cache_busting_to_params(params=params)
# Extend default headers
headers = self.add_default_headers(headers)
# We always send a user-agent header
headers.update({'User-Agent': self._user_agent()})
# Indicate that we support gzip and deflate compression
headers.update({'Accept-Encoding': 'gzip,deflate'})
port = int(self.port)
if port not in (80, 443):
headers.update({'Host': "%s:%d" % (self.host, port)})
else:
headers.update({'Host': self.host})
if data:
data = self.encode_data(data)
params, headers = self.pre_connect_hook(params, headers)
if params:
if '?' in action:
url = '&'.join((action, urlencode(params, doseq=True)))
else:
url = '?'.join((action, urlencode(params, doseq=True)))
else:
url = action
# IF connection has not yet been established
if self.connection is None:
self.connect()
try:
# @TODO: Should we just pass File object as body to request method
# instead of dealing with splitting and sending the file ourselves?
if raw:
self.connection.prepared_request(method=method,
url=url,
body=data,
headers=headers,
stream=stream)
else:
if retry_enabled:
retry_request = retry(timeout=self.timeout,
retry_delay=self.retry_delay,
backoff=self.backoff)
retry_request(self.connection.request)(method=method,
url=url,
body=data,
headers=headers,
stream=stream)
else:
self.connection.request(method=method,
url=url,
body=data,
headers=headers,
stream=stream)
except socket.gaierror:
e = sys.exc_info()[1]
message = str(e)
errno = getattr(e, 'errno', None)
if errno == -5:
# Throw a more-friendly exception on "no address associated
# with hostname" error. This error could simpli indicate that
# "host" Connection class attribute is set to an incorrect
# value
class_name = self.__class__.__name__
msg = ('%s. Perhaps "host" Connection class attribute '
'(%s.connection) is set to an invalid, non-hostname '
'value (%s)?' % (message, class_name, self.host))
raise socket.gaierror(msg)
self.reset_context()
raise e
except ssl.SSLError:
e = sys.exc_info()[1]
self.reset_context()
raise ssl.SSLError(str(e))
if raw:
responseCls = self.rawResponseCls
kwargs = {
'connection': self,
'response': self.connection.getresponse()
}
else:
responseCls = self.responseCls
kwargs = {
'connection': self,
'response': self.connection.getresponse()
}
try:
response = responseCls(**kwargs)
finally:
# Always reset the context after the request has completed
self.reset_context()
return response
def request(self,
action,
params=None,
data=None,
headers=None,
method='GET',
raw=False):
"""
Request a given `action`.
Basically a wrapper around the connection
object's `request` that does some helpful pre-processing.
:type action: ``str``
:param action: A path. This can include arguments. If included,
any extra parameters are appended to the existing ones.
:type params: ``dict``
:param params: Optional mapping of additional parameters to send. If
None, leave as an empty ``dict``.
:type data: ``unicode``
:param data: A body of data to send with the request.
:type headers: ``dict``
:param headers: Extra headers to add to the request
None, leave as an empty ``dict``.
:type method: ``str``
:param method: An HTTP method such as "GET" or "POST".
:type raw: ``bool``
:param raw: True to perform a "raw" request aka only send the headers
and use the rawResponseCls class. This is used with
storage API when uploading a file.
:return: An :class:`Response` instance.
:rtype: :class:`Response` instance
"""
if params is None:
params = {}
else:
params = copy.copy(params)
if headers is None:
headers = {}
else:
headers = copy.copy(headers)
retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
False) or RETRY_FAILED_HTTP_REQUESTS
action = self.morph_action_hook(action)
self.action = action
self.method = method
# Extend default parameters
params = self.add_default_params(params)
# Add cache busting parameters (if enabled)
if self.cache_busting and method == 'GET':
params = self._add_cache_busting_to_params(params=params)
# Extend default headers
headers = self.add_default_headers(headers)
# We always send a user-agent header
headers.update({'User-Agent': self._user_agent()})
# Indicate that we support gzip and deflate compression
headers.update({'Accept-Encoding': 'gzip,deflate'})
port = int(self.port)
if port not in (80, 443):
headers.update({'Host': "%s:%d" % (self.host, port)})
else:
headers.update({'Host': self.host})
if data:
data = self.encode_data(data)
headers['Content-Length'] = str(len(data))
elif method.upper() in ['POST', 'PUT'] and not raw:
# Only send Content-Length 0 with POST and PUT request.
#
# Note: Content-Length is not added when using "raw" mode means
# means that headers are upfront and the body is sent at some point
# later on. With raw mode user can specify Content-Length with
# "data" not being set.
headers['Content-Length'] = '0'
params, headers = self.pre_connect_hook(params, headers)
if params:
if '?' in action:
url = '&'.join((action, urlencode(params, doseq=True)))
else:
url = '?'.join((action, urlencode(params, doseq=True)))
else:
url = action
# Removed terrible hack...this a less-bad hack that doesn't execute a
# request twice, but it's still a hack.
self.connect()
try:
# @TODO: Should we just pass File object as body to request method
# instead of dealing with splitting and sending the file ourselves?
if raw:
self.connection.putrequest(method, url)
for key, value in list(headers.items()):
self.connection.putheader(key, str(value))
self.connection.endheaders()
else:
if retry_enabled:
retry_request = retry(timeout=self.timeout,
retry_delay=self.retry_delay,
backoff=self.backoff)
retry_request(self.connection.request)(method=method,
url=url,
body=data,
headers=headers)
else:
self.connection.request(method=method,
url=url,
body=data,
headers=headers)
except ssl.SSLError:
e = sys.exc_info()[1]
self.reset_context()
raise ssl.SSLError(str(e))
if raw:
responseCls = self.rawResponseCls
kwargs = {'connection': self}
else:
responseCls = self.responseCls
kwargs = {
'connection': self,
'response': self.connection.getresponse()
}
try:
response = responseCls(**kwargs)
finally:
# Always reset the context after the request has completed
self.reset_context()
return response
def request(self, action, params=None, data=None, headers=None,
method='GET', raw=False):
"""
Request a given `action`.
Basically a wrapper around the connection
object's `request` that does some helpful pre-processing.
:type action: ``str``
:param action: A path. This can include arguments. If included,
any extra parameters are appended to the existing ones.
:type params: ``dict``
:param params: Optional mapping of additional parameters to send. If
None, leave as an empty ``dict``.
:type data: ``unicode``
:param data: A body of data to send with the request.
:type headers: ``dict``
:param headers: Extra headers to add to the request
None, leave as an empty ``dict``.
:type method: ``str``
:param method: An HTTP method such as "GET" or "POST".
:type raw: ``bool``
:param raw: True to perform a "raw" request aka only send the headers
and use the rawResponseCls class. This is used with
storage API when uploading a file.
:return: An :class:`Response` instance.
:rtype: :class:`Response` instance
"""
if params is None:
params = {}
else:
params = copy.copy(params)
if headers is None:
headers = {}
else:
headers = copy.copy(headers)
retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
False) or RETRY_FAILED_HTTP_REQUESTS
action = self.morph_action_hook(action)
self.action = action
self.method = method
# Extend default parameters
params = self.add_default_params(params)
# Add cache busting parameters (if enabled)
if self.cache_busting and method == 'GET':
params = self._add_cache_busting_to_params(params=params)
# Extend default headers
headers = self.add_default_headers(headers)
# We always send a user-agent header
headers.update({'User-Agent': self._user_agent()})
# Indicate that we support gzip and deflate compression
headers.update({'Accept-Encoding': 'gzip,deflate'})
port = int(self.port)
if port not in (80, 443):
headers.update({'Host': "%s:%d" % (self.host, port)})
else:
headers.update({'Host': self.host})
if data:
data = self.encode_data(data)
headers['Content-Length'] = str(len(data))
elif method.upper() in ['POST', 'PUT'] and not raw:
# Only send Content-Length 0 with POST and PUT request.
#
# Note: Content-Length is not added when using "raw" mode means
# means that headers are upfront and the body is sent at some point
# later on. With raw mode user can specify Content-Length with
# "data" not being set.
headers['Content-Length'] = '0'
params, headers = self.pre_connect_hook(params, headers)
if params:
if '?' in action:
url = '&'.join((action, urlencode(params, doseq=True)))
else:
url = '?'.join((action, urlencode(params, doseq=True)))
else:
url = action
# Removed terrible hack...this a less-bad hack that doesn't execute a
# request twice, but it's still a hack.
self.connect()
try:
# @TODO: Should we just pass File object as body to request method
# instead of dealing with splitting and sending the file ourselves?
if raw:
self.connection.putrequest(method, url)
for key, value in list(headers.items()):
self.connection.putheader(key, str(value))
self.connection.endheaders()
else:
if retry_enabled:
retry_request = retry(timeout=self.timeout,
retry_delay=self.retry_delay,
backoff=self.backoff)
retry_request(self.connection.request)(method=method,
url=url,
body=data,
headers=headers)
else:
self.connection.request(method=method, url=url, body=data,
headers=headers)
except ssl.SSLError:
e = sys.exc_info()[1]
self.reset_context()
raise ssl.SSLError(str(e))
if raw:
responseCls = self.rawResponseCls
kwargs = {'connection': self}
else:
responseCls = self.responseCls
kwargs = {'connection': self,
'response': self.connection.getresponse()}
try:
response = responseCls(**kwargs)
finally:
# Always reset the context after the request has completed
self.reset_context()
return response
def request(self, action, params=None, data=None, headers=None,
method='GET', raw=False, stream=False):
"""
Request a given `action`.
Basically a wrapper around the connection
object's `request` that does some helpful pre-processing.
:type action: ``str``
:param action: A path. This can include arguments. If included,
any extra parameters are appended to the existing ones.
:type params: ``dict``
:param params: Optional mapping of additional parameters to send. If
None, leave as an empty ``dict``.
:type data: ``unicode``
:param data: A body of data to send with the request.
:type headers: ``dict``
:param headers: Extra headers to add to the request
None, leave as an empty ``dict``.
:type method: ``str``
:param method: An HTTP method such as "GET" or "POST".
:type raw: ``bool``
:param raw: True to perform a "raw" request aka only send the headers
and use the rawResponseCls class. This is used with
storage API when uploading a file.
:type stream: ``bool``
:param stream: True to return an iterator in Response.iter_content
and allow streaming of the response data
(for downloading large files)
:return: An :class:`Response` instance.
:rtype: :class:`Response` instance
"""
if params is None:
params = {}
else:
params = copy.copy(params)
if headers is None:
headers = {}
else:
headers = copy.copy(headers)
retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
False) or RETRY_FAILED_HTTP_REQUESTS
action = self.morph_action_hook(action)
self.action = action
self.method = method
self.data = data
# Extend default parameters
params = self.add_default_params(params)
# Add cache busting parameters (if enabled)
if self.cache_busting and method == 'GET':
params = self._add_cache_busting_to_params(params=params)
# Extend default headers
headers = self.add_default_headers(headers)
# We always send a user-agent header
headers.update({'User-Agent': self._user_agent()})
# Indicate that we support gzip and deflate compression
headers.update({'Accept-Encoding': 'gzip,deflate'})
port = int(self.port)
if port not in (80, 443):
headers.update({'Host': "%s:%d" % (self.host, port)})
else:
headers.update({'Host': self.host})
if data:
data = self.encode_data(data)
params, headers = self.pre_connect_hook(params, headers)
if params:
if '?' in action:
url = '&'.join((action, urlencode(params, doseq=True)))
else:
url = '?'.join((action, urlencode(params, doseq=True)))
else:
url = action
# IF connection has not yet been established
if self.connection is None:
self.connect()
try:
# @TODO: Should we just pass File object as body to request method
# instead of dealing with splitting and sending the file ourselves?
if raw:
self.connection.prepared_request(
method=method,
url=url,
body=data,
headers=headers,
stream=stream)
else:
if retry_enabled:
retry_request = retry(timeout=self.timeout,
retry_delay=self.retry_delay,
backoff=self.backoff)
retry_request(self.connection.request)(method=method,
url=url,
body=data,
headers=headers,
stream=stream)
else:
self.connection.request(method=method, url=url, body=data,
headers=headers, stream=stream)
except socket.gaierror:
e = sys.exc_info()[1]
message = str(e)
errno = getattr(e, 'errno', None)
if errno == -5:
# Throw a more-friendly exception on "no address associated
# with hostname" error. This error could simpli indicate that
# "host" Connection class attribute is set to an incorrect
# value
class_name = self.__class__.__name__
msg = ('%s. Perhaps "host" Connection class attribute '
'(%s.connection) is set to an invalid, non-hostname '
'value (%s)?' %
(message, class_name, self.host))
raise socket.gaierror(msg)
self.reset_context()
raise e
except ssl.SSLError:
e = sys.exc_info()[1]
self.reset_context()
raise ssl.SSLError(str(e))
if raw:
responseCls = self.rawResponseCls
kwargs = {'connection': self,
'response': self.connection.getresponse()}
else:
responseCls = self.responseCls
kwargs = {'connection': self,
'response': self.connection.getresponse()}
try:
response = responseCls(**kwargs)
finally:
# Always reset the context after the request has completed
self.reset_context()
return response