def test_retry_with_backoff(self): con = Connection() con.connection = Mock() connect_method = "libcloud.common.base.Connection.request" with patch(connect_method) as mock_connect: mock_connect.__name__ = "mock_connect" with self.assertRaises(socket.gaierror): mock_connect.side_effect = socket.gaierror("") retry_request = retry(timeout=2, retry_delay=0.1, backoff=1) retry_request(con.request)(action="/") self.assertGreater(mock_connect.call_count, 1, "Retry logic failed")
def test_retry_with_backoff(self): con = Connection() con.connection = Mock() connect_method = 'libcloud.common.base.Connection.request' with patch(connect_method) as mock_connect: mock_connect.__name__ = 'mock_connect' with self.assertRaises(socket.gaierror): mock_connect.side_effect = socket.gaierror('') retry_request = retry(timeout=0.2, retry_delay=0.1, backoff=1) retry_request(con.request)(action='/') self.assertGreater(mock_connect.call_count, 1, 'Retry logic failed')
def test_retry_with_timeout(self): con = Connection() con.connection = Mock() connect_method = 'libcloud.common.base.Connection.request' with patch(connect_method) as mock_connect: mock_connect.__name__ = 'mock_connect' with self.assertRaises(socket.gaierror): mock_connect.side_effect = socket.gaierror('') retry_request = retry(timeout=2, retry_delay=.1, backoff=1) retry_request(con.request)(action='/') self.assertGreater(mock_connect.call_count, 1, 'Retry logic failed')
def request(self, action, params=None, data=None, headers=None, method='GET', raw=False, stream=False): """ Request a given `action`. Basically a wrapper around the connection object's `request` that does some helpful pre-processing. :type action: ``str`` :param action: A path. This can include arguments. If included, any extra parameters are appended to the existing ones. :type params: ``dict`` :param params: Optional mapping of additional parameters to send. If None, leave as an empty ``dict``. :type data: ``unicode`` :param data: A body of data to send with the request. :type headers: ``dict`` :param headers: Extra headers to add to the request None, leave as an empty ``dict``. :type method: ``str`` :param method: An HTTP method such as "GET" or "POST". :type raw: ``bool`` :param raw: True to perform a "raw" request aka only send the headers and use the rawResponseCls class. This is used with storage API when uploading a file. :type stream: ``bool`` :param stream: True to return an iterator in Response.iter_content and allow streaming of the response data (for downloading large files) :return: An :class:`Response` instance. :rtype: :class:`Response` instance """ if params is None: params = {} else: params = copy.copy(params) if headers is None: headers = {} else: headers = copy.copy(headers) retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS', False) or RETRY_FAILED_HTTP_REQUESTS action = self.morph_action_hook(action) self.action = action self.method = method self.data = data # Extend default parameters params = self.add_default_params(params) # Add cache busting parameters (if enabled) if self.cache_busting and method == 'GET': params = self._add_cache_busting_to_params(params=params) # Extend default headers headers = self.add_default_headers(headers) # We always send a user-agent header headers.update({'User-Agent': self._user_agent()}) # Indicate that we support gzip and deflate compression headers.update({'Accept-Encoding': 'gzip,deflate'}) port = int(self.port) if port not in (80, 443): headers.update({'Host': "%s:%d" % (self.host, port)}) else: headers.update({'Host': self.host}) if data: data = self.encode_data(data) params, headers = self.pre_connect_hook(params, headers) if params: if '?' in action: url = '&'.join((action, urlencode(params, doseq=True))) else: url = '?'.join((action, urlencode(params, doseq=True))) else: url = action # IF connection has not yet been established if self.connection is None: self.connect() try: # @TODO: Should we just pass File object as body to request method # instead of dealing with splitting and sending the file ourselves? if raw: self.connection.prepared_request(method=method, url=url, body=data, headers=headers, stream=stream) else: if retry_enabled: retry_request = retry(timeout=self.timeout, retry_delay=self.retry_delay, backoff=self.backoff) retry_request(self.connection.request)(method=method, url=url, body=data, headers=headers, stream=stream) else: self.connection.request(method=method, url=url, body=data, headers=headers, stream=stream) except socket.gaierror: e = sys.exc_info()[1] message = str(e) errno = getattr(e, 'errno', None) if errno == -5: # Throw a more-friendly exception on "no address associated # with hostname" error. This error could simpli indicate that # "host" Connection class attribute is set to an incorrect # value class_name = self.__class__.__name__ msg = ('%s. Perhaps "host" Connection class attribute ' '(%s.connection) is set to an invalid, non-hostname ' 'value (%s)?' % (message, class_name, self.host)) raise socket.gaierror(msg) self.reset_context() raise e except ssl.SSLError: e = sys.exc_info()[1] self.reset_context() raise ssl.SSLError(str(e)) if raw: responseCls = self.rawResponseCls kwargs = { 'connection': self, 'response': self.connection.getresponse() } else: responseCls = self.responseCls kwargs = { 'connection': self, 'response': self.connection.getresponse() } try: response = responseCls(**kwargs) finally: # Always reset the context after the request has completed self.reset_context() return response
def request(self, action, params=None, data=None, headers=None, method='GET', raw=False): """ Request a given `action`. Basically a wrapper around the connection object's `request` that does some helpful pre-processing. :type action: ``str`` :param action: A path. This can include arguments. If included, any extra parameters are appended to the existing ones. :type params: ``dict`` :param params: Optional mapping of additional parameters to send. If None, leave as an empty ``dict``. :type data: ``unicode`` :param data: A body of data to send with the request. :type headers: ``dict`` :param headers: Extra headers to add to the request None, leave as an empty ``dict``. :type method: ``str`` :param method: An HTTP method such as "GET" or "POST". :type raw: ``bool`` :param raw: True to perform a "raw" request aka only send the headers and use the rawResponseCls class. This is used with storage API when uploading a file. :return: An :class:`Response` instance. :rtype: :class:`Response` instance """ if params is None: params = {} else: params = copy.copy(params) if headers is None: headers = {} else: headers = copy.copy(headers) retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS', False) or RETRY_FAILED_HTTP_REQUESTS action = self.morph_action_hook(action) self.action = action self.method = method # Extend default parameters params = self.add_default_params(params) # Add cache busting parameters (if enabled) if self.cache_busting and method == 'GET': params = self._add_cache_busting_to_params(params=params) # Extend default headers headers = self.add_default_headers(headers) # We always send a user-agent header headers.update({'User-Agent': self._user_agent()}) # Indicate that we support gzip and deflate compression headers.update({'Accept-Encoding': 'gzip,deflate'}) port = int(self.port) if port not in (80, 443): headers.update({'Host': "%s:%d" % (self.host, port)}) else: headers.update({'Host': self.host}) if data: data = self.encode_data(data) headers['Content-Length'] = str(len(data)) elif method.upper() in ['POST', 'PUT'] and not raw: # Only send Content-Length 0 with POST and PUT request. # # Note: Content-Length is not added when using "raw" mode means # means that headers are upfront and the body is sent at some point # later on. With raw mode user can specify Content-Length with # "data" not being set. headers['Content-Length'] = '0' params, headers = self.pre_connect_hook(params, headers) if params: if '?' in action: url = '&'.join((action, urlencode(params, doseq=True))) else: url = '?'.join((action, urlencode(params, doseq=True))) else: url = action # Removed terrible hack...this a less-bad hack that doesn't execute a # request twice, but it's still a hack. self.connect() try: # @TODO: Should we just pass File object as body to request method # instead of dealing with splitting and sending the file ourselves? if raw: self.connection.putrequest(method, url) for key, value in list(headers.items()): self.connection.putheader(key, str(value)) self.connection.endheaders() else: if retry_enabled: retry_request = retry(timeout=self.timeout, retry_delay=self.retry_delay, backoff=self.backoff) retry_request(self.connection.request)(method=method, url=url, body=data, headers=headers) else: self.connection.request(method=method, url=url, body=data, headers=headers) except ssl.SSLError: e = sys.exc_info()[1] self.reset_context() raise ssl.SSLError(str(e)) if raw: responseCls = self.rawResponseCls kwargs = {'connection': self} else: responseCls = self.responseCls kwargs = { 'connection': self, 'response': self.connection.getresponse() } try: response = responseCls(**kwargs) finally: # Always reset the context after the request has completed self.reset_context() return response
def request(self, action, params=None, data=None, headers=None, method='GET', raw=False): """ Request a given `action`. Basically a wrapper around the connection object's `request` that does some helpful pre-processing. :type action: ``str`` :param action: A path. This can include arguments. If included, any extra parameters are appended to the existing ones. :type params: ``dict`` :param params: Optional mapping of additional parameters to send. If None, leave as an empty ``dict``. :type data: ``unicode`` :param data: A body of data to send with the request. :type headers: ``dict`` :param headers: Extra headers to add to the request None, leave as an empty ``dict``. :type method: ``str`` :param method: An HTTP method such as "GET" or "POST". :type raw: ``bool`` :param raw: True to perform a "raw" request aka only send the headers and use the rawResponseCls class. This is used with storage API when uploading a file. :return: An :class:`Response` instance. :rtype: :class:`Response` instance """ if params is None: params = {} else: params = copy.copy(params) if headers is None: headers = {} else: headers = copy.copy(headers) retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS', False) or RETRY_FAILED_HTTP_REQUESTS action = self.morph_action_hook(action) self.action = action self.method = method # Extend default parameters params = self.add_default_params(params) # Add cache busting parameters (if enabled) if self.cache_busting and method == 'GET': params = self._add_cache_busting_to_params(params=params) # Extend default headers headers = self.add_default_headers(headers) # We always send a user-agent header headers.update({'User-Agent': self._user_agent()}) # Indicate that we support gzip and deflate compression headers.update({'Accept-Encoding': 'gzip,deflate'}) port = int(self.port) if port not in (80, 443): headers.update({'Host': "%s:%d" % (self.host, port)}) else: headers.update({'Host': self.host}) if data: data = self.encode_data(data) headers['Content-Length'] = str(len(data)) elif method.upper() in ['POST', 'PUT'] and not raw: # Only send Content-Length 0 with POST and PUT request. # # Note: Content-Length is not added when using "raw" mode means # means that headers are upfront and the body is sent at some point # later on. With raw mode user can specify Content-Length with # "data" not being set. headers['Content-Length'] = '0' params, headers = self.pre_connect_hook(params, headers) if params: if '?' in action: url = '&'.join((action, urlencode(params, doseq=True))) else: url = '?'.join((action, urlencode(params, doseq=True))) else: url = action # Removed terrible hack...this a less-bad hack that doesn't execute a # request twice, but it's still a hack. self.connect() try: # @TODO: Should we just pass File object as body to request method # instead of dealing with splitting and sending the file ourselves? if raw: self.connection.putrequest(method, url) for key, value in list(headers.items()): self.connection.putheader(key, str(value)) self.connection.endheaders() else: if retry_enabled: retry_request = retry(timeout=self.timeout, retry_delay=self.retry_delay, backoff=self.backoff) retry_request(self.connection.request)(method=method, url=url, body=data, headers=headers) else: self.connection.request(method=method, url=url, body=data, headers=headers) except ssl.SSLError: e = sys.exc_info()[1] self.reset_context() raise ssl.SSLError(str(e)) if raw: responseCls = self.rawResponseCls kwargs = {'connection': self} else: responseCls = self.responseCls kwargs = {'connection': self, 'response': self.connection.getresponse()} try: response = responseCls(**kwargs) finally: # Always reset the context after the request has completed self.reset_context() return response
def request(self, action, params=None, data=None, headers=None, method='GET', raw=False, stream=False): """ Request a given `action`. Basically a wrapper around the connection object's `request` that does some helpful pre-processing. :type action: ``str`` :param action: A path. This can include arguments. If included, any extra parameters are appended to the existing ones. :type params: ``dict`` :param params: Optional mapping of additional parameters to send. If None, leave as an empty ``dict``. :type data: ``unicode`` :param data: A body of data to send with the request. :type headers: ``dict`` :param headers: Extra headers to add to the request None, leave as an empty ``dict``. :type method: ``str`` :param method: An HTTP method such as "GET" or "POST". :type raw: ``bool`` :param raw: True to perform a "raw" request aka only send the headers and use the rawResponseCls class. This is used with storage API when uploading a file. :type stream: ``bool`` :param stream: True to return an iterator in Response.iter_content and allow streaming of the response data (for downloading large files) :return: An :class:`Response` instance. :rtype: :class:`Response` instance """ if params is None: params = {} else: params = copy.copy(params) if headers is None: headers = {} else: headers = copy.copy(headers) retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS', False) or RETRY_FAILED_HTTP_REQUESTS action = self.morph_action_hook(action) self.action = action self.method = method self.data = data # Extend default parameters params = self.add_default_params(params) # Add cache busting parameters (if enabled) if self.cache_busting and method == 'GET': params = self._add_cache_busting_to_params(params=params) # Extend default headers headers = self.add_default_headers(headers) # We always send a user-agent header headers.update({'User-Agent': self._user_agent()}) # Indicate that we support gzip and deflate compression headers.update({'Accept-Encoding': 'gzip,deflate'}) port = int(self.port) if port not in (80, 443): headers.update({'Host': "%s:%d" % (self.host, port)}) else: headers.update({'Host': self.host}) if data: data = self.encode_data(data) params, headers = self.pre_connect_hook(params, headers) if params: if '?' in action: url = '&'.join((action, urlencode(params, doseq=True))) else: url = '?'.join((action, urlencode(params, doseq=True))) else: url = action # IF connection has not yet been established if self.connection is None: self.connect() try: # @TODO: Should we just pass File object as body to request method # instead of dealing with splitting and sending the file ourselves? if raw: self.connection.prepared_request( method=method, url=url, body=data, headers=headers, stream=stream) else: if retry_enabled: retry_request = retry(timeout=self.timeout, retry_delay=self.retry_delay, backoff=self.backoff) retry_request(self.connection.request)(method=method, url=url, body=data, headers=headers, stream=stream) else: self.connection.request(method=method, url=url, body=data, headers=headers, stream=stream) except socket.gaierror: e = sys.exc_info()[1] message = str(e) errno = getattr(e, 'errno', None) if errno == -5: # Throw a more-friendly exception on "no address associated # with hostname" error. This error could simpli indicate that # "host" Connection class attribute is set to an incorrect # value class_name = self.__class__.__name__ msg = ('%s. Perhaps "host" Connection class attribute ' '(%s.connection) is set to an invalid, non-hostname ' 'value (%s)?' % (message, class_name, self.host)) raise socket.gaierror(msg) self.reset_context() raise e except ssl.SSLError: e = sys.exc_info()[1] self.reset_context() raise ssl.SSLError(str(e)) if raw: responseCls = self.rawResponseCls kwargs = {'connection': self, 'response': self.connection.getresponse()} else: responseCls = self.responseCls kwargs = {'connection': self, 'response': self.connection.getresponse()} try: response = responseCls(**kwargs) finally: # Always reset the context after the request has completed self.reset_context() return response