Ejemplo n.º 1
0
    def test_retry_with_backoff(self):
        con = Connection()
        con.connection = Mock()
        connect_method = "libcloud.common.base.Connection.request"

        with patch(connect_method) as mock_connect:
            mock_connect.__name__ = "mock_connect"
            with self.assertRaises(socket.gaierror):
                mock_connect.side_effect = socket.gaierror("")
                retry_request = retry(timeout=2, retry_delay=0.1, backoff=1)
                retry_request(con.request)(action="/")

            self.assertGreater(mock_connect.call_count, 1, "Retry logic failed")
Ejemplo n.º 2
0
    def test_retry_with_backoff(self):
        con = Connection()
        con.connection = Mock()
        connect_method = 'libcloud.common.base.Connection.request'

        with patch(connect_method) as mock_connect:
            mock_connect.__name__ = 'mock_connect'
            with self.assertRaises(socket.gaierror):
                mock_connect.side_effect = socket.gaierror('')
                retry_request = retry(timeout=0.2, retry_delay=0.1, backoff=1)
                retry_request(con.request)(action='/')

            self.assertGreater(mock_connect.call_count, 1,
                               'Retry logic failed')
Ejemplo n.º 3
0
    def test_retry_with_timeout(self):
        con = Connection()
        con.connection = Mock()
        connect_method = 'libcloud.common.base.Connection.request'

        with patch(connect_method) as mock_connect:
            mock_connect.__name__ = 'mock_connect'
            with self.assertRaises(socket.gaierror):
                mock_connect.side_effect = socket.gaierror('')
                retry_request = retry(timeout=2, retry_delay=.1,
                                      backoff=1)
                retry_request(con.request)(action='/')

            self.assertGreater(mock_connect.call_count, 1,
                               'Retry logic failed')
Ejemplo n.º 4
0
    def request(self,
                action,
                params=None,
                data=None,
                headers=None,
                method='GET',
                raw=False,
                stream=False):
        """
        Request a given `action`.

        Basically a wrapper around the connection
        object's `request` that does some helpful pre-processing.

        :type action: ``str``
        :param action: A path. This can include arguments. If included,
            any extra parameters are appended to the existing ones.

        :type params: ``dict``
        :param params: Optional mapping of additional parameters to send. If
            None, leave as an empty ``dict``.

        :type data: ``unicode``
        :param data: A body of data to send with the request.

        :type headers: ``dict``
        :param headers: Extra headers to add to the request
            None, leave as an empty ``dict``.

        :type method: ``str``
        :param method: An HTTP method such as "GET" or "POST".

        :type raw: ``bool``
        :param raw: True to perform a "raw" request aka only send the headers
                     and use the rawResponseCls class. This is used with
                     storage API when uploading a file.

        :type stream: ``bool``
        :param stream: True to return an iterator in Response.iter_content
                    and allow streaming of the response data
                    (for downloading large files)

        :return: An :class:`Response` instance.
        :rtype: :class:`Response` instance

        """
        if params is None:
            params = {}
        else:
            params = copy.copy(params)

        if headers is None:
            headers = {}
        else:
            headers = copy.copy(headers)

        retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
                                       False) or RETRY_FAILED_HTTP_REQUESTS

        action = self.morph_action_hook(action)
        self.action = action
        self.method = method
        self.data = data

        # Extend default parameters
        params = self.add_default_params(params)

        # Add cache busting parameters (if enabled)
        if self.cache_busting and method == 'GET':
            params = self._add_cache_busting_to_params(params=params)

        # Extend default headers
        headers = self.add_default_headers(headers)

        # We always send a user-agent header
        headers.update({'User-Agent': self._user_agent()})

        # Indicate that we support gzip and deflate compression
        headers.update({'Accept-Encoding': 'gzip,deflate'})

        port = int(self.port)

        if port not in (80, 443):
            headers.update({'Host': "%s:%d" % (self.host, port)})
        else:
            headers.update({'Host': self.host})

        if data:
            data = self.encode_data(data)

        params, headers = self.pre_connect_hook(params, headers)

        if params:
            if '?' in action:
                url = '&'.join((action, urlencode(params, doseq=True)))
            else:
                url = '?'.join((action, urlencode(params, doseq=True)))
        else:
            url = action

        # IF connection has not yet been established
        if self.connection is None:
            self.connect()

        try:
            # @TODO: Should we just pass File object as body to request method
            # instead of dealing with splitting and sending the file ourselves?
            if raw:
                self.connection.prepared_request(method=method,
                                                 url=url,
                                                 body=data,
                                                 headers=headers,
                                                 stream=stream)
            else:
                if retry_enabled:
                    retry_request = retry(timeout=self.timeout,
                                          retry_delay=self.retry_delay,
                                          backoff=self.backoff)
                    retry_request(self.connection.request)(method=method,
                                                           url=url,
                                                           body=data,
                                                           headers=headers,
                                                           stream=stream)
                else:
                    self.connection.request(method=method,
                                            url=url,
                                            body=data,
                                            headers=headers,
                                            stream=stream)
        except socket.gaierror:
            e = sys.exc_info()[1]
            message = str(e)
            errno = getattr(e, 'errno', None)

            if errno == -5:
                # Throw a more-friendly exception on "no address associated
                # with hostname" error. This error could simpli indicate that
                # "host" Connection class attribute is set to an incorrect
                # value
                class_name = self.__class__.__name__
                msg = ('%s. Perhaps "host" Connection class attribute '
                       '(%s.connection) is set to an invalid, non-hostname '
                       'value (%s)?' % (message, class_name, self.host))
                raise socket.gaierror(msg)
            self.reset_context()
            raise e
        except ssl.SSLError:
            e = sys.exc_info()[1]
            self.reset_context()
            raise ssl.SSLError(str(e))

        if raw:
            responseCls = self.rawResponseCls
            kwargs = {
                'connection': self,
                'response': self.connection.getresponse()
            }
        else:
            responseCls = self.responseCls
            kwargs = {
                'connection': self,
                'response': self.connection.getresponse()
            }

        try:
            response = responseCls(**kwargs)
        finally:
            # Always reset the context after the request has completed
            self.reset_context()

        return response
Ejemplo n.º 5
0
    def request(self,
                action,
                params=None,
                data=None,
                headers=None,
                method='GET',
                raw=False):
        """
        Request a given `action`.

        Basically a wrapper around the connection
        object's `request` that does some helpful pre-processing.

        :type action: ``str``
        :param action: A path. This can include arguments. If included,
            any extra parameters are appended to the existing ones.

        :type params: ``dict``
        :param params: Optional mapping of additional parameters to send. If
            None, leave as an empty ``dict``.

        :type data: ``unicode``
        :param data: A body of data to send with the request.

        :type headers: ``dict``
        :param headers: Extra headers to add to the request
            None, leave as an empty ``dict``.

        :type method: ``str``
        :param method: An HTTP method such as "GET" or "POST".

        :type raw: ``bool``
        :param raw: True to perform a "raw" request aka only send the headers
                     and use the rawResponseCls class. This is used with
                     storage API when uploading a file.

        :return: An :class:`Response` instance.
        :rtype: :class:`Response` instance

        """
        if params is None:
            params = {}
        else:
            params = copy.copy(params)

        if headers is None:
            headers = {}
        else:
            headers = copy.copy(headers)

        retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
                                       False) or RETRY_FAILED_HTTP_REQUESTS

        action = self.morph_action_hook(action)
        self.action = action
        self.method = method

        # Extend default parameters
        params = self.add_default_params(params)

        # Add cache busting parameters (if enabled)
        if self.cache_busting and method == 'GET':
            params = self._add_cache_busting_to_params(params=params)

        # Extend default headers
        headers = self.add_default_headers(headers)

        # We always send a user-agent header
        headers.update({'User-Agent': self._user_agent()})

        # Indicate that we support gzip and deflate compression
        headers.update({'Accept-Encoding': 'gzip,deflate'})

        port = int(self.port)

        if port not in (80, 443):
            headers.update({'Host': "%s:%d" % (self.host, port)})
        else:
            headers.update({'Host': self.host})

        if data:
            data = self.encode_data(data)
            headers['Content-Length'] = str(len(data))
        elif method.upper() in ['POST', 'PUT'] and not raw:
            # Only send Content-Length 0 with POST and PUT request.
            #
            # Note: Content-Length is not added when using "raw" mode means
            # means that headers are upfront and the body is sent at some point
            # later on. With raw mode user can specify Content-Length with
            # "data" not being set.
            headers['Content-Length'] = '0'

        params, headers = self.pre_connect_hook(params, headers)

        if params:
            if '?' in action:
                url = '&'.join((action, urlencode(params, doseq=True)))
            else:
                url = '?'.join((action, urlencode(params, doseq=True)))
        else:
            url = action

        # Removed terrible hack...this a less-bad hack that doesn't execute a
        # request twice, but it's still a hack.
        self.connect()
        try:
            # @TODO: Should we just pass File object as body to request method
            # instead of dealing with splitting and sending the file ourselves?
            if raw:
                self.connection.putrequest(method, url)

                for key, value in list(headers.items()):
                    self.connection.putheader(key, str(value))

                self.connection.endheaders()
            else:
                if retry_enabled:
                    retry_request = retry(timeout=self.timeout,
                                          retry_delay=self.retry_delay,
                                          backoff=self.backoff)
                    retry_request(self.connection.request)(method=method,
                                                           url=url,
                                                           body=data,
                                                           headers=headers)
                else:
                    self.connection.request(method=method,
                                            url=url,
                                            body=data,
                                            headers=headers)
        except ssl.SSLError:
            e = sys.exc_info()[1]
            self.reset_context()
            raise ssl.SSLError(str(e))

        if raw:
            responseCls = self.rawResponseCls
            kwargs = {'connection': self}
        else:
            responseCls = self.responseCls
            kwargs = {
                'connection': self,
                'response': self.connection.getresponse()
            }

        try:
            response = responseCls(**kwargs)
        finally:
            # Always reset the context after the request has completed
            self.reset_context()

        return response
Ejemplo n.º 6
0
    def request(self, action, params=None, data=None, headers=None,
                method='GET', raw=False):
        """
        Request a given `action`.

        Basically a wrapper around the connection
        object's `request` that does some helpful pre-processing.

        :type action: ``str``
        :param action: A path. This can include arguments. If included,
            any extra parameters are appended to the existing ones.

        :type params: ``dict``
        :param params: Optional mapping of additional parameters to send. If
            None, leave as an empty ``dict``.

        :type data: ``unicode``
        :param data: A body of data to send with the request.

        :type headers: ``dict``
        :param headers: Extra headers to add to the request
            None, leave as an empty ``dict``.

        :type method: ``str``
        :param method: An HTTP method such as "GET" or "POST".

        :type raw: ``bool``
        :param raw: True to perform a "raw" request aka only send the headers
                     and use the rawResponseCls class. This is used with
                     storage API when uploading a file.

        :return: An :class:`Response` instance.
        :rtype: :class:`Response` instance

        """
        if params is None:
            params = {}
        else:
            params = copy.copy(params)

        if headers is None:
            headers = {}
        else:
            headers = copy.copy(headers)

        retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
                                       False) or RETRY_FAILED_HTTP_REQUESTS

        action = self.morph_action_hook(action)
        self.action = action
        self.method = method

        # Extend default parameters
        params = self.add_default_params(params)

        # Add cache busting parameters (if enabled)
        if self.cache_busting and method == 'GET':
            params = self._add_cache_busting_to_params(params=params)

        # Extend default headers
        headers = self.add_default_headers(headers)

        # We always send a user-agent header
        headers.update({'User-Agent': self._user_agent()})

        # Indicate that we support gzip and deflate compression
        headers.update({'Accept-Encoding': 'gzip,deflate'})

        port = int(self.port)

        if port not in (80, 443):
            headers.update({'Host': "%s:%d" % (self.host, port)})
        else:
            headers.update({'Host': self.host})

        if data:
            data = self.encode_data(data)
            headers['Content-Length'] = str(len(data))
        elif method.upper() in ['POST', 'PUT'] and not raw:
            # Only send Content-Length 0 with POST and PUT request.
            #
            # Note: Content-Length is not added when using "raw" mode means
            # means that headers are upfront and the body is sent at some point
            # later on. With raw mode user can specify Content-Length with
            # "data" not being set.
            headers['Content-Length'] = '0'

        params, headers = self.pre_connect_hook(params, headers)

        if params:
            if '?' in action:
                url = '&'.join((action, urlencode(params, doseq=True)))
            else:
                url = '?'.join((action, urlencode(params, doseq=True)))
        else:
            url = action

        # Removed terrible hack...this a less-bad hack that doesn't execute a
        # request twice, but it's still a hack.
        self.connect()
        try:
            # @TODO: Should we just pass File object as body to request method
            # instead of dealing with splitting and sending the file ourselves?
            if raw:
                self.connection.putrequest(method, url)

                for key, value in list(headers.items()):
                    self.connection.putheader(key, str(value))

                self.connection.endheaders()
            else:
                if retry_enabled:
                    retry_request = retry(timeout=self.timeout,
                                          retry_delay=self.retry_delay,
                                          backoff=self.backoff)
                    retry_request(self.connection.request)(method=method,
                                                           url=url,
                                                           body=data,
                                                           headers=headers)
                else:
                    self.connection.request(method=method, url=url, body=data,
                                            headers=headers)
        except ssl.SSLError:
            e = sys.exc_info()[1]
            self.reset_context()
            raise ssl.SSLError(str(e))

        if raw:
            responseCls = self.rawResponseCls
            kwargs = {'connection': self}
        else:
            responseCls = self.responseCls
            kwargs = {'connection': self,
                      'response': self.connection.getresponse()}

        try:
            response = responseCls(**kwargs)
        finally:
            # Always reset the context after the request has completed
            self.reset_context()

        return response
Ejemplo n.º 7
0
    def request(self, action, params=None, data=None, headers=None,
                method='GET', raw=False, stream=False):
        """
        Request a given `action`.

        Basically a wrapper around the connection
        object's `request` that does some helpful pre-processing.

        :type action: ``str``
        :param action: A path. This can include arguments. If included,
            any extra parameters are appended to the existing ones.

        :type params: ``dict``
        :param params: Optional mapping of additional parameters to send. If
            None, leave as an empty ``dict``.

        :type data: ``unicode``
        :param data: A body of data to send with the request.

        :type headers: ``dict``
        :param headers: Extra headers to add to the request
            None, leave as an empty ``dict``.

        :type method: ``str``
        :param method: An HTTP method such as "GET" or "POST".

        :type raw: ``bool``
        :param raw: True to perform a "raw" request aka only send the headers
                     and use the rawResponseCls class. This is used with
                     storage API when uploading a file.

        :type stream: ``bool``
        :param stream: True to return an iterator in Response.iter_content
                    and allow streaming of the response data
                    (for downloading large files)

        :return: An :class:`Response` instance.
        :rtype: :class:`Response` instance

        """
        if params is None:
            params = {}
        else:
            params = copy.copy(params)

        if headers is None:
            headers = {}
        else:
            headers = copy.copy(headers)

        retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
                                       False) or RETRY_FAILED_HTTP_REQUESTS

        action = self.morph_action_hook(action)
        self.action = action
        self.method = method
        self.data = data

        # Extend default parameters
        params = self.add_default_params(params)

        # Add cache busting parameters (if enabled)
        if self.cache_busting and method == 'GET':
            params = self._add_cache_busting_to_params(params=params)

        # Extend default headers
        headers = self.add_default_headers(headers)

        # We always send a user-agent header
        headers.update({'User-Agent': self._user_agent()})

        # Indicate that we support gzip and deflate compression
        headers.update({'Accept-Encoding': 'gzip,deflate'})

        port = int(self.port)

        if port not in (80, 443):
            headers.update({'Host': "%s:%d" % (self.host, port)})
        else:
            headers.update({'Host': self.host})

        if data:
            data = self.encode_data(data)

        params, headers = self.pre_connect_hook(params, headers)

        if params:
            if '?' in action:
                url = '&'.join((action, urlencode(params, doseq=True)))
            else:
                url = '?'.join((action, urlencode(params, doseq=True)))
        else:
            url = action

        # IF connection has not yet been established
        if self.connection is None:
            self.connect()

        try:
            # @TODO: Should we just pass File object as body to request method
            # instead of dealing with splitting and sending the file ourselves?
            if raw:
                self.connection.prepared_request(
                    method=method,
                    url=url,
                    body=data,
                    headers=headers,
                    stream=stream)
            else:
                if retry_enabled:
                    retry_request = retry(timeout=self.timeout,
                                          retry_delay=self.retry_delay,
                                          backoff=self.backoff)
                    retry_request(self.connection.request)(method=method,
                                                           url=url,
                                                           body=data,
                                                           headers=headers,
                                                           stream=stream)
                else:
                    self.connection.request(method=method, url=url, body=data,
                                            headers=headers, stream=stream)
        except socket.gaierror:
            e = sys.exc_info()[1]
            message = str(e)
            errno = getattr(e, 'errno', None)

            if errno == -5:
                # Throw a more-friendly exception on "no address associated
                # with hostname" error. This error could simpli indicate that
                # "host" Connection class attribute is set to an incorrect
                # value
                class_name = self.__class__.__name__
                msg = ('%s. Perhaps "host" Connection class attribute '
                       '(%s.connection) is set to an invalid, non-hostname '
                       'value (%s)?' %
                       (message, class_name, self.host))
                raise socket.gaierror(msg)
            self.reset_context()
            raise e
        except ssl.SSLError:
            e = sys.exc_info()[1]
            self.reset_context()
            raise ssl.SSLError(str(e))

        if raw:
            responseCls = self.rawResponseCls
            kwargs = {'connection': self,
                      'response': self.connection.getresponse()}
        else:
            responseCls = self.responseCls
            kwargs = {'connection': self,
                      'response': self.connection.getresponse()}

        try:
            response = responseCls(**kwargs)
        finally:
            # Always reset the context after the request has completed
            self.reset_context()

        return response