Esempio n. 1
0
    def build_response_into(self, response, req, urllib3_resp):
        """Same as requests.adapters.HTTPAdapter.build_response, but writes
        into a provided requests.Response object instead of creating a new one.
        """
        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(urllib3_resp, 'status', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(
            getattr(urllib3_resp, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = urllib3_resp
        response.reason = response.raw.reason

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, urllib3_resp)

        # Give the Response some context.
        response.request = req
        response.connection = self
Esempio n. 2
0
    def handle_auth(self, r, **kwargs):
        """Takes the given response and tries SAML auth, if needed."""

        if r.status_code not in [ 401, 403 ]:
          return r

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        r.content
        r.raw.release_conn()
        prep = r.request.copy()
        extract_cookies_to_jar(prep._cookies, r.request, r.raw)
        cookie_dict = requests.utils.dict_from_cookiejar(prep._cookies)

        self.session.auth = None
        self.session.get_sso_cookie()
        self.session.auth = self

        prep.prepare_cookies(self.session.cookies)

        _r = r.connection.send(prep, **kwargs)
        _r.history.append(r)
        _r.request = prep

        return _r
Esempio n. 3
0
    def handle_auth(self, response, **kwargs):
        '''Takes the given response and tries SAML auth, if needed.'''

        if response.status_code not in [401, 403]:
            return response

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content  # pylint: disable=pointless-statement
        response.raw.release_conn()
        prep = response.request.copy()

        # PreparedResponse does not have a method to get _cookies.
        # This is reuse of code from python requests.auth
        # pylint: disable=protected-access
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)

        self.session.auth = None
        self.session.get_sso_cookie()
        self.session.auth = self

        prep.prepare_cookies(self.session.cookies)

        new_response = response.connection.send(prep, **kwargs)
        new_response.history.append(response)
        new_response.request = prep

        return new_response
Esempio n. 4
0
    def build_response(self, request, resp):
        """
        Builds a Requests' response object.  This emulates most of the logic of
        the standard fuction but deals with the lack of the ``.headers``
        property on the HTTP20Response object.
        """
        response = Response()

        response.status_code = resp.status
        response.headers = CaseInsensitiveDict(resp.getheaders())
        response.raw = resp
        response.reason = resp.reason
        response.encoding = get_encoding_from_headers(response.headers)

        extract_cookies_to_jar(response.cookies, request, response)

        if isinstance(request.url, bytes):
            response.url = request.url.decode('utf-8')
        else:
            response.url = request.url

        response.request = request
        response.connection = self

        # One last horrible patch: Requests expects its raw responses to have a
        # release_conn method, which I don't. We should monkeypatch a no-op on.
        resp.release_conn = lambda: None

        return response
Esempio n. 5
0
    def handle_401(self, response, repo, **kwargs):
        """Fetch Bearer token and retry."""
        if response.status_code != requests.codes.unauthorized:
            return response

        auth_info = response.headers.get('www-authenticate', '')

        if 'bearer' not in auth_info.lower():
            return response

        self._token_cache[repo] = self._get_token(auth_info, repo)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        # This pattern was inspired by the source code of requests.auth.HTTPDigestAuth
        response.content
        response.close()
        retry_request = response.request.copy()
        extract_cookies_to_jar(retry_request._cookies, response.request, response.raw)
        retry_request.prepare_cookies(retry_request._cookies)

        self._set_header(retry_request, repo)
        retry_response = response.connection.send(retry_request, **kwargs)
        retry_response.history.append(response)
        retry_response.request = retry_request

        return retry_response
Esempio n. 6
0
    def handle_500(self, r, **kwargs):
        if not self.enable_retry_on_500 \
           or r.status_code < 500 or r.status_code >= 600:
            return r

        num_500_calls = getattr(self, 'num_500_calls', 1)

        if num_500_calls < 4:
            logger.debug("Got %s status, retrying" % r.status_code)
            self.num_500_calls += 1

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            if "cookies" in prep.headers:
                extract_cookies_to_jar(prep.headers.get("cookies", None),
                                       r.request, r.raw)
                prep.prepare_cookies(prep._cookies)

            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        self.num_500_calls = 1
        return r
Esempio n. 7
0
    def handle_auth(self, response, **kwargs):
        '''Takes the given response and tries SAML auth, if needed.'''

        if response.status_code not in [401, 403]:
            return response

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content # pylint: disable=pointless-statement
        response.raw.release_conn()
        prep = response.request.copy()

        # PreparedResponse does not have a method to get _cookies.
        # This is reuse of code from python requests.auth
        # pylint: disable=protected-access
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)

        self.session.auth = None
        self.session.get_sso_cookie()
        self.session.auth = self

        prep.prepare_cookies(self.session.cookies)

        new_response = response.connection.send(prep, **kwargs)
        new_response.history.append(response)
        new_response.request = prep

        return new_response
Esempio n. 8
0
    def build_response(self, req, resp):
        """Builds a :class:`Response <requests.Response>` object from a urllib3
        response. This should not be called from user code, and is only exposed
        for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`

        :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
        :param resp: The urllib3 response object.
        """
        response = Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, 'status', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = resp
        response.reason = response.raw.reason

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, resp)

        # Give the Response some context.
        response.request = req
        response.connection = self

        return response
Esempio n. 9
0
    def handle_401(self, response, repo, **kwargs):
        """Fetch Bearer token and retry."""
        if response.status_code != requests.codes.unauthorized:
            return response

        auth_info = response.headers.get('www-authenticate', '')

        if 'bearer' not in auth_info.lower():
            return response

        self._token_cache[repo] = self._get_token(auth_info, repo)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        # This pattern was inspired by the source code of requests.auth.HTTPDigestAuth
        response.content
        response.close()
        retry_request = response.request.copy()
        extract_cookies_to_jar(retry_request._cookies, response.request,
                               response.raw)
        retry_request.prepare_cookies(retry_request._cookies)

        self._set_header(retry_request, repo)
        retry_response = response.connection.send(retry_request, **kwargs)
        retry_response.history.append(response)
        retry_response.request = retry_request

        return retry_response
Esempio n. 10
0
    def handle_401(self, r, **kwargs):
        """Resends a request with auth headers, if needed."""

        www_authenticate = r.headers.get('www-authenticate', '').lower()

        if 'basic' in www_authenticate:
            if self.pos is not None:
                r.request.body.seek(self.pos)

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.raw.release_conn()
            prep = r.request.copy()
            if not hasattr(prep, '_cookies'):
                prep._cookies = RequestsCookieJar()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            self.auth = HTTPBasicAuth(self.username, self.password)
            prep = self.auth(prep)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        if 'digest' in www_authenticate:
            self.auth = HTTPDigestAuth(self.username, self.password)
            # Digest auth would resend the request by itself. We can take a
            # shortcut here.
            return self.auth.handle_401(r, **kwargs)
Esempio n. 11
0
    def to_requests_response(self):
        """Returns an instance of `requests.Response` based on this response.

        Returns:
            request.Response: the generated response.
        """

        # Make sure that body is at position 0 before returning
        self.body.seek(0)

        urllib3_response = URLLib3Rresponse(
            body=self.body,
            headers=self.headers,
            status=self.http_code,
            request_method=self.request.method,
            reason=self.reason,
            preload_content=False
        )

        response = RequestResponse()
        response.request = self.request
        response.raw = urllib3_response
        response.status_code = self.http_code
        response.reason = self.reason
        response.headers = CaseInsensitiveDict(response.raw.headers)
        response.encoding = get_encoding_from_headers(response.headers)

        extract_cookies_to_jar(response.cookies, self.request, urllib3_response)

        if isinstance(self.request.url, six.binary_type):
            response.url = self.request.url.decode("utf-8")
        else:
            response.url = self.request.url

        return response
Esempio n. 12
0
    def http_fetch(self, url, fetch):
        """
        HTTP fetcher
        """
        start_time = time.time()

        def handle_error(x):
            BaseCrawler.handle_error(url, start_time, x)

        # making requests
        while True:
            try:
                response = yield gen.maybe_future(self._request(url, fetch))
            except HTTPError as e:
                if e.response:
                    response = e.response
                else:
                    raise gen.Return(handle_error(e))
            except Exception as e:
                raise gen.Return(handle_error(e))

            cookies.extract_cookies_to_jar(self._cookies, response.request,
                                           response)

            error = self._prepare_response(response.status_code, url)
            if error is not None:
                raise gen.Return(handle_error(error))
            self.gen_result(url=response.url or url,
                            code=response.status_code,
                            headers=dict(response.headers),
                            cookies=self._cookies.get_dict(),
                            content=response.content or '',
                            start_time=start_time)
Esempio n. 13
0
    def handle_407(self, r, **kwargs):
        if r.status_code == 407 and self.stale_rejects < 2:
            pat = re.compile(r'digest ', flags=re.IGNORECASE)
            if "proxy-authenticate" not in r.headers:
                raise IOError(
                    "proxy server violated RFC 7235:"
                    "407 response MUST contain header proxy-authenticate")
    
            self.chal = parse_dict_header(
                             pat.sub('', r.headers['proxy-authenticate'], count=1))
    
            if 'Proxy-Authorization' in r.request.headers and 'stale' in self.chal:
                if self.chal['stale'].lower() == 'true': # try again
                    self.stale_rejects += 1
                elif self.chal['stale'].lower() == 'false':
                    raise IOError("User or password is invalid")
    
            r.content
            r.raw.release_conn()
            
            prep = r.request.copy()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)
    
            prep.headers['Proxy-Authorization'] = self.build_digest_header(
            prep.method, prep.url)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r
        else: # give up authenticate
            return r
Esempio n. 14
0
    def build_response(self, request, resp):
        """
        Builds a Requests' response object.  This emulates most of the logic of
        the standard fuction but deals with the lack of the ``.headers``
        property on the HTTP20Response object.
        """
        response = Response()

        response.status_code = resp.status
        response.headers = CaseInsensitiveDict(resp.getheaders())
        response.raw = resp
        response.reason = resp.reason
        response.encoding = get_encoding_from_headers(response.headers)

        extract_cookies_to_jar(response.cookies, request, response)
        response.url = request.url

        response.request = request
        response.connection = self

        # One last horrible patch: Requests expects its raw responses to have a
        # release_conn method, which I don't. We should monkeypatch a no-op on.
        resp.release_conn = lambda: None

        return response
Esempio n. 15
0
    def handle_401(self, r, **kwargs):
        """Resends a request with auth headers, if needed."""

        www_authenticate = r.headers.get('www-authenticate', '').lower()

        if 'basic' in www_authenticate:
            if self.pos is not None:
                r.request.body.seek(self.pos)

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.raw.release_conn()
            prep = r.request.copy()
            if not hasattr(prep, '_cookies'):
                prep._cookies = RequestsCookieJar()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            self.auth = HTTPBasicAuth(self.username, self.password)
            prep = self.auth(prep)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        if 'digest' in www_authenticate:
            self.auth = HTTPDigestAuth(self.username, self.password)
            # Digest auth would resend the request by itself. We can take a
            # shortcut here.
            return self.auth.handle_401(r, **kwargs)
Esempio n. 16
0
    def build_response(self, req, resp):
        """Builds a :class:`Response <requests.Response>` object from a urllib3
        response. This should not be called from user code, and is only exposed
        for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`

        :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
        :param resp: The urllib3 response object.
        """
        response = Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, 'status', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = resp
        response.reason = response.raw.reason

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, resp)

        # Give the Response some context.
        response.request = req
        response.connection = self

        return response
Esempio n. 17
0
    def handle_401(self, response, repo, **kwargs):
        """Fetch Bearer token and retry."""
        if response.status_code != requests.codes.unauthorized:
            return response

        auth_info = response.headers.get('www-authenticate', '')

        if 'bearer' not in auth_info.lower():
            return response

        self._token_cache[repo] = self._get_token(auth_info, repo)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        # This pattern was inspired by the source code of requests.auth.HTTPDigestAuth
        response.content    # pylint: disable=pointless-statement; is a property
        response.close()
        retry_request = response.request.copy()
        extract_cookies_to_jar(retry_request._cookies, response.request, response.raw)
        retry_request.prepare_cookies(retry_request._cookies)

        self._set_header(retry_request, repo)
        retry_response = response.connection.send(retry_request, **kwargs)
        retry_response.history.append(response)
        retry_response.request = retry_request
        # when quay returns 401 even after we have token,
        # repository doesn't exist at all, so we will treat it as 404
        if retry_response.status_code == requests.codes.unauthorized:
            retry_response.status_code = requests.codes.not_found

        return retry_response
Esempio n. 18
0
    def http_response_to_response(self, http_response, prepared_request):
        """
        transform a WSGIResponse into a requests's Response model
        :param django.http.response.HttpResponse http_response: the http response send by django view
        :return: the requests's Response model corresponding to the http_response
        :rtype: Response
        """
        response = Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(http_response, 'status_code', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(http_response._headers, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = http_response
        response.reason = response.raw.reason_phrase
        response._content = http_response.content
        req = prepared_request

        if isinstance(req.url, bytes):  # pragma: no cover
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, response)

        # Give the Response some context.
        response.request = req
        response.connection = self

        return response
Esempio n. 19
0
 def deserialize(self):
     """Turn a serialized interaction into a Response."""
     r = util.deserialize_response(self.json['response'])
     r.request = util.deserialize_prepared_request(self.json['request'])
     extract_cookies_to_jar(r.cookies, r.request, r.raw)
     self.recorded_at = datetime.strptime(self.json['recorded_at'],
                                          '%Y-%m-%dT%H:%M:%S')
     self.recorded_response = r
Esempio n. 20
0
 def deserialize(self):
     """Turn a serialized interaction into a Response."""
     r = deserialize_response(self.json['response'])
     r.request = deserialize_prepared_request(self.json['request'])
     extract_cookies_to_jar(r.cookies, r.request, r.raw)
     self.recorded_at = datetime.strptime(
         self.json['recorded_at'], '%Y-%m-%dT%H:%M:%S'
     )
     self.recorded_response = r
Esempio n. 21
0
 def start_response(status, headers, exc_info=None):
     headers = make_headers(headers)
     response.status_code = int(status.split(" ")[0])
     response.reason = responses.get(response.status_code, "Unknown Status Code")
     response.headers = headers
     resp._original_response.msg = headers
     extract_cookies_to_jar(response.cookies, request, resp)
     response.encoding = get_encoding_from_headers(response.headers)
     response.elapsed = datetime.datetime.utcnow() - start
     self._log(response)
Esempio n. 22
0
    def send(self, request, **kwargs) -> UpgradeResponse:
        kwargs.setdefault('stream', self.stream)
        kwargs.setdefault('verify', self.verify)
        kwargs.setdefault('cert', self.cert)
        kwargs.setdefault('proxies', self.proxies)

        if isinstance(request, requests.Request):
            raise ValueError('You can only send PreparedRequests.')
        if isinstance(request, HttpRequest):
            request.check()
        allow_redirects = kwargs.pop('allow_redirects', True)
        hooks = request.hooks

        adapter = self.get_adapter(url=request.url)

        start = preferred_clock()

        r = adapter.send(request, **kwargs)

        elapsed = preferred_clock() - start
        r.elapsed = timedelta(seconds=elapsed)

        # this function not change response and result params
        if len(self._dispatch_hooks) > 0:
            dispatch(self._dispatch_hooks, hooks, r, **kwargs)

        # dispatch hook will change response and request params
        r = dispatch_hook('response', hooks, r, **kwargs)

        if r.history:
            for resp in r.history:
                extract_cookies_to_jar(self.cookies, resp.request, resp.raw)

        extract_cookies_to_jar(self.cookies, request, r.raw)

        gen = self.resolve_redirects(r, request, **kwargs)

        history = [resp for resp in gen] if allow_redirects else []

        if history:
            history.insert(0, r)
            r = history.pop()
            r.history = history

        if not allow_redirects:
            try:
                r._next = next(
                    self.resolve_redirects(r,
                                           request,
                                           yield_requests=True,
                                           **kwargs))
            except StopIteration:
                pass

        return r
Esempio n. 23
0
    def handle_407(response, **kwargs):
        """
        Prompts the user for the proxy username and password and modifies the
        proxy in the session object to include it.

        This method is modeled after
          * requests.auth.HTTPDigestAuth.handle_401()
          * requests.auth.HTTPProxyAuth
          * the previous conda.fetch.handle_proxy_407()

        It both adds 'username:password' to the proxy URL, as well as adding a
        'Proxy-Authorization' header.  If any of this is incorrect, please file an issue.

        """
        # kwargs = {'verify': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False,
        #           'timeout': (3.05, 60)}

        if response.status_code != 407:
            return response

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content
        response.close()

        proxies = kwargs.pop('proxies')

        proxy_scheme = urlparse(response.url).scheme
        if proxy_scheme not in proxies:
            raise ProxyError(
                dals("""
            Could not find a proxy for %r. See
            %s/docs/html#configure-conda-for-use-behind-a-proxy-server
            for more information on how to configure proxies.
            """ % (proxy_scheme, CONDA_HOMEPAGE_URL)))

        # fix-up proxy_url with username & password
        proxy_url = proxies[proxy_scheme]
        username, password = get_proxy_username_and_pass(proxy_scheme)
        proxy_url = add_username_and_password(proxy_url, username, password)
        proxy_authorization_header = _basic_auth_str(username, password)
        proxies[proxy_scheme] = proxy_url
        kwargs['proxies'] = proxies

        prep = response.request.copy()
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)
        prep.prepare_cookies(prep._cookies)
        prep.headers['Proxy-Authorization'] = proxy_authorization_header

        _response = response.connection.send(prep, **kwargs)
        _response.history.append(response)
        _response.request = prep

        return _response
Esempio n. 24
0
    def handle_407(response, **kwargs):
        """
        Prompts the user for the proxy username and password and modifies the
        proxy in the session object to include it.

        This method is modeled after
          * requests.auth.HTTPDigestAuth.handle_401()
          * requests.auth.HTTPProxyAuth
          * the previous conda.fetch.handle_proxy_407()

        It both adds 'username:password' to the proxy URL, as well as adding a
        'Proxy-Authorization' header.  If any of this is incorrect, please file an issue.

        """
        # kwargs = {'verify': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False,
        #           'timeout': (3.05, 60)}

        if response.status_code != 407:
            return response

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content
        response.close()

        proxies = kwargs.pop('proxies')

        proxy_scheme = urlparse(response.url).scheme
        if proxy_scheme not in proxies:
            raise ProxyError(dals("""
            Could not find a proxy for %r. See
            %s/docs/html#configure-conda-for-use-behind-a-proxy-server
            for more information on how to configure proxies.
            """ % (proxy_scheme, CONDA_HOMEPAGE_URL)))

        # fix-up proxy_url with username & password
        proxy_url = proxies[proxy_scheme]
        username, password = get_proxy_username_and_pass(proxy_scheme)
        proxy_url = add_username_and_password(proxy_url, username, password)
        proxy_authorization_header = _basic_auth_str(username, password)
        proxies[proxy_scheme] = proxy_url
        kwargs['proxies'] = proxies

        prep = response.request.copy()
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)
        prep.prepare_cookies(prep._cookies)
        prep.headers['Proxy-Authorization'] = proxy_authorization_header

        _response = response.connection.send(prep, **kwargs)
        _response.history.append(response)
        _response.request = prep

        return _response
Esempio n. 25
0
    def handle_401(self, response, **kwargs):
        """
        Takes the response authenticates and resends if neccissary
        :return: The final response to the authenticated request
        :rtype: requests.Response
        """
        # If response is not 401 do not auth and return response
        if not response.status_code == 401:
            self._thread_local.auth_attempted = False
            return response

        auth_header = response.headers.get('www-authenticate', '')

        # if the response auth header is not a bearer challenge do not auth and return response
        if not HttpBearerChallenge.is_bearer_challenge(auth_header):
            self._thread_local.auth_attempted = False
            return response

        # If we've already attempted to auth for this request once, do not auth and return response
        if self._thread_local.auth_attempted:
            self._thread_local.auth_attempted = False
            return response

        # Otherwise authenticate and retry the request
        self._thread_local.auth_attempted = True

        if self._thread_local.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            response.request.body.seek(self._thread_local.pos)

        # add the challenge to the cache
        challenge = HttpBearerChallenge(response.request.url, auth_header)
        ChallengeCache.set_challenge_for_url(response.request.url, challenge)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content
        response.close()

        # copy the request to resend
        prep = response.request.copy()
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)
        prep.prepare_cookies(prep._cookies)

        # setup the auth header on the copied request
        self.set_authorization_header(prep, challenge)

        # resend the request with proper authentication
        _response = response.connection.send(prep, **kwargs)
        _response.history.append(response)
        _response.request = prep
        return _response
    def handle_401(self, response, **kwargs):
        """
        Takes the response authenticates and resends if neccissary
        :return: The final response to the authenticated request
        :rtype: requests.Response
        """
        # If response is not 401 do not auth and return response
        if not response.status_code == 401:
            self._thread_local.auth_attempted = False
            return response

        auth_header = response.headers.get('www-authenticate', '')

        # if the response auth header is not a bearer challenge do not auth and return response
        if not HttpBearerChallenge.is_bearer_challenge(auth_header):
            self._thread_local.auth_attempted = False
            return response

        # If we've already attempted to auth for this request once, do not auth and return response
        if self._thread_local.auth_attempted:
            self._thread_local.auth_attempted = False
            return response

        # Otherwise authenticate and retry the request
        self._thread_local.auth_attempted = True

        if self._thread_local.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            response.request.body.seek(self._thread_local.pos)

        # add the challenge to the cache
        challenge = HttpBearerChallenge(response.request.url, auth_header)
        ChallengeCache.set_challenge_for_url(response.request.url, challenge)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content
        response.close()

        # copy the request to resend
        prep = response.request.copy()
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)
        prep.prepare_cookies(prep._cookies)

        # setup the auth header on the copied request
        self.set_authorization_header(prep, challenge)

        # resend the request with proper authentication
        _response = response.connection.send(prep, **kwargs)
        _response.history.append(response)
        _response.request = prep
        return _response
Esempio n. 27
0
 def build_response(self, req, resp):
     response = PlexResponse()
     response.status_code = getattr(resp, 'status', None)
     response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
     response.encoding = get_encoding_from_headers(response.headers)
     response.raw = resp
     response.reason = response.raw.reason
     if isinstance(req.url, bytes):
         response.url = req.url.decode('utf-8')
     else:
         response.url = req.url
     extract_cookies_to_jar(response.cookies, req, resp)
     response.request = req
     response.connection = self
     return response
Esempio n. 28
0
def merge_cookies_from_response(cookie_jar, http_response, url):
    p = models.PreparedRequest()
    p.prepare(url=url)
    cookies.extract_cookies_to_jar(cookie_jar, p, http_response)

    # cookie_list_str = http_response.info().getlist('set-cookie')
    # cookie_jar._now = int(time.time())
    # cookie_set = cookiejar.parse_ns_headers(cookie_list_str)
    # cookie_tuples = cookie_jar._normalized_cookie_tuples(cookie_set)
    #
    # for tup in cookie_tuples:
    #     cookie = mark_cookie(tup)
    #     if cookie:
    #         cookie_jar.set_cookie(cookie)

    return cookie_jar
Esempio n. 29
0
    def build_response(self, req: BitexPreparedRequest,
                       resp: HTTPResponse) -> BitexResponse:
        """Build a :class:`BitexResponse` from the given `req` and `resp`.

        The method is largely identical to :meth:`HTTPAdapter.build_response`,
        and only differs in the class type used when constructing a response.

        This class is taken firstly from any valid plugin that supplies an
        adequate class for the exchange that was queried (as stated in
        :attr:`BitexPreparedRequest.exchange`), or :mod:`bitex-framework` 's own default
        :class:`BitexResponse` class.

        :param BitexPreparedRequest req:
            The :class:`BitexPreparedRequest` used to generate the response.
        :param HTTPResponse resp: The urllib3 response object.
        """
        if req.exchange in list_loaded_plugins():
            response = list_loaded_plugins()[req.exchange]["Response"]()
        else:
            response = BitexResponse()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, "status", None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = resp
        response.reason = response.raw.reason

        if isinstance(req.url, bytes):
            response.url = req.url.decode("utf-8")
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, resp)

        # Give the Response some context.
        response.request = req
        response.connection = self

        return response
Esempio n. 30
0
    def handle_407(self, r, **kwargs):
        """Handle HTTP 407 only once, otherwise give up

        :param r: current response
        :returns: responses, along with the new response
        """
        if r.status_code == 407 and self.stale_rejects < 2:
            s_auth = r.headers.get("proxy-authenticate")
            if s_auth is None:
                raise IOError(
                    "proxy server violated RFC 7235:"
                    "407 response MUST contain header proxy-authenticate")
            elif not self._pat.match(s_auth):
                return r

            self.chal = utils.parse_dict_header(
                self._pat.sub('', s_auth, count=1))

            # if we present the customuser/passwd and still get rejected
            # http://tools.ietf.org/html/rfc2617#section-3.2.1
            if ('Proxy-Authorization' in r.request.headers and
                    'stale' in self.chal):
                if self.chal['stale'].lower() == 'true':  # try again
                    self.stale_rejects += 1
                # wrong customuser/passwd
                elif self.chal['stale'].lower() == 'false':
                    raise IOError("User or password is invalid")

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            prep.headers['Proxy-Authorization'] = self.build_digest_header(
                prep.method, prep.url)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r
        else:  # give up authenticate
            return r
Esempio n. 31
0
    def handle_407(self, r, **kwargs):
        """Handle HTTP 407 only once, otherwise give up

        :param r: current response
        :returns: responses, along with the new response
        """
        if r.status_code == 407 and self.stale_rejects < 2:
            s_auth = r.headers.get("proxy-authenticate")
            if s_auth is None:
                raise IOError(
                    "proxy server violated RFC 7235:"
                    "407 response MUST contain header proxy-authenticate")
            elif not self._pat.match(s_auth):
                return r

            self.chal = utils.parse_dict_header(
                self._pat.sub('', s_auth, count=1))

            # if we present the user/passwd and still get rejected
            # http://tools.ietf.org/html/rfc2617#section-3.2.1
            if ('Proxy-Authorization' in r.request.headers and
                    'stale' in self.chal):
                if self.chal['stale'].lower() == 'true':  # try again
                    self.stale_rejects += 1
                # wrong user/passwd
                elif self.chal['stale'].lower() == 'false':
                    raise IOError("User or password is invalid")

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            prep.headers['Proxy-Authorization'] = self.build_digest_header(
                prep.method, prep.url)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r
        else:  # give up authenticate
            return r
Esempio n. 32
0
    def handle_401(self, r, **kwargs):
        """
        Takes the given response and tries digest-auth, if needed.

        :rtype: requests.Response
        """

        # If response is not 4xx, do not auth
        # See https://github.com/psf/requests/issues/3772
        if not 400 <= r.status_code < 500:
            self._thread_local.num_401_calls = 1
            return r

        if self._thread_local.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            r.request.body.seek(self._thread_local.pos)
        s_auth = r.headers.get('www-authenticate', '')

        if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2:

            self._thread_local.num_401_calls += 1
            pat = re.compile(r'digest ', flags=re.IGNORECASE)
            self._thread_local.chal = parse_dict_header(
                pat.sub('', s_auth, count=1))

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            prep.headers['Authorization'] = self.build_digest_header(
                prep.method, prep.url)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        self._thread_local.num_401_calls = 1
        return r
Esempio n. 33
0
    def handle_401(self, r, **kwargs):
        if r.status_code == 401:
            LOG.debug('Got 401. Trying to reauth...')
            self.token = self.https_auth()
            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            prep.headers['Authorization'] = 'Bearer %s' % self.token
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r
        return r
Esempio n. 34
0
    def _handle_basic_auth_407(self, r, kwargs):
        if self.pos is not None:
            r.request.body.seek(self.pos)

        r.content
        r.raw.release_conn()
        prep = r.request.copy()
        if not hasattr(prep, "_cookies"):
            prep._cookies = cookies.RequestsCookieJar()
        cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
        prep.prepare_cookies(prep._cookies)

        self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username,
                                             self.proxy_password)
        prep = self.proxy_auth(prep)
        _r = r.connection.send(prep, **kwargs)
        _r.history.append(r)
        _r.request = prep

        return _r
Esempio n. 35
0
    def _handle_basic_auth_407(self, r, kwargs):
        if self.pos is not None:
            r.request.body.seek(self.pos)

        r.content
        r.raw.release_conn()
        prep = r.request.copy()
        if not hasattr(prep, '_cookies'):
            prep._cookies = cookies.RequestsCookieJar()
        cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
        prep.prepare_cookies(prep._cookies)

        self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username,
                                             self.proxy_password)
        prep = self.proxy_auth(prep)
        _r = r.connection.send(prep, **kwargs)
        _r.history.append(r)
        _r.request = prep

        return _r
Esempio n. 36
0
    def _handle_basic_auth_401(self, r, kwargs):
        if self.pos is not None:
            r.request.body.seek(self.pos)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        r.content
        r.raw.release_conn()
        prep = r.request.copy()
        if not hasattr(prep, '_cookies'):
            prep._cookies = cookies.RequestsCookieJar()
        cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
        prep.prepare_cookies(prep._cookies)

        self.auth = auth.HTTPBasicAuth(self.username, self.password)
        prep = self.auth(prep)
        _r = r.connection.send(prep, **kwargs)
        _r.history.append(r)
        _r.request = prep

        return _r
Esempio n. 37
0
    def _handle_basic_auth_401(self, r, kwargs):
        if self.pos is not None:
            r.request.body.seek(self.pos)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        r.content
        r.raw.release_conn()
        prep = r.request.copy()
        if not hasattr(prep, "_cookies"):
            prep._cookies = cookies.RequestsCookieJar()
        cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
        prep.prepare_cookies(prep._cookies)

        self.auth = auth.HTTPBasicAuth(self.username, self.password)
        prep = self.auth(prep)
        _r = r.connection.send(prep, **kwargs)
        _r.history.append(r)
        _r.request = prep

        return _r
Esempio n. 38
0
    def persist_cookies(self, r):
        """
        From requests/sessions.py, Session.send()

        Session.send() 方法会首先 dispatch_hook 然后再 extract_cookies_to_jar

        在该项目中,对于返回信息异常的请求,在 hooks 校验时会将错误抛出,send() 之后的处理将不会执行。
        遇到的错误往往是 SystemException / TipsException ,而这些客户端认为是错误的情况,
        对于服务器端来说并不是错误请求,服务器端在该次请求结束后可能会要求 Set-Cookies
        但是由于 send() 在 dispatch_hook 时遇到错误而中止,导致后面的 extract_cookies_to_jar
        未能调用,因此 Cookies 并未更新。下一次再请求服务器的时候,就会遇到会话过期的情况。

        在这种情况下,需要在捕获错误后手动更新 cookies 以确保能够保持会话

        """
        if r.history:

            # If the hooks create history then we want those cookies too
            for resp in r.history:
                extract_cookies_to_jar(self._session.cookies, resp.request, resp.raw)

        extract_cookies_to_jar(self._session.cookies, r.request, r.raw)
    def _check_and_login(
            self,
            response,
            **kwargs
    ):
        if not is_keycloak_login_url(response.url):
            return response

        post_url = parse_post_url(response.text)
        if not post_url:
            raise Exception("TODO")

        if self._thread_local.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            response.request.body.seek(self._thread_local.pos)

        # Content was consumed earlier when we parsed for post url

        # Release the original connection to allow our new request to reuse the
        #   same one.
        response.close()

        prep = response.request.copy()
        extract_cookies_to_jar(prep._cookies, response.request, response.raw)
        prep.prepare_cookies(prep._cookies)
        prep.prepare_body(data=self._get_creds(), files=None)

        prep.method = "POST"
        prep.url = post_url

        _r = response.connection.send(prep, **kwargs)
        _r.history.append(response)
        _r.request = prep

        # TODO should do a full retry of the first request

        return _r
Esempio n. 40
0
    def handle_421(self, r, **kwargs):
        """
        Given a response, if is 421 parses out the
        Attestation-Challenge header and tries again
        with an attestation. Will try once.
        """

        if self.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            r.request.body.seek(self.pos)

        num_421_calls = getattr(self, 'num_421_calls', 1)
        attestation_challenge = r.headers.get('attestation-challenge', '')

        if r.status_code == 421 and num_421_calls < 2:
            setattr(self, 'num_421_calls', num_421_calls + 1)

            pat = re.compile(r'^attestation ', flags=re.IGNORECASE)
            self.chal = parse_dict_header(pat.sub('', attestation_challenge, count=1))

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.raw.release_conn()
            prep = r.request.copy()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            prep.headers['Attestation'] = self.build_attestation_header(prep)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        setattr(self, 'num_421_calls', 1)
        return r
Esempio n. 41
0
    def handle_401(self, r, **kwargs):
        """Takes the given response and tries using fresh OAuth tokens,
        if needed."""

        if r.status_code != 401:
            return r

        if self.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            r.request.body.seek(self.pos)

        num_401_calls = getattr(self, 'num_401_calls', 1)

        if num_401_calls < 2:
            self.num_401_calls += 1
            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            if "cookies" in prep.headers:
                extract_cookies_to_jar(
                    prep.headers.get("cookies", None), r.request, r.raw)
                prep.prepare_cookies(prep._cookies)

            self.refresh()
            prep.headers['Authorization'] = self.access_token
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        self.num_401_calls = 1
        return r
Esempio n. 42
0
    def send(self, request, **kwargs):
        """Send a given PreparedRequest."""
        # Set defaults that the hooks can utilize to ensure they always have
        # the correct parameters to reproduce the previous request.
        kwargs.setdefault('stream', self.stream)
        kwargs.setdefault('verify', self.verify)
        kwargs.setdefault('cert', self.cert)
        kwargs.setdefault('proxies', self.proxies)

        # It's possible that users might accidentally send a Request object.
        # Guard against that specific failure case.
        if not isinstance(request, PreparedRequest):
            raise ValueError('You can only send PreparedRequests.')

        checked_urls = set()
        while request.url in self.redirect_cache:
            checked_urls.add(request.url)
            new_url = self.redirect_cache.get(request.url)
            if new_url in checked_urls:
                break
            request.url = new_url

        # Set up variables needed for resolve_redirects and dispatching of hooks
        allow_redirects = kwargs.pop('allow_redirects', True)
        stream = kwargs.get('stream')
        hooks = request.hooks

        # Get the appropriate adapter to use

        # Start time (approximately) of the request
        start = datetime.utcnow()

        # Send the request
        r = adapter.send(request, **kwargs)

        # Total elapsed time of the request (approximately)
        r.elapsed = datetime.utcnow() - start

        # Response manipulation hooks
        r = dispatch_hook('response', hooks, r, **kwargs)

        # Persist cookies
        if r.history:

            # If the hooks create history then we want those cookies too
            for resp in r.history:
                extract_cookies_to_jar(self.cookies, resp.request, resp.raw)

        extract_cookies_to_jar(self.cookies, request, r.raw)

        # Redirect resolving generator.
        gen = self.resolve_redirects(r, request, **kwargs)

        # Resolve redirects if allowed.
        history = [resp for resp in gen] if allow_redirects else []

        # Shuffle things around if there's history.
        if history:
            # Insert the first (original) request at the start
            history.insert(0, r)
            # Get the last request made
            r = history.pop()
            r.history = history

        if not stream:
            r.content

        return r
Esempio n. 43
0
 def deserialize(self):
     """Turn a serialized interaction into a Response."""
     r = util.deserialize_response(self.data['response'])
     r.request = util.deserialize_prepared_request(self.data['request'])
     extract_cookies_to_jar(r.cookies, r.request, r.raw)
     return r
    def _handle_401(self, response, **kwargs):
        """
        Takes the response authenticates and resends if neccissary
        :return: The final response to the authenticated request
        :rtype: requests.Response
        """
        # If response is not 401 do not auth and return response
        if not response.status_code == 401:
            self._thread_local.auth_attempted = False
            return response

        # If we've already attempted to auth for this request once, do not auth and return response
        if self._thread_local.auth_attempted:
            self._thread_local.auth_attempted = False
            return response

        auth_header = response.headers.get('www-authenticate', '')

        # Otherwise authenticate and retry the request
        self._thread_local.auth_attempted = True

        # parse the challenge
        challenge = HttpChallenge(response.request.url, auth_header, response.headers)

        # bearer and PoP are the only authentication schemes supported at this time
        # if the response auth header is not a bearer challenge or pop challange do not auth and return response
        if not (challenge.is_bearer_challenge() or challenge.is_pop_challenge()):
            self._thread_local.auth_attempted = False
            return response

        # add the challenge to the cache
        ChallengeCache.set_challenge_for_url(response.request.url, challenge)

        # Consume content and release the original connection
        # to allow our new request to reuse the same one.
        response.content
        response.close()

        # copy the request to resend
        prep = response.request.copy()

        if self._thread_local.orig_body is not None:
            # replace the body with the saved body
            prep.prepare_body(data=self._thread_local.orig_body, files=None)

        extract_cookies_to_jar(prep._cookies, response.request, response.raw)
        prep.prepare_cookies(prep._cookies)

        security = self._get_message_security(prep, challenge)

        # auth and protect the prepped request message
        security.protect_request(prep)

        # resend the request with proper authentication and message protection
        _response = response.connection.send(prep, **kwargs)
        _response.history.append(response)
        _response.request = prep

        # unprotected the response
        security.unprotect_response(_response)

        return _response
Esempio n. 45
0
 def deserialize(self):
     """Turn a serialized interaction into a Response."""
     r = util.deserialize_response(self.json['response'])
     r.request = util.deserialize_prepared_request(self.json['request'])
     extract_cookies_to_jar(r.cookies, r.request, r.raw)
     return r
Esempio n. 46
0
    def http_fetch(self, url, fetch):
        """
        HTTP fetcher
        """
        start_time = time.time()

        def handle_error(x):
            BaseCrawler.handle_error(url, start_time, x)

        max_redirects = self.max_redirects
        # we will handle redirects by hand to capture cookies
        fetch['follow_redirects'] = False

        # making requests
        while True:
            try:
                request = tornado.httpclient.HTTPRequest(**fetch)
                # if cookie already in header, get_cookie_header wouldn't work
                old_cookie_header = request.headers.get('Cookie')
                if old_cookie_header:
                    del request.headers['Cookie']
                cookie_header = cookies.get_cookie_header(
                    self._cookies, request)
                if cookie_header:
                    request.headers['Cookie'] = cookie_header
                elif old_cookie_header:
                    request.headers['Cookie'] = old_cookie_header
            except Exception as e:
                self.exception(e)
                raise gen.Return(handle_error(e))

            try:
                response = yield gen.maybe_future(
                    self.http_client.fetch(request))
            except tornado.httpclient.HTTPError as e:
                if e.response:
                    response = e.response
                else:
                    raise gen.Return(handle_error(e))
            except Exception as e:
                raise gen.Return(handle_error(e))
            cookies.extract_cookies_to_jar(self._cookies, response.request,
                                           response)

            if response.code in (301, 302, 303,
                                 307) and response.headers.get('Location'):
                if max_redirects <= 0:
                    error = CDSpiderCrawlerBadRequest(
                        599, 'Maximum (%d) redirects followed' %
                        fetch.get('max_redirects', 5), response)
                    raise gen.Return(handle_error(error))
                if response.code in (302, 303):
                    fetch['method'] = 'GET'
                    if 'body' in fetch:
                        del fetch['body']
                fetch['url'] = utils.quote_chinese(
                    urljoin(fetch['url'], response.headers['Location']))
                fetch['request_timeout'] -= time.time() - start_time
                if fetch['request_timeout'] < 0:
                    fetch['request_timeout'] = 0.1
                max_redirects -= 1
                continue
            else:
                error = self._prepare_response(response.code, url)
                if error is not None:
                    raise gen.Return(handle_error(error))
            self.gen_result(url=response.effective_url or url,
                            code=response.code,
                            headers=dict(response.headers),
                            cookies=self._cookies.get_dict(),
                            content=response.body or '',
                            start_time=start_time,
                            error=response.error)
Esempio n. 47
0
    def build_response(self, request, resp):
        """
        Builds a Requests' response object.  This emulates most of the logic of
        the standard fuction but deals with the lack of the ``.headers``
        property on the HTTP20Response object.

        Additionally, this function builds in a number of features that are
        purely for HTTPie. This is to allow maximum compatibility with what
        urllib3 does, so that HTTPie doesn't fall over when it uses us.
        """
        response = Response()

        response.status_code = resp.status
        response.headers = CaseInsensitiveDict(resp.headers.iter_raw())
        response.raw = resp
        response.reason = resp.reason
        response.encoding = get_encoding_from_headers(response.headers)

        extract_cookies_to_jar(response.cookies, request, response)
        response.url = request.url

        response.request = request
        response.connection = self

        # First horrible patch: Requests expects its raw responses to have a
        # release_conn method, which I don't. We should monkeypatch a no-op on.
        resp.release_conn = lambda: None

        # Next, add the things HTTPie needs. It needs the following things:
        #
        # - The `raw` object has a property called `_original_response` that is
        #   a `httplib` response object.
        # - `raw._original_response` has three simple properties: `version`,
        #   `status`, `reason`.
        # - `raw._original_response.version` has one of three values: `9`,
        #   `10`, `11`.
        # - `raw._original_response.msg` exists.
        # - `raw._original_response.msg._headers` exists and is an iterable of
        #   two-tuples.
        #
        # We fake this out. Most of this exists on our response object already,
        # and the rest can be faked.
        #
        # All of this exists for httpie, which I don't have any tests for,
        # so I'm not going to bother adding test coverage for it.
        class FakeOriginalResponse(object):  # pragma: no cover
            def __init__(self, headers):
                self._headers = headers

            def get_all(self, name, default=None):
                values = []

                for n, v in self._headers:
                    if n == name.lower():
                        values.append(v)

                if not values:
                    return default

                return values

            def getheaders(self, name):
                return self.get_all(name, [])


        response.raw._original_response = orig = FakeOriginalResponse(None)
        orig.version = 20
        orig.status = resp.status
        orig.reason = resp.reason
        orig.msg = FakeOriginalResponse(resp.headers.iter_raw())

        return response