def build_response_into(self, response, req, urllib3_resp): """Same as requests.adapters.HTTPAdapter.build_response, but writes into a provided requests.Response object instead of creating a new one. """ # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(urllib3_resp, 'status', None) # Make headers case-insensitive. response.headers = CaseInsensitiveDict( getattr(urllib3_resp, 'headers', {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) response.raw = urllib3_resp response.reason = response.raw.reason if isinstance(req.url, bytes): response.url = req.url.decode('utf-8') else: response.url = req.url # Add new cookies from the server. extract_cookies_to_jar(response.cookies, req, urllib3_resp) # Give the Response some context. response.request = req response.connection = self
def handle_auth(self, r, **kwargs): """Takes the given response and tries SAML auth, if needed.""" if r.status_code not in [ 401, 403 ]: return r # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.raw.release_conn() prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) cookie_dict = requests.utils.dict_from_cookiejar(prep._cookies) self.session.auth = None self.session.get_sso_cookie() self.session.auth = self prep.prepare_cookies(self.session.cookies) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r
def handle_auth(self, response, **kwargs): '''Takes the given response and tries SAML auth, if needed.''' if response.status_code not in [401, 403]: return response # Consume content and release the original connection # to allow our new request to reuse the same one. response.content # pylint: disable=pointless-statement response.raw.release_conn() prep = response.request.copy() # PreparedResponse does not have a method to get _cookies. # This is reuse of code from python requests.auth # pylint: disable=protected-access extract_cookies_to_jar(prep._cookies, response.request, response.raw) self.session.auth = None self.session.get_sso_cookie() self.session.auth = self prep.prepare_cookies(self.session.cookies) new_response = response.connection.send(prep, **kwargs) new_response.history.append(response) new_response.request = prep return new_response
def build_response(self, request, resp): """ Builds a Requests' response object. This emulates most of the logic of the standard fuction but deals with the lack of the ``.headers`` property on the HTTP20Response object. """ response = Response() response.status_code = resp.status response.headers = CaseInsensitiveDict(resp.getheaders()) response.raw = resp response.reason = resp.reason response.encoding = get_encoding_from_headers(response.headers) extract_cookies_to_jar(response.cookies, request, response) if isinstance(request.url, bytes): response.url = request.url.decode('utf-8') else: response.url = request.url response.request = request response.connection = self # One last horrible patch: Requests expects its raw responses to have a # release_conn method, which I don't. We should monkeypatch a no-op on. resp.release_conn = lambda: None return response
def handle_401(self, response, repo, **kwargs): """Fetch Bearer token and retry.""" if response.status_code != requests.codes.unauthorized: return response auth_info = response.headers.get('www-authenticate', '') if 'bearer' not in auth_info.lower(): return response self._token_cache[repo] = self._get_token(auth_info, repo) # Consume content and release the original connection # to allow our new request to reuse the same one. # This pattern was inspired by the source code of requests.auth.HTTPDigestAuth response.content response.close() retry_request = response.request.copy() extract_cookies_to_jar(retry_request._cookies, response.request, response.raw) retry_request.prepare_cookies(retry_request._cookies) self._set_header(retry_request, repo) retry_response = response.connection.send(retry_request, **kwargs) retry_response.history.append(response) retry_response.request = retry_request return retry_response
def handle_500(self, r, **kwargs): if not self.enable_retry_on_500 \ or r.status_code < 500 or r.status_code >= 600: return r num_500_calls = getattr(self, 'num_500_calls', 1) if num_500_calls < 4: logger.debug("Got %s status, retrying" % r.status_code) self.num_500_calls += 1 # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.close() prep = r.request.copy() if "cookies" in prep.headers: extract_cookies_to_jar(prep.headers.get("cookies", None), r.request, r.raw) prep.prepare_cookies(prep._cookies) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r self.num_500_calls = 1 return r
def build_response(self, req, resp): """Builds a :class:`Response <requests.Response>` object from a urllib3 response. This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>` :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response. :param resp: The urllib3 response object. """ response = Response() # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(resp, 'status', None) # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) response.raw = resp response.reason = response.raw.reason if isinstance(req.url, bytes): response.url = req.url.decode('utf-8') else: response.url = req.url # Add new cookies from the server. extract_cookies_to_jar(response.cookies, req, resp) # Give the Response some context. response.request = req response.connection = self return response
def handle_401(self, r, **kwargs): """Resends a request with auth headers, if needed.""" www_authenticate = r.headers.get('www-authenticate', '').lower() if 'basic' in www_authenticate: if self.pos is not None: r.request.body.seek(self.pos) # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.raw.release_conn() prep = r.request.copy() if not hasattr(prep, '_cookies'): prep._cookies = RequestsCookieJar() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) self.auth = HTTPBasicAuth(self.username, self.password) prep = self.auth(prep) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r if 'digest' in www_authenticate: self.auth = HTTPDigestAuth(self.username, self.password) # Digest auth would resend the request by itself. We can take a # shortcut here. return self.auth.handle_401(r, **kwargs)
def to_requests_response(self): """Returns an instance of `requests.Response` based on this response. Returns: request.Response: the generated response. """ # Make sure that body is at position 0 before returning self.body.seek(0) urllib3_response = URLLib3Rresponse( body=self.body, headers=self.headers, status=self.http_code, request_method=self.request.method, reason=self.reason, preload_content=False ) response = RequestResponse() response.request = self.request response.raw = urllib3_response response.status_code = self.http_code response.reason = self.reason response.headers = CaseInsensitiveDict(response.raw.headers) response.encoding = get_encoding_from_headers(response.headers) extract_cookies_to_jar(response.cookies, self.request, urllib3_response) if isinstance(self.request.url, six.binary_type): response.url = self.request.url.decode("utf-8") else: response.url = self.request.url return response
def http_fetch(self, url, fetch): """ HTTP fetcher """ start_time = time.time() def handle_error(x): BaseCrawler.handle_error(url, start_time, x) # making requests while True: try: response = yield gen.maybe_future(self._request(url, fetch)) except HTTPError as e: if e.response: response = e.response else: raise gen.Return(handle_error(e)) except Exception as e: raise gen.Return(handle_error(e)) cookies.extract_cookies_to_jar(self._cookies, response.request, response) error = self._prepare_response(response.status_code, url) if error is not None: raise gen.Return(handle_error(error)) self.gen_result(url=response.url or url, code=response.status_code, headers=dict(response.headers), cookies=self._cookies.get_dict(), content=response.content or '', start_time=start_time)
def handle_407(self, r, **kwargs): if r.status_code == 407 and self.stale_rejects < 2: pat = re.compile(r'digest ', flags=re.IGNORECASE) if "proxy-authenticate" not in r.headers: raise IOError( "proxy server violated RFC 7235:" "407 response MUST contain header proxy-authenticate") self.chal = parse_dict_header( pat.sub('', r.headers['proxy-authenticate'], count=1)) if 'Proxy-Authorization' in r.request.headers and 'stale' in self.chal: if self.chal['stale'].lower() == 'true': # try again self.stale_rejects += 1 elif self.chal['stale'].lower() == 'false': raise IOError("User or password is invalid") r.content r.raw.release_conn() prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) prep.headers['Proxy-Authorization'] = self.build_digest_header( prep.method, prep.url) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r else: # give up authenticate return r
def build_response(self, request, resp): """ Builds a Requests' response object. This emulates most of the logic of the standard fuction but deals with the lack of the ``.headers`` property on the HTTP20Response object. """ response = Response() response.status_code = resp.status response.headers = CaseInsensitiveDict(resp.getheaders()) response.raw = resp response.reason = resp.reason response.encoding = get_encoding_from_headers(response.headers) extract_cookies_to_jar(response.cookies, request, response) response.url = request.url response.request = request response.connection = self # One last horrible patch: Requests expects its raw responses to have a # release_conn method, which I don't. We should monkeypatch a no-op on. resp.release_conn = lambda: None return response
def handle_401(self, response, repo, **kwargs): """Fetch Bearer token and retry.""" if response.status_code != requests.codes.unauthorized: return response auth_info = response.headers.get('www-authenticate', '') if 'bearer' not in auth_info.lower(): return response self._token_cache[repo] = self._get_token(auth_info, repo) # Consume content and release the original connection # to allow our new request to reuse the same one. # This pattern was inspired by the source code of requests.auth.HTTPDigestAuth response.content # pylint: disable=pointless-statement; is a property response.close() retry_request = response.request.copy() extract_cookies_to_jar(retry_request._cookies, response.request, response.raw) retry_request.prepare_cookies(retry_request._cookies) self._set_header(retry_request, repo) retry_response = response.connection.send(retry_request, **kwargs) retry_response.history.append(response) retry_response.request = retry_request # when quay returns 401 even after we have token, # repository doesn't exist at all, so we will treat it as 404 if retry_response.status_code == requests.codes.unauthorized: retry_response.status_code = requests.codes.not_found return retry_response
def http_response_to_response(self, http_response, prepared_request): """ transform a WSGIResponse into a requests's Response model :param django.http.response.HttpResponse http_response: the http response send by django view :return: the requests's Response model corresponding to the http_response :rtype: Response """ response = Response() # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(http_response, 'status_code', None) # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(http_response._headers, 'headers', {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) response.raw = http_response response.reason = response.raw.reason_phrase response._content = http_response.content req = prepared_request if isinstance(req.url, bytes): # pragma: no cover response.url = req.url.decode('utf-8') else: response.url = req.url # Add new cookies from the server. extract_cookies_to_jar(response.cookies, req, response) # Give the Response some context. response.request = req response.connection = self return response
def deserialize(self): """Turn a serialized interaction into a Response.""" r = util.deserialize_response(self.json['response']) r.request = util.deserialize_prepared_request(self.json['request']) extract_cookies_to_jar(r.cookies, r.request, r.raw) self.recorded_at = datetime.strptime(self.json['recorded_at'], '%Y-%m-%dT%H:%M:%S') self.recorded_response = r
def deserialize(self): """Turn a serialized interaction into a Response.""" r = deserialize_response(self.json['response']) r.request = deserialize_prepared_request(self.json['request']) extract_cookies_to_jar(r.cookies, r.request, r.raw) self.recorded_at = datetime.strptime( self.json['recorded_at'], '%Y-%m-%dT%H:%M:%S' ) self.recorded_response = r
def start_response(status, headers, exc_info=None): headers = make_headers(headers) response.status_code = int(status.split(" ")[0]) response.reason = responses.get(response.status_code, "Unknown Status Code") response.headers = headers resp._original_response.msg = headers extract_cookies_to_jar(response.cookies, request, resp) response.encoding = get_encoding_from_headers(response.headers) response.elapsed = datetime.datetime.utcnow() - start self._log(response)
def send(self, request, **kwargs) -> UpgradeResponse: kwargs.setdefault('stream', self.stream) kwargs.setdefault('verify', self.verify) kwargs.setdefault('cert', self.cert) kwargs.setdefault('proxies', self.proxies) if isinstance(request, requests.Request): raise ValueError('You can only send PreparedRequests.') if isinstance(request, HttpRequest): request.check() allow_redirects = kwargs.pop('allow_redirects', True) hooks = request.hooks adapter = self.get_adapter(url=request.url) start = preferred_clock() r = adapter.send(request, **kwargs) elapsed = preferred_clock() - start r.elapsed = timedelta(seconds=elapsed) # this function not change response and result params if len(self._dispatch_hooks) > 0: dispatch(self._dispatch_hooks, hooks, r, **kwargs) # dispatch hook will change response and request params r = dispatch_hook('response', hooks, r, **kwargs) if r.history: for resp in r.history: extract_cookies_to_jar(self.cookies, resp.request, resp.raw) extract_cookies_to_jar(self.cookies, request, r.raw) gen = self.resolve_redirects(r, request, **kwargs) history = [resp for resp in gen] if allow_redirects else [] if history: history.insert(0, r) r = history.pop() r.history = history if not allow_redirects: try: r._next = next( self.resolve_redirects(r, request, yield_requests=True, **kwargs)) except StopIteration: pass return r
def handle_407(response, **kwargs): """ Prompts the user for the proxy username and password and modifies the proxy in the session object to include it. This method is modeled after * requests.auth.HTTPDigestAuth.handle_401() * requests.auth.HTTPProxyAuth * the previous conda.fetch.handle_proxy_407() It both adds 'username:password' to the proxy URL, as well as adding a 'Proxy-Authorization' header. If any of this is incorrect, please file an issue. """ # kwargs = {'verify': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False, # 'timeout': (3.05, 60)} if response.status_code != 407: return response # Consume content and release the original connection # to allow our new request to reuse the same one. response.content response.close() proxies = kwargs.pop('proxies') proxy_scheme = urlparse(response.url).scheme if proxy_scheme not in proxies: raise ProxyError( dals(""" Could not find a proxy for %r. See %s/docs/html#configure-conda-for-use-behind-a-proxy-server for more information on how to configure proxies. """ % (proxy_scheme, CONDA_HOMEPAGE_URL))) # fix-up proxy_url with username & password proxy_url = proxies[proxy_scheme] username, password = get_proxy_username_and_pass(proxy_scheme) proxy_url = add_username_and_password(proxy_url, username, password) proxy_authorization_header = _basic_auth_str(username, password) proxies[proxy_scheme] = proxy_url kwargs['proxies'] = proxies prep = response.request.copy() extract_cookies_to_jar(prep._cookies, response.request, response.raw) prep.prepare_cookies(prep._cookies) prep.headers['Proxy-Authorization'] = proxy_authorization_header _response = response.connection.send(prep, **kwargs) _response.history.append(response) _response.request = prep return _response
def handle_407(response, **kwargs): """ Prompts the user for the proxy username and password and modifies the proxy in the session object to include it. This method is modeled after * requests.auth.HTTPDigestAuth.handle_401() * requests.auth.HTTPProxyAuth * the previous conda.fetch.handle_proxy_407() It both adds 'username:password' to the proxy URL, as well as adding a 'Proxy-Authorization' header. If any of this is incorrect, please file an issue. """ # kwargs = {'verify': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False, # 'timeout': (3.05, 60)} if response.status_code != 407: return response # Consume content and release the original connection # to allow our new request to reuse the same one. response.content response.close() proxies = kwargs.pop('proxies') proxy_scheme = urlparse(response.url).scheme if proxy_scheme not in proxies: raise ProxyError(dals(""" Could not find a proxy for %r. See %s/docs/html#configure-conda-for-use-behind-a-proxy-server for more information on how to configure proxies. """ % (proxy_scheme, CONDA_HOMEPAGE_URL))) # fix-up proxy_url with username & password proxy_url = proxies[proxy_scheme] username, password = get_proxy_username_and_pass(proxy_scheme) proxy_url = add_username_and_password(proxy_url, username, password) proxy_authorization_header = _basic_auth_str(username, password) proxies[proxy_scheme] = proxy_url kwargs['proxies'] = proxies prep = response.request.copy() extract_cookies_to_jar(prep._cookies, response.request, response.raw) prep.prepare_cookies(prep._cookies) prep.headers['Proxy-Authorization'] = proxy_authorization_header _response = response.connection.send(prep, **kwargs) _response.history.append(response) _response.request = prep return _response
def handle_401(self, response, **kwargs): """ Takes the response authenticates and resends if neccissary :return: The final response to the authenticated request :rtype: requests.Response """ # If response is not 401 do not auth and return response if not response.status_code == 401: self._thread_local.auth_attempted = False return response auth_header = response.headers.get('www-authenticate', '') # if the response auth header is not a bearer challenge do not auth and return response if not HttpBearerChallenge.is_bearer_challenge(auth_header): self._thread_local.auth_attempted = False return response # If we've already attempted to auth for this request once, do not auth and return response if self._thread_local.auth_attempted: self._thread_local.auth_attempted = False return response # Otherwise authenticate and retry the request self._thread_local.auth_attempted = True if self._thread_local.pos is not None: # Rewind the file position indicator of the body to where # it was to resend the request. response.request.body.seek(self._thread_local.pos) # add the challenge to the cache challenge = HttpBearerChallenge(response.request.url, auth_header) ChallengeCache.set_challenge_for_url(response.request.url, challenge) # Consume content and release the original connection # to allow our new request to reuse the same one. response.content response.close() # copy the request to resend prep = response.request.copy() extract_cookies_to_jar(prep._cookies, response.request, response.raw) prep.prepare_cookies(prep._cookies) # setup the auth header on the copied request self.set_authorization_header(prep, challenge) # resend the request with proper authentication _response = response.connection.send(prep, **kwargs) _response.history.append(response) _response.request = prep return _response
def build_response(self, req, resp): response = PlexResponse() response.status_code = getattr(resp, 'status', None) response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) response.encoding = get_encoding_from_headers(response.headers) response.raw = resp response.reason = response.raw.reason if isinstance(req.url, bytes): response.url = req.url.decode('utf-8') else: response.url = req.url extract_cookies_to_jar(response.cookies, req, resp) response.request = req response.connection = self return response
def merge_cookies_from_response(cookie_jar, http_response, url): p = models.PreparedRequest() p.prepare(url=url) cookies.extract_cookies_to_jar(cookie_jar, p, http_response) # cookie_list_str = http_response.info().getlist('set-cookie') # cookie_jar._now = int(time.time()) # cookie_set = cookiejar.parse_ns_headers(cookie_list_str) # cookie_tuples = cookie_jar._normalized_cookie_tuples(cookie_set) # # for tup in cookie_tuples: # cookie = mark_cookie(tup) # if cookie: # cookie_jar.set_cookie(cookie) return cookie_jar
def build_response(self, req: BitexPreparedRequest, resp: HTTPResponse) -> BitexResponse: """Build a :class:`BitexResponse` from the given `req` and `resp`. The method is largely identical to :meth:`HTTPAdapter.build_response`, and only differs in the class type used when constructing a response. This class is taken firstly from any valid plugin that supplies an adequate class for the exchange that was queried (as stated in :attr:`BitexPreparedRequest.exchange`), or :mod:`bitex-framework` 's own default :class:`BitexResponse` class. :param BitexPreparedRequest req: The :class:`BitexPreparedRequest` used to generate the response. :param HTTPResponse resp: The urllib3 response object. """ if req.exchange in list_loaded_plugins(): response = list_loaded_plugins()[req.exchange]["Response"]() else: response = BitexResponse() # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(resp, "status", None) # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, "headers", {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) response.raw = resp response.reason = response.raw.reason if isinstance(req.url, bytes): response.url = req.url.decode("utf-8") else: response.url = req.url # Add new cookies from the server. extract_cookies_to_jar(response.cookies, req, resp) # Give the Response some context. response.request = req response.connection = self return response
def handle_407(self, r, **kwargs): """Handle HTTP 407 only once, otherwise give up :param r: current response :returns: responses, along with the new response """ if r.status_code == 407 and self.stale_rejects < 2: s_auth = r.headers.get("proxy-authenticate") if s_auth is None: raise IOError( "proxy server violated RFC 7235:" "407 response MUST contain header proxy-authenticate") elif not self._pat.match(s_auth): return r self.chal = utils.parse_dict_header( self._pat.sub('', s_auth, count=1)) # if we present the customuser/passwd and still get rejected # http://tools.ietf.org/html/rfc2617#section-3.2.1 if ('Proxy-Authorization' in r.request.headers and 'stale' in self.chal): if self.chal['stale'].lower() == 'true': # try again self.stale_rejects += 1 # wrong customuser/passwd elif self.chal['stale'].lower() == 'false': raise IOError("User or password is invalid") # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.close() prep = r.request.copy() cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) prep.headers['Proxy-Authorization'] = self.build_digest_header( prep.method, prep.url) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r else: # give up authenticate return r
def handle_407(self, r, **kwargs): """Handle HTTP 407 only once, otherwise give up :param r: current response :returns: responses, along with the new response """ if r.status_code == 407 and self.stale_rejects < 2: s_auth = r.headers.get("proxy-authenticate") if s_auth is None: raise IOError( "proxy server violated RFC 7235:" "407 response MUST contain header proxy-authenticate") elif not self._pat.match(s_auth): return r self.chal = utils.parse_dict_header( self._pat.sub('', s_auth, count=1)) # if we present the user/passwd and still get rejected # http://tools.ietf.org/html/rfc2617#section-3.2.1 if ('Proxy-Authorization' in r.request.headers and 'stale' in self.chal): if self.chal['stale'].lower() == 'true': # try again self.stale_rejects += 1 # wrong user/passwd elif self.chal['stale'].lower() == 'false': raise IOError("User or password is invalid") # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.close() prep = r.request.copy() cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) prep.headers['Proxy-Authorization'] = self.build_digest_header( prep.method, prep.url) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r else: # give up authenticate return r
def handle_401(self, r, **kwargs): """ Takes the given response and tries digest-auth, if needed. :rtype: requests.Response """ # If response is not 4xx, do not auth # See https://github.com/psf/requests/issues/3772 if not 400 <= r.status_code < 500: self._thread_local.num_401_calls = 1 return r if self._thread_local.pos is not None: # Rewind the file position indicator of the body to where # it was to resend the request. r.request.body.seek(self._thread_local.pos) s_auth = r.headers.get('www-authenticate', '') if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2: self._thread_local.num_401_calls += 1 pat = re.compile(r'digest ', flags=re.IGNORECASE) self._thread_local.chal = parse_dict_header( pat.sub('', s_auth, count=1)) # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.close() prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) prep.headers['Authorization'] = self.build_digest_header( prep.method, prep.url) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r self._thread_local.num_401_calls = 1 return r
def handle_401(self, r, **kwargs): if r.status_code == 401: LOG.debug('Got 401. Trying to reauth...') self.token = self.https_auth() # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.close() prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) prep.headers['Authorization'] = 'Bearer %s' % self.token _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r return r
def _handle_basic_auth_407(self, r, kwargs): if self.pos is not None: r.request.body.seek(self.pos) r.content r.raw.release_conn() prep = r.request.copy() if not hasattr(prep, "_cookies"): prep._cookies = cookies.RequestsCookieJar() cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username, self.proxy_password) prep = self.proxy_auth(prep) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r
def _handle_basic_auth_407(self, r, kwargs): if self.pos is not None: r.request.body.seek(self.pos) r.content r.raw.release_conn() prep = r.request.copy() if not hasattr(prep, '_cookies'): prep._cookies = cookies.RequestsCookieJar() cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username, self.proxy_password) prep = self.proxy_auth(prep) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r
def _handle_basic_auth_401(self, r, kwargs): if self.pos is not None: r.request.body.seek(self.pos) # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.raw.release_conn() prep = r.request.copy() if not hasattr(prep, '_cookies'): prep._cookies = cookies.RequestsCookieJar() cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) self.auth = auth.HTTPBasicAuth(self.username, self.password) prep = self.auth(prep) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r
def _handle_basic_auth_401(self, r, kwargs): if self.pos is not None: r.request.body.seek(self.pos) # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.raw.release_conn() prep = r.request.copy() if not hasattr(prep, "_cookies"): prep._cookies = cookies.RequestsCookieJar() cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) self.auth = auth.HTTPBasicAuth(self.username, self.password) prep = self.auth(prep) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r
def persist_cookies(self, r): """ From requests/sessions.py, Session.send() Session.send() 方法会首先 dispatch_hook 然后再 extract_cookies_to_jar 在该项目中,对于返回信息异常的请求,在 hooks 校验时会将错误抛出,send() 之后的处理将不会执行。 遇到的错误往往是 SystemException / TipsException ,而这些客户端认为是错误的情况, 对于服务器端来说并不是错误请求,服务器端在该次请求结束后可能会要求 Set-Cookies 但是由于 send() 在 dispatch_hook 时遇到错误而中止,导致后面的 extract_cookies_to_jar 未能调用,因此 Cookies 并未更新。下一次再请求服务器的时候,就会遇到会话过期的情况。 在这种情况下,需要在捕获错误后手动更新 cookies 以确保能够保持会话 """ if r.history: # If the hooks create history then we want those cookies too for resp in r.history: extract_cookies_to_jar(self._session.cookies, resp.request, resp.raw) extract_cookies_to_jar(self._session.cookies, r.request, r.raw)
def _check_and_login( self, response, **kwargs ): if not is_keycloak_login_url(response.url): return response post_url = parse_post_url(response.text) if not post_url: raise Exception("TODO") if self._thread_local.pos is not None: # Rewind the file position indicator of the body to where # it was to resend the request. response.request.body.seek(self._thread_local.pos) # Content was consumed earlier when we parsed for post url # Release the original connection to allow our new request to reuse the # same one. response.close() prep = response.request.copy() extract_cookies_to_jar(prep._cookies, response.request, response.raw) prep.prepare_cookies(prep._cookies) prep.prepare_body(data=self._get_creds(), files=None) prep.method = "POST" prep.url = post_url _r = response.connection.send(prep, **kwargs) _r.history.append(response) _r.request = prep # TODO should do a full retry of the first request return _r
def handle_421(self, r, **kwargs): """ Given a response, if is 421 parses out the Attestation-Challenge header and tries again with an attestation. Will try once. """ if self.pos is not None: # Rewind the file position indicator of the body to where # it was to resend the request. r.request.body.seek(self.pos) num_421_calls = getattr(self, 'num_421_calls', 1) attestation_challenge = r.headers.get('attestation-challenge', '') if r.status_code == 421 and num_421_calls < 2: setattr(self, 'num_421_calls', num_421_calls + 1) pat = re.compile(r'^attestation ', flags=re.IGNORECASE) self.chal = parse_dict_header(pat.sub('', attestation_challenge, count=1)) # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.raw.release_conn() prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) prep.headers['Attestation'] = self.build_attestation_header(prep) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r setattr(self, 'num_421_calls', 1) return r
def handle_401(self, r, **kwargs): """Takes the given response and tries using fresh OAuth tokens, if needed.""" if r.status_code != 401: return r if self.pos is not None: # Rewind the file position indicator of the body to where # it was to resend the request. r.request.body.seek(self.pos) num_401_calls = getattr(self, 'num_401_calls', 1) if num_401_calls < 2: self.num_401_calls += 1 # Consume content and release the original connection # to allow our new request to reuse the same one. r.content r.close() prep = r.request.copy() if "cookies" in prep.headers: extract_cookies_to_jar( prep.headers.get("cookies", None), r.request, r.raw) prep.prepare_cookies(prep._cookies) self.refresh() prep.headers['Authorization'] = self.access_token _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep return _r self.num_401_calls = 1 return r
def send(self, request, **kwargs): """Send a given PreparedRequest.""" # Set defaults that the hooks can utilize to ensure they always have # the correct parameters to reproduce the previous request. kwargs.setdefault('stream', self.stream) kwargs.setdefault('verify', self.verify) kwargs.setdefault('cert', self.cert) kwargs.setdefault('proxies', self.proxies) # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. if not isinstance(request, PreparedRequest): raise ValueError('You can only send PreparedRequests.') checked_urls = set() while request.url in self.redirect_cache: checked_urls.add(request.url) new_url = self.redirect_cache.get(request.url) if new_url in checked_urls: break request.url = new_url # Set up variables needed for resolve_redirects and dispatching of hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') hooks = request.hooks # Get the appropriate adapter to use # Start time (approximately) of the request start = datetime.utcnow() # Send the request r = adapter.send(request, **kwargs) # Total elapsed time of the request (approximately) r.elapsed = datetime.utcnow() - start # Response manipulation hooks r = dispatch_hook('response', hooks, r, **kwargs) # Persist cookies if r.history: # If the hooks create history then we want those cookies too for resp in r.history: extract_cookies_to_jar(self.cookies, resp.request, resp.raw) extract_cookies_to_jar(self.cookies, request, r.raw) # Redirect resolving generator. gen = self.resolve_redirects(r, request, **kwargs) # Resolve redirects if allowed. history = [resp for resp in gen] if allow_redirects else [] # Shuffle things around if there's history. if history: # Insert the first (original) request at the start history.insert(0, r) # Get the last request made r = history.pop() r.history = history if not stream: r.content return r
def deserialize(self): """Turn a serialized interaction into a Response.""" r = util.deserialize_response(self.data['response']) r.request = util.deserialize_prepared_request(self.data['request']) extract_cookies_to_jar(r.cookies, r.request, r.raw) return r
def _handle_401(self, response, **kwargs): """ Takes the response authenticates and resends if neccissary :return: The final response to the authenticated request :rtype: requests.Response """ # If response is not 401 do not auth and return response if not response.status_code == 401: self._thread_local.auth_attempted = False return response # If we've already attempted to auth for this request once, do not auth and return response if self._thread_local.auth_attempted: self._thread_local.auth_attempted = False return response auth_header = response.headers.get('www-authenticate', '') # Otherwise authenticate and retry the request self._thread_local.auth_attempted = True # parse the challenge challenge = HttpChallenge(response.request.url, auth_header, response.headers) # bearer and PoP are the only authentication schemes supported at this time # if the response auth header is not a bearer challenge or pop challange do not auth and return response if not (challenge.is_bearer_challenge() or challenge.is_pop_challenge()): self._thread_local.auth_attempted = False return response # add the challenge to the cache ChallengeCache.set_challenge_for_url(response.request.url, challenge) # Consume content and release the original connection # to allow our new request to reuse the same one. response.content response.close() # copy the request to resend prep = response.request.copy() if self._thread_local.orig_body is not None: # replace the body with the saved body prep.prepare_body(data=self._thread_local.orig_body, files=None) extract_cookies_to_jar(prep._cookies, response.request, response.raw) prep.prepare_cookies(prep._cookies) security = self._get_message_security(prep, challenge) # auth and protect the prepped request message security.protect_request(prep) # resend the request with proper authentication and message protection _response = response.connection.send(prep, **kwargs) _response.history.append(response) _response.request = prep # unprotected the response security.unprotect_response(_response) return _response
def deserialize(self): """Turn a serialized interaction into a Response.""" r = util.deserialize_response(self.json['response']) r.request = util.deserialize_prepared_request(self.json['request']) extract_cookies_to_jar(r.cookies, r.request, r.raw) return r
def http_fetch(self, url, fetch): """ HTTP fetcher """ start_time = time.time() def handle_error(x): BaseCrawler.handle_error(url, start_time, x) max_redirects = self.max_redirects # we will handle redirects by hand to capture cookies fetch['follow_redirects'] = False # making requests while True: try: request = tornado.httpclient.HTTPRequest(**fetch) # if cookie already in header, get_cookie_header wouldn't work old_cookie_header = request.headers.get('Cookie') if old_cookie_header: del request.headers['Cookie'] cookie_header = cookies.get_cookie_header( self._cookies, request) if cookie_header: request.headers['Cookie'] = cookie_header elif old_cookie_header: request.headers['Cookie'] = old_cookie_header except Exception as e: self.exception(e) raise gen.Return(handle_error(e)) try: response = yield gen.maybe_future( self.http_client.fetch(request)) except tornado.httpclient.HTTPError as e: if e.response: response = e.response else: raise gen.Return(handle_error(e)) except Exception as e: raise gen.Return(handle_error(e)) cookies.extract_cookies_to_jar(self._cookies, response.request, response) if response.code in (301, 302, 303, 307) and response.headers.get('Location'): if max_redirects <= 0: error = CDSpiderCrawlerBadRequest( 599, 'Maximum (%d) redirects followed' % fetch.get('max_redirects', 5), response) raise gen.Return(handle_error(error)) if response.code in (302, 303): fetch['method'] = 'GET' if 'body' in fetch: del fetch['body'] fetch['url'] = utils.quote_chinese( urljoin(fetch['url'], response.headers['Location'])) fetch['request_timeout'] -= time.time() - start_time if fetch['request_timeout'] < 0: fetch['request_timeout'] = 0.1 max_redirects -= 1 continue else: error = self._prepare_response(response.code, url) if error is not None: raise gen.Return(handle_error(error)) self.gen_result(url=response.effective_url or url, code=response.code, headers=dict(response.headers), cookies=self._cookies.get_dict(), content=response.body or '', start_time=start_time, error=response.error)
def build_response(self, request, resp): """ Builds a Requests' response object. This emulates most of the logic of the standard fuction but deals with the lack of the ``.headers`` property on the HTTP20Response object. Additionally, this function builds in a number of features that are purely for HTTPie. This is to allow maximum compatibility with what urllib3 does, so that HTTPie doesn't fall over when it uses us. """ response = Response() response.status_code = resp.status response.headers = CaseInsensitiveDict(resp.headers.iter_raw()) response.raw = resp response.reason = resp.reason response.encoding = get_encoding_from_headers(response.headers) extract_cookies_to_jar(response.cookies, request, response) response.url = request.url response.request = request response.connection = self # First horrible patch: Requests expects its raw responses to have a # release_conn method, which I don't. We should monkeypatch a no-op on. resp.release_conn = lambda: None # Next, add the things HTTPie needs. It needs the following things: # # - The `raw` object has a property called `_original_response` that is # a `httplib` response object. # - `raw._original_response` has three simple properties: `version`, # `status`, `reason`. # - `raw._original_response.version` has one of three values: `9`, # `10`, `11`. # - `raw._original_response.msg` exists. # - `raw._original_response.msg._headers` exists and is an iterable of # two-tuples. # # We fake this out. Most of this exists on our response object already, # and the rest can be faked. # # All of this exists for httpie, which I don't have any tests for, # so I'm not going to bother adding test coverage for it. class FakeOriginalResponse(object): # pragma: no cover def __init__(self, headers): self._headers = headers def get_all(self, name, default=None): values = [] for n, v in self._headers: if n == name.lower(): values.append(v) if not values: return default return values def getheaders(self, name): return self.get_all(name, []) response.raw._original_response = orig = FakeOriginalResponse(None) orig.version = 20 orig.status = resp.status orig.reason = resp.reason orig.msg = FakeOriginalResponse(resp.headers.iter_raw()) return response