def build_response_into(self, response, req, urllib3_resp):
"""Same as requests.adapters.HTTPAdapter.build_response, but writes
into a provided requests.Response object instead of creating a new one.
"""
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(urllib3_resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(
getattr(urllib3_resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = urllib3_resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, urllib3_resp)
# Give the Response some context.
response.request = req
response.connection = self
def handle_auth(self, r, **kwargs):
"""Takes the given response and tries SAML auth, if needed."""
if r.status_code not in [ 401, 403 ]:
return r
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
cookie_dict = requests.utils.dict_from_cookiejar(prep._cookies)
self.session.auth = None
self.session.get_sso_cookie()
self.session.auth = self
prep.prepare_cookies(self.session.cookies)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def handle_auth(self, response, **kwargs):
'''Takes the given response and tries SAML auth, if needed.'''
if response.status_code not in [401, 403]:
return response
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content # pylint: disable=pointless-statement
response.raw.release_conn()
prep = response.request.copy()
# PreparedResponse does not have a method to get _cookies.
# This is reuse of code from python requests.auth
# pylint: disable=protected-access
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
self.session.auth = None
self.session.get_sso_cookie()
self.session.auth = self
prep.prepare_cookies(self.session.cookies)
new_response = response.connection.send(prep, **kwargs)
new_response.history.append(response)
new_response.request = prep
return new_response
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.getheaders())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
if isinstance(request.url, bytes):
response.url = request.url.decode('utf-8')
else:
response.url = request.url
response.request = request
response.connection = self
# One last horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
return response
def handle_401(self, response, repo, **kwargs):
"""Fetch Bearer token and retry."""
if response.status_code != requests.codes.unauthorized:
return response
auth_info = response.headers.get('www-authenticate', '')
if 'bearer' not in auth_info.lower():
return response
self._token_cache[repo] = self._get_token(auth_info, repo)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
# This pattern was inspired by the source code of requests.auth.HTTPDigestAuth
response.content
response.close()
retry_request = response.request.copy()
extract_cookies_to_jar(retry_request._cookies, response.request, response.raw)
retry_request.prepare_cookies(retry_request._cookies)
self._set_header(retry_request, repo)
retry_response = response.connection.send(retry_request, **kwargs)
retry_response.history.append(response)
retry_response.request = retry_request
return retry_response
def handle_500(self, r, **kwargs):
if not self.enable_retry_on_500 \
or r.status_code < 500 or r.status_code >= 600:
return r
num_500_calls = getattr(self, 'num_500_calls', 1)
if num_500_calls < 4:
logger.debug("Got %s status, retrying" % r.status_code)
self.num_500_calls += 1
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
if "cookies" in prep.headers:
extract_cookies_to_jar(prep.headers.get("cookies", None),
r.request, r.raw)
prep.prepare_cookies(prep._cookies)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self.num_500_calls = 1
return r
def handle_auth(self, response, **kwargs):
'''Takes the given response and tries SAML auth, if needed.'''
if response.status_code not in [401, 403]:
return response
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content # pylint: disable=pointless-statement
response.raw.release_conn()
prep = response.request.copy()
# PreparedResponse does not have a method to get _cookies.
# This is reuse of code from python requests.auth
# pylint: disable=protected-access
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
self.session.auth = None
self.session.get_sso_cookie()
self.session.auth = self
prep.prepare_cookies(self.session.cookies)
new_response = response.connection.send(prep, **kwargs)
new_response.history.append(response)
new_response.request = prep
return new_response
def build_response(self, req, resp):
"""Builds a :class:`Response <requests.Response>` object from a urllib3
response. This should not be called from user code, and is only exposed
for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
:param resp: The urllib3 response object.
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def handle_401(self, response, repo, **kwargs):
"""Fetch Bearer token and retry."""
if response.status_code != requests.codes.unauthorized:
return response
auth_info = response.headers.get('www-authenticate', '')
if 'bearer' not in auth_info.lower():
return response
self._token_cache[repo] = self._get_token(auth_info, repo)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
# This pattern was inspired by the source code of requests.auth.HTTPDigestAuth
response.content
response.close()
retry_request = response.request.copy()
extract_cookies_to_jar(retry_request._cookies, response.request,
response.raw)
retry_request.prepare_cookies(retry_request._cookies)
self._set_header(retry_request, repo)
retry_response = response.connection.send(retry_request, **kwargs)
retry_response.history.append(response)
retry_response.request = retry_request
return retry_response
def handle_401(self, r, **kwargs):
"""Resends a request with auth headers, if needed."""
www_authenticate = r.headers.get('www-authenticate', '').lower()
if 'basic' in www_authenticate:
if self.pos is not None:
r.request.body.seek(self.pos)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = RequestsCookieJar()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.auth = HTTPBasicAuth(self.username, self.password)
prep = self.auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
if 'digest' in www_authenticate:
self.auth = HTTPDigestAuth(self.username, self.password)
# Digest auth would resend the request by itself. We can take a
# shortcut here.
return self.auth.handle_401(r, **kwargs)
def to_requests_response(self):
"""Returns an instance of `requests.Response` based on this response.
Returns:
request.Response: the generated response.
"""
# Make sure that body is at position 0 before returning
self.body.seek(0)
urllib3_response = URLLib3Rresponse(
body=self.body,
headers=self.headers,
status=self.http_code,
request_method=self.request.method,
reason=self.reason,
preload_content=False
)
response = RequestResponse()
response.request = self.request
response.raw = urllib3_response
response.status_code = self.http_code
response.reason = self.reason
response.headers = CaseInsensitiveDict(response.raw.headers)
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, self.request, urllib3_response)
if isinstance(self.request.url, six.binary_type):
response.url = self.request.url.decode("utf-8")
else:
response.url = self.request.url
return response
def http_fetch(self, url, fetch):
"""
HTTP fetcher
"""
start_time = time.time()
def handle_error(x):
BaseCrawler.handle_error(url, start_time, x)
# making requests
while True:
try:
response = yield gen.maybe_future(self._request(url, fetch))
except HTTPError as e:
if e.response:
response = e.response
else:
raise gen.Return(handle_error(e))
except Exception as e:
raise gen.Return(handle_error(e))
cookies.extract_cookies_to_jar(self._cookies, response.request,
response)
error = self._prepare_response(response.status_code, url)
if error is not None:
raise gen.Return(handle_error(error))
self.gen_result(url=response.url or url,
code=response.status_code,
headers=dict(response.headers),
cookies=self._cookies.get_dict(),
content=response.content or '',
start_time=start_time)
def handle_407(self, r, **kwargs):
if r.status_code == 407 and self.stale_rejects < 2:
pat = re.compile(r'digest ', flags=re.IGNORECASE)
if "proxy-authenticate" not in r.headers:
raise IOError(
"proxy server violated RFC 7235:"
"407 response MUST contain header proxy-authenticate")
self.chal = parse_dict_header(
pat.sub('', r.headers['proxy-authenticate'], count=1))
if 'Proxy-Authorization' in r.request.headers and 'stale' in self.chal:
if self.chal['stale'].lower() == 'true': # try again
self.stale_rejects += 1
elif self.chal['stale'].lower() == 'false':
raise IOError("User or password is invalid")
r.content
r.raw.release_conn()
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
else: # give up authenticate
return r
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.getheaders())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
response.url = request.url
response.request = request
response.connection = self
# One last horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
return response
def handle_401(self, r, **kwargs):
"""Resends a request with auth headers, if needed."""
www_authenticate = r.headers.get('www-authenticate', '').lower()
if 'basic' in www_authenticate:
if self.pos is not None:
r.request.body.seek(self.pos)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = RequestsCookieJar()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.auth = HTTPBasicAuth(self.username, self.password)
prep = self.auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
if 'digest' in www_authenticate:
self.auth = HTTPDigestAuth(self.username, self.password)
# Digest auth would resend the request by itself. We can take a
# shortcut here.
return self.auth.handle_401(r, **kwargs)
def build_response(self, req, resp):
"""Builds a :class:`Response <requests.Response>` object from a urllib3
response. This should not be called from user code, and is only exposed
for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
:param resp: The urllib3 response object.
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def handle_401(self, response, repo, **kwargs):
"""Fetch Bearer token and retry."""
if response.status_code != requests.codes.unauthorized:
return response
auth_info = response.headers.get('www-authenticate', '')
if 'bearer' not in auth_info.lower():
return response
self._token_cache[repo] = self._get_token(auth_info, repo)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
# This pattern was inspired by the source code of requests.auth.HTTPDigestAuth
response.content # pylint: disable=pointless-statement; is a property
response.close()
retry_request = response.request.copy()
extract_cookies_to_jar(retry_request._cookies, response.request, response.raw)
retry_request.prepare_cookies(retry_request._cookies)
self._set_header(retry_request, repo)
retry_response = response.connection.send(retry_request, **kwargs)
retry_response.history.append(response)
retry_response.request = retry_request
# when quay returns 401 even after we have token,
# repository doesn't exist at all, so we will treat it as 404
if retry_response.status_code == requests.codes.unauthorized:
retry_response.status_code = requests.codes.not_found
return retry_response
def http_response_to_response(self, http_response, prepared_request):
"""
transform a WSGIResponse into a requests's Response model
:param django.http.response.HttpResponse http_response: the http response send by django view
:return: the requests's Response model corresponding to the http_response
:rtype: Response
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(http_response, 'status_code', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(http_response._headers, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = http_response
response.reason = response.raw.reason_phrase
response._content = http_response.content
req = prepared_request
if isinstance(req.url, bytes): # pragma: no cover
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, response)
# Give the Response some context.
response.request = req
response.connection = self
return response
def deserialize(self):
"""Turn a serialized interaction into a Response."""
r = util.deserialize_response(self.json['response'])
r.request = util.deserialize_prepared_request(self.json['request'])
extract_cookies_to_jar(r.cookies, r.request, r.raw)
self.recorded_at = datetime.strptime(self.json['recorded_at'],
'%Y-%m-%dT%H:%M:%S')
self.recorded_response = r
def deserialize(self):
"""Turn a serialized interaction into a Response."""
r = deserialize_response(self.json['response'])
r.request = deserialize_prepared_request(self.json['request'])
extract_cookies_to_jar(r.cookies, r.request, r.raw)
self.recorded_at = datetime.strptime(
self.json['recorded_at'], '%Y-%m-%dT%H:%M:%S'
)
self.recorded_response = r
def start_response(status, headers, exc_info=None):
headers = make_headers(headers)
response.status_code = int(status.split(" ")[0])
response.reason = responses.get(response.status_code, "Unknown Status Code")
response.headers = headers
resp._original_response.msg = headers
extract_cookies_to_jar(response.cookies, request, resp)
response.encoding = get_encoding_from_headers(response.headers)
response.elapsed = datetime.datetime.utcnow() - start
self._log(response)
def send(self, request, **kwargs) -> UpgradeResponse:
kwargs.setdefault('stream', self.stream)
kwargs.setdefault('verify', self.verify)
kwargs.setdefault('cert', self.cert)
kwargs.setdefault('proxies', self.proxies)
if isinstance(request, requests.Request):
raise ValueError('You can only send PreparedRequests.')
if isinstance(request, HttpRequest):
request.check()
allow_redirects = kwargs.pop('allow_redirects', True)
hooks = request.hooks
adapter = self.get_adapter(url=request.url)
start = preferred_clock()
r = adapter.send(request, **kwargs)
elapsed = preferred_clock() - start
r.elapsed = timedelta(seconds=elapsed)
# this function not change response and result params
if len(self._dispatch_hooks) > 0:
dispatch(self._dispatch_hooks, hooks, r, **kwargs)
# dispatch hook will change response and request params
r = dispatch_hook('response', hooks, r, **kwargs)
if r.history:
for resp in r.history:
extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
extract_cookies_to_jar(self.cookies, request, r.raw)
gen = self.resolve_redirects(r, request, **kwargs)
history = [resp for resp in gen] if allow_redirects else []
if history:
history.insert(0, r)
r = history.pop()
r.history = history
if not allow_redirects:
try:
r._next = next(
self.resolve_redirects(r,
request,
yield_requests=True,
**kwargs))
except StopIteration:
pass
return r
def handle_407(response, **kwargs):
"""
Prompts the user for the proxy username and password and modifies the
proxy in the session object to include it.
This method is modeled after
* requests.auth.HTTPDigestAuth.handle_401()
* requests.auth.HTTPProxyAuth
* the previous conda.fetch.handle_proxy_407()
It both adds 'username:password' to the proxy URL, as well as adding a
'Proxy-Authorization' header. If any of this is incorrect, please file an issue.
"""
# kwargs = {'verify': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False,
# 'timeout': (3.05, 60)}
if response.status_code != 407:
return response
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content
response.close()
proxies = kwargs.pop('proxies')
proxy_scheme = urlparse(response.url).scheme
if proxy_scheme not in proxies:
raise ProxyError(
dals("""
Could not find a proxy for %r. See
%s/docs/html#configure-conda-for-use-behind-a-proxy-server
for more information on how to configure proxies.
""" % (proxy_scheme, CONDA_HOMEPAGE_URL)))
# fix-up proxy_url with username & password
proxy_url = proxies[proxy_scheme]
username, password = get_proxy_username_and_pass(proxy_scheme)
proxy_url = add_username_and_password(proxy_url, username, password)
proxy_authorization_header = _basic_auth_str(username, password)
proxies[proxy_scheme] = proxy_url
kwargs['proxies'] = proxies
prep = response.request.copy()
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = proxy_authorization_header
_response = response.connection.send(prep, **kwargs)
_response.history.append(response)
_response.request = prep
return _response
def handle_407(response, **kwargs):
"""
Prompts the user for the proxy username and password and modifies the
proxy in the session object to include it.
This method is modeled after
* requests.auth.HTTPDigestAuth.handle_401()
* requests.auth.HTTPProxyAuth
* the previous conda.fetch.handle_proxy_407()
It both adds 'username:password' to the proxy URL, as well as adding a
'Proxy-Authorization' header. If any of this is incorrect, please file an issue.
"""
# kwargs = {'verify': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False,
# 'timeout': (3.05, 60)}
if response.status_code != 407:
return response
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content
response.close()
proxies = kwargs.pop('proxies')
proxy_scheme = urlparse(response.url).scheme
if proxy_scheme not in proxies:
raise ProxyError(dals("""
Could not find a proxy for %r. See
%s/docs/html#configure-conda-for-use-behind-a-proxy-server
for more information on how to configure proxies.
""" % (proxy_scheme, CONDA_HOMEPAGE_URL)))
# fix-up proxy_url with username & password
proxy_url = proxies[proxy_scheme]
username, password = get_proxy_username_and_pass(proxy_scheme)
proxy_url = add_username_and_password(proxy_url, username, password)
proxy_authorization_header = _basic_auth_str(username, password)
proxies[proxy_scheme] = proxy_url
kwargs['proxies'] = proxies
prep = response.request.copy()
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = proxy_authorization_header
_response = response.connection.send(prep, **kwargs)
_response.history.append(response)
_response.request = prep
return _response
def handle_401(self, response, **kwargs):
"""
Takes the response authenticates and resends if neccissary
:return: The final response to the authenticated request
:rtype: requests.Response
"""
# If response is not 401 do not auth and return response
if not response.status_code == 401:
self._thread_local.auth_attempted = False
return response
auth_header = response.headers.get('www-authenticate', '')
# if the response auth header is not a bearer challenge do not auth and return response
if not HttpBearerChallenge.is_bearer_challenge(auth_header):
self._thread_local.auth_attempted = False
return response
# If we've already attempted to auth for this request once, do not auth and return response
if self._thread_local.auth_attempted:
self._thread_local.auth_attempted = False
return response
# Otherwise authenticate and retry the request
self._thread_local.auth_attempted = True
if self._thread_local.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
response.request.body.seek(self._thread_local.pos)
# add the challenge to the cache
challenge = HttpBearerChallenge(response.request.url, auth_header)
ChallengeCache.set_challenge_for_url(response.request.url, challenge)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content
response.close()
# copy the request to resend
prep = response.request.copy()
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
prep.prepare_cookies(prep._cookies)
# setup the auth header on the copied request
self.set_authorization_header(prep, challenge)
# resend the request with proper authentication
_response = response.connection.send(prep, **kwargs)
_response.history.append(response)
_response.request = prep
return _response
def handle_401(self, response, **kwargs):
"""
Takes the response authenticates and resends if neccissary
:return: The final response to the authenticated request
:rtype: requests.Response
"""
# If response is not 401 do not auth and return response
if not response.status_code == 401:
self._thread_local.auth_attempted = False
return response
auth_header = response.headers.get('www-authenticate', '')
# if the response auth header is not a bearer challenge do not auth and return response
if not HttpBearerChallenge.is_bearer_challenge(auth_header):
self._thread_local.auth_attempted = False
return response
# If we've already attempted to auth for this request once, do not auth and return response
if self._thread_local.auth_attempted:
self._thread_local.auth_attempted = False
return response
# Otherwise authenticate and retry the request
self._thread_local.auth_attempted = True
if self._thread_local.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
response.request.body.seek(self._thread_local.pos)
# add the challenge to the cache
challenge = HttpBearerChallenge(response.request.url, auth_header)
ChallengeCache.set_challenge_for_url(response.request.url, challenge)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content
response.close()
# copy the request to resend
prep = response.request.copy()
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
prep.prepare_cookies(prep._cookies)
# setup the auth header on the copied request
self.set_authorization_header(prep, challenge)
# resend the request with proper authentication
_response = response.connection.send(prep, **kwargs)
_response.history.append(response)
_response.request = prep
return _response
def build_response(self, req, resp):
response = PlexResponse()
response.status_code = getattr(resp, 'status', None)
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
extract_cookies_to_jar(response.cookies, req, resp)
response.request = req
response.connection = self
return response
def merge_cookies_from_response(cookie_jar, http_response, url):
p = models.PreparedRequest()
p.prepare(url=url)
cookies.extract_cookies_to_jar(cookie_jar, p, http_response)
# cookie_list_str = http_response.info().getlist('set-cookie')
# cookie_jar._now = int(time.time())
# cookie_set = cookiejar.parse_ns_headers(cookie_list_str)
# cookie_tuples = cookie_jar._normalized_cookie_tuples(cookie_set)
#
# for tup in cookie_tuples:
# cookie = mark_cookie(tup)
# if cookie:
# cookie_jar.set_cookie(cookie)
return cookie_jar
def build_response(self, req: BitexPreparedRequest,
resp: HTTPResponse) -> BitexResponse:
"""Build a :class:`BitexResponse` from the given `req` and `resp`.
The method is largely identical to :meth:`HTTPAdapter.build_response`,
and only differs in the class type used when constructing a response.
This class is taken firstly from any valid plugin that supplies an
adequate class for the exchange that was queried (as stated in
:attr:`BitexPreparedRequest.exchange`), or :mod:`bitex-framework` 's own default
:class:`BitexResponse` class.
:param BitexPreparedRequest req:
The :class:`BitexPreparedRequest` used to generate the response.
:param HTTPResponse resp: The urllib3 response object.
"""
if req.exchange in list_loaded_plugins():
response = list_loaded_plugins()[req.exchange]["Response"]()
else:
response = BitexResponse()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, "status", None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode("utf-8")
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def handle_407(self, r, **kwargs):
"""Handle HTTP 407 only once, otherwise give up
:param r: current response
:returns: responses, along with the new response
"""
if r.status_code == 407 and self.stale_rejects < 2:
s_auth = r.headers.get("proxy-authenticate")
if s_auth is None:
raise IOError(
"proxy server violated RFC 7235:"
"407 response MUST contain header proxy-authenticate")
elif not self._pat.match(s_auth):
return r
self.chal = utils.parse_dict_header(
self._pat.sub('', s_auth, count=1))
# if we present the customuser/passwd and still get rejected
# http://tools.ietf.org/html/rfc2617#section-3.2.1
if ('Proxy-Authorization' in r.request.headers and
'stale' in self.chal):
if self.chal['stale'].lower() == 'true': # try again
self.stale_rejects += 1
# wrong customuser/passwd
elif self.chal['stale'].lower() == 'false':
raise IOError("User or password is invalid")
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
else: # give up authenticate
return r
def handle_407(self, r, **kwargs):
"""Handle HTTP 407 only once, otherwise give up
:param r: current response
:returns: responses, along with the new response
"""
if r.status_code == 407 and self.stale_rejects < 2:
s_auth = r.headers.get("proxy-authenticate")
if s_auth is None:
raise IOError(
"proxy server violated RFC 7235:"
"407 response MUST contain header proxy-authenticate")
elif not self._pat.match(s_auth):
return r
self.chal = utils.parse_dict_header(
self._pat.sub('', s_auth, count=1))
# if we present the user/passwd and still get rejected
# http://tools.ietf.org/html/rfc2617#section-3.2.1
if ('Proxy-Authorization' in r.request.headers and
'stale' in self.chal):
if self.chal['stale'].lower() == 'true': # try again
self.stale_rejects += 1
# wrong user/passwd
elif self.chal['stale'].lower() == 'false':
raise IOError("User or password is invalid")
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
else: # give up authenticate
return r
def handle_401(self, r, **kwargs):
"""
Takes the given response and tries digest-auth, if needed.
:rtype: requests.Response
"""
# If response is not 4xx, do not auth
# See https://github.com/psf/requests/issues/3772
if not 400 <= r.status_code < 500:
self._thread_local.num_401_calls = 1
return r
if self._thread_local.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
r.request.body.seek(self._thread_local.pos)
s_auth = r.headers.get('www-authenticate', '')
if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2:
self._thread_local.num_401_calls += 1
pat = re.compile(r'digest ', flags=re.IGNORECASE)
self._thread_local.chal = parse_dict_header(
pat.sub('', s_auth, count=1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self._thread_local.num_401_calls = 1
return r
def handle_401(self, r, **kwargs):
if r.status_code == 401:
LOG.debug('Got 401. Trying to reauth...')
self.token = self.https_auth()
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Authorization'] = 'Bearer %s' % self.token
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
return r
def _handle_basic_auth_407(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, "_cookies"):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username,
self.proxy_password)
prep = self.proxy_auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_basic_auth_407(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username,
self.proxy_password)
prep = self.proxy_auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_basic_auth_401(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.auth = auth.HTTPBasicAuth(self.username, self.password)
prep = self.auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_basic_auth_401(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, "_cookies"):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.auth = auth.HTTPBasicAuth(self.username, self.password)
prep = self.auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def persist_cookies(self, r):
"""
From requests/sessions.py, Session.send()
Session.send() 方法会首先 dispatch_hook 然后再 extract_cookies_to_jar
在该项目中，对于返回信息异常的请求，在 hooks 校验时会将错误抛出，send() 之后的处理将不会执行。
遇到的错误往往是 SystemException / TipsException ，而这些客户端认为是错误的情况，
对于服务器端来说并不是错误请求，服务器端在该次请求结束后可能会要求 Set-Cookies
但是由于 send() 在 dispatch_hook 时遇到错误而中止，导致后面的 extract_cookies_to_jar
未能调用，因此 Cookies 并未更新。下一次再请求服务器的时候，就会遇到会话过期的情况。
在这种情况下，需要在捕获错误后手动更新 cookies 以确保能够保持会话
"""
if r.history:
# If the hooks create history then we want those cookies too
for resp in r.history:
extract_cookies_to_jar(self._session.cookies, resp.request, resp.raw)
extract_cookies_to_jar(self._session.cookies, r.request, r.raw)
def _check_and_login(
self,
response,
**kwargs
):
if not is_keycloak_login_url(response.url):
return response
post_url = parse_post_url(response.text)
if not post_url:
raise Exception("TODO")
if self._thread_local.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
response.request.body.seek(self._thread_local.pos)
# Content was consumed earlier when we parsed for post url
# Release the original connection to allow our new request to reuse the
# same one.
response.close()
prep = response.request.copy()
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
prep.prepare_cookies(prep._cookies)
prep.prepare_body(data=self._get_creds(), files=None)
prep.method = "POST"
prep.url = post_url
_r = response.connection.send(prep, **kwargs)
_r.history.append(response)
_r.request = prep
# TODO should do a full retry of the first request
return _r
def handle_421(self, r, **kwargs):
"""
Given a response, if is 421 parses out the
Attestation-Challenge header and tries again
with an attestation. Will try once.
"""
if self.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
r.request.body.seek(self.pos)
num_421_calls = getattr(self, 'num_421_calls', 1)
attestation_challenge = r.headers.get('attestation-challenge', '')
if r.status_code == 421 and num_421_calls < 2:
setattr(self, 'num_421_calls', num_421_calls + 1)
pat = re.compile(r'^attestation ', flags=re.IGNORECASE)
self.chal = parse_dict_header(pat.sub('', attestation_challenge, count=1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Attestation'] = self.build_attestation_header(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
setattr(self, 'num_421_calls', 1)
return r
def handle_401(self, r, **kwargs):
"""Takes the given response and tries using fresh OAuth tokens,
if needed."""
if r.status_code != 401:
return r
if self.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
r.request.body.seek(self.pos)
num_401_calls = getattr(self, 'num_401_calls', 1)
if num_401_calls < 2:
self.num_401_calls += 1
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
if "cookies" in prep.headers:
extract_cookies_to_jar(
prep.headers.get("cookies", None), r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.refresh()
prep.headers['Authorization'] = self.access_token
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self.num_401_calls = 1
return r
def send(self, request, **kwargs):
"""Send a given PreparedRequest."""
# Set defaults that the hooks can utilize to ensure they always have
# the correct parameters to reproduce the previous request.
kwargs.setdefault('stream', self.stream)
kwargs.setdefault('verify', self.verify)
kwargs.setdefault('cert', self.cert)
kwargs.setdefault('proxies', self.proxies)
# It's possible that users might accidentally send a Request object.
# Guard against that specific failure case.
if not isinstance(request, PreparedRequest):
raise ValueError('You can only send PreparedRequests.')
checked_urls = set()
while request.url in self.redirect_cache:
checked_urls.add(request.url)
new_url = self.redirect_cache.get(request.url)
if new_url in checked_urls:
break
request.url = new_url
# Set up variables needed for resolve_redirects and dispatching of hooks
allow_redirects = kwargs.pop('allow_redirects', True)
stream = kwargs.get('stream')
hooks = request.hooks
# Get the appropriate adapter to use
# Start time (approximately) of the request
start = datetime.utcnow()
# Send the request
r = adapter.send(request, **kwargs)
# Total elapsed time of the request (approximately)
r.elapsed = datetime.utcnow() - start
# Response manipulation hooks
r = dispatch_hook('response', hooks, r, **kwargs)
# Persist cookies
if r.history:
# If the hooks create history then we want those cookies too
for resp in r.history:
extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
extract_cookies_to_jar(self.cookies, request, r.raw)
# Redirect resolving generator.
gen = self.resolve_redirects(r, request, **kwargs)
# Resolve redirects if allowed.
history = [resp for resp in gen] if allow_redirects else []
# Shuffle things around if there's history.
if history:
# Insert the first (original) request at the start
history.insert(0, r)
# Get the last request made
r = history.pop()
r.history = history
if not stream:
r.content
return r
def deserialize(self):
"""Turn a serialized interaction into a Response."""
r = util.deserialize_response(self.data['response'])
r.request = util.deserialize_prepared_request(self.data['request'])
extract_cookies_to_jar(r.cookies, r.request, r.raw)
return r
def _handle_401(self, response, **kwargs):
"""
Takes the response authenticates and resends if neccissary
:return: The final response to the authenticated request
:rtype: requests.Response
"""
# If response is not 401 do not auth and return response
if not response.status_code == 401:
self._thread_local.auth_attempted = False
return response
# If we've already attempted to auth for this request once, do not auth and return response
if self._thread_local.auth_attempted:
self._thread_local.auth_attempted = False
return response
auth_header = response.headers.get('www-authenticate', '')
# Otherwise authenticate and retry the request
self._thread_local.auth_attempted = True
# parse the challenge
challenge = HttpChallenge(response.request.url, auth_header, response.headers)
# bearer and PoP are the only authentication schemes supported at this time
# if the response auth header is not a bearer challenge or pop challange do not auth and return response
if not (challenge.is_bearer_challenge() or challenge.is_pop_challenge()):
self._thread_local.auth_attempted = False
return response
# add the challenge to the cache
ChallengeCache.set_challenge_for_url(response.request.url, challenge)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
response.content
response.close()
# copy the request to resend
prep = response.request.copy()
if self._thread_local.orig_body is not None:
# replace the body with the saved body
prep.prepare_body(data=self._thread_local.orig_body, files=None)
extract_cookies_to_jar(prep._cookies, response.request, response.raw)
prep.prepare_cookies(prep._cookies)
security = self._get_message_security(prep, challenge)
# auth and protect the prepped request message
security.protect_request(prep)
# resend the request with proper authentication and message protection
_response = response.connection.send(prep, **kwargs)
_response.history.append(response)
_response.request = prep
# unprotected the response
security.unprotect_response(_response)
return _response
def deserialize(self):
"""Turn a serialized interaction into a Response."""
r = util.deserialize_response(self.json['response'])
r.request = util.deserialize_prepared_request(self.json['request'])
extract_cookies_to_jar(r.cookies, r.request, r.raw)
return r
def http_fetch(self, url, fetch):
"""
HTTP fetcher
"""
start_time = time.time()
def handle_error(x):
BaseCrawler.handle_error(url, start_time, x)
max_redirects = self.max_redirects
# we will handle redirects by hand to capture cookies
fetch['follow_redirects'] = False
# making requests
while True:
try:
request = tornado.httpclient.HTTPRequest(**fetch)
# if cookie already in header, get_cookie_header wouldn't work
old_cookie_header = request.headers.get('Cookie')
if old_cookie_header:
del request.headers['Cookie']
cookie_header = cookies.get_cookie_header(
self._cookies, request)
if cookie_header:
request.headers['Cookie'] = cookie_header
elif old_cookie_header:
request.headers['Cookie'] = old_cookie_header
except Exception as e:
self.exception(e)
raise gen.Return(handle_error(e))
try:
response = yield gen.maybe_future(
self.http_client.fetch(request))
except tornado.httpclient.HTTPError as e:
if e.response:
response = e.response
else:
raise gen.Return(handle_error(e))
except Exception as e:
raise gen.Return(handle_error(e))
cookies.extract_cookies_to_jar(self._cookies, response.request,
response)
if response.code in (301, 302, 303,
307) and response.headers.get('Location'):
if max_redirects <= 0:
error = CDSpiderCrawlerBadRequest(
599, 'Maximum (%d) redirects followed' %
fetch.get('max_redirects', 5), response)
raise gen.Return(handle_error(error))
if response.code in (302, 303):
fetch['method'] = 'GET'
if 'body' in fetch:
del fetch['body']
fetch['url'] = utils.quote_chinese(
urljoin(fetch['url'], response.headers['Location']))
fetch['request_timeout'] -= time.time() - start_time
if fetch['request_timeout'] < 0:
fetch['request_timeout'] = 0.1
max_redirects -= 1
continue
else:
error = self._prepare_response(response.code, url)
if error is not None:
raise gen.Return(handle_error(error))
self.gen_result(url=response.effective_url or url,
code=response.code,
headers=dict(response.headers),
cookies=self._cookies.get_dict(),
content=response.body or '',
start_time=start_time,
error=response.error)
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
Additionally, this function builds in a number of features that are
purely for HTTPie. This is to allow maximum compatibility with what
urllib3 does, so that HTTPie doesn't fall over when it uses us.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.headers.iter_raw())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
response.url = request.url
response.request = request
response.connection = self
# First horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
# Next, add the things HTTPie needs. It needs the following things:
#
# - The `raw` object has a property called `_original_response` that is
# a `httplib` response object.
# - `raw._original_response` has three simple properties: `version`,
# `status`, `reason`.
# - `raw._original_response.version` has one of three values: `9`,
# `10`, `11`.
# - `raw._original_response.msg` exists.
# - `raw._original_response.msg._headers` exists and is an iterable of
# two-tuples.
#
# We fake this out. Most of this exists on our response object already,
# and the rest can be faked.
#
# All of this exists for httpie, which I don't have any tests for,
# so I'm not going to bother adding test coverage for it.
class FakeOriginalResponse(object): # pragma: no cover
def __init__(self, headers):
self._headers = headers
def get_all(self, name, default=None):
values = []
for n, v in self._headers:
if n == name.lower():
values.append(v)
if not values:
return default
return values
def getheaders(self, name):
return self.get_all(name, [])
response.raw._original_response = orig = FakeOriginalResponse(None)
orig.version = 20
orig.status = resp.status
orig.reason = resp.reason
orig.msg = FakeOriginalResponse(resp.headers.iter_raw())
return response