def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=conf.timeout if 'timeout' in conf else None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies),
cookies or conf.cookie if 'cookie' in conf else None)
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(
headers, conf.http_headers if 'http_headers' in conf else {}),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or conf.proxies if 'proxies' in conf else {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
conf = CONF.get("requests", {})
if timeout is None and "timeout" in conf:
timeout = conf["timeout"]
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies
or (conf.cookie if "cookie" in conf else None))
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers,
conf["headers"] if "headers" in conf else {}),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or (conf["proxies"] if "proxies" in conf else {})
settings = self.merged_environment_settings(prep.url, proxies, stream,
verify, cert)
send_kwargs = {
"timeout": timeout,
"allow_redirects": allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == "ISO-8859-1":
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def session_request(self, method, url,
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
timeout=10,
allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None):
# Create the Request.
merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies),
cookies)
default_header = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
settings = self.merge_environment_settings(
prep.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
'timeout': TIMEOUT,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def prepare_request(self, request):
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for
transmission and returns it. The :class:`PreparedRequest` has settings
merged from the :class:`Request <Request>` instance and those of the
:class:`Session`.
:param request: :class:`Request` instance to prepare with this
session's settings.
:rtype: requests.PreparedRequest
"""
cookies = request.cookies or {}
# Bootstrap CookieJar.
if not isinstance(cookies, cookielib.CookieJar):
cookies = cookiejar_from_dict(cookies)
# Merge with session cookies
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies)
# Set environment's basic authentication if not explicitly set.
auth = request.auth
if self.trust_env and not auth and not self.auth:
auth = get_netrc_auth(request.url)
p = CuPreparedRequest()
p.prepare(
method=request.method.upper(),
url=request.url,
files=request.files,
data=request.data,
json=request.json,
headers=merge_setting(request.headers,
self.headers,
dict_class=CaseInsensitiveDict),
params=merge_setting(request.params, self.params),
auth=merge_setting(auth, self.auth),
cookies=merged_cookies,
hooks=merge_hooks(request.hooks, self.hooks),
)
return p
def session_request(self, method, url,
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
timeout=None,
allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None):
# Create the Request.
merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies),
cookies)
default_header = {
"User-Agent": conf.agent,
"Connection": "close"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
raw = ''
p = urlparse(url)
_headers = copy.deepcopy(prep.headers)
if "Host" not in _headers:
_headers["Host"] = p.netloc
if prep.body:
raw = "{}\n{}\n\n{}\n\n".format(
prep.method + ' ' + prep.url + ' HTTP/1.1',
'\n'.join('{}: {}'.format(k, v) for k, v in _headers.items()),
prep.body)
else:
raw = "{}\n{}\n\n".format(
prep.method + ' ' + prep.url + ' HTTP/1.1',
'\n'.join('{}: {}'.format(k, v) for k, v in _headers.items()))
proxies = proxies or {}
if conf["proxy_config_bool"] and not proxies:
proxies = conf["proxy"]
if "socks4" in proxies.keys():
_tmp_str = "socks4://" + proxies["socks4"]
_tmp_proxy = {
"http": _tmp_str,
"https": _tmp_str
}
proxies = _tmp_proxy
elif "socks5" in proxies.keys():
_tmp_str = "socks5://" + proxies["socks5"]
_tmp_proxy = {
"http": _tmp_str,
"https": _tmp_str
}
proxies = _tmp_proxy
# prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore')
# fix https://github.com/boy-hack/w13scan/issues/64
settings = self.merge_environment_settings(
prep.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
'timeout': timeout or conf["timeout"],
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
setattr(resp, 'reqinfo', raw)
return resp
async def send(self, request, **kwargs):
"""Send a given PreparedRequest.
:rtype: requests.Response
"""
allow_redirects = kwargs.pop('allow_redirects', True)
if not allow_redirects:
return await self._send(request, **kwargs)
history = []
while True:
resp = await self._send(request, **kwargs)
resp.history = history[:]
history.append(resp)
if not resp.is_redirect:
return resp
# Release the connection back into the pool.
await resp.close()
if len(history) > self.max_redirects:
raise TooManyRedirects('Exceeded %s redirects.' %
self.max_redirects,
response=resp)
next_request = request.copy()
next_request.url = self._get_next_url(resp)
next_request.method = self._get_next_method(resp)
logger.debug(
f'Redirect to: {next_request.method} {next_request.url}')
headers = next_request.headers
# https://github.com/requests/requests/issues/1084
if resp.status_code not in (307, 308):
# https://github.com/requests/requests/issues/3490
purged_headers = ('Content-Length', 'Content-Type',
'Transfer-Encoding')
for header in purged_headers:
next_request.headers.pop(header, None)
next_request.body = None
# Attempt to rewind consumed file-like object.
should_rewind = (('Content-Length' in headers
or 'Transfer-Encoding' in headers)
and isinstance(next_request.body,
(MultipartBody, StreamBody)))
if should_rewind:
logger.debug(
f'Rewind request body for redirection: {next_request}')
next_request.body.rewind()
try:
del headers['Cookie']
except KeyError:
pass
# Extract any cookies sent on the response to the cookiejar
# in the new request. Because we've mutated our copied prepared
# request, use the old one that we haven't yet touched.
extract_cookies_to_jar(next_request._cookies, request, resp.raw)
merge_cookies(next_request._cookies, self.cookies)
next_request.prepare_cookies(next_request._cookies)
self.rebuild_auth(next_request, resp)
# Override the original request.
request = next_request
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies)
default_header = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Connection": "close"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
raw = ''
if prep.body:
raw = "{}\n{}\n\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()),
prep.body)
else:
raw = "{}\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()))
proxies = proxies or {}
if conf["proxy_config_bool"] and not proxies:
proxies = conf["proxy"]
# prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore')
# fix https://github.com/boy-hack/w13scan/issues/64
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout or conf["timeout"],
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
if not getattr(resp, 'raw', None):
resp.raw = raw
else:
tmp = resp.raw
resp.raw = raw
setattr(resp, 'raw2', tmp)
return resp
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# In order to remove headers that are set to None
def _merge_retain_none(request_setting,
session_setting,
dict_class=OrderedDict):
if session_setting is None:
return request_setting
if request_setting is None:
return session_setting
# Bypass if not a dictionary (e.g. verify)
if not (isinstance(session_setting, Mapping)
and isinstance(request_setting, Mapping)):
return request_setting
merged_setting = dict_class(to_key_val_list(session_setting))
merged_setting.update(to_key_val_list(request_setting))
return merged_setting
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies
or (conf.cookie if 'cookie' in conf else None))
if conf.random_agent:
conf.http_headers[HTTP_HEADER.USER_AGENT] = choice(conf.agents)
req = Request(
method=method.upper(),
url=url,
headers=_merge_retain_none(
headers, conf.http_headers if 'http_headers' in conf else {}),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
# proxies = proxies or (conf.proxies if 'proxies' in conf else {})
if proxies is None:
proxies = conf.proxies if 'proxies' in conf else {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
timeout = timeout or conf.get("timeout", None)
if timeout:
timeout = float(timeout)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies)
default_header = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
raw = ''
if prep.body:
raw = "{}\n{}\n\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()),
prep.body)
else:
raw = "{}\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()))
proxies = proxies or {}
if PROXY_CONFIG_BOOL and not proxies:
proxies = PROXY_CONFIG
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout or TIMEOUT,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
resp.raw = raw
return resp
def _hook(self, rsp, **kwargs):
from requests.sessions import merge_cookies
self._cookies = merge_cookies(rsp.cookies, self._cookies)
