def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=conf.timeout if 'timeout' in conf else None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies or conf.cookie if 'cookie' in conf else None) req = Request( method=method.upper(), url=url, headers=merge_setting( headers, conf.http_headers if 'http_headers' in conf else {}), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or conf.proxies if 'proxies' in conf else {} settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): conf = CONF.get("requests", {}) if timeout is None and "timeout" in conf: timeout = conf["timeout"] merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies or (conf.cookie if "cookie" in conf else None)) req = Request( method=method.upper(), url=url, headers=merge_setting(headers, conf["headers"] if "headers" in conf else {}), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or (conf["proxies"] if "proxies" in conf else {}) settings = self.merged_environment_settings(prep.url, proxies, stream, verify, cert) send_kwargs = { "timeout": timeout, "allow_redirects": allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == "ISO-8859-1": encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=10, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or {} settings = self.merge_environment_settings( prep.url, proxies, stream, verify, cert ) # Send the request. send_kwargs = { 'timeout': TIMEOUT, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def prepare_request(self, request): """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it. The :class:`PreparedRequest` has settings merged from the :class:`Request <Request>` instance and those of the :class:`Session`. :param request: :class:`Request` instance to prepare with this session's settings. :rtype: requests.PreparedRequest """ cookies = request.cookies or {} # Bootstrap CookieJar. if not isinstance(cookies, cookielib.CookieJar): cookies = cookiejar_from_dict(cookies) # Merge with session cookies merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) # Set environment's basic authentication if not explicitly set. auth = request.auth if self.trust_env and not auth and not self.auth: auth = get_netrc_auth(request.url) p = CuPreparedRequest() p.prepare( method=request.method.upper(), url=request.url, files=request.files, data=request.data, json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, hooks=merge_hooks(request.hooks, self.hooks), ) return p
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": conf.agent, "Connection": "close" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) raw = '' p = urlparse(url) _headers = copy.deepcopy(prep.headers) if "Host" not in _headers: _headers["Host"] = p.netloc if prep.body: raw = "{}\n{}\n\n{}\n\n".format( prep.method + ' ' + prep.url + ' HTTP/1.1', '\n'.join('{}: {}'.format(k, v) for k, v in _headers.items()), prep.body) else: raw = "{}\n{}\n\n".format( prep.method + ' ' + prep.url + ' HTTP/1.1', '\n'.join('{}: {}'.format(k, v) for k, v in _headers.items())) proxies = proxies or {} if conf["proxy_config_bool"] and not proxies: proxies = conf["proxy"] if "socks4" in proxies.keys(): _tmp_str = "socks4://" + proxies["socks4"] _tmp_proxy = { "http": _tmp_str, "https": _tmp_str } proxies = _tmp_proxy elif "socks5" in proxies.keys(): _tmp_str = "socks5://" + proxies["socks5"] _tmp_proxy = { "http": _tmp_str, "https": _tmp_str } proxies = _tmp_proxy # prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore') # fix https://github.com/boy-hack/w13scan/issues/64 settings = self.merge_environment_settings( prep.url, proxies, stream, verify, cert ) # Send the request. send_kwargs = { 'timeout': timeout or conf["timeout"], 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding setattr(resp, 'reqinfo', raw) return resp
async def send(self, request, **kwargs): """Send a given PreparedRequest. :rtype: requests.Response """ allow_redirects = kwargs.pop('allow_redirects', True) if not allow_redirects: return await self._send(request, **kwargs) history = [] while True: resp = await self._send(request, **kwargs) resp.history = history[:] history.append(resp) if not resp.is_redirect: return resp # Release the connection back into the pool. await resp.close() if len(history) > self.max_redirects: raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) next_request = request.copy() next_request.url = self._get_next_url(resp) next_request.method = self._get_next_method(resp) logger.debug( f'Redirect to: {next_request.method} {next_request.url}') headers = next_request.headers # https://github.com/requests/requests/issues/1084 if resp.status_code not in (307, 308): # https://github.com/requests/requests/issues/3490 purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding') for header in purged_headers: next_request.headers.pop(header, None) next_request.body = None # Attempt to rewind consumed file-like object. should_rewind = (('Content-Length' in headers or 'Transfer-Encoding' in headers) and isinstance(next_request.body, (MultipartBody, StreamBody))) if should_rewind: logger.debug( f'Rewind request body for redirection: {next_request}') next_request.body.rewind() try: del headers['Cookie'] except KeyError: pass # Extract any cookies sent on the response to the cookiejar # in the new request. Because we've mutated our copied prepared # request, use the old one that we haven't yet touched. extract_cookies_to_jar(next_request._cookies, request, resp.raw) merge_cookies(next_request._cookies, self.cookies) next_request.prepare_cookies(next_request._cookies) self.rebuild_auth(next_request, resp) # Override the original request. request = next_request
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", "Connection": "close" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) raw = '' if prep.body: raw = "{}\n{}\n\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()), prep.body) else: raw = "{}\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items())) proxies = proxies or {} if conf["proxy_config_bool"] and not proxies: proxies = conf["proxy"] # prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore') # fix https://github.com/boy-hack/w13scan/issues/64 settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout or conf["timeout"], 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding if not getattr(resp, 'raw', None): resp.raw = raw else: tmp = resp.raw resp.raw = raw setattr(resp, 'raw2', tmp) return resp
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # In order to remove headers that are set to None def _merge_retain_none(request_setting, session_setting, dict_class=OrderedDict): if session_setting is None: return request_setting if request_setting is None: return session_setting # Bypass if not a dictionary (e.g. verify) if not (isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping)): return request_setting merged_setting = dict_class(to_key_val_list(session_setting)) merged_setting.update(to_key_val_list(request_setting)) return merged_setting # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies or (conf.cookie if 'cookie' in conf else None)) if conf.random_agent: conf.http_headers[HTTP_HEADER.USER_AGENT] = choice(conf.agents) req = Request( method=method.upper(), url=url, headers=_merge_retain_none( headers, conf.http_headers if 'http_headers' in conf else {}), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) # proxies = proxies or (conf.proxies if 'proxies' in conf else {}) if proxies is None: proxies = conf.proxies if 'proxies' in conf else {} settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) timeout = timeout or conf.get("timeout", None) if timeout: timeout = float(timeout) # Send the request. send_kwargs = { 'timeout': timeout, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) raw = '' if prep.body: raw = "{}\n{}\n\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()), prep.body) else: raw = "{}\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items())) proxies = proxies or {} if PROXY_CONFIG_BOOL and not proxies: proxies = PROXY_CONFIG settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout or TIMEOUT, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding resp.raw = raw return resp
def _hook(self, rsp, **kwargs): from requests.sessions import merge_cookies self._cookies = merge_cookies(rsp.cookies, self._cookies)