def prepare_request(self, request): cookies = request.cookies or {} # Bootstrap CookieJar. if not isinstance(cookies, cookielib.CookieJar): cookies = cookiejar_from_dict(cookies) # Merge with session cookies merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) # Set environment's basic authentication if not explicitly set. # auth = request.auth # if self.trust_env and not auth and not self.auth: # auth = get_netrc_auth(request.url) p = PreparedRequest() p.prepare( method=request.method.upper(), url=request.url, files=request.files, data=request.data, json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(request.auth, self.auth), cookies=merged_cookies, hooks=merge_hooks(request.hooks, self.hooks), ) return p
def merge_environment_settings(self, url, proxies, stream, verify, cert): """ Check the environment and merge it with some settings. :rtype: dict """ # Gather clues from the surrounding environment. # if self.trust_env: # # Set environment's proxies. # no_proxy = proxies.get('no_proxy') if proxies is not None else None # env_proxies = get_environ_proxies(url, no_proxy=no_proxy) # for (k, v) in env_proxies.items(): # proxies.setdefault(k, v) # # # Look for requests environment configuration and be compatible # # with cURL. # if verify is True or verify is None: # verify = (os.environ.get('REQUESTS_CA_BUNDLE') or # os.environ.get('CURL_CA_BUNDLE')) # Merge all the kwargs. proxies = merge_setting(proxies, self.proxies) stream = merge_setting(stream, self.stream) verify = merge_setting(verify, self.verify) cert = merge_setting(cert, self.cert) return { 'verify': verify, 'proxies': proxies, 'stream': stream, 'cert': cert }
def request(self, method=None, url=None, data=None, headers=None, expected_status=None, wrapper=None, raw_response=False, params=None, serializer=None, host=None, **kwargs): """Main method for data transformation and transferring Parameters: method(str): http method name host(str): optional. Parameter for overriding self.host url(str): optional. Parameter for overriding self.url data(any): optional. Parameter for overriding self.model. Could be Model type or Python type(dict, list, etc.) headers(dict): optional. Parameter for merging with Session.headers expected_status(int): optional. Status for response assertion. If None assertion will be skipped wrapper(callable): default sefl.model.wrap. Callable object for wrapping deserialized response.content raw_response(bool): default False. If True, method returns requests.Response object. In this case all transformations and checks will be ignored params(dict): optional. requsts.Session.get params parameter serializer(str): optional. Parameter for overriding self.serializer **kwargs: requests.Session.request method kwargs Returns: prepared response(usually Model instance) """ serializer = serializer or self.serializer url = self.prepare_url(host, url) data, headers = self.prepare_request(data=data, headers=headers, serializer=serializer) _response = self.session.request(method=method, url=url, data=data, headers=headers, params=params, **kwargs) response = self.prepare_response( response=_response, method=method, url=url, expected_status=expected_status, wrapper=wrapper, headers=merge_setting(headers, self.session.headers, dict_class=CaseInsensitiveDict), params=merge_setting(params, self.session.params), data=data, raw_response=raw_response, serializer=serializer) return response
def request(url, method, session=None, **kw): """ Makes an HTTP request following redirects. """ import os user_agent = os.environ.get('WEX_USER_AGENT') if session is None: session = requests.Session() session.stream = True session.headers = {'User-Agent': user_agent} if user_agent else None decode_content = kw.get('decode_content', True) proxies = kw.get('proxies', None) headers = merge_setting(method.args.get('headers'), kw.get('headers')) context = kw.get('context', {}) auth = merge_setting(method.args.get('auth'), kw.get('auth')) params = merge_setting(method.args.get('params'), kw.get('params')) timeout = merge_setting(method.args.get('timeout'), kw.get('timeout', DEFAULT_TIMEOUT)) # requests will not accept a list, but we might get this from # our JSON method fragment dict so lets convert it to a tuple if isinstance(timeout, list): timeout = tuple(timeout) response = session.request( method.name, url, allow_redirects=False, cookies=method.args.get('cookies', None), data=method.args.get('data', None), headers=headers, params=params, proxies=proxies, timeout=timeout, auth=auth, ) if 'params' in kw: response.url = remove_url_params(response.url, kw['params']) yield readable_from_response(response, url, decode_content, context) redirects = session.resolve_redirects(response, response.request, proxies=proxies, stream=True, timeout=timeout) for redirect in redirects: yield readable_from_response(redirect, url, decode_content, context)
def prepare_request(self, request: BitexRequest) -> BitexPreparedRequest: """Prepare a :class:`BitexPreparedRequest` object for transmission. This implementation extends :class:`requests.Session.prepare_request` by making a call to :data:`bitex.list_loaded_plugins` and checking if we have any plugins that may provide a custom :class:`BitexPreparedRequest` class. """ cookies = request.cookies or {} # Bootstrap CookieJar. if not isinstance(cookies, cookielib.CookieJar): cookies = cookiejar_from_dict(cookies) # Merge with session cookies session_cookies = merge_cookies(RequestsCookieJar(), self.cookies) merged_cookies = merge_cookies(session_cookies, cookies) # Set environment's basic authentication if not explicitly set. auth = request.auth if self.trust_env and not auth and not self.auth: auth = get_netrc_auth(request.url) # Inject any custom classes for handling the exchange stated in the # BitexRequest object. custom_classes = list_loaded_plugins().get(request.exchange, None) if custom_classes: p = custom_classes["PreparedRequest"](request.exchange) # Only use the custom auth class if no auth object was # provided explicitly. Otherwise we would overwrite user-specified # auth objects passed to self.request. if not self.auth and request.private: self.auth = custom_classes["Auth"](self.key, self.secret) else: p = BitexPreparedRequest(request.exchange) p.prepare( method=request.method.upper(), url=request.url, files=request.files, data=request.data, json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, hooks=merge_hooks(request.hooks, self.hooks), ) return p
def __init__(self, auth=None, settings=None, headers=None, *args, **kwargs): """ Initialize the Base API client functions :param auth: optional authentication tuple or object for request :param settings: optional :class:`BigStashAPISettings` instance to use. :param headers: optional dict of HTTP Headers """ self.settings = settings or BigStashAPISettings() if 'base_url' not in self.settings: raise TypeError("Must provide base_url setting") self._base_url = self.settings['base_url'].rstrip('/') self._headers = DEFAULT_HEADERS if headers is not None: self._headers = merge_setting( headers, self._headers, dict_class=CaseInsensitiveDict) self._auth = None if 'auth' is not None: self._auth = auth # setup requests session self._session = self._setup_session() super(BigStashAPIBase, self).__init__(*args, **kwargs)
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=conf.timeout if 'timeout' in conf else None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies or conf.cookie if 'cookie' in conf else None) req = Request( method=method.upper(), url=url, headers=merge_setting( headers, conf.http_headers if 'http_headers' in conf else {}), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or conf.proxies if 'proxies' in conf else {} settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def init_server(self, version=TAXII_VER_2_0): """ Initializes a server in the requested version :param version: taxii version key (either 2.0 or 2.1) """ server_url = urljoin(self.base_url) self._conn = _HTTPConnection(verify=self.verify, proxies=self.proxies, version=version, auth=self.auth, cert=self.crt) if self.auth_header: # add auth_header to the session object self._conn.session.headers = ( # type: ignore[attr-defined] merge_setting( self._conn.session.headers, # type: ignore[attr-defined] {self.auth_header: self.auth_key}, dict_class=CaseInsensitiveDict, ), ) if version is TAXII_VER_2_0: self.server = v20.Server( server_url, verify=self.verify, proxies=self.proxies, conn=self._conn, ) else: self.server = v21.Server( server_url, verify=self.verify, proxies=self.proxies, conn=self._conn, )
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): conf = CONF.get("requests", {}) if timeout is None and "timeout" in conf: timeout = conf["timeout"] merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies or (conf.cookie if "cookie" in conf else None)) req = Request( method=method.upper(), url=url, headers=merge_setting(headers, conf["headers"] if "headers" in conf else {}), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or (conf["proxies"] if "proxies" in conf else {}) settings = self.merged_environment_settings(prep.url, proxies, stream, verify, cert) send_kwargs = { "timeout": timeout, "allow_redirects": allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == "ISO-8859-1": encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def update_session(self, alias, headers=None, cookies=None): """Updates HTTP Session Headers and Cookies. Session will be identified using the ``alias`` name. Dictionary of ``headers`` and ``cookies`` to be updated and merged into session data. """ session = self._cache.switch(alias) session.headers = merge_setting(headers, session.headers) session.cookies = merge_cookies(session.cookies, cookies)
def update_session(self, alias, headers=None, cookies=None): """Update Session Headers: update a HTTP Session Headers ``alias`` Robot Framework alias to identify the session ``headers`` Dictionary of headers merge into session """ session = self._cache.switch(alias) session.headers = merge_setting(headers, session.headers) session.cookies = merge_cookies(session.cookies, cookies)
def prepare_request(self, request): """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it. The :class:`PreparedRequest` has settings merged from the :class:`Request <Request>` instance and those of the :class:`Session`. :param request: :class:`Request` instance to prepare with this session's settings. :rtype: requests.PreparedRequest """ cookies = request.cookies or {} # Bootstrap CookieJar. if not isinstance(cookies, cookielib.CookieJar): cookies = cookiejar_from_dict(cookies) # Merge with session cookies merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) # Set environment's basic authentication if not explicitly set. auth = request.auth if self.trust_env and not auth and not self.auth: auth = get_netrc_auth(request.url) p = CuPreparedRequest() p.prepare( method=request.method.upper(), url=request.url, files=request.files, data=request.data, json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, hooks=merge_hooks(request.hooks, self.hooks), ) return p
def prepare_request(self, request): """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it. The :class:`PreparedRequest` has settings merged from the :class:`Request <Request>` instance and those of the :class:`Session`. :param request: :class:`Request` instance to prepare with this session's settings. """ cookies = request.cookies or {} # Bootstrap CookieJar. if not isinstance(cookies, cookielib.CookieJar): cookies = cookiejar_from_dict(cookies) # Merge with session cookies merged_cookies = RequestsCookieJar() merged_cookies.update(self.cookies) merged_cookies.update(cookies) # Set environment's basic authentication if not explicitly set. auth = request.auth if self.trust_env and not auth and not self.auth: auth = get_netrc_auth(request.url) p = PreparedRequest() p.prepare( method=request.method.upper(), url=request.url, files=request.files, data=request.data, json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, hooks=merge_hooks(request.hooks, self.hooks), ) return p
def request(url, method, session=None, **kw): """ Makes an HTTP request following redirects. """ import os user_agent = os.environ.get('WEX_USER_AGENT') if session is None: session = requests.Session() session.stream = True session.headers = {'User-Agent': user_agent} if user_agent else None decode_content = kw.get('decode_content', True) proxies = kw.get('proxies', None) headers = merge_setting(method.args.get('headers'), kw.get('headers')) context = kw.get('context', {}) auth = merge_setting(method.args.get('auth'), kw.get('auth')) response = session.request( method.name, url, allow_redirects=False, cookies=method.args.get('cookies', None), data=method.args.get('data', None), headers=headers, params=method.args.get('params', None), proxies=proxies, timeout=timeout, auth=auth, ) yield readable_from_response(response, url, decode_content, context) redirects = session.resolve_redirects(response, response.request, proxies=proxies, stream=True, timeout=timeout) for redirect in redirects: yield readable_from_response(redirect, url, decode_content, context)
def _setup_session(self): s = Session() if 'verify' in self.settings: # default is True s.verify = self.settings['verify'] if 'trust_env' in self.settings: s.trust_env = self.settings['trust_env'] if self._auth is not None: s.auth = self._auth s.headers = merge_setting( # add our headers to requests' default set self._headers, s.headers, dict_class=CaseInsensitiveDict) return s
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=10, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or {} settings = self.merge_environment_settings( prep.url, proxies, stream, verify, cert ) # Send the request. send_kwargs = { 'timeout': TIMEOUT, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding return resp
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) raw = '' if prep.body: raw = "{}\n{}\n\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()), prep.body) else: raw = "{}\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items())) proxies = proxies or {} if PROXY_CONFIG_BOOL and not proxies: proxies = PROXY_CONFIG settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout or TIMEOUT, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding resp.raw = raw return resp
def request(self, method, url=None, # Request parameters data=None, params=None, auth=None, hooks=None, # Session parameters timeout=None, proxies=None, verify=None, cert=None): """Constructs a :class:`Request <Request>`, prepares it and sends it. Returns :class:`Response <Response>` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True by default. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ method = str(method) # Create the request. req = requests.Request( method = method.upper(), url = url, data = data or {}, params = params or {}, auth = auth, hooks = hooks, ) # Prepare the request. preq = self.prepare_request(req) proxies = merge_setting(proxies, self.proxies) verify = merge_setting(verify, self.verify) cert = merge_setting(cert, self.cert) # Send the request. send_kwargs = { 'timeout': timeout, 'proxies': proxies, 'verify': verify, 'cert': cert, } resp = self.send(preq, **send_kwargs) return resp
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", "Connection": "close" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) raw = '' if prep.body: raw = "{}\n{}\n\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()), prep.body) else: raw = "{}\n{}".format( prep.method + ' ' + prep.url, '\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items())) proxies = proxies or {} if conf["proxy_config_bool"] and not proxies: proxies = conf["proxy"] # prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore') # fix https://github.com/boy-hack/w13scan/issues/64 settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout or conf["timeout"], 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding if not getattr(resp, 'raw', None): resp.raw = raw else: tmp = resp.raw resp.raw = raw setattr(resp, 'raw2', tmp) return resp
def __call__(self, r): r.headers = merge_setting(self.defaultHeaders(r), r.headers, dict_class=CaseInsensitiveDict) r.headers["X-Ca-Signature-Headers"], self.tempHeader = self._format_headers(r) r.headers["X-Ca-Signature"] = self.sign.get_sign(self.buildSignStr(r), self._appSecret) return r
def session_request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None): # Create the Request. merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies), cookies) default_header = { "User-Agent": conf.agent, "Connection": "close" } req = Request( method=method.upper(), url=url, headers=merge_setting(headers, default_header), files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=merged_cookies, hooks=hooks, ) prep = self.prepare_request(req) raw = '' p = urlparse(url) _headers = copy.deepcopy(prep.headers) if "Host" not in _headers: _headers["Host"] = p.netloc if prep.body: raw = "{}\n{}\n\n{}\n\n".format( prep.method + ' ' + prep.url + ' HTTP/1.1', '\n'.join('{}: {}'.format(k, v) for k, v in _headers.items()), prep.body) else: raw = "{}\n{}\n\n".format( prep.method + ' ' + prep.url + ' HTTP/1.1', '\n'.join('{}: {}'.format(k, v) for k, v in _headers.items())) proxies = proxies or {} if conf["proxy_config_bool"] and not proxies: proxies = conf["proxy"] if "socks4" in proxies.keys(): _tmp_str = "socks4://" + proxies["socks4"] _tmp_proxy = { "http": _tmp_str, "https": _tmp_str } proxies = _tmp_proxy elif "socks5" in proxies.keys(): _tmp_str = "socks5://" + proxies["socks5"] _tmp_proxy = { "http": _tmp_str, "https": _tmp_str } proxies = _tmp_proxy # prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore') # fix https://github.com/boy-hack/w13scan/issues/64 settings = self.merge_environment_settings( prep.url, proxies, stream, verify, cert ) # Send the request. send_kwargs = { 'timeout': timeout or conf["timeout"], 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) if resp.encoding == 'ISO-8859-1': encodings = get_encodings_from_content(resp.text) if encodings: encoding = encodings[0] else: encoding = resp.apparent_encoding resp.encoding = encoding setattr(resp, 'reqinfo', raw) return resp