def HTTPRequest(url, method='GET', **kwargs): url = getClearURL(url) method = method.upper() headers = kwargs.pop('headers', {}) cookies = kwargs.pop('cookies', {}) params = kwargs.pop('params', {}) bypass = kwargs.pop('bypass', True) timeout = kwargs.pop('timeout', None) allow_redirects = kwargs.pop('allow_redirects', True) proxies = {} if Prefs['proxy_enable']: proxy = '%s://%s:%s' % (Prefs['proxy_type'], Prefs['proxy_ip'], Prefs['proxy_port']) proxies = { 'http': proxy, 'https': proxy, } if 'User-Agent' not in headers: headers['User-Agent'] = getUserAgent() if params: method = 'POST' Log('Requesting %s "%s"' % (method, url)) req = None try: req = requests.request(method, url, proxies=proxies, headers=headers, cookies=cookies, data=params, timeout=timeout, verify=False, allow_redirects=allow_redirects) except: req = FakeResponse(None, url, 418, None) req_bypass = None if not req.ok and bypass: if req.status_code == 403 or req.status_code == 503: req_bypass = HTTPBypass(url, method, proxies=proxies, headers=headers, cookies=cookies, params=params) if req_bypass: req = req_bypass req.encoding = 'UTF-8' if Prefs['debug_enable']: saveRequest(url, req) return req
def HTTPRequest(url, method='GET', **kwargs): url = getClearURL(url) method = method.upper() headers = kwargs.pop('headers', {}) cookies = kwargs.pop('cookies', {}) params = kwargs.pop('params', {}) bypass = kwargs.pop('bypass', True) timeout = kwargs.pop('timeout', None) errors = kwargs.pop('errors', 'ignore') sleep = kwargs.pop('sleep', 0) allow_redirects = kwargs.pop('allow_redirects', True) proxies = {} if 'User-Agent' not in headers: headers['User-Agent'] = getUserAgent() if params: method = 'POST' Log('UTILS:: Requesting %s "%s"' % (method, url)) req = None try: req = requests.request(method, url, proxies=proxies, headers=headers, cookies=cookies, data=params, timeout=timeout, verify=False, allow_redirects=allow_redirects) Log('UTILS:: Req is OK? %s - Req.status: %s', req.ok, req.status_code) except Exception as e: req = FakeResponse(None, url, 418, None) Log('UTILS:: Sending fake response following exception %s', e) req_bypass = None if not req.ok and bypass: if req.status_code == 403 or req.status_code == 503: Log('UTILS:: Switching to HTTPBypass for %s', url) req_bypass = HTTPBypass(url, method, proxies=proxies, headers=headers, cookies=cookies, params=params) if req_bypass: req = req_bypass req.encoding = 'UTF-8' Log('UTILS:: Returning req: %s', req) return req
def HTTPRequest(url, method='GET', **kwargs): url = getClearURL(url) method = method.upper() headers = kwargs.pop('headers', {}) cookies = kwargs.pop('cookies', {}) params = kwargs.pop('params', {}) bypass = kwargs.pop('bypass', True) allow_redirects = kwargs.pop('allow_redirects', True) proxies = {} if Prefs['proxy_enable']: proxy = '%s://%s:%s' % (Prefs['proxy_type'], Prefs['proxy_ip'], Prefs['proxy_port']) proxies = { 'http': proxy, 'https': proxy, } if 'User-Agent' not in headers: headers['User-Agent'] = getUserAgent() if params: method = 'POST' Log('Requesting %s "%s"' % (method, url)) req = None try: req = requests.request(method, url, proxies=proxies, headers=headers, cookies=cookies, data=params, verify=False, allow_redirects=allow_redirects) except: req = FakeResponse(None, url, 418, None) req_bypass = None if not req.ok and bypass: if req.status_code == 403 or req.status_code == 503: Log('%d: trying to bypass with CloudScraper' % req.status_code) try: req_bypass = bypassCloudflare(url, method, proxies=proxies, headers=headers, cookies=cookies, params=params) if not req_bypass.ok: raise Exception(req.status_code) except Exception as e: Log('CloudScraper error: %s' % e) Log('Trying through ReqBIN') req_bypass = reqBinRequest(url, method, proxies=proxies, headers=headers, cookies=cookies, params=params) if req_bypass: req = req_bypass req.encoding = 'UTF-8' if Prefs['debug_enable']: saveRequest(url, req) return req