def HTTPRequest(url, method='GET', **kwargs):
url = getClearURL(url)
method = method.upper()
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
bypass = kwargs.pop('bypass', True)
timeout = kwargs.pop('timeout', None)
allow_redirects = kwargs.pop('allow_redirects', True)
proxies = {}
if Prefs['proxy_enable']:
proxy = '%s://%s:%s' % (Prefs['proxy_type'], Prefs['proxy_ip'],
Prefs['proxy_port'])
proxies = {
'http': proxy,
'https': proxy,
}
if 'User-Agent' not in headers:
headers['User-Agent'] = getUserAgent()
if params:
method = 'POST'
Log('Requesting %s "%s"' % (method, url))
req = None
try:
req = requests.request(method,
url,
proxies=proxies,
headers=headers,
cookies=cookies,
data=params,
timeout=timeout,
verify=False,
allow_redirects=allow_redirects)
except:
req = FakeResponse(None, url, 418, None)
req_bypass = None
if not req.ok and bypass:
if req.status_code == 403 or req.status_code == 503:
req_bypass = HTTPBypass(url,
method,
proxies=proxies,
headers=headers,
cookies=cookies,
params=params)
if req_bypass:
req = req_bypass
req.encoding = 'UTF-8'
if Prefs['debug_enable']:
saveRequest(url, req)
return req
def HTTPRequest(url, method='GET', **kwargs):
url = getClearURL(url)
method = method.upper()
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
bypass = kwargs.pop('bypass', True)
timeout = kwargs.pop('timeout', None)
errors = kwargs.pop('errors', 'ignore')
sleep = kwargs.pop('sleep', 0)
allow_redirects = kwargs.pop('allow_redirects', True)
proxies = {}
if 'User-Agent' not in headers:
headers['User-Agent'] = getUserAgent()
if params:
method = 'POST'
Log('UTILS:: Requesting %s "%s"' % (method, url))
req = None
try:
req = requests.request(method, url, proxies=proxies, headers=headers, cookies=cookies, data=params, timeout=timeout, verify=False, allow_redirects=allow_redirects)
Log('UTILS:: Req is OK? %s - Req.status: %s', req.ok, req.status_code)
except Exception as e:
req = FakeResponse(None, url, 418, None)
Log('UTILS:: Sending fake response following exception %s', e)
req_bypass = None
if not req.ok and bypass:
if req.status_code == 403 or req.status_code == 503:
Log('UTILS:: Switching to HTTPBypass for %s', url)
req_bypass = HTTPBypass(url, method, proxies=proxies, headers=headers, cookies=cookies, params=params)
if req_bypass:
req = req_bypass
req.encoding = 'UTF-8'
Log('UTILS:: Returning req: %s', req)
return req
def HTTPRequest(url, method='GET', **kwargs):
url = getClearURL(url)
method = method.upper()
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
bypass = kwargs.pop('bypass', True)
allow_redirects = kwargs.pop('allow_redirects', True)
proxies = {}
if Prefs['proxy_enable']:
proxy = '%s://%s:%s' % (Prefs['proxy_type'], Prefs['proxy_ip'],
Prefs['proxy_port'])
proxies = {
'http': proxy,
'https': proxy,
}
if 'User-Agent' not in headers:
headers['User-Agent'] = getUserAgent()
if params:
method = 'POST'
Log('Requesting %s "%s"' % (method, url))
req = None
try:
req = requests.request(method,
url,
proxies=proxies,
headers=headers,
cookies=cookies,
data=params,
verify=False,
allow_redirects=allow_redirects)
except:
req = FakeResponse(None, url, 418, None)
req_bypass = None
if not req.ok and bypass:
if req.status_code == 403 or req.status_code == 503:
Log('%d: trying to bypass with CloudScraper' % req.status_code)
try:
req_bypass = bypassCloudflare(url,
method,
proxies=proxies,
headers=headers,
cookies=cookies,
params=params)
if not req_bypass.ok:
raise Exception(req.status_code)
except Exception as e:
Log('CloudScraper error: %s' % e)
Log('Trying through ReqBIN')
req_bypass = reqBinRequest(url,
method,
proxies=proxies,
headers=headers,
cookies=cookies,
params=params)
if req_bypass:
req = req_bypass
req.encoding = 'UTF-8'
if Prefs['debug_enable']:
saveRequest(url, req)
return req
