def HTTPRequest(url, method='GET', **kwargs):
url = getClearURL(url)
method = method.upper()
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
bypass = kwargs.pop('bypass', True)
timeout = kwargs.pop('timeout', None)
allow_redirects = kwargs.pop('allow_redirects', True)
proxies = {}
if Prefs['proxy_enable']:
proxy = '%s://%s:%s' % (Prefs['proxy_type'], Prefs['proxy_ip'],
Prefs['proxy_port'])
proxies = {
'http': proxy,
'https': proxy,
}
if 'User-Agent' not in headers:
headers['User-Agent'] = getUserAgent()
if params:
method = 'POST'
Log('Requesting %s "%s"' % (method, url))
req = None
try:
req = requests.request(method,
url,
proxies=proxies,
headers=headers,
cookies=cookies,
data=params,
timeout=timeout,
verify=False,
allow_redirects=allow_redirects)
except:
req = FakeResponse(None, url, 418, None)
req_bypass = None
if not req.ok and bypass:
if req.status_code == 403 or req.status_code == 503:
req_bypass = HTTPBypass(url,
method,
proxies=proxies,
headers=headers,
cookies=cookies,
params=params)
if req_bypass:
req = req_bypass
req.encoding = 'UTF-8'
if Prefs['debug_enable']:
saveRequest(url, req)
return req
def flareSolverrRequest(url, method, **kwargs):
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
if method not in ['GET', 'POST']:
return None
req_params = {
'cmd': 'request.%s' % method.lower(),
'url': url,
'userAgent': headers['User-Agent'] if 'User-Agent' in headers else getUserAgent(),
'maxTimeout': 60000,
'headers': json.dumps(headers),
}
if method == 'POST':
req_params['postData'] = json.dumps(params)
req = HTTPRequest('%s/v1' % Prefs['flaresolverr_endpoint'], headers={'Content-Type': 'application/json'}, params=json.dumps(req_params), timeout=60, bypass=False)
if req.ok:
data = req.json()['solution']
headers = data['headers']
headers['User-Agent'] = data['userAgent']
cookies = {cookie['name']: cookie['value'] for cookie in data['cookies']}
return FakeResponse(req, url, int(data['headers']['status']), data['response'], headers, cookies)
return None
def HTTPRequest(url, method='GET', **kwargs):
url = getClearURL(url)
method = method.upper()
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
bypass = kwargs.pop('bypass', True)
timeout = kwargs.pop('timeout', None)
errors = kwargs.pop('errors', 'ignore')
sleep = kwargs.pop('sleep', 0)
allow_redirects = kwargs.pop('allow_redirects', True)
proxies = {}
if 'User-Agent' not in headers:
headers['User-Agent'] = getUserAgent()
if params:
method = 'POST'
Log('UTILS:: Requesting %s "%s"' % (method, url))
req = None
try:
req = requests.request(method, url, proxies=proxies, headers=headers, cookies=cookies, data=params, timeout=timeout, verify=False, allow_redirects=allow_redirects)
Log('UTILS:: Req is OK? %s - Req.status: %s', req.ok, req.status_code)
except Exception as e:
req = FakeResponse(None, url, 418, None)
Log('UTILS:: Sending fake response following exception %s', e)
req_bypass = None
if not req.ok and bypass:
if req.status_code == 403 or req.status_code == 503:
Log('UTILS:: Switching to HTTPBypass for %s', url)
req_bypass = HTTPBypass(url, method, proxies=proxies, headers=headers, cookies=cookies, params=params)
if req_bypass:
req = req_bypass
req.encoding = 'UTF-8'
Log('UTILS:: Returning req: %s', req)
return req
def reqBinRequest(url, method, **kwargs):
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
proxies = kwargs.pop('proxies', {})
params = kwargs.pop('params', {})
if cookies and 'Cookie' not in headers:
cookie = '; '.join(['%s=%s' % (key, cookies[key]) for key in cookies])
headers['Cookie'] = cookie
req_headers = '\n'.join(
['%s: %s' % (key, headers[key]) for key in headers])
for node in ['US', 'DE']:
req_data = {
'method': method,
'url': url,
'headers': req_headers,
'apiNode': node,
'idnUrl': url
}
if method == 'POST':
if headers['Content-Type'] == 'application/json':
req_data['contentType'] = 'JSON'
req_data['content'] = params
else:
req_data['contentType'] = 'URLENCODED'
req_data['content'] = '&'.join(
['%s=%s' % (key, params[key]) for key in params])
req_params = json.dumps({
'id': 0,
'json': json.dumps(req_data),
'deviceId': '',
'sessionId': ''
})
req = HTTPRequest('https://api.reqbin.com/api/v1/requests',
headers={'Content-Type': 'application/json'},
params=req_params,
proxies=proxies,
bypass=False)
if req.ok:
data = req.json()
return FakeResponse(req, url, int(data['StatusCode']),
data['Content'])
return None
def HTTPRequest(url, method='GET', **kwargs):
url = getClearURL(url)
method = method.upper()
headers = kwargs.pop('headers', {})
cookies = kwargs.pop('cookies', {})
params = kwargs.pop('params', {})
bypass = kwargs.pop('bypass', True)
allow_redirects = kwargs.pop('allow_redirects', True)
proxies = {}
if Prefs['proxy_enable']:
proxy = '%s://%s:%s' % (Prefs['proxy_type'], Prefs['proxy_ip'],
Prefs['proxy_port'])
proxies = {
'http': proxy,
'https': proxy,
}
if 'User-Agent' not in headers:
headers['User-Agent'] = getUserAgent()
if params:
method = 'POST'
Log('Requesting %s "%s"' % (method, url))
req = None
try:
req = requests.request(method,
url,
proxies=proxies,
headers=headers,
cookies=cookies,
data=params,
verify=False,
allow_redirects=allow_redirects)
except:
req = FakeResponse(None, url, 418, None)
req_bypass = None
if not req.ok and bypass:
if req.status_code == 403 or req.status_code == 503:
Log('%d: trying to bypass with CloudScraper' % req.status_code)
try:
req_bypass = bypassCloudflare(url,
method,
proxies=proxies,
headers=headers,
cookies=cookies,
params=params)
if not req_bypass.ok:
raise Exception(req.status_code)
except Exception as e:
Log('CloudScraper error: %s' % e)
Log('Trying through ReqBIN')
req_bypass = reqBinRequest(url,
method,
proxies=proxies,
headers=headers,
cookies=cookies,
params=params)
if req_bypass:
req = req_bypass
req.encoding = 'UTF-8'
if Prefs['debug_enable']:
saveRequest(url, req)
return req