def getHtml(url, referer='', hdr=None, NoCookie=None, data=None):
try:
if not hdr:
req = Request(url, data, headers)
else:
req = Request(url, data, hdr)
if len(referer) > 1:
req.add_header('Referer', referer)
if data:
req.add_header('Content-Length', len(data))
response = urlopen(req, timeout=60)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
f.close()
else:
data = response.read()
if not NoCookie:
# Cope with problematic timestamp values on RPi on OpenElec 4.2.1
try:
cj.save(cookiePath)
except: pass
response.close()
except urllib2.HTTPError as e:
data = e.read()
if e.code == 503 and 'cf-browser-verification' in data:
data = cloudflare.solve(url,cj, USER_AGENT)
return data
def _solve_http_errors(url, error):
global _cookie_jar
data = error.read()
if error.code == 503 and 'cf-browser-verification' in data:
data = cloudflare.solve(url, _cookie_jar, UA)
error.close()
return data
def _solve_http_errors(url, error):
global _cookie_jar
data = error.read()
if error.code == 503 and 'cf-browser-verification' in data:
data = cloudflare.solve(url, _cookie_jar, UA)
error.close()
return data
def postHtml(url, form_data={}, headers={}, compression=True, NoCookie=None):
try:
_user_agent = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 ' + \
'(KHTML, like Gecko) Chrome/13.0.782.99 Safari/535.1'
req = urllib2.Request(url)
if form_data:
form_data = urllib.urlencode(form_data)
req = urllib2.Request(url, form_data)
req.add_header('User-Agent', _user_agent)
for k, v in headers.items():
req.add_header(k, v)
if compression:
req.add_header('Accept-Encoding', 'gzip')
response = urllib2.urlopen(req)
data = response.read()
if not NoCookie:
try:
cj.save(cookiePath)
except: pass
response.close()
except urllib2.HTTPError as e:
data = e.read()
if e.code == 503 and 'cf-browser-verification' in data:
data = cloudflare.solve(url,cj, USER_AGENT,form_data=form_data)
else:
raise urllib2.HTTPError()
except Exception as e:
if 'SSL23_GET_SERVER_HELLO' in str(e):
print 'Oh oh, Python version to old - update to Krypton or FTMC'
raise urllib2.HTTPError()
else:
print 'Oh oh, It looks like this website is down.'
raise urllib2.HTTPError()
return None
return data
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None):
try:
if not hdr:
req = Request(url, data, headers)
else:
req = Request(url, data, hdr)
if len(referer) > 1:
req.add_header('Referer', referer)
if data:
req.add_header('Content-Length', len(data))
response = urlopen(req, timeout=60)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
f.close()
else:
data = response.read()
if not NoCookie:
# Cope with problematic timestamp values on RPi on OpenElec 4.2.1
try:
cj.save(cookiePath)
except: pass
response.close()
except urllib2.HTTPError as e:
data = e.read()
if e.code == 503 and 'cf-browser-verification' in data:
data = cloudflare.solve(url,cj, USER_AGENT)
else:
raise urllib2.HTTPError()
return data
def _solve_http_errors(url, err):
global _cookie_jar
import cloudflare
data = err.read()
if err.code == 503 and 'cf-browser-verification' in data:
data = cloudflare.solve(url, _cookie_jar, UA)
err.close()
else:
err.close() # throw exception
raise err
return data
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None):
try:
if data:
data = urllib.urlencode(data)
if not hdr:
req = Request(url, data, headers)
else:
req = Request(url, data, hdr)
if len(referer) > 1:
req.add_header('Referer', referer)
if data:
req.add_header('Content-Length', len(data))
response = urlopen(req, timeout=60)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
result = f.read()
f.close()
else:
result = response.read()
if not NoCookie:
# Cope with problematic timestamp values on RPi on OpenElec 4.2.1
try:
cj.save(cookiePath)
except:
pass
response.close()
except urllib2.HTTPError as e:
result = e.read()
if e.code == 503 and 'cf-browser-verification' in result:
result = cloudflare.solve(url, cj, USER_AGENT)
else:
raise
except Exception as e:
if 'SSL23_GET_SERVER_HELLO' in str(e):
notify('Oh oh',
'Python version to old - update to Krypton or FTMC')
raise
else:
notify('Oh oh', 'It looks like this website is down.')
raise
return None
if 'sucuri_cloudproxy_js' in result:
headers['Cookie'] = get_sucuri_cookie(result)
result = getHtml(url, referer, hdr=headers)
return result
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None):
try:
if not hdr:
req = Request(url, data, headers)
else:
req = Request(url, data, hdr)
if len(referer) > 1:
req.add_header('Referer', referer)
if data:
req.add_header('Content-Length', len(data))
response = urlopen(req, timeout=60)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
result = f.read()
f.close()
else:
result = response.read()
if not NoCookie:
# Cope with problematic timestamp values on RPi on OpenElec 4.2.1
try:
cj.save(cookiePath)
except:
pass
response.close()
except urllib2.HTTPError as e:
result = e.read()
if e.code == 503 and 'cf-browser-verification' in result:
result = cloudflare.solve(url,cj, USER_AGENT)
else:
raise
except Exception as e:
if 'SSL23_GET_SERVER_HELLO' in str(e):
notify('Oh oh','Python version to old - update to Krypton or FTMC')
raise
else:
notify('Oh oh','It looks like this website is down.')
raise
return None
if 'sucuri_cloudproxy_js' in result:
headers['Cookie'] = get_sucuri_cookie(result)
result = getHtml(url, referer, hdr=headers)
return result
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None):
try:
if data:
data = urllib.urlencode(data)
if not hdr:
hdr = headers
req = Request(url, data, hdr)
if len(referer) > 1:
req.add_header('Referer', referer)
if data:
req.add_header('Content-Length', len(data))
response = urlopen(req, timeout=60)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
result = f.read()
f.close()
else:
result = response.read()
if not NoCookie:
# Cope with problematic timestamp values on RPi on OpenElec 4.2.1
try:
cj.save(cookiePath)
except:
pass
except urllib2.HTTPError as e:
result = e.read()
if e.code == 429 and 'ccapimg' in result:
keyname = re.search('key=([^"]+)"', result).group(1)
img = captcha_lib.write_img(referer + 'ccapimg?key=' + keyname)
solution = captcha_lib.get_response(img, y=225)
form_data = {'secimgkey': (None, keyname), 'secimginp': (None, solution)}
hdr = headers
if 'Referer' in hdr:
del hdr['Referer']
hdr['referer'] = url
try:
resp = requests.post(url, files=form_data, headers=hdr)
resp.encoding = 'ISO-8859-1'
result = resp.text
finally:
if resp: resp.close()
return result
elif e.code == 503 and 'cf-browser-verification' in result:
result = cloudflare.solve(url,cj, USER_AGENT)
else:
notify('HTTPError', str(e))
raise
except Exception as e:
if 'SSL23_GET_SERVER_HELLO' in str(e):
notify('Oh oh','Python version to old - update to Krypton or FTMC')
raise
else:
notify('Oh oh','It looks like this website is down.')
raise
return None
finally:
if response: response.close()
if 'sucuri_cloudproxy_js' in result:
headers['Cookie'] = get_sucuri_cookie(result)
result = getHtml(url, referer, hdr=headers)
return result
