def getHtml(url, referer='', hdr=None, NoCookie=None, data=None): try: if not hdr: req = Request(url, data, headers) else: req = Request(url, data, hdr) if len(referer) > 1: req.add_header('Referer', referer) if data: req.add_header('Content-Length', len(data)) response = urlopen(req, timeout=60) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO( response.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() f.close() else: data = response.read() if not NoCookie: # Cope with problematic timestamp values on RPi on OpenElec 4.2.1 try: cj.save(cookiePath) except: pass response.close() except urllib2.HTTPError as e: data = e.read() if e.code == 503 and 'cf-browser-verification' in data: data = cloudflare.solve(url,cj, USER_AGENT) return data
def _solve_http_errors(url, error): global _cookie_jar data = error.read() if error.code == 503 and 'cf-browser-verification' in data: data = cloudflare.solve(url, _cookie_jar, UA) error.close() return data
def postHtml(url, form_data={}, headers={}, compression=True, NoCookie=None): try: _user_agent = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 ' + \ '(KHTML, like Gecko) Chrome/13.0.782.99 Safari/535.1' req = urllib2.Request(url) if form_data: form_data = urllib.urlencode(form_data) req = urllib2.Request(url, form_data) req.add_header('User-Agent', _user_agent) for k, v in headers.items(): req.add_header(k, v) if compression: req.add_header('Accept-Encoding', 'gzip') response = urllib2.urlopen(req) data = response.read() if not NoCookie: try: cj.save(cookiePath) except: pass response.close() except urllib2.HTTPError as e: data = e.read() if e.code == 503 and 'cf-browser-verification' in data: data = cloudflare.solve(url,cj, USER_AGENT,form_data=form_data) else: raise urllib2.HTTPError() except Exception as e: if 'SSL23_GET_SERVER_HELLO' in str(e): print 'Oh oh, Python version to old - update to Krypton or FTMC' raise urllib2.HTTPError() else: print 'Oh oh, It looks like this website is down.' raise urllib2.HTTPError() return None return data
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None): try: if not hdr: req = Request(url, data, headers) else: req = Request(url, data, hdr) if len(referer) > 1: req.add_header('Referer', referer) if data: req.add_header('Content-Length', len(data)) response = urlopen(req, timeout=60) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO( response.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() f.close() else: data = response.read() if not NoCookie: # Cope with problematic timestamp values on RPi on OpenElec 4.2.1 try: cj.save(cookiePath) except: pass response.close() except urllib2.HTTPError as e: data = e.read() if e.code == 503 and 'cf-browser-verification' in data: data = cloudflare.solve(url,cj, USER_AGENT) else: raise urllib2.HTTPError() return data
def _solve_http_errors(url, err): global _cookie_jar import cloudflare data = err.read() if err.code == 503 and 'cf-browser-verification' in data: data = cloudflare.solve(url, _cookie_jar, UA) err.close() else: err.close() # throw exception raise err return data
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None): try: if data: data = urllib.urlencode(data) if not hdr: req = Request(url, data, headers) else: req = Request(url, data, hdr) if len(referer) > 1: req.add_header('Referer', referer) if data: req.add_header('Content-Length', len(data)) response = urlopen(req, timeout=60) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) result = f.read() f.close() else: result = response.read() if not NoCookie: # Cope with problematic timestamp values on RPi on OpenElec 4.2.1 try: cj.save(cookiePath) except: pass response.close() except urllib2.HTTPError as e: result = e.read() if e.code == 503 and 'cf-browser-verification' in result: result = cloudflare.solve(url, cj, USER_AGENT) else: raise except Exception as e: if 'SSL23_GET_SERVER_HELLO' in str(e): notify('Oh oh', 'Python version to old - update to Krypton or FTMC') raise else: notify('Oh oh', 'It looks like this website is down.') raise return None if 'sucuri_cloudproxy_js' in result: headers['Cookie'] = get_sucuri_cookie(result) result = getHtml(url, referer, hdr=headers) return result
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None): try: if not hdr: req = Request(url, data, headers) else: req = Request(url, data, hdr) if len(referer) > 1: req.add_header('Referer', referer) if data: req.add_header('Content-Length', len(data)) response = urlopen(req, timeout=60) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO( response.read()) f = gzip.GzipFile(fileobj=buf) result = f.read() f.close() else: result = response.read() if not NoCookie: # Cope with problematic timestamp values on RPi on OpenElec 4.2.1 try: cj.save(cookiePath) except: pass response.close() except urllib2.HTTPError as e: result = e.read() if e.code == 503 and 'cf-browser-verification' in result: result = cloudflare.solve(url,cj, USER_AGENT) else: raise except Exception as e: if 'SSL23_GET_SERVER_HELLO' in str(e): notify('Oh oh','Python version to old - update to Krypton or FTMC') raise else: notify('Oh oh','It looks like this website is down.') raise return None if 'sucuri_cloudproxy_js' in result: headers['Cookie'] = get_sucuri_cookie(result) result = getHtml(url, referer, hdr=headers) return result
def getHtml(url, referer='', hdr=None, NoCookie=None, data=None): try: if data: data = urllib.urlencode(data) if not hdr: hdr = headers req = Request(url, data, hdr) if len(referer) > 1: req.add_header('Referer', referer) if data: req.add_header('Content-Length', len(data)) response = urlopen(req, timeout=60) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO( response.read()) f = gzip.GzipFile(fileobj=buf) result = f.read() f.close() else: result = response.read() if not NoCookie: # Cope with problematic timestamp values on RPi on OpenElec 4.2.1 try: cj.save(cookiePath) except: pass except urllib2.HTTPError as e: result = e.read() if e.code == 429 and 'ccapimg' in result: keyname = re.search('key=([^"]+)"', result).group(1) img = captcha_lib.write_img(referer + 'ccapimg?key=' + keyname) solution = captcha_lib.get_response(img, y=225) form_data = {'secimgkey': (None, keyname), 'secimginp': (None, solution)} hdr = headers if 'Referer' in hdr: del hdr['Referer'] hdr['referer'] = url try: resp = requests.post(url, files=form_data, headers=hdr) resp.encoding = 'ISO-8859-1' result = resp.text finally: if resp: resp.close() return result elif e.code == 503 and 'cf-browser-verification' in result: result = cloudflare.solve(url,cj, USER_AGENT) else: notify('HTTPError', str(e)) raise except Exception as e: if 'SSL23_GET_SERVER_HELLO' in str(e): notify('Oh oh','Python version to old - update to Krypton or FTMC') raise else: notify('Oh oh','It looks like this website is down.') raise return None finally: if response: response.close() if 'sucuri_cloudproxy_js' in result: headers['Cookie'] = get_sucuri_cookie(result) result = getHtml(url, referer, hdr=headers) return result