def get_page(self, url, encoding, query=None, data=None, range=None, tout=None, head=False, referer=None, header=None): if self.terminate: raise task.killbill #python cant handle unicode urlencoding so needs to get dirty below. if not query is None: query = urllib.urlencode( dict([k, v.encode('utf-8') if isinstance(v, unicode) else v] for k, v in query.items())) url = url + "?" + query if not data is None: data = urllib.urlencode( dict([k, v.encode('utf-8') if isinstance(v, unicode) else v] for k, v in data.items())) #change timeout if tout is None: tout = getdefaulttimeout() headers = {'Accept-encoding': 'gzip'} if not referer is None: headers["Referer"] = referer if not header is None: for k, v in header.iteritems(): headers[k] = v if head == True: req = HeadRequest(url, headers=headers) else: if not range is None: headers["Range"] = "bytes=%d-%d" % (range) req = urllib2.Request(url, headers=headers) response = cloudfare.ddos_open(self.opener, req, data, timeout=tout) if head: return response if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) stream = f.read() else: stream = response.read() if encoding is None: #binary data src = stream else: #unicode data src = unicode(unescape(stream.decode(encoding, "ignore"))) return src
def get_page(self,url,encoding,query=None,data=None,range=None,tout=None,head=False,referer=None,header=None,tunnel="disabled",forcetunnel=False,cache=None,throttle=2,title=None): if self.terminate: raise task.killbill tid=self.throttle.id(url,query,referer,header,data) if head or range:tunnel="disabled" if throttle==True:throttle=0 if not throttle==False and isinstance(throttle,(int,float)) and not head and not range and self.throttle.check(tid,throttle): stream=self.throttle.get(tid) else: #python cant handle unicode urlencoding so needs to get dirty below. if not query is None: query=uniurlenc(query) url=url+"?"+query if not data is None and isinstance(data,dict): data=uniurlenc(data) #change timeout if tout is None: tout=int(float(addon.getSetting("tout"))) headers={'Accept-encoding':'gzip'} if not referer is None : headers["Referer"]=referer if not header is None : for k,v in header.iteritems(): headers[k]=v tmode="disabled" if head==True: req=http.HeadRequest(url,headers=headers) else: if not range is None : headers["Range"]="bytes=%d-%d"%(range) req=urllib2.Request(url,headers=headers) if not head: tmode=self.tunnel.set_tunnel(tunnel,force=forcetunnel) req=self.tunnel.pre(req,tmode,self.cj) response = cloudfare.ddos_open(url,self.opener, req, data,tout,self.cj,self.cfagents,self.cflocks,self.tunnel,tmode) self.tunnel.cook(self.cj,self.cj.make_cookies(response,req),tmode) if head :return response stream=cloudfare.readzip(response,title) stream=self.tunnel.post(stream,tmode) if isinstance(throttle,(int,float)) and not head and not range: self.throttle.do(tid,stream) if encoding is None: #binary data src=stream else: #unicode data src=unicode(unescape(stream.decode(encoding,"ignore"))) return src
def get_page(self,url,encoding,query=None,data=None,range=None,tout=None,head=False,referer=None,header=None): if self.terminate: raise task.killbill #python cant handle unicode urlencoding so needs to get dirty below. if not query is None: query=urllib.urlencode (dict ([k, v.encode('utf-8') if isinstance (v, unicode) else v] for k, v in query.items())) url=url+"?"+query if not data is None: data=urllib.urlencode (dict ([k, v.encode('utf-8') if isinstance (v, unicode) else v] for k, v in data.items())) #change timeout if tout is None: tout=getdefaulttimeout() headers={'Accept-encoding':'gzip'} if not referer is None : headers["Referer"]=referer if not header is None : for k,v in header.iteritems(): headers[k]=v if head==True: req=HeadRequest(url,headers=headers) else: if not range is None : headers["Range"]="bytes=%d-%d"%(range) req=urllib2.Request(url,headers=headers) response = cloudfare.ddos_open(self.opener, req, data,timeout=tout) if head : return response if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) stream = f.read() else: stream=response.read() if encoding is None: #binary data src=stream else: #unicode data src=unicode(unescape(stream.decode(encoding,"ignore"))) return src
def get_page( self, url, encoding, query=None, data=None, range=None, tout=None, head=False, referer=None, header=None ): if self.terminate: raise task.killbill # python cant handle unicode urlencoding so needs to get dirty below. if not query is None: query = urlencode(dict([k, v.encode("utf-8") if isinstance(v, unicode) else v] for k, v in query.items())) url = url + "?" + query if not data is None: data = urlencode(dict([k, v.encode("utf-8") if isinstance(v, unicode) else v] for k, v in data.items())) # change timeout if tout is None: tout = int(float(addon.getSetting("tout"))) headers = {"Accept-encoding": "gzip"} if not referer is None: headers["Referer"] = referer if not header is None: for k, v in header.iteritems(): headers[k] = v if head == True: req = HeadRequest(url, headers=headers) else: if not range is None: headers["Range"] = "bytes=%d-%d" % (range) req = urllib2.Request(url, headers=headers) response = cloudfare.ddos_open(self.opener, req, data, timeout=tout) if head: return response stream = cloudfare.readzip(response) if encoding is None: # binary data src = stream else: # unicode data src = unicode(unescape(stream.decode(encoding, "ignore"))) return src