def get_page(self,
url,
encoding,
query=None,
data=None,
range=None,
tout=None,
head=False,
referer=None,
header=None):
if self.terminate:
raise task.killbill
#python cant handle unicode urlencoding so needs to get dirty below.
if not query is None:
query = urllib.urlencode(
dict([k, v.encode('utf-8') if isinstance(v, unicode) else v]
for k, v in query.items()))
url = url + "?" + query
if not data is None:
data = urllib.urlencode(
dict([k, v.encode('utf-8') if isinstance(v, unicode) else v]
for k, v in data.items()))
#change timeout
if tout is None:
tout = getdefaulttimeout()
headers = {'Accept-encoding': 'gzip'}
if not referer is None: headers["Referer"] = referer
if not header is None:
for k, v in header.iteritems():
headers[k] = v
if head == True:
req = HeadRequest(url, headers=headers)
else:
if not range is None: headers["Range"] = "bytes=%d-%d" % (range)
req = urllib2.Request(url, headers=headers)
response = cloudfare.ddos_open(self.opener, req, data, timeout=tout)
if head:
return response
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
stream = f.read()
else:
stream = response.read()
if encoding is None:
#binary data
src = stream
else:
#unicode data
src = unicode(unescape(stream.decode(encoding, "ignore")))
return src
def get_page(self,url,encoding,query=None,data=None,range=None,tout=None,head=False,referer=None,header=None,tunnel="disabled",forcetunnel=False,cache=None,throttle=2,title=None):
if self.terminate:
raise task.killbill
tid=self.throttle.id(url,query,referer,header,data)
if head or range:tunnel="disabled"
if throttle==True:throttle=0
if not throttle==False and isinstance(throttle,(int,float)) and not head and not range and self.throttle.check(tid,throttle):
stream=self.throttle.get(tid)
else:
#python cant handle unicode urlencoding so needs to get dirty below.
if not query is None:
query=uniurlenc(query)
url=url+"?"+query
if not data is None and isinstance(data,dict):
data=uniurlenc(data)
#change timeout
if tout is None:
tout=int(float(addon.getSetting("tout")))
headers={'Accept-encoding':'gzip'}
if not referer is None : headers["Referer"]=referer
if not header is None :
for k,v in header.iteritems():
headers[k]=v
tmode="disabled"
if head==True:
req=http.HeadRequest(url,headers=headers)
else:
if not range is None : headers["Range"]="bytes=%d-%d"%(range)
req=urllib2.Request(url,headers=headers)
if not head:
tmode=self.tunnel.set_tunnel(tunnel,force=forcetunnel)
req=self.tunnel.pre(req,tmode,self.cj)
response = cloudfare.ddos_open(url,self.opener, req, data,tout,self.cj,self.cfagents,self.cflocks,self.tunnel,tmode)
self.tunnel.cook(self.cj,self.cj.make_cookies(response,req),tmode)
if head :return response
stream=cloudfare.readzip(response,title)
stream=self.tunnel.post(stream,tmode)
if isinstance(throttle,(int,float)) and not head and not range:
self.throttle.do(tid,stream)
if encoding is None:
#binary data
src=stream
else:
#unicode data
src=unicode(unescape(stream.decode(encoding,"ignore")))
return src
def get_page(self,url,encoding,query=None,data=None,range=None,tout=None,head=False,referer=None,header=None):
if self.terminate:
raise task.killbill
#python cant handle unicode urlencoding so needs to get dirty below.
if not query is None:
query=urllib.urlencode (dict ([k, v.encode('utf-8') if isinstance (v, unicode) else v] for k, v in query.items()))
url=url+"?"+query
if not data is None:
data=urllib.urlencode (dict ([k, v.encode('utf-8') if isinstance (v, unicode) else v] for k, v in data.items()))
#change timeout
if tout is None:
tout=getdefaulttimeout()
headers={'Accept-encoding':'gzip'}
if not referer is None : headers["Referer"]=referer
if not header is None :
for k,v in header.iteritems():
headers[k]=v
if head==True:
req=HeadRequest(url,headers=headers)
else:
if not range is None : headers["Range"]="bytes=%d-%d"%(range)
req=urllib2.Request(url,headers=headers)
response = cloudfare.ddos_open(self.opener, req, data,timeout=tout)
if head :
return response
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
stream = f.read()
else:
stream=response.read()
if encoding is None:
#binary data
src=stream
else:
#unicode data
src=unicode(unescape(stream.decode(encoding,"ignore")))
return src
def get_page(
self, url, encoding, query=None, data=None, range=None, tout=None, head=False, referer=None, header=None
):
if self.terminate:
raise task.killbill
# python cant handle unicode urlencoding so needs to get dirty below.
if not query is None:
query = urlencode(dict([k, v.encode("utf-8") if isinstance(v, unicode) else v] for k, v in query.items()))
url = url + "?" + query
if not data is None:
data = urlencode(dict([k, v.encode("utf-8") if isinstance(v, unicode) else v] for k, v in data.items()))
# change timeout
if tout is None:
tout = int(float(addon.getSetting("tout")))
headers = {"Accept-encoding": "gzip"}
if not referer is None:
headers["Referer"] = referer
if not header is None:
for k, v in header.iteritems():
headers[k] = v
if head == True:
req = HeadRequest(url, headers=headers)
else:
if not range is None:
headers["Range"] = "bytes=%d-%d" % (range)
req = urllib2.Request(url, headers=headers)
response = cloudfare.ddos_open(self.opener, req, data, timeout=tout)
if head:
return response
stream = cloudfare.readzip(response)
if encoding is None:
# binary data
src = stream
else:
# unicode data
src = unicode(unescape(stream.decode(encoding, "ignore")))
return src
