class HtmlFetchThread(threading.Thread):
def __init__(self, proxy,url):
threading.Thread.__init__(self)
self.html=''
self.spider = ProxySpider(name='duokan',host='localhost')
self.proxy =proxy
self.url =url
def run(self):
self.html=self.spider.fetch(url=self.url, proxy=self.proxy, nb_retries=0, retry_delay=0.5)
def does_function(self):
trycount=1
data=""
while trycount<=3:
try:
f=self.opener.open(self.request)
data=f.read()
break
except:
time.sleep(0.5)
trycount=trycount+1
if trycount>3:
print "bad"
return None
else:
return data
def __init__(self, proxy,url):
threading.Thread.__init__(self)
self.html=''
self.spider = ProxySpider(name='duokan',host='localhost')
self.proxy =proxy
self.url =url
