def download(self):
'''
Private function
Download thread.
'''
out.printstr("Download thread started.\n")
while True:
url = self.get_url()
if url == None:
break
#Download url
while True:
request = urllib2.Request(url[0])
request.add_header('User-Agent', self.agent)
try:
try:
out.printstr("\nGetting %s"%(url[0]))
response = urllib2.urlopen(request,timeout = self.timeout)
except urllib2.URLError,e:
out.printerr(e)
out.printstr("Download failed retrying...\n")
continue
data = response.read()
if len(data) == 0:
out.printstr("Download failed retrying...\n")
continue
except Exception:
out.printstr("Download failed retrying...\n")
continue
break
def analyse(self):
'''
Private function
Thread function.It will be called in __init__().Don't call it manually.
'''
exec "from analyser.%s import *"%(self.analyser)
#Get first page
request = urllib2.Request(self.url)
request.add_header('User-Agent', self.agent)
try:
out.printstr("\nGetting %s"%(self.url))
response = urllib2.urlopen(request,timeout=5)
except Exception,e:
out.printerr(e)
self.end = True
self.buff_lock.acquire()
self.buff_lock.notifyAll()
self.buff_lock.release()
exit(-1)
#Initialize analyser
data = response.read()
out.printstr("%i bytes downloaded.\nAnalysing..."%(len(data)))
try:
analy = analyser(data,self.args)
except Exception,e:
out.printerr(e)
self.end = True
self.buff_lock.acquire()
self.buff_lock.notifyAll()
self.buff_lock.release()
exit(-1)
try:
data = analy.get_data(0)
except Exception,e:
out.printerr(e)
self.end = True
self.buff_lock.acquire()
self.buff_lock.notifyAll()
self.buff_lock.release()
exit(-1)
self.write_buf(data)
#Analyse
if analy.allow_multi_thread():
#The analyser support multi-thread download
#Get args
try:
timeout = int(self.args["t"])
except KeyError:
timeout = 5
try: