Ejemplo n.º 1
0
 def run(self):
     while self.isRun:
         try:
             do, args = self.work_queue.get(block=False)
             do(args)
             self.work_queue.task_done()
         except Exception, e:
             if str(type(e)) != "<class 'Queue.Empty'>":
                 error_log(trace_back())
             break
Ejemplo n.º 2
0
 def urlopen(self,url,method = 'get',data = {},info = {},timeout = 30): 
     _url = ''
     if method == "post":
         query = urllib.urlencode(data)
         _url = url
         url = url + "?" + query
         
            
     if self.cache:            
         if self.checkUrlCacheExits(url):                
             return self.getCacheContent(url)
     
     if self.opener is None:
         self.setOpener()   
         
     v = {}
     
     for k in info:
         v[k]  = info[k]
     
     v['url']     = url    
     v['local']    = self.getUrlCacheFile(url)        
     v['headers'] = ''
     v['cache']   = False
     v['body']    = ''
     
     self.setUrlCache(url,v)
                            
     try:
         if method == "get":                
             req = urllib2.Request(url)
         else:
             req = urllib2.Request(_url,query)
                         
         req.add_header("User-Agent", self.user_agent)
         r = self.opener.open(req,timeout = timeout)
     except urllib2.HTTPError, e:
         self.delUrlCache(url)
         error_log(url+"\n"+trace_back()+"\n")
         return None
Ejemplo n.º 3
0
                            
     try:
         if method == "get":                
             req = urllib2.Request(url)
         else:
             req = urllib2.Request(_url,query)
                         
         req.add_header("User-Agent", self.user_agent)
         r = self.opener.open(req,timeout = timeout)
     except urllib2.HTTPError, e:
         self.delUrlCache(url)
         error_log(url+"\n"+trace_back()+"\n")
         return None
     except Exception , e:
         self.delUrlCache(url)
         error_log(url+"\n"+trace_back()+"\n")
         return None   
          
     self.saveCookie()
     
     v['headers'] = dict(r.headers)  
     v['body']    = r.read()
     
     self.setUrlCache(url,v)
     r.close()
     return v 
 
 def setUrlCache(self,url,v,level = 3):
     #if self.cache == False:
     #    return        
     vv = {}