def run(self): myLogger.info("worker %s begin to run" % (self.name)) myLogger.info("my manager's inQueue size [%d]" % (self.inQueue.qsize())) self.hungerly = False while True: try: # get get = None try: get = self.inQueue.get(timeout=1) self.hungerly = False except Queue.Empty, e: self.hungerly = True myLogger.debug('thread [%s] is hungerly now' % (self.name)) if self.m.shouldExit == True: myLogger.debug('thread [%s] is exiting' % (self.name)) break if self.hungerly == False: # do myLogger.info("thread [%s], get task[%s] from queue" % (self.name, get.id)) time.sleep(3) # put self.doOneTask(get) else: time.sleep(2) except Exception, e: util.printException() myLogger.error('sth wrong[%s] in thread [%s]' % (e, self.name)) get.status = 'failed' if get.msg != '': get.msg = util.exprException()
def download(self, url, to, postdata={}, timeout=60, times=3, safeFetch=True): myLogger.info("fetcher download from [%s] to [%s]" % (url, to)) ret = None while times!=0: status='' if safeFetch == True: resp = self.safeFetch(url, postdata, timeout) else: resp, status = self.fetch(url, postdata, timeout) if util.mkdir(os.path.dirname(to)) == False: times -= 1 ret = None continue if status == 'OK' or (status == '' and resp != None): # fetch ok CHUNK = 1024 * 1024 * 5 with open(to, 'wb') as f: ret = True try: while True: chunk = resp.read(CHUNK) if not chunk: break f.write(chunk) except Exception, e: msg = util.exprException() util.printException() ret = None if ret == True: break times -= 1