def run(self):
myLogger.info("worker %s begin to run" % (self.name))
myLogger.info("my manager's inQueue size [%d]" % (self.inQueue.qsize()))
self.hungerly = False
while True:
try:
# get
get = None
try:
get = self.inQueue.get(timeout=1)
self.hungerly = False
except Queue.Empty, e:
self.hungerly = True
myLogger.debug('thread [%s] is hungerly now' % (self.name))
if self.m.shouldExit == True:
myLogger.debug('thread [%s] is exiting' % (self.name))
break
if self.hungerly == False:
# do
myLogger.info("thread [%s], get task[%s] from queue" % (self.name, get.id))
time.sleep(3)
# put
self.doOneTask(get)
else:
time.sleep(2)
except Exception, e:
util.printException()
myLogger.error('sth wrong[%s] in thread [%s]' % (e, self.name))
get.status = 'failed'
if get.msg != '':
get.msg = util.exprException()
def download(self, url, to, postdata={}, timeout=60, times=3, safeFetch=True):
myLogger.info("fetcher download from [%s] to [%s]" % (url, to))
ret = None
while times!=0:
status=''
if safeFetch == True:
resp = self.safeFetch(url, postdata, timeout)
else:
resp, status = self.fetch(url, postdata, timeout)
if util.mkdir(os.path.dirname(to)) == False:
times -= 1
ret = None
continue
if status == 'OK' or (status == '' and resp != None): # fetch ok
CHUNK = 1024 * 1024 * 5
with open(to, 'wb') as f:
ret = True
try:
while True:
chunk = resp.read(CHUNK)
if not chunk:
break
f.write(chunk)
except Exception, e:
msg = util.exprException()
util.printException()
ret = None
if ret == True:
break
times -= 1
def run(self):
myLogger.info("worker %s begin to run" % (self.name))
myLogger.info("my manager's inQueue size [%d]" %
(self.inQueue.qsize()))
self.hungerly = False
while True:
try:
# get
get = None
try:
get = self.inQueue.get(timeout=1)
self.hungerly = False
except Queue.Empty, e:
self.hungerly = True
myLogger.debug('thread [%s] is hungerly now' % (self.name))
if self.m.shouldExit == True:
myLogger.debug('thread [%s] is exiting' % (self.name))
break
if self.hungerly == False:
# do
myLogger.info("thread [%s], get task[%s] from queue" %
(self.name, get.id))
time.sleep(3)
# put
self.doOneTask(get)
else:
time.sleep(2)
except Exception, e:
util.printException()
myLogger.error('sth wrong[%s] in thread [%s]' % (e, self.name))
get.status = 'failed'
if get.msg != '':
get.msg = util.exprException()
