def run(self):
while True:
if not self.runValue.value:
print "%s stops" % self.name
break
self.signalget()
start_t = time.time()
self.ctask = self.taskqueue.get()
if self.ctask.empty:
time.sleep(10)
continue
end_t = time.time()
self.log_wait_task(end_t - start_t)
self.log_get_task()
start_t = time.time()
c = Crawler().create(self.ctask.type, self.ctask.key, self.ctask.data)
if c:
try:
c.crawl()
success = True
logger.info("CRAWL SUCCEED - <%s> %s" % (self.taskqueue.queueid, c))
end_t = time.time()
self.log_done_task(end_t - start_t)
except Exception:
msg = get_exception_info()
success = False
logger.error("CRAWL FAILED - <%s> %s, %s" % (self.taskqueue.queueid, c, msg))
else:
logger.error("CRAWL FAILED - <%s> %s" % (self.taskqueue.queueid, self.ctask))
success = False
Scheduler.finish(self.ctask.type, self.ctask.key, c.data if c else {}, success)
def run(self):
while True:
if not self.runValue.value:
print "%s stops" % self.name
break
self.signalget()
start_t = time.time()
self.ctask = self.taskqueue.get()
if self.ctask.empty:
time.sleep(10)
continue
end_t = time.time()
self.log_wait_task(end_t - start_t)
self.log_get_task()
start_t = time.time()
c = Crawler().create(self.ctask.type, self.ctask.key,
self.ctask.data)
if c:
try:
c.crawl()
success = True
logger.info("CRAWL SUCCEED - <%s> %s" %
(self.taskqueue.queueid, c))
end_t = time.time()
self.log_done_task(end_t - start_t)
except Exception:
msg = get_exception_info()
success = False
logger.error("CRAWL FAILED - <%s> %s, %s" %
(self.taskqueue.queueid, c, msg))
else:
logger.error("CRAWL FAILED - <%s> %s" %
(self.taskqueue.queueid, self.ctask))
success = False
Scheduler.finish(self.ctask.type, self.ctask.key,
c.data if c else {}, success)
