def crawl(self):
for url in self.seeds:
try:
log.debug("开始采集[%s]的文章"%url);
self.fetchrss(url)
except Exception as e:
log.debug("采集[%s]的文章发生异常"%url);
log.exception(e)
def crawl(self):
for url in self.seeds:
try:
log.debug("开始采集[%s]的文章" % url)
self.fetchrss(url)
except Exception as e:
log.debug("采集[%s]的文章发生异常" % url)
log.exception(e)
def run(self):
queue = self.queue
while True:
try:
task = queue.get()
log.info("worker执行"+str(task))
task._execute()
except Exception as e:
log.exception(e)
def shutdown(self):
for worker in self.workers:
try:
worker._stop();
except Exception as e:
log.exception(e)
