def crawl(self): for url in self.seeds: try: log.debug("开始采集[%s]的文章"%url); self.fetchrss(url) except Exception as e: log.debug("采集[%s]的文章发生异常"%url); log.exception(e)
def crawl(self): for url in self.seeds: try: log.debug("开始采集[%s]的文章" % url) self.fetchrss(url) except Exception as e: log.debug("采集[%s]的文章发生异常" % url) log.exception(e)
def run(self): queue = self.queue while True: try: task = queue.get() log.info("worker执行"+str(task)) task._execute() except Exception as e: log.exception(e)
def shutdown(self): for worker in self.workers: try: worker._stop(); except Exception as e: log.exception(e)