class CommonUpdate: def __init__(self): self.dao = DaoService() def update_website(self): Log.init_log() logging.debug("网站内容更新开始") categories = self.get_categories() for category in categories: self.update_category(category) logging.debug("网站内容更新完成") def update_hot_blog(self): search = HotSearchCore() ids = [6, 7] search.hot_search(ids) #查询所有二级目录 def get_categories(self): return self.dao.query_categories() #更新一个目录 def update_category(self, category): category_id = category.category_id blog_num = category.update_blog_num crawlers = self.dao.query_tcrawlers_category(category_id, blog_num) if not crawlers: return for crawler in crawlers: try: logging.debug('执行') logging.info(crawler.name) self.crawler_run(crawler) next_run_time = self.get_next_run_time(crawler.interval) self.dao.update_time_t_crawler(crawler.id, next_run_time) except Exception as err: logging.error('执行出错') logging.error(traceback.format_exc()) logging.error( err) next_run_time = self.get_next_run_time(category.interval) self.dao.update_time_category(category_id, next_run_time) #执行爬虫 def crawler_run(self, crawler): search = SearchCore(crawler) search.search() #获得下次执行时间 def get_next_run_time(self, interval): now = DateUtils.now() next_run_time = DateUtils.add_day(now, interval) return DateUtils.date_of_str_format(next_run_time, DateUtils.STYLE_MYSQL)