def saveWebsiteDelaytime(): """ @summary: 保存网站爬取延迟到数据库中 """ try: for website_id in Cache.keys(cache.websiteDelay_dict): delaytime = Cache.getDict(cache.websiteDelay_dict, website_id) db.saveDelay_time(website_id, delaytime) except Exception as e: log.logMsg(LogType.error, "[saveWebsiteDelaytime] %s" % (repr(e)))
def show_delay_time(): """ @summary: 显示各网站的爬取延迟 """ records = [] keys = Cache.keys(cache.websiteDelay_dict) or [] for website_id in keys: record = mysql.Mysql.queryWebsiteUrl(website_id) # id,url,xpath,detail,delay_time records.append((record[0][0], record[0][3] or record[0][1], Cache.getDict(cache.websiteDelay_dict, website_id))) headers = ["id", "url", "delay-time(s)"] print(tabulate(records, headers=headers))
def resetDelay_time(): """ @summary: 重置各网站的爬取延迟 """ db = None try: db = mysql.Mysql() for website_id in Cache.keys(cache.websiteDelay_dict): record = Cache.getDict(cache.websiteDelay_dict, website_id) Cache.setDict(cache.websiteDelay_dict, website_id, (record[0], 0)) db.saveDelay_time(website_id, 0) except Exception as e: log.logMsg(LogType.error, "[saveWebsiteDelaytime] %s" % (repr(e))) finally: if db: db.dispose()