Beispiel #1
0
 def initWebsite_delay_dict(self, record):
     """
     @summary: 初始化网站的等待更新时间
     :param record: 网站记录(id, url, xpath, delay_time)
     :return:
     """
     if not Cache.keyExist(cache.websiteDelay_dict, record[0]):
         Cache.setDict(cache.websiteDelay_dict, record[0], record[-1])
Beispiel #2
0
 def putRecord(self, record):
     """
     @summary: 把record添加到正在等待的网站队列中
     """
     website_id, website_url, xpath = record[:3]
     if not Cache.listItemExist(cache.workingWebsite_list, website_id) and \
             not Cache.keyExist(cache.unrecognized_websiteUrl_dict, website_id):
         Cache.appendList(cache.workingWebsite_list, website_id)
         Cache.putQueue(cache.websiteUrl_queue, (website_id, website_url, xpath))
         sleep_time = Cache.getDict(cache.websiteDelay_dict, website_id)
         for i in range(int(sleep_time)):
             if global_EXIT: return
             time.sleep(1)
         Cache.removeList(cache.workingWebsite_list, website_id)