def detach(subscribe, beat_sync=False): """ @param subscribe: @param beat_sync: 是否立即删除, True:立即删除,False:节拍同步,随ddt删除 @return: """ from faker import Faker from urllib.parse import urlparse # 清洗出订阅中的token token = urlparse(subscribe).path r = RedisClient().get_driver() # 遍历所有任务类型 for task in CRAWLER_SEQUENCE: # 遍历某种类型的链接池 for sub in r.hgetall(REDIS_SECRET_KEY.format(task)).items(): # 匹配用户token if token == urlparse(sub[0]).path: # 若节拍同步,立即移除订阅 if beat_sync: r.hdel(REDIS_SECRET_KEY.format(task), sub[0]) logger.debug(f'>> Detach -> {sub[0]}') # 否则将订阅过期时间标记为过期,该链接将随下一波任一节点的ddt任务被删除 else: r.hset(REDIS_SECRET_KEY.format(task), sub[0], str(Faker().past_datetime())) break
def to_redis(): r = RedisClient().get_driver() for docker in Middleware.cache_redis_queue.items(): key_name = REDIS_SECRET_KEY.format(docker[0]) if docker[-1]: r.hset(key_name, mapping=docker[-1]) # logger.success(f">> PUSH -> Redis") for k in Middleware.cache_redis_queue.keys(): Middleware.cache_redis_queue[k] = {}
def register_auth(self, sckey): """ :param sckey: db_token :return: """ sckey_path = self.SCKEY_PATH_ROOT.format(sckey) driver = RedisClient().get_driver() driver.hset(sckey_path, key="CREATE", value=str(datetime.now(TIME_ZONE_CN))) driver.hset(sckey_path, key="READ", value="None") driver.hset(sckey_path, key="UPDATE", value="None") driver.hset(sckey_path, key="DELETE", value="True") driver.hset(sckey_path, key="SURVIVE", value="True")