def all(self) -> List[Proxy]: ''' 获取所有代理 :return: 代理列表 ''' return convert_proxy_or_proxies( self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
def all(self) -> List[Proxy]: """ get all proxies :return: list of proxies """ return convert_proxy_or_proxies( self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
def all(self) -> List[Proxy]: """ 获取全部代理 :return: 全部代理列表 """ return convert_proxy_or_proxies( self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
def batch(self, start, end) -> List[Proxy]: """ get batch of proxies :param start: start index :param end: end index :return: list of proxies """ return convert_proxy_or_proxies(self.db.zrevrange(REDIS_KEY, start, end - 1))
def random(self) -> Proxy: """ 随机获取有效代理,首先尝试获取最高分数代理,如果不存在,按照排名获取,否则异常 :return: 随机代理 """ # try to get proxy with max score proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error raise PoolEmptyException
def random(self) -> Proxy: """ get random proxy firstly try to get proxy with max score if not exists, try to get proxy by rank if not exists, raise error :return: proxy like 8.8.8.8:88 """ proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) raise PoolEmptyException
def batch(self, cursor, count) -> List[Proxy]: """ get batch of proxies :param oursor: soan cursor :param count: soan count """ cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count) return cursor, convert_proxy_or_proxies([i[0] for i in proxies])
def batch(self, cursor, count): """ get batch of proxies :param cursor: scan cursor :param count: scan count :return: (cursor, list of proxies) """ cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count) return cursor, convert_proxy_or_proxies([i[0] for i in proxies])
def batch(self, start, end) -> List[Proxy]: ''' 获取一批特定区间中的代理 :param start: 代理的起始索引 :param end: 代理的结束索引 :return: 代理列表 ''' return convert_proxy_or_proxies( self.db.zrevrange(REDIS_KEY, start, end - 1))
def random(self) -> Proxy: ''' 获取随机代理 1. 最开始,尝试获取最大分数的代理 2. 如果不存在, 尝试通过排行榜获取 3. 如果还是不存在,报错 :return: proxy, like 8.8.8.8:888 ''' # 1. 尝试获取最大分数的代理 proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # 没有的话,通过排名获取 proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # 如果都没有, 报错 raise PoolEmptyException
def count(self) -> int: """ get count of proxies :return: count, int """ return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))