def all(self) -> List[Proxy]:
'''
获取所有代理
:return: 代理列表
'''
return convert_proxy_or_proxies(
self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
def all(self) -> List[Proxy]:
"""
get all proxies
:return: list of proxies
"""
return convert_proxy_or_proxies(
self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
def all(self) -> List[Proxy]:
"""
获取全部代理
:return: 全部代理列表
"""
return convert_proxy_or_proxies(
self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
def batch(self, start, end) -> List[Proxy]:
"""
get batch of proxies
:param start: start index
:param end: end index
:return: list of proxies
"""
return convert_proxy_or_proxies(self.db.zrevrange(REDIS_KEY, start, end - 1))
def random(self) -> Proxy:
"""
随机获取有效代理，首先尝试获取最高分数代理，如果不存在，按照排名获取，否则异常
:return: 随机代理
"""
# try to get proxy with max score
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX,
PROXY_SCORE_MAX)
if len(proxies):
return convert_proxy_or_proxies(choice(proxies))
# else get proxy by rank
proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN,
PROXY_SCORE_MAX)
if len(proxies):
return convert_proxy_or_proxies(choice(proxies))
# else raise error
raise PoolEmptyException
def random(self) -> Proxy:
"""
get random proxy
firstly try to get proxy with max score
if not exists, try to get proxy by rank
if not exists, raise error
:return: proxy like 8.8.8.8:88
"""
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX)
if len(proxies):
return convert_proxy_or_proxies(choice(proxies))
proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)
if len(proxies):
return convert_proxy_or_proxies(choice(proxies))
raise PoolEmptyException
def batch(self, cursor, count) -> List[Proxy]:
"""
get batch of proxies
:param oursor: soan cursor
:param count: soan count
"""
cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count)
return cursor, convert_proxy_or_proxies([i[0] for i in proxies])
def batch(self, cursor, count):
"""
get batch of proxies
:param cursor: scan cursor
:param count: scan count
:return: (cursor, list of proxies)
"""
cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count)
return cursor, convert_proxy_or_proxies([i[0] for i in proxies])
def batch(self, start, end) -> List[Proxy]:
'''
获取一批特定区间中的代理
:param start: 代理的起始索引
:param end: 代理的结束索引
:return: 代理列表
'''
return convert_proxy_or_proxies(
self.db.zrevrange(REDIS_KEY, start, end - 1))
def random(self) -> Proxy:
'''
获取随机代理
1. 最开始，尝试获取最大分数的代理
2. 如果不存在， 尝试通过排行榜获取
3. 如果还是不存在，报错
:return: proxy, like 8.8.8.8:888
'''
# 1. 尝试获取最大分数的代理
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX,
PROXY_SCORE_MAX)
if len(proxies):
return convert_proxy_or_proxies(choice(proxies))
# 没有的话，通过排名获取
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN,
PROXY_SCORE_MAX)
if len(proxies):
return convert_proxy_or_proxies(choice(proxies))
# 如果都没有, 报错
raise PoolEmptyException
def count(self) -> int:
"""
get count of proxies
:return: count, int
"""
return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX))
