class Getter:
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_less_threshold(self):
"""
判断是否达到了代理池限制
"""
if self.redis.available_count(
) <= POOL_UPPER_AV_THRESHOLD or self.redis.count(
) <= POOL_UPPER_THRESHOLD:
return True
else:
return False
def run(self):
print('获取器开始执行')
if self.is_less_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
# 获取代理
proxies = self.crawler.get_proxies(callback)
sys.stdout.flush()
for proxy in proxies:
self.redis.add(proxy)
