class GetNewProxy(object):
#得到新的代理ip
def __init__(self,max):
self._max_count=max
self._redis=RedisClient()
self._tester=ValidityTester()
self._getter=FreeProxyGetter()
def is_over(self):
#是否超过总数
print(self._redis.len())
if self._redis.len()>=self._max_count:
return True
else:
return False
def add_new_proxy(self):
#增加新的代理ip到代理池中
print('Add and get new proxy')
while not self.is_over():
for callback in range(self._getter.__CrawlFuncCount__):
# print(self._getter.__CrawlFunc__[callback])
self._tester.set_raw_proxies(self._getter.get_raw_proxies(self._getter.__CrawlFunc__[callback]))
self._tester.check_some_proxies()
if self.is_over():
print('IP is enough, waiting to be used')
break
def addnewproxy(wait_time=POOL_LEN_CHECK_TIME,min_count=POOL_LOWER_THRESHOLD):
#增加新的代理ip
g = GetNewProxy(POOL_UPPER_THRESHOLD)
redis=RedisClient()
while True:
if redis.len()<min_count:
print('Add new Proxies')
g.add_new_proxy()
time.sleep(wait_time)
def checkproxypool(wait_time=VALID_CHECK_TIME):
#检查原有的代理池
print('Checking pool')
tester = ValidityTester()
redis=RedisClient()
while True:
halfcount = int(redis.len() / 2)
if halfcount==0:
time.sleep(wait_time)
continue
halfproxies = redis.cget(halfcount)
tester.set_raw_proxies(halfproxies)
tester.check_some_proxies()
time.sleep(wait_time)
