class GetNewProxy(object): #得到新的代理ip def __init__(self,max): self._max_count=max self._redis=RedisClient() self._tester=ValidityTester() self._getter=FreeProxyGetter() def is_over(self): #是否超过总数 print(self._redis.len()) if self._redis.len()>=self._max_count: return True else: return False def add_new_proxy(self): #增加新的代理ip到代理池中 print('Add and get new proxy') while not self.is_over(): for callback in range(self._getter.__CrawlFuncCount__): # print(self._getter.__CrawlFunc__[callback]) self._tester.set_raw_proxies(self._getter.get_raw_proxies(self._getter.__CrawlFunc__[callback])) self._tester.check_some_proxies() if self.is_over(): print('IP is enough, waiting to be used') break
def addnewproxy(wait_time=POOL_LEN_CHECK_TIME,min_count=POOL_LOWER_THRESHOLD): #增加新的代理ip g = GetNewProxy(POOL_UPPER_THRESHOLD) redis=RedisClient() while True: if redis.len()<min_count: print('Add new Proxies') g.add_new_proxy() time.sleep(wait_time)
def checkproxypool(wait_time=VALID_CHECK_TIME): #检查原有的代理池 print('Checking pool') tester = ValidityTester() redis=RedisClient() while True: halfcount = int(redis.len() / 2) if halfcount==0: time.sleep(wait_time) continue halfproxies = redis.cget(halfcount) tester.set_raw_proxies(halfproxies) tester.check_some_proxies() time.sleep(wait_time)