class Proxy:
def __init__(self):
self.fetcher = ProxyFetcher('https',
strategy='greedy',
redis_args=args)
self.pools = self.fetcher.pool
self.used = 1
# self.usedConn = StrictRedis(**args)
def get_ip(self):
self.used += 1
# log.info("pid: {} , used: {}, length: {}, ***** {} %".format(os.getpid(), self.used, len(self.pools),
# self.used / (len(self.pools) + 1) * 100))
if self.used > len(self.pools) / 3:
# self.fetcher = ProxyFetcher('http', strategy='greedy', redis_args=args)
self.pools = self.fetcher.get_available_proxies(self.fetcher.conn)
self.used = 0
# log.info("""
# ******************************
# ****** {} use ********
# ******************************
# """.format(str(len(self.pools))))
return random.choice(self.pools)
# use = self.usedConn.srandmember('haipproxy:all').decode()
# log.info(use)
# return use
def remove(self, url):
self.fetcher.delete_proxy(url)
self.pools.remove(url)
class Proxy:
def __init__(self):
self.fetcher = ProxyFetcher('https',
strategy='greedy',
redis_args=args)
self.pools = self.fetcher.get_proxies()
self.used = 1
def get_ip(self):
self.used += 1
log.info("pid: {} , used: {}, length: {}, ***** {} %".format(
os.getpid(), self.used, len(self.pools),
self.used / (len(self.pools) + 1) * 100))
if self.used > len(self.pools) / 10:
# self.fetcher = ProxyFetcher('http', strategy='greedy', redis_args=args)
self.pools = self.fetcher.get_proxies()
self.used = 0
log.info("""
******************************
****** {} ********
******************************
""".format(str(len(self.pools))))
return random.choice(self.pools)
def remove(self, url):
self.fetcher.delete_proxy(url)
self.pools.remove(url)