def __init__(self, crawler):
print(
'[' +
time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(time.time())) +
']', "【init】")
self.proxy = None # 默认代理为空，使用真实IP
self.db = RedisClient()
def run(self):
print('Ip processing running')
RedisClient().flush()
valid_process = Process(target=Schedule.valid_proxies)
check_process = Process(target=Schedule.check_pool)
valid_process.start()
check_process.start()
def check_pool(cycle=POOL_LEN_CHECK_CYCLE,
min=POOL_LOWER_THRESHOLD,
max=POOL_UPPER_THRESHOLD):
conn = RedisClient()
adder = ProxyPoolAdder(max)
while True:
if conn.zset_len <= min:
adder.add_to_zset()
time.sleep(cycle)
def control_add_proxy():
proxy_min = PROXY_MIN
conn = RedisClient()
addr = PoolAdder()
cycle = CYCLE_TIME
while True:
if conn.count() < PROXY_MAX:
addr.add_to_redis()
time.sleep(cycle)
def valid_proxies(cycle=VALID_CHECK_CYCLE):
conn = RedisClient()
tester = ValidityTester()
while True:
print("check valid proxies")
count = int(conn.zset_len / 2)
if count == 0:
print("add new valid proxies")
time.sleep(cycle)
continue
proxies = conn.get(count)
tester.set_raw_proxies(proxies)
tester.test()
time.sleep(cycle)
def control_test_proxy():
conn = RedisClient()
cycle = CYCLE_TIME
tester = ValidityTester()
while True:
print('Refreshing ip')
count = int(0.5 * conn.count())
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_test_procies(raw_proxies)
tester.test()
time.sleep(cycle)
def __init__(self):
self.redis = RedisClient()
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def get_conn():
if not hasattr(g, 'redis'):
g.redis = RedisClient()
return g.redis
from ProxyPool.db import RedisClient
conn = RedisClient()
def set(proxy):
result = conn.add(proxy)
print(proxy)
print('录入成功' if result else '录入失败')
def scan():
print('请输入代理, 输入exit退出读入')
while True:
proxy = input()
if proxy == 'exit':
break
set(proxy)
if __name__ == '__main__':
scan()
def __init__(self):
self.proxyData = RedisClient()
self.logger = logging.getLogger(__name__)
def get_proxy():
cli = RedisClient()
return cli.random()
def __init__(self, threshold):
self._conn = RedisClient()
self._crawl = FreeProxyGetter()
self._tester = ValidityTester()
self._threshold = threshold
self.logger = logging.getLogger(__name__)
def __init__(self):
self.proxy_max = PROXY_MAX
self.tester = ValidityTester()
self.getter = FreeProxyGetter()
self.conn = RedisClient()
def set_test_procies(self, proxies):
self.test_proxies = proxies
self.conn = RedisClient()
def __init__(self):
self._raw_proxies = None
self._available_proxies = []
self._conn = RedisClient()
self.logger = logging.getLogger(__name__)