def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
"""
# 实例化redis客户端
conn = RedisClient()
# 实例化ip有效性测试器
tester = ValidityTester()
while True:
print('Refreshing ip')
# 获取redis数据库列表中一半的个数
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
# 等待一个周期时间
time.sleep(cycle)
continue
# 获取redis数据库列表中一半的代理
raw_proxies = conn.get(count)
# 添加到为测试有效性列表中
tester.set_raw_proxies(raw_proxies)
# 测试所用代理
tester.test()
# 等待一个周期时间
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
conn = RedisClient()
tester = ValidityTester()
while True:
print('Refresing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""从redis里面获取一半的代理
"""
conn = RedisClient()
tester = VaildityTester()
while True:
Logger.log_high('Refreshing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
Logger.log_normal('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
"""
conn = RedisClient()
tester = ValidityTester()#创建实例
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)#调用conn = RedisClient()中get方法，get方法是获取代理
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): # 定时检测器
"""
Get half of proxies which in redis
"""
conn = RedisClient() # redis连接对象
tester = ValidityTester() # 检测代理是否可用
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len) # 取出前一半的代理
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
"""
conn = RedisClient() # redis连接对象
tester = ValidityTester() # 代理检测对象
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len) # 需要从redis中取出一半的代理地址
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count) # 从redis中取出一半的代理地址,返回列表
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
获取队列中一半的代理进行可用性测试！
"""
conn = RedisClient()
tester = ValidityTester()
while True:
print('测试redis队列代理可用性：')
count = int(0.5 * conn.queue_len)
if count == 0:
print('代理池为空或者全部测试完毕！')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
"""
conn = RedisClient()
tester = ValidityTester()
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
"""
conn = RedisClient() #Redis连接对象
tester = ValidityTester()
while True:
print('刷新代理池中...')
count = int(0.3 * conn.queue_len) #从左侧拿出一半的代理,只剩一个时,看做0个
if count == 0: #如果队列长度不够了
print('等待添加代理中...')
time.sleep(cycle) #设置暂时睡眠,等待添加
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies) #调用函数添加,raw_proxies设置为类变量
tester.test() #检测代理是否可用
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
从redis取出代理进行异步检测，将可用代理重新放回redis数据库列表中的右侧，保证代理的定时更新
"""
conn = RedisClient()
tester = ValidityTester()
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): # VALID_CHECK_CYCLE是在setting中定义好
"""
Get half of proxies which in redis
"""
conn = RedisClient()
tester = ValidityTester()
while True:
print('Refreshing ip')
# 从redis拿出一半的数据进行检测
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(
cycle=VALID_CHECK_CYCLE): # VALID_CHECK_CYCLE 时间的参数 setting中配置
"""
Get half of proxies which in redis
定时检测器 检测redis数据库ip是否有效
"""
conn = RedisClient() # 连接redis数据库
tester = ValidityTester() # 检测代理是否可用的类
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len) # 取出队列长度一般的代理
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE):
"""
Get half of proxies which in redis
"""
conn = RedisClient()
tester = ValidityTester()
raw_proxies = []
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
time.sleep(30 * 60)
raw_proxies = conn.get()
# raw_proxies.append(raw_proxie)
tester.set_raw_proxies(raw_proxies)
tester.test()
time.sleep(cycle)
def valid_proxy(
cycle=VALID_CHECK_CYCLE
): # classmethod与staticmethod的区别是1、classmethod下的函数要加（cls）, staticmethod不需要 /
""" # 2、classmethod的方法可以继承到子类的属性，staitcmethod不能继承子类的任何属性。
Get half of proxies which in redis # 所以，多用classmethod少用staticmethod
"""
conn = RedisClient()
tester = ValidityTester()
while True:
print('Refreshing ip')
count = int(0.5 * conn.queue_len)
if count == 0:
print('Waiting for adding')
time.sleep(cycle)
continue
raw_proxies = conn.get(count)
tester.set_raw_proxies(raw_proxies)
tester.test()
# with open('proxies.txt', 'a') as f: #增加功能：把代理写入txt文件
# for proxies in proxy:
# f.write(proxy + '\n')
time.sleep(cycle)
class RedisClientTestCase(unittest.TestCase):
def setUp(self):
self._conn = RedisClient()
def tearDown(self):
self._conn.flush()
def test_put_and_pop(self):
self._conn.put("label")
assert self._conn.pop() == "label"
def test_put_many(self):
self._conn.put_many(['a', 'b'])
assert self._conn.pop() == "a"
assert self._conn.pop() == "b"
def test_len(self):
self._conn.put_many(['a', 'b', 'c'])
assert self._conn.queue_len == 3
def test_get(self):
self._conn.put_many(['a', 'b', 'c', 'd'])
_ = self._conn.get(2)
assert self._conn.queue_len == 2
