def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis """ # 实例化redis客户端 conn = RedisClient() # 实例化ip有效性测试器 tester = ValidityTester() while True: print('Refreshing ip') # 获取redis数据库列表中一半的个数 count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') # 等待一个周期时间 time.sleep(cycle) continue # 获取redis数据库列表中一半的代理 raw_proxies = conn.get(count) # 添加到为测试有效性列表中 tester.set_raw_proxies(raw_proxies) # 测试所用代理 tester.test() # 等待一个周期时间 time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): conn = RedisClient() tester = ValidityTester() while True: print('Refresing ip') count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """从redis里面获取一半的代理 """ conn = RedisClient() tester = VaildityTester() while True: Logger.log_high('Refreshing ip') count = int(0.5 * conn.queue_len) if count == 0: Logger.log_normal('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis """ conn = RedisClient() tester = ValidityTester()#创建实例 while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count)#调用conn = RedisClient()中get方法,get方法是获取代理 tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): # 定时检测器 """ Get half of proxies which in redis """ conn = RedisClient() # redis连接对象 tester = ValidityTester() # 检测代理是否可用 while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) # 取出前一半的代理 if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis """ conn = RedisClient() # redis连接对象 tester = ValidityTester() # 代理检测对象 while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) # 需要从redis中取出一半的代理地址 if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) # 从redis中取出一半的代理地址,返回列表 tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ 获取队列中一半的代理进行可用性测试! """ conn = RedisClient() tester = ValidityTester() while True: print('测试redis队列代理可用性:') count = int(0.5 * conn.queue_len) if count == 0: print('代理池为空或者全部测试完毕!') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis """ conn = RedisClient() tester = ValidityTester() while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis """ conn = RedisClient() #Redis连接对象 tester = ValidityTester() while True: print('刷新代理池中...') count = int(0.3 * conn.queue_len) #从左侧拿出一半的代理,只剩一个时,看做0个 if count == 0: #如果队列长度不够了 print('等待添加代理中...') time.sleep(cycle) #设置暂时睡眠,等待添加 continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) #调用函数添加,raw_proxies设置为类变量 tester.test() #检测代理是否可用 time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis 从redis取出代理进行异步检测,将可用代理重新放回redis数据库列表中的右侧,保证代理的定时更新 """ conn = RedisClient() tester = ValidityTester() while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): # VALID_CHECK_CYCLE是在setting中定义好 """ Get half of proxies which in redis """ conn = RedisClient() tester = ValidityTester() while True: print('Refreshing ip') # 从redis拿出一半的数据进行检测 count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy( cycle=VALID_CHECK_CYCLE): # VALID_CHECK_CYCLE 时间的参数 setting中配置 """ Get half of proxies which in redis 定时检测器 检测redis数据库ip是否有效 """ conn = RedisClient() # 连接redis数据库 tester = ValidityTester() # 检测代理是否可用的类 while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) # 取出队列长度一般的代理 if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy(cycle=VALID_CHECK_CYCLE): """ Get half of proxies which in redis """ conn = RedisClient() tester = ValidityTester() raw_proxies = [] while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue time.sleep(30 * 60) raw_proxies = conn.get() # raw_proxies.append(raw_proxie) tester.set_raw_proxies(raw_proxies) tester.test() time.sleep(cycle)
def valid_proxy( cycle=VALID_CHECK_CYCLE ): # classmethod与staticmethod的区别是1、classmethod下的函数要加(cls), staticmethod不需要 / """ # 2、classmethod的方法可以继承到子类的属性,staitcmethod不能继承子类的任何属性。 Get half of proxies which in redis # 所以,多用classmethod少用staticmethod """ conn = RedisClient() tester = ValidityTester() while True: print('Refreshing ip') count = int(0.5 * conn.queue_len) if count == 0: print('Waiting for adding') time.sleep(cycle) continue raw_proxies = conn.get(count) tester.set_raw_proxies(raw_proxies) tester.test() # with open('proxies.txt', 'a') as f: #增加功能:把代理写入txt文件 # for proxies in proxy: # f.write(proxy + '\n') time.sleep(cycle)
class RedisClientTestCase(unittest.TestCase): def setUp(self): self._conn = RedisClient() def tearDown(self): self._conn.flush() def test_put_and_pop(self): self._conn.put("label") assert self._conn.pop() == "label" def test_put_many(self): self._conn.put_many(['a', 'b']) assert self._conn.pop() == "a" assert self._conn.pop() == "b" def test_len(self): self._conn.put_many(['a', 'b', 'c']) assert self._conn.queue_len == 3 def test_get(self): self._conn.put_many(['a', 'b', 'c', 'd']) _ = self._conn.get(2) assert self._conn.queue_len == 2