class Test(): def __init__(self): self.redis = RedisClient() def test_single(self, proxy): ip = json.loads(proxy) ip_type = ip['type'] ip = ip['ip'] try: response = get_response(URL, ip) if response.status_code in VALID_CODE: print(ip, ip_type, '正常') self.redis.max(proxy) else: print(ip, ip_type, '-1') self.redis.decrease(proxy) except Exception as e: print(e) print(ip, ip_type, '-1') self.redis.decrease(proxy) def run(self): proxies = self.redis.all() q = queue.Queue(10) for proxy in proxies: if not q.full(): q.put(proxy) if not q.empty(): proxy = q.get() t = threading.Thread(target=self.test_single, args=(proxy, )) t.start()
class Tester(object): def __init__(self): self.redis = RedisClient() async def test_single_proxy(self,proxy): """ 测试单个代理 :param proxy: 单个代理 :return: """ conn=aiohttp.TCPConnector(verify_ssl=False) async with aiohttp.ClientSession(connector=conn) as session: try: if isinstance(proxy,bytes): proxy=proxy.decode('utf-8') real_proxy='http://'+proxy print('正在测试',proxy) async with session.get(TEST_URL,proxy=real_proxy,timeout=15) as response: if response.status in VALID_STATUS_CODES: self.redis.max(proxy) print('代理可用',proxy) else: self.redis.decrease(proxy) print('请求响应码不合法',proxy) except (ClientError,ClientConnectorError,TimeoutError,AttributeError): self.redis.decrease(proxy) print('代理请求失败',proxy) def run(self): """ 测试主函数 :return: None """ print('测试器开始执行') try: proxies=self.redis.all() loop=asyncio.get_event_loop() # 批量测试 for i in range(0,len(proxies),BATCH_TEST_SIZE): test_proxies=proxies[i:i+BATCH_TEST_SIZE] tasks=[self.test_single_proxy(proxy) for proxy in test_proxies] loop.run_until_complete(asyncio.wait(tasks)) time.sleep(5) except Exception as e: print('测试器发生错误',e.args)
class Check(): def __init__(self): self.redis = RedisClient() async def check_single(self, proxy): conn = aiohttp.TCPConnector(verify_ssl=False) async with aiohttp.ClientSession(connector=conn) as session: try: if isinstance(proxy, bytes): proxy = proxy.decode("utf-8") real_proxy = "http://" + proxy check_url = CHECK_URL[random.randint(0, len(CHECK_URL) - 1)] print("checking: ", real_proxy, check_url) async with session.get(check_url, proxy=real_proxy, timeout=10) as response: score = self.redis.get_score(proxy) if response.status in VALID_STATUS_CODES: if score != 100: #设置到50分 self.redis.max(proxy) print("%s can be use 50 score" % proxy) else: print("1:Not valid code:%s decrease" % proxy) #付费代理检测到是51分则直接删除 if socore == 51: self.redis.delproxy(proxy) else: self.redis.decrease(proxy) except (TimeoutError, OSError, ServerDisconnectedError, ClientResponseError) as e: print("2:Catch Error error:%s %s decrese" % (str(e), proxy)) self.redis.decrease(proxy) def check(self): try: proxies = self.redis.all() loop = asyncio.get_event_loop() for index in range(0, len(proxies), CHECK_SIZE): check_proxies = proxies[index:index + CHECK_SIZE] tasks = [self.check_single(proxy) for proxy in check_proxies] #把多个写成放进一个事件循环 loop.run_until_complete(asyncio.wait(tasks)) except Exception as e: print("check err: %s" % str(e))