class Test():
def __init__(self):
self.redis = RedisClient()
def test_single(self, proxy):
ip = json.loads(proxy)
ip_type = ip['type']
ip = ip['ip']
try:
response = get_response(URL, ip)
if response.status_code in VALID_CODE:
print(ip, ip_type, '正常')
self.redis.max(proxy)
else:
print(ip, ip_type, '-1')
self.redis.decrease(proxy)
except Exception as e:
print(e)
print(ip, ip_type, '-1')
self.redis.decrease(proxy)
def run(self):
proxies = self.redis.all()
q = queue.Queue(10)
for proxy in proxies:
if not q.full():
q.put(proxy)
if not q.empty():
proxy = q.get()
t = threading.Thread(target=self.test_single, args=(proxy, ))
t.start()
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self,proxy):
"""
测试单个代理
:param proxy: 单个代理
:return:
"""
conn=aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy,bytes):
proxy=proxy.decode('utf-8')
real_proxy='http://'+proxy
print('正在测试',proxy)
async with session.get(TEST_URL,proxy=real_proxy,timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用',proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法',proxy)
except (ClientError,ClientConnectorError,TimeoutError,AttributeError):
self.redis.decrease(proxy)
print('代理请求失败',proxy)
def run(self):
"""
测试主函数
:return: None
"""
print('测试器开始执行')
try:
proxies=self.redis.all()
loop=asyncio.get_event_loop()
# 批量测试
for i in range(0,len(proxies),BATCH_TEST_SIZE):
test_proxies=proxies[i:i+BATCH_TEST_SIZE]
tasks=[self.test_single_proxy(proxy) for proxy in test_proxies]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('测试器发生错误',e.args)
class Check():
def __init__(self):
self.redis = RedisClient()
async def check_single(self, proxy):
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode("utf-8")
real_proxy = "http://" + proxy
check_url = CHECK_URL[random.randint(0, len(CHECK_URL) - 1)]
print("checking: ", real_proxy, check_url)
async with session.get(check_url, proxy=real_proxy,
timeout=10) as response:
score = self.redis.get_score(proxy)
if response.status in VALID_STATUS_CODES:
if score != 100:
#设置到50分
self.redis.max(proxy)
print("%s can be use 50 score" % proxy)
else:
print("1:Not valid code:%s decrease" % proxy)
#付费代理检测到是51分则直接删除
if socore == 51:
self.redis.delproxy(proxy)
else:
self.redis.decrease(proxy)
except (TimeoutError, OSError, ServerDisconnectedError,
ClientResponseError) as e:
print("2:Catch Error error:%s %s decrese" % (str(e), proxy))
self.redis.decrease(proxy)
def check(self):
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
for index in range(0, len(proxies), CHECK_SIZE):
check_proxies = proxies[index:index + CHECK_SIZE]
tasks = [self.check_single(proxy) for proxy in check_proxies]
#把多个写成放进一个事件循环
loop.run_until_complete(asyncio.wait(tasks))
except Exception as e:
print("check err: %s" % str(e))
