def request_url_by_requests(url, proxies):
"""
Request a url by requests
:param url:
:param proxies:
:return:
"""
headers = {'User-Agent': get_random_user_agent()}
try:
res = requests.get(url,
headers=headers,
timeout=CONFIG.TEST_URL['timeout'],
proxies=proxies)
res.raise_for_status()
except requests.exceptions.ConnectTimeout as e:
logger.error(type='Request timeout!',
message=proxies.get('http', ) + ' Request ' + url +
' failed')
res = None
except Exception as e:
logger.error(type='Request failed!',
message=proxies.get('http', ) + ' Request ' + url +
' failed')
res = None
return res
async def valid_proxy(proxy, nums=1):
if nums > 3:
await db_client.delete(proxy)
logger.error(type='无效代理', message="{0} 已丢弃".format(proxy))
else:
ip, port = proxy.split(':')
isOk = await get_proxy_info(ip, port)
if not isOk:
logger.error(type='无效代理',
message="{0}：第 {1} 次重试".format(proxy, nums))
await valid_proxy(proxy, nums=nums + 1)
else:
logger.info(type='有效代理', message="{0} 有效".format(proxy))
async def valid_proxy(proxy, nums=1):
if nums > 5:
await db_client.delete(proxy)
logger.error(type='Invalid proxy', message="{0} had been abandoned".format(proxy))
return False
else:
ip, port = proxy.split(':')
is_ok = await get_proxy_info(ip, port)
if not is_ok:
logger.error(type='Invalid proxy', message="{0}：retry times = {1}".format(proxy, nums))
res = await valid_proxy(proxy, nums=nums + 1)
return res
else:
logger.info(type='Valid proxy', message="{0} is valid".format(proxy))
return True