def get_page(url, options={}):
headers = dict(base_header, **options)
Logger.log_normal('Getting %s' % url)
try:
r = requests.get(url, headers=headers)
Logger.log_high('Getting result %s %s' % (url, r.status_code))
if r.status_code == 200:
return r.text
except ConnectionError:
Logger.log_fail('Crawling Failed %s' % url)
return None
def test(self):
"""
异步检测_raw_proxies中的全部代理
"""
Logger.log_normal('VaildityTester is working')
try:
loop = asyncio.get_event_loop()
tasks = [
self.test_single_proxy(proxy) for proxy in self._raw_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
except ValueError:
Logger.log_fail('Async Error')