def get_page(url, options={}): headers = dict(base_header, **options) Logger.log_normal('Getting %s' % url) try: r = requests.get(url, headers=headers) Logger.log_high('Getting result %s %s' % (url, r.status_code)) if r.status_code == 200: return r.text except ConnectionError: Logger.log_fail('Crawling Failed %s' % url) return None
def test(self): """ 异步检测_raw_proxies中的全部代理 """ Logger.log_normal('VaildityTester is working') try: loop = asyncio.get_event_loop() tasks = [ self.test_single_proxy(proxy) for proxy in self._raw_proxies ] loop.run_until_complete(asyncio.wait(tasks)) except ValueError: Logger.log_fail('Async Error')