def session(self): if not self._session or self._session.closed: if certifi: context = ssl.create_default_context(cafile=certifi.where()) connector = ProxyConnector(ssl_context=context) else: connector = ProxyConnector() self._session = aiohttp.ClientSession( connector=connector, request_class=ProxyClientRequest, json_serialize=self.json_serialize) return self._session
def __init__(self): self.connector = ProxyConnector() self.session = aiohttp.ClientSession( connector=self.connector, request_class=ProxyClientRequest, timeout=aiohttp.ClientTimeout(total=4)) self.proxies = None if config["Proxy"]["PROXY_TYPE"] != "None": if config["Proxy"]["PROXY_TYPE"].lower() == "http": if config["Proxy"]["PROXY_USERNAME"] != "" and config["Proxy"][ "PROXY_PASSWORD"] != "": self.proxies = "http://" + config["Proxy"][ "PROXY_USERNAME"] + ":" + config["Proxy"][ "PROXY_PASSWORD"] + "@" + config["Proxy"][ "PROXY_ADDRESS"] + ":" + config["Proxy"][ "PROXY_PORT"] else: self.proxies = "http://" + config["Proxy"][ "PROXY_ADDRESS"] + ":" + config["Proxy"]["PROXY_PORT"] elif config["Proxy"]["PROXY_TYPE"].lower() == "socks5": if config["Proxy"]["PROXY_USERNAME"] != "" and config["Proxy"][ "PROXY_PASSWORD"] != "": self.proxies = "socks5://" + config["Proxy"][ "PROXY_USERNAME"] + ":" + config["Proxy"][ "PROXY_PASSWORD"] + "@" + config["Proxy"][ "PROXY_ADDRESS"] + ":" + config["Proxy"][ "PROXY_PORT"] else: self.proxies = "socks5://" + config["Proxy"][ "PROXY_ADDRESS"] + ":" + config["Proxy"]["PROXY_PORT"]
async def async_request_aio_http(arr_handle): timeout = aiohttp.ClientTimeout(total=30) auth = Socks5Auth(login='', password='') connector = ProxyConnector(verify_ssl=False) socks_api = "" auth = "" async with aiohttp.ClientSession(connector=connector, request_class=ProxyClientRequest, timeout=timeout) as session: tasks = [] for item in arr_handle: url = item['url'] data_post = {} task = asyncio.ensure_future( fetch("GET", url, data_post, session, socks_api, auth)) tasks.append(task) responses = await asyncio.gather(*tasks) await session.close() return responses
async def test_https_connect(loop): async def handler(request): return web.Response(text='Test message') here = os.path.join(os.path.dirname(__file__), '..', 'tests') keyfile = os.path.join(here, 'sample.key') certfile = os.path.join(here, 'sample.crt') sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) sslcontext.load_cert_chain(certfile, keyfile) ws = RawTestServer(handler, scheme='https', host='127.0.0.1', loop=loop) await ws.start_server(loop=loop, ssl=sslcontext) v_fp = (b'0\x9a\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd' b'\xcb~7U\x14D@L' b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x8b') inv_fp = (b'0\x9d\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd' b'\xcb~7U\x14D@L' b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x9e') async with FakeSocks4Srv(loop) as srv: v_conn = ProxyConnector(loop=loop, remote_resolve=False, fingerprint=v_fp) inv_conn = ProxyConnector(loop=loop, remote_resolve=False, fingerprint=inv_fp) async with aiohttp.ClientSession( connector=v_conn, loop=loop, request_class=ProxyClientRequest) as ses: proxy = 'socks4://127.0.0.1:{}'.format(srv.port) async with ses.get(ws.make_url('/'), proxy=proxy) as resp: assert resp.status == 200 assert (await resp.text()) == 'Test message' async with aiohttp.ClientSession( connector=inv_conn, loop=loop, request_class=ProxyClientRequest) as ses: proxy = 'socks4://127.0.0.1:{}'.format(srv.port) with pytest.raises(aiohttp.ServerFingerprintMismatch): async with ses.get(ws.make_url('/'), proxy=proxy) as resp: assert resp.status == 200
async def run(self): # TODO: rework proxy async with ClientSession( cookie_jar=DummyCookieJar(), connector=ProxyConnector(), request_class=ProxyClientRequest, ) as session: tasks = self.prepare_tasks(session) return await asyncio.gather(*tasks)
async def __work(): session = aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) on_start() results = await asyncio.gather(self.loop.create_task( worker(session, *worker_args, **worker_kwargs)), return_exceptions=True) on_end(results[0]) await session.close()
async def __aiter__(self): # TODO: rework proxy async with ClientSession( cookie_jar=DummyCookieJar(), connector=ProxyConnector(), request_class=ProxyClientRequest, ) as session: tasks = self.prepare_tasks(session) for result in asyncio.as_completed(tasks): yield await result
async def download(url): connector = ProxyConnector() socks = None async with aiohttp.ClientSession( connector=connector, request_class=ProxyClientRequest) as session: ret, status = await session_get(session, url, socks) if 'window.location.href' in ret and len(ret) < 1000: url = ret.split("window.location.href='")[1].split("'")[0] ret, status = await session_get(session, url, socks) return ret, status
def __init__(self): self.connector = ProxyConnector() self.session = aiohttp.ClientSession(connector=self.connector, request_class=ProxyClientRequest, timeout=aiohttp.ClientTimeout(total=4)) self.proxies = None if config["Proxy"]["PROXY_TYPE"] != "None": self.proxies = { "http": config["Proxy"]["PROXY_ADDRESS"], "https": config["Proxy"]["PROXY_ADDRESS"] }
async def get_visible_ip(proxy_url: str, ip_judge: str = 'http://ifconfig.me/ip') -> str: try: async with ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) as client: async with client.get(ip_judge, proxy=proxy_url, timeout=5) as response: return await response.text() except (ClientError, TimeoutError): raise ServerConnectionError
async def is_available_to_resource(self, resource: str) -> bool: try: async with ClientSession( connector=ProxyConnector(), request_class=ProxyClientRequest) as client: async with client.get(resource, proxy=self.url, timeout=5) as response: if response.status != 200: return False except (ClientError, asyncio.TimeoutError): return False return True
async def download(url): connector = ProxyConnector() if DEPLOY_MODE == "dev": socks = None elif DEPLOY_MODE == "Prod": socks = random.choice(socks5_address_prod) async with aiohttp.ClientSession( connector=connector, request_class=ProxyClientRequest) as session: ret, status = await session_get(session, url, socks) if 'window.location.href' in ret and len(ret) < 1000: url = ret.split("window.location.href='")[1].split("'")[0] ret, status = await session_get(session, url, socks) return ret, status
async def init_app(loop=None): app = web.Application(middlewares=[], loop=loop) conn = ProxyConnector(remote_resolve=False) app['client_session'] = aiohttp.ClientSession( connector=conn, request_class=ProxyClientRequest, raise_for_status=True) app['check_semaphore'] = asyncio.Semaphore(settings.CHECK_SEMAPHORE) app['parse_semaphore'] = asyncio.Semaphore(settings.PARSE_SEMAPHORE) app['db'] = await init_db() app.router.add_get("/", get_proxies) return app
async def test_http_connect(loop): async def handler(request): return web.Response(text='Test message') async with RawTestServer(handler, host='127.0.0.1', loop=loop) as ws: async with FakeSocks4Srv(loop) as srv: conn = ProxyConnector(loop=loop, remote_resolve=False) async with aiohttp.ClientSession( connector=conn, loop=loop, request_class=ProxyClientRequest) as ses: proxy = 'socks4://127.0.0.1:{}'.format(srv.port) async with ses.get(ws.make_url('/'), proxy=proxy) as resp: assert resp.status == 200 assert (await resp.text()) == 'Test message'
async def __work(): semaphore = asyncio.Semaphore(max_concurrent_workers) session = aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) async def __task(element: Union[Proxy, Any]): async with semaphore: return await worker(session, element, *worker_args, **worker_kwargs) on_start() tasks = [ self.loop.create_task(__task(element)) for element in elements ] results = await asyncio.gather(*tasks, return_exceptions=True) await session.close() on_end(results)
async def get_url(url, headers): content = False conn = ProxyConnector() try: async with aiohttp.ClientSession( connector=conn, request_class=ProxyClientRequest) as session: async with session.get(url, proxy='socks5://127.0.0.1:1080', headers=headers) as resp: if resp.status == 200: content = await resp.text() else: print('failed to get {}, status {}'.format( url, resp.status)) except Exception as e: print('failed to get {}, error {}'.format(url, e)) return content
async def get_proxy_ping(proxy_url: str, ping_target: str = 'https://google.com') -> int: conn = ProxyConnector() start_time = time.process_time() try: async with ClientSession(connector=conn, request_class=ProxyClientRequest) as client: async with client.get(ping_target, proxy=proxy_url, timeout=5) as response: end_time = time.process_time() if response.status != 200: ping = None else: ping = int((end_time - start_time) * 1000) except (ClientError, TimeoutError, ValueError): ping = None return ping
async def run(self): """ HTTP session initialization :return: None """ conn = ProxyConnector() if not self.proxies: self.proxies = [] if isinstance(self.proxies, str): self.proxies = [self.proxies] async with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as self.session: try: async for ret in self.startup(): if isinstance(ret, Request): await self.push_request(ret) except TypeError: logger.error('self.startup() returned in the wrong way, please use yield to return data.') async for req in self._get_request(): await self.request_handler(req)
async def fetch(url_list, proxy, proxy_user, proxy_password, loop): # 请求及解析 requests_task_list = [] # 构造代理 if proxy_user and proxy_password: proxy_auth = aiohttp.BasicAuth(proxy_user, proxy_password) else: proxy_auth = None # 创建连接对象,多个连接由同一个对象发出请求,减少创建对象资源 if proxy and 'socks5' in proxy: # socks5代理需要额外处理 connector, request_class = ProxyConnector(enable_cleanup_closed=True), ProxyClientRequest client = ClientSession(connector=connector, request_class=request_class) else: connector = aiohttp.TCPConnector(enable_cleanup_closed=True) client = ClientSession(connector=connector) async with client: # 请求 for url in url_list: task = loop.create_task(requests_page(client, url, proxy, proxy_auth)) requests_task_list.append(task) await asyncio.wait(requests_task_list) # 解析任务并保存 save_file_task_list = [] requests_failed_url_list = [] for i, task_future in enumerate(requests_task_list): html, url = task_future.result() if html: # save_file_task = loop.create_task(save_html_to_local_file(url, html)) url_md5 = get_token(url) html = parse_html(html) redis_help.hset(html_dict_key, url_md5, html) print(f'requests {url} successful') else: requests_failed_url_list.append(url) print(f'requests {url} error')
async def test_single_proxy(self, proxy): """ 测试单个代理 :param proxy: :return: """ #auth = Socks5Auth(login=None, password=None), connector = ProxyConnector() socks = 'socks5://' + proxy print("测试代理{0}".format(proxy)) async with aiohttp.ClientSession( connector=connector, request_class=ProxyClientRequest) as session: try: async with session.get('http://spys.one', proxy=socks) as resp: # proxy_auth=auth #print(await resp.text()) if resp.status == 200: self.useproxy.append(proxy) print("存入代理{0}".format(proxy)) except: #aiohttp.ProxyConnectionError: #print('connection problem') pass
async def check(self, proxy): possible_exceptions = ( aiohttp.client_exceptions.ClientProxyConnectionError, concurrent.futures._base.TimeoutError, aiohttp.client_exceptions.ClientOSError, aiohttp.client_exceptions.ClientHttpProxyError, aiohttp.client_exceptions.ServerDisconnectedError, aiohttp.client_exceptions.ClientResponseError, aiosocksy.errors.SocksError, aiohttp.client_exceptions.InvalidURL, aiohttp.client_exceptions.ClientPayloadError) start_time = time.time() try: is_port_open = await check_port_open(proxy.host, proxy.port) if not is_port_open: raise concurrent.futures._base.TimeoutError( 'Could not open connection') async with async_timeout.timeout(self.timeout): async with aiohttp.ClientSession( connector=ProxyConnector(verify_ssl=False, limit=0), request_class=ProxyClientRequest, conn_timeout=self.timeout, read_timeout=self.timeout) as session: async with session.get( self.url, proxy=str(proxy), headers=random_session()['headers']) as response: content = await response.read() # self.logger.debug('Got response [{}]: {} bytes'.format(response.status, len(content))) #DELETE_DEBUG result = response except possible_exceptions as e: result = e delta_time = time.time() - start_time is_passed = True is_banned = False if isinstance(result, aiohttp.client_reqrep.ClientResponse): if self.check_xpath: any_xpath_worked = False try: doc = lxml.html.fromstring(content) if settings.SHOW_RESPONSE_BODY: self.logger.info(content) for xpath in self.check_xpath: xpath_result = doc.xpath(xpath) if xpath_result: any_xpath_worked = True if xpath_result and isinstance( xpath, xpath_check.BanXPathCheck): is_banned = True except (lxml.etree.ParserError, lxml.etree.XMLSyntaxError): pass except lxml.etree.XPathEvalError as e: self.logger.warning('XPath is wrong: {}'.format(xpath)) raise e if not any_xpath_worked: is_passed = False # self.logger.debug('No any xpath worked for proxy {} on url {} ({}):'.format(proxy, self.url, ", ".join(self.check_xpath))) #DELETE_DEBUG if self.status and int(result.status) not in self.status: self.logger.debug( '{} not passed status code check on {}. {} got, but {} expected' .format(proxy, self.url, result.status, self.status)) is_passed = False else: is_passed = False if isinstance(result, aiohttp.client_reqrep.ClientResponse): status = int(result.status) else: status = None check_result = CheckResult() check_result.proxy = proxy check_result.is_passed = is_passed check_result.is_banned = is_banned check_result.check = self check_result.time = delta_time check_result.done_at = datetime.datetime.utcnow() check_result.status = status if isinstance(result, BaseException): check_result.error = str(result) get_session().add(check_result) await proxy.on_check_executed() error = '' if isinstance(result, BaseException): error = ', error: {}'.format(str(result)) self.logger.info( 'Finished check (is passed: {}) for {} on {} by {:0.3f} s{}'. format(is_passed, proxy, self.url, delta_time, error)) return check_result
def __init__(self, token: base.String, loop: Optional[Union[asyncio.BaseEventLoop, asyncio.AbstractEventLoop]] = None, connections_limit: Optional[base.Integer] = 10, proxy: Optional[base.String] = None, proxy_auth: Optional[aiohttp.BasicAuth] = None, validate_token: Optional[base.Boolean] = True, parse_mode=None): """ Instructions how to get Bot token is found here: https://core.telegram.org/bots#3-how-do-i-create-a-bot :param token: token from @BotFather :type token: :obj:`str` :param loop: event loop :type loop: Optional Union :obj:`asyncio.BaseEventLoop`, :obj:`asyncio.AbstractEventLoop` :param connections_limit: connections limit for aiohttp.ClientSession :type connections_limit: :obj:`int` :param proxy: HTTP proxy URL :type proxy: :obj:`str` :param proxy_auth: Authentication information :type proxy_auth: Optional :obj:`aiohttp.BasicAuth` :param validate_token: Validate token. :type validate_token: :obj:`bool` :param parse_mode: You can set default parse mode :type parse_mode: :obj:`str` :raise: when token is invalid throw an :obj:`aiogram.utils.exceptions.ValidationError` """ # Authentication if validate_token: api.check_token(token) self.__token = token # Proxy settings self.proxy = proxy self.proxy_auth = proxy_auth # Asyncio loop instance if loop is None: loop = asyncio.get_event_loop() self.loop = loop # aiohttp main session ssl_context = ssl.create_default_context(cafile=certifi.where()) if isinstance(proxy, str) and proxy.startswith('socks5://'): from aiosocksy.connector import ProxyClientRequest, ProxyConnector connector = ProxyConnector(limit=connections_limit, ssl_context=ssl_context, loop=self.loop) request_class = ProxyClientRequest else: connector = aiohttp.TCPConnector(limit=connections_limit, ssl_context=ssl_context, loop=self.loop) request_class = aiohttp.ClientRequest self.session = aiohttp.ClientSession(connector=connector, request_class=request_class, loop=self.loop, json_serialize=json.dumps) # Temp sessions self._temp_sessions = [] # Data stored in bot instance self._data = {} self.parse_mode = parse_mode