async def load_github_main(): auth5 = aiosocks.Socks5Auth("proxyuser1", password="******") ba = aiohttp.BasicAuth("login") # remote resolve conn = ProxyConnector(remote_resolve=True) # or locale resolve conn = ProxyConnector(remote_resolve=False) try: with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as session: # socks5 proxy async with session.get( "http://github.com/", proxy="socks5://127.0.0.1:1080", proxy_auth=auth5 ) as resp: if resp.status == 200: print(await resp.text()) # http proxy async with session.get( "http://github.com/", proxy="http://127.0.0.1:8080", proxy_auth=ba ) as resp: if resp.status == 200: print(await resp.text()) except aiohttp.ClientProxyConnectionError: # connection problem print("communication problem") except aiohttp.ClientConnectorError: # ssl error, certificate error, etc print("ssl error, certificate error, etc") except aiosocks.SocksError: # communication problem print("communication problem")
async def test_resolve_host_fail(loop, remote_resolve): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') with mock.patch('aiosocks.connector.create_connection', make_mocked_coro((tr, proto))): req = ProxyClientRequest( 'GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://proxy.example')) connector = ProxyConnector(loop=loop, remote_resolve=remote_resolve) connector._resolve_host = make_mocked_coro(raise_exception=OSError()) with pytest.raises(aiohttp.ClientConnectorError): await connector.connect(req)
async def test_resolve_host_fail(loop, remote_resolve): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') with mock.patch('aiosocks.connector.create_connection', make_mocked_coro((tr, proto))): req = ProxyClientRequest('GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://proxy.example')) connector = ProxyConnector(loop=loop, remote_resolve=remote_resolve) connector._resolve_host = make_mocked_coro(raise_exception=OSError()) with pytest.raises(aiohttp.ClientConnectorError): await connector.connect(req, [], ClientTimeout())
async def start_urls(): start_url = 'http://t66y.com/thread0806.php?fid=16' async with aiohttp.ClientSession(connector= ProxyConnector(),request_class=ProxyClientRequest) as session: async with session.get(start_url, proxy = socks5,) as response: if response.status == 200: await get_title_url(await response.text(encoding = 'GBK'))
async def async_check(self, proxy): start_time = time.time() try: ip, port = proxy.split(":") pr_connector = ProxyConnector(remote_resolve=True, verify_ssl=False) with aiohttp.ClientSession(connector=pr_connector, request_class=ProxyClientRequest) as session, \ aiohttp.Timeout(self.__timeout): async with session.get('http://www.httpbin.org/get?show_env=1', proxy="socks5://%s:%s" % (ip, port)) as resp: await resp.json() result_dict = { 'ip': ip, 'port': port, 'result': True, 'elapsed': time.time() - start_time, 'exc': None } except BaseException as exc: result_dict = { 'ip': None, 'port': None, 'result': False, 'elapsed': -1, 'exc': exc } self.__check_counter += 1 if self.__check_counter % math.floor( self.__check_proxy_count / 100) == 0: next(self.__check_progress_bar) return result_dict
async def urlopen(url: str, bauth_tuple=None, loop=None) -> str: bauth = None if bauth_tuple: login, password = bauth_tuple bauth = aiohttp.BasicAuth(login, password=password, encoding='latin1') auth5 = aiosocks.Socks5Auth( 'proxyuser1', password='******') # type: aiosocks.Socks5Auth conn = ProxyConnector(remote_resolve=True, loop=loop) # type: ProxyConnector try: with aiohttp.ClientSession(connector=conn, auth=bauth, request_class=ProxyClientRequest) as session: async with session.get(url, # Always connects through Tor. proxy='socks5://127.0.0.1:9050', proxy_auth=auth5) as resp: if resp.status == 200: return await resp.text() else: raise SocksHTTPError( "HTTP response not OK: {}".format(resp.status)) except aiohttp.ClientProxyConnectionError: # connection problem pass except aiosocks.SocksError: # communication problem pass return "" # Should never happen
async def request(method, url, **kwargs): session_kwargs = {} if 'proxy' in kwargs and kwargs['proxy'].startswith('socks'): session_kwargs['connector'] = ProxyConnector(remote_resolve=False) session_kwargs['request_class'] = ProxyClientRequest if 'cookies' in kwargs: session_kwargs['cookies'] = kwargs['cookies'] del kwargs['cookies'] if 'timeout' not in kwargs: kwargs['timeout'] = 10 # headers={'User-Agent': get_random_user_agent()} if 'headers' not in kwargs: kwargs['headers'] = {'User-Agent': get_random_user_agent()} elif 'User-Agent' not in kwargs['headers']: kwargs['headers']['User-Agent'] = get_random_user_agent() if 'override_session' in kwargs: session = kwargs['override_session'] del kwargs['override_session'] async with session.request(method, url, **kwargs) as response: return await Response.from_aiohttp_response(response) async with aiohttp.ClientSession(**session_kwargs) as session: async with session.request(method, url, **kwargs) as response: return await Response.from_aiohttp_response(response)
async def request(method, url, **kwargs): session_kwargs = {} if "proxy" in kwargs and kwargs["proxy"].startswith("socks"): session_kwargs["connector"] = ProxyConnector(remote_resolve=False) session_kwargs["request_class"] = ProxyClientRequest if "cookies" in kwargs: session_kwargs["cookies"] = kwargs["cookies"] del kwargs["cookies"] if "timeout" not in kwargs: kwargs["timeout"] = 10 # headers={'User-Agent': get_random_user_agent()} if "headers" not in kwargs: kwargs["headers"] = {"User-Agent": get_random_user_agent()} elif "User-Agent" not in kwargs["headers"]: kwargs["headers"]["User-Agent"] = get_random_user_agent() if "override_session" in kwargs: session = kwargs["override_session"] del kwargs["override_session"] async with session.request(method, url, **kwargs) as response: return await Response.from_aiohttp_response(response) async with aiohttp.ClientSession(**session_kwargs) as session: async with session.request(method, url, **kwargs) as response: return await Response.from_aiohttp_response(response)
async def Attack(): if Functions.YokaiMode() == True: async with ClientSession(connector=ProxyConnector(remote_resolve=True,limit=None), request_class=ProxyClientRequest) as session: await ensure_future(Yokai(session)) else: async with ClientSession(connector=TCPConnector(limit=None)) as session: await ensure_future(Direct(session))
async def test_connect_locale_resolve(loop): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') with mock.patch('aiosocks.connector.create_connection', make_mocked_coro((tr, proto))): req = ProxyClientRequest('GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://proxy.example')) connector = ProxyConnector(loop=loop, remote_resolve=False) connector._resolve_host = make_mocked_coro([mock.MagicMock()]) conn = await connector.connect(req, [], ClientTimeout()) assert connector._resolve_host.call_count == 2 assert conn.protocol is proto conn.close()
async def test_connect_locale_resolve(loop): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') with mock.patch('aiosocks.connector.create_connection', make_mocked_coro((tr, proto))): req = ProxyClientRequest( 'GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://proxy.example')) connector = ProxyConnector(loop=loop, remote_resolve=False) connector._resolve_host = make_mocked_coro([mock.MagicMock()]) conn = await connector.connect(req) assert connector._resolve_host.call_count == 2 assert conn.protocol is proto conn.close()
async def test_https_connect(loop): async def handler(request): return web.Response(text='Test message') here = os.path.join(os.path.dirname(__file__), '..', 'tests') keyfile = os.path.join(here, 'sample.key') certfile = os.path.join(here, 'sample.crt') sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) sslcontext.load_cert_chain(certfile, keyfile) ws = RawTestServer(handler, scheme='https', host='127.0.0.1', loop=loop) await ws.start_server(loop=loop, ssl=sslcontext) v_fp = (b'0\x9a\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd' b'\xcb~7U\x14D@L' b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x8b') inv_fp = (b'0\x9d\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd' b'\xcb~7U\x14D@L' b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x9e') async with FakeSocks4Srv(loop) as srv: v_conn = ProxyConnector(loop=loop, remote_resolve=False, fingerprint=v_fp) inv_conn = ProxyConnector(loop=loop, remote_resolve=False, fingerprint=inv_fp) async with aiohttp.ClientSession( connector=v_conn, loop=loop, request_class=ProxyClientRequest) as ses: proxy = 'socks4://127.0.0.1:{}'.format(srv.port) async with ses.get(ws.make_url('/'), proxy=proxy) as resp: assert resp.status == 200 assert (await resp.text()) == 'Test message' async with aiohttp.ClientSession( connector=inv_conn, loop=loop, request_class=ProxyClientRequest) as ses: proxy = 'socks4://127.0.0.1:{}'.format(srv.port) with pytest.raises(aiohttp.ServerFingerprintMismatch): async with ses.get(ws.make_url('/'), proxy=proxy) as resp: assert resp.status == 200
async def get_content_from_urls(loop, urls): conn = ProxyConnector(remote_resolve=True) with aiohttp.ClientSession(loop=loop, connector=conn, request_class=ProxyClientRequest) as session: return await asyncio.gather( *[fetch(session, url.url) for url in urls], return_exceptions=True # default is false, that would raise )
def test_all(xserver_ip): conn = ProxyConnector(remote_resolve=True) shared_session = ClientSession(connector=conn, request_class=ProxyClientRequest) for key in get_servers(xserver_ip): server_info = get_server_info(xserver_ip, key) test_single_server(shared_session, server_info, xserver_ip) print('*******************************************') shared_session.close()
async def main(): tasks = [] conn = ProxyConnector(remote_resolve=True) async with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as session: for i in range(1, 5): task = asyncio.ensure_future(fetch(session, str(i))) tasks.append(task) responses = await asyncio.gather(*tasks)
async def test_connect_proxy_domain(): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') with mock.patch('aiosocks.connector.create_connection', make_mocked_coro((tr, proto))): loop_mock = mock.Mock() req = ProxyClientRequest( 'GET', URL('http://python.org'), loop=loop_mock, proxy=URL('socks5://proxy.example')) connector = ProxyConnector(loop=loop_mock) connector._resolve_host = make_mocked_coro([mock.MagicMock()]) conn = await connector.connect(req) assert connector._resolve_host.call_count == 1 assert conn.protocol is proto conn.close()
def __init__(self): self.user_agent = UserAgent().random self.timeout = 60 if HttpClient._aiohttp_connector is None: HttpClient._aiohttp_connector = ProxyConnector( remote_resolve=True, limit=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS, limit_per_host=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS_PER_HOST, ) self.proxy_address = None
def __init__(self, url=None, request_type="GET", timeout=None): if BaseChecker.aiohttp_connector is None: BaseChecker.aiohttp_connector = ProxyConnector( remote_resolve=True, limit=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS, limit_per_host=settings. NUMBER_OF_SIMULTANEOUS_REQUESTS_PER_HOST, ) self.request_type = request_type self.timeout = timeout if timeout is not None else settings.PROXY_CHECKING_TIMEOUT self.url = url
async def find_imgs(page_url): ''' :param page_url:page_url :return: ''' async with aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) as session: async with session.get(page_url, proxy=socks5, ) as response: bsobj = BeautifulSoup(await response.text(encoding = 'GBK'), 'lxml') for url in bsobj.find_all('input', {'src': True}): new_imgae_url = str(url['src']) await download(new_imgae_url)
async def crawler(urls_q, proxies_q, proxies_q_good): async with create_engine_async(maxsize=treads, timeout=6000, **connection) as db_client: async with aiohttp.ClientSession( connector=ProxyConnector(), request_class=ProxyClientRequest) as http_client: parser_jobs = [ asyncio.ensure_future( worker(urls_q, proxies_q, proxies_q_good, db_client, http_client)) for _ in range(treads) ] await asyncio.gather(*parser_jobs)
async def test_proxy_connect_http(loop): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') loop_mock = mock.Mock() loop_mock.getaddrinfo = make_mocked_coro([ [0, 0, 0, 0, ['127.0.0.1', 1080]]]) loop_mock.create_connection = make_mocked_coro((tr, proto)) req = ProxyClientRequest( 'GET', URL('http://python.org'), loop=loop, proxy=URL('http://127.0.0.1')) connector = ProxyConnector(loop=loop_mock) await connector.connect(req)
def get_connector(self, socks, limit=400): try: return self.socks_connector if socks else self.connector except AttributeError: if socks: self.socks_connector = ProxyConnector(limit=limit, loop=self.loop, verify_ssl=False) return self.socks_connector self.connector = TCPConnector(limit=limit, loop=self.loop, verify_ssl=False) return self.connector
async def test_proxy_negotiate_fail(loop): loop_mock = mock.Mock() loop_mock.getaddrinfo = make_mocked_coro( [[0, 0, 0, 0, ['127.0.0.1', 1080]]]) with mock.patch('aiosocks.connector.create_connection', make_mocked_coro(raise_exception=aiosocks.SocksError())): req = ProxyClientRequest( 'GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://127.0.0.1')) connector = ProxyConnector(loop=loop_mock) with pytest.raises(aiosocks.SocksError): await connector.connect(req)
async def http_fetch(self, url): with await self.semaphore: if self.socks_proxy: connector = ProxyConnector(remote_resolve=True) request_class = ProxyClientRequest else: connector = GoogleDirectConnector(self.google_ip) request_class = aiohttp.client_reqrep.ClientRequest with aiohttp.ClientSession(loop=self.loop, connector=connector, request_class=request_class) as session: async with session.get(url, proxy=self.socks_proxy, headers=self.headers) as resp: result = await resp.read() return result
async def worker(ipq): global counter async with aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) as http_client: while ipq.qsize(): ip = ipq.get() proxy = f'socks5://{ip}' counter += 1 # print('Scan:', proxy, counter) try: resp = await http_client.get(url, proxy=proxy, timeout=4) assert resp.status == 200 print(proxy, counter) except Exception as e: # print(type(e), e) pass
async def test_proxy_connect_fail(loop, exc): loop_mock = mock.Mock() loop_mock.getaddrinfo = make_mocked_coro( [[0, 0, 0, 0, ['127.0.0.1', 1080]]]) cc_coro = make_mocked_coro(raise_exception=exc[0]()) with mock.patch('aiosocks.connector.create_connection', cc_coro): req = ProxyClientRequest('GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://127.0.0.1')) connector = ProxyConnector(loop=loop_mock) with pytest.raises(exc[1]): await connector.connect(req, [], ClientTimeout())
async def worker(urls_q, proxies_q, proxies_q_good): while urls_q.qsize(): # Get proxy from Queue if not proxies_q_good.empty(): proxy = await proxies_q_good.get() else: if not proxies_q.empty(): proxy = await proxies_q.get() else: await asyncio.sleep(1) continue # Get data from Queue key = await urls_q.get() url = base_link.format(quote(key)) print(proxy.row, '-->', 'url', '| WPQ:', proxies_q_good.qsize(), '| APQ:', proxies_q.qsize()) # Make http request with SOCKS proxy try: async with aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest, conn_timeout=120) as http_client: async with http_client.get(url, headers=headers, proxy=proxy.socks5) as resp: assert resp.status == 200 code = await resp.text() assert 'body' in code except Exception as e: print(type(e), e, '[worker, http_client.Exception]') await urls_q.put(key) continue # If proxy is working put it into good (working) proxies Queue again proxies_q_good.put_nowait(proxy) # Create dictionary data, to save it into database try: sites = get_sites(code, key) if sites: with open(result_file, 'a', encoding='utf-8') as result: for site in sites: result.write('{}\t{}\t{}\n'.format(*site)) # result.write('{}\n'.format(URL(site[1]).host)) except Exception as e: print(type(e), e, '[data_formatting Exception]') continue
async def test_http_connect(loop): async def handler(request): return web.Response(text='Test message') async with RawTestServer(handler, host='127.0.0.1', loop=loop) as ws: async with FakeSocks4Srv(loop) as srv: conn = ProxyConnector(loop=loop, remote_resolve=False) async with aiohttp.ClientSession( connector=conn, loop=loop, request_class=ProxyClientRequest) as ses: proxy = 'socks4://127.0.0.1:{}'.format(srv.port) async with ses.get(ws.make_url('/'), proxy=proxy) as resp: assert resp.status == 200 assert (await resp.text()) == 'Test message'
async def worker(ipq): async with aiohttp.ClientSession( connector=ProxyConnector(), request_class=ProxyClientRequest) as http_client: while ipq.qsize(): ip = await ipq.get() proxy = f'socks5://{ip}' # print('Scan:', proxy) try: resp = await http_client.get(_judge, proxy=proxy, timeout=4) assert resp.status == 200 # print(ip) with open('proxies_working.txt', 'a', encoding='utf-8') as f: f.write(ip + '\n') except Exception as e: # print(type(e), e) pass
async def _request(self, proxy_address, timeout) -> tuple: checker_result = CheckerResult() if self.url is None: raise Exception() headers = {'User-Agent': async_requests.get_random_user_agent()} conn = ProxyConnector(remote_resolve=True) with aiohttp.ClientSession( connector=conn, request_class=ProxyClientRequest) as session: async with session.request( self.request_type, self.url, proxy=proxy_address, timeout=timeout, headers=headers) as \ response: is_working = await self._check(response, checker_result) return is_working, checker_result
async def request(method, url, **kwargs): session_kwargs = {} if 'proxy' in kwargs and kwargs['proxy'].startswith('socks'): session_kwargs['connector'] = ProxyConnector(remote_resolve=False) session_kwargs['request_class'] = ProxyClientRequest if 'cookies'in kwargs: session_kwargs['cookies'] = kwargs['cookies'] del kwargs['cookies'] if 'timeout' not in kwargs: kwargs['timeout'] = 10 async with aiohttp.ClientSession(**session_kwargs) as session: async with session.request(method, url, **kwargs) as response: status = response.status text = await response.text() return Response(status, text)
async def test_connect_proxy_ip(loop): tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol') with mock.patch('aiosocks.connector.create_connection', make_mocked_coro((tr, proto))): loop.getaddrinfo = make_mocked_coro([[0, 0, 0, 0, ['127.0.0.1', 1080]]]) req = ProxyClientRequest('GET', URL('http://python.org'), loop=loop, proxy=URL('socks5://proxy.org')) connector = ProxyConnector(loop=loop) conn = await connector.connect(req, [], ClientTimeout()) assert loop.getaddrinfo.called assert conn.protocol is proto conn.close()