async def load_github_main():
auth5 = aiosocks.Socks5Auth("proxyuser1", password="******")
ba = aiohttp.BasicAuth("login")
# remote resolve
conn = ProxyConnector(remote_resolve=True)
# or locale resolve
conn = ProxyConnector(remote_resolve=False)
try:
with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as session:
# socks5 proxy
async with session.get(
"http://github.com/", proxy="socks5://127.0.0.1:1080", proxy_auth=auth5
) as resp:
if resp.status == 200:
print(await resp.text())
# http proxy
async with session.get(
"http://github.com/", proxy="http://127.0.0.1:8080", proxy_auth=ba
) as resp:
if resp.status == 200:
print(await resp.text())
except aiohttp.ClientProxyConnectionError:
# connection problem
print("communication problem")
except aiohttp.ClientConnectorError:
# ssl error, certificate error, etc
print("ssl error, certificate error, etc")
except aiosocks.SocksError:
# communication problem
print("communication problem")
async def test_resolve_host_fail(loop, remote_resolve):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro((tr, proto))):
req = ProxyClientRequest(
'GET', URL('http://python.org'), loop=loop,
proxy=URL('socks5://proxy.example'))
connector = ProxyConnector(loop=loop, remote_resolve=remote_resolve)
connector._resolve_host = make_mocked_coro(raise_exception=OSError())
with pytest.raises(aiohttp.ClientConnectorError):
await connector.connect(req)
async def test_resolve_host_fail(loop, remote_resolve):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro((tr, proto))):
req = ProxyClientRequest('GET',
URL('http://python.org'),
loop=loop,
proxy=URL('socks5://proxy.example'))
connector = ProxyConnector(loop=loop, remote_resolve=remote_resolve)
connector._resolve_host = make_mocked_coro(raise_exception=OSError())
with pytest.raises(aiohttp.ClientConnectorError):
await connector.connect(req, [], ClientTimeout())
async def start_urls():
start_url = 'http://t66y.com/thread0806.php?fid=16'
async with aiohttp.ClientSession(connector= ProxyConnector(),request_class=ProxyClientRequest) as session:
async with session.get(start_url, proxy = socks5,) as response:
if response.status == 200:
await get_title_url(await response.text(encoding = 'GBK'))
async def async_check(self, proxy):
start_time = time.time()
try:
ip, port = proxy.split(":")
pr_connector = ProxyConnector(remote_resolve=True,
verify_ssl=False)
with aiohttp.ClientSession(connector=pr_connector, request_class=ProxyClientRequest) as session, \
aiohttp.Timeout(self.__timeout):
async with session.get('http://www.httpbin.org/get?show_env=1',
proxy="socks5://%s:%s" %
(ip, port)) as resp:
await resp.json()
result_dict = {
'ip': ip,
'port': port,
'result': True,
'elapsed': time.time() - start_time,
'exc': None
}
except BaseException as exc:
result_dict = {
'ip': None,
'port': None,
'result': False,
'elapsed': -1,
'exc': exc
}
self.__check_counter += 1
if self.__check_counter % math.floor(
self.__check_proxy_count / 100) == 0:
next(self.__check_progress_bar)
return result_dict
async def urlopen(url: str, bauth_tuple=None, loop=None) -> str:
bauth = None
if bauth_tuple:
login, password = bauth_tuple
bauth = aiohttp.BasicAuth(login, password=password, encoding='latin1')
auth5 = aiosocks.Socks5Auth(
'proxyuser1', password='******') # type: aiosocks.Socks5Auth
conn = ProxyConnector(remote_resolve=True,
loop=loop) # type: ProxyConnector
try:
with aiohttp.ClientSession(connector=conn, auth=bauth,
request_class=ProxyClientRequest) as session:
async with session.get(url, # Always connects through Tor.
proxy='socks5://127.0.0.1:9050',
proxy_auth=auth5) as resp:
if resp.status == 200:
return await resp.text()
else:
raise SocksHTTPError(
"HTTP response not OK: {}".format(resp.status))
except aiohttp.ClientProxyConnectionError:
# connection problem
pass
except aiosocks.SocksError:
# communication problem
pass
return "" # Should never happen
async def request(method, url, **kwargs):
session_kwargs = {}
if 'proxy' in kwargs and kwargs['proxy'].startswith('socks'):
session_kwargs['connector'] = ProxyConnector(remote_resolve=False)
session_kwargs['request_class'] = ProxyClientRequest
if 'cookies' in kwargs:
session_kwargs['cookies'] = kwargs['cookies']
del kwargs['cookies']
if 'timeout' not in kwargs:
kwargs['timeout'] = 10
# headers={'User-Agent': get_random_user_agent()}
if 'headers' not in kwargs:
kwargs['headers'] = {'User-Agent': get_random_user_agent()}
elif 'User-Agent' not in kwargs['headers']:
kwargs['headers']['User-Agent'] = get_random_user_agent()
if 'override_session' in kwargs:
session = kwargs['override_session']
del kwargs['override_session']
async with session.request(method, url, **kwargs) as response:
return await Response.from_aiohttp_response(response)
async with aiohttp.ClientSession(**session_kwargs) as session:
async with session.request(method, url, **kwargs) as response:
return await Response.from_aiohttp_response(response)
async def request(method, url, **kwargs):
session_kwargs = {}
if "proxy" in kwargs and kwargs["proxy"].startswith("socks"):
session_kwargs["connector"] = ProxyConnector(remote_resolve=False)
session_kwargs["request_class"] = ProxyClientRequest
if "cookies" in kwargs:
session_kwargs["cookies"] = kwargs["cookies"]
del kwargs["cookies"]
if "timeout" not in kwargs:
kwargs["timeout"] = 10
# headers={'User-Agent': get_random_user_agent()}
if "headers" not in kwargs:
kwargs["headers"] = {"User-Agent": get_random_user_agent()}
elif "User-Agent" not in kwargs["headers"]:
kwargs["headers"]["User-Agent"] = get_random_user_agent()
if "override_session" in kwargs:
session = kwargs["override_session"]
del kwargs["override_session"]
async with session.request(method, url, **kwargs) as response:
return await Response.from_aiohttp_response(response)
async with aiohttp.ClientSession(**session_kwargs) as session:
async with session.request(method, url, **kwargs) as response:
return await Response.from_aiohttp_response(response)
async def Attack():
if Functions.YokaiMode() == True:
async with ClientSession(connector=ProxyConnector(remote_resolve=True,limit=None), request_class=ProxyClientRequest) as session:
await ensure_future(Yokai(session))
else:
async with ClientSession(connector=TCPConnector(limit=None)) as session:
await ensure_future(Direct(session))
async def test_connect_locale_resolve(loop):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro((tr, proto))):
req = ProxyClientRequest('GET',
URL('http://python.org'),
loop=loop,
proxy=URL('socks5://proxy.example'))
connector = ProxyConnector(loop=loop, remote_resolve=False)
connector._resolve_host = make_mocked_coro([mock.MagicMock()])
conn = await connector.connect(req, [], ClientTimeout())
assert connector._resolve_host.call_count == 2
assert conn.protocol is proto
conn.close()
async def test_connect_locale_resolve(loop):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro((tr, proto))):
req = ProxyClientRequest(
'GET', URL('http://python.org'), loop=loop,
proxy=URL('socks5://proxy.example'))
connector = ProxyConnector(loop=loop, remote_resolve=False)
connector._resolve_host = make_mocked_coro([mock.MagicMock()])
conn = await connector.connect(req)
assert connector._resolve_host.call_count == 2
assert conn.protocol is proto
conn.close()
async def test_https_connect(loop):
async def handler(request):
return web.Response(text='Test message')
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
keyfile = os.path.join(here, 'sample.key')
certfile = os.path.join(here, 'sample.crt')
sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
sslcontext.load_cert_chain(certfile, keyfile)
ws = RawTestServer(handler, scheme='https', host='127.0.0.1', loop=loop)
await ws.start_server(loop=loop, ssl=sslcontext)
v_fp = (b'0\x9a\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd'
b'\xcb~7U\x14D@L'
b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x8b')
inv_fp = (b'0\x9d\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd'
b'\xcb~7U\x14D@L'
b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x9e')
async with FakeSocks4Srv(loop) as srv:
v_conn = ProxyConnector(loop=loop,
remote_resolve=False,
fingerprint=v_fp)
inv_conn = ProxyConnector(loop=loop,
remote_resolve=False,
fingerprint=inv_fp)
async with aiohttp.ClientSession(
connector=v_conn, loop=loop,
request_class=ProxyClientRequest) as ses:
proxy = 'socks4://127.0.0.1:{}'.format(srv.port)
async with ses.get(ws.make_url('/'), proxy=proxy) as resp:
assert resp.status == 200
assert (await resp.text()) == 'Test message'
async with aiohttp.ClientSession(
connector=inv_conn, loop=loop,
request_class=ProxyClientRequest) as ses:
proxy = 'socks4://127.0.0.1:{}'.format(srv.port)
with pytest.raises(aiohttp.ServerFingerprintMismatch):
async with ses.get(ws.make_url('/'), proxy=proxy) as resp:
assert resp.status == 200
async def get_content_from_urls(loop, urls):
conn = ProxyConnector(remote_resolve=True)
with aiohttp.ClientSession(loop=loop,
connector=conn,
request_class=ProxyClientRequest) as session:
return await asyncio.gather(
*[fetch(session, url.url) for url in urls],
return_exceptions=True # default is false, that would raise
)
def test_all(xserver_ip):
conn = ProxyConnector(remote_resolve=True)
shared_session = ClientSession(connector=conn, request_class=ProxyClientRequest)
for key in get_servers(xserver_ip):
server_info = get_server_info(xserver_ip, key)
test_single_server(shared_session, server_info, xserver_ip)
print('*******************************************')
shared_session.close()
async def main():
tasks = []
conn = ProxyConnector(remote_resolve=True)
async with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as session:
for i in range(1, 5):
task = asyncio.ensure_future(fetch(session, str(i)))
tasks.append(task)
responses = await asyncio.gather(*tasks)
async def test_connect_proxy_domain():
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro((tr, proto))):
loop_mock = mock.Mock()
req = ProxyClientRequest(
'GET', URL('http://python.org'), loop=loop_mock,
proxy=URL('socks5://proxy.example'))
connector = ProxyConnector(loop=loop_mock)
connector._resolve_host = make_mocked_coro([mock.MagicMock()])
conn = await connector.connect(req)
assert connector._resolve_host.call_count == 1
assert conn.protocol is proto
conn.close()
def __init__(self):
self.user_agent = UserAgent().random
self.timeout = 60
if HttpClient._aiohttp_connector is None:
HttpClient._aiohttp_connector = ProxyConnector(
remote_resolve=True,
limit=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS,
limit_per_host=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS_PER_HOST,
)
self.proxy_address = None
def __init__(self, url=None, request_type="GET", timeout=None):
if BaseChecker.aiohttp_connector is None:
BaseChecker.aiohttp_connector = ProxyConnector(
remote_resolve=True,
limit=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS,
limit_per_host=settings.
NUMBER_OF_SIMULTANEOUS_REQUESTS_PER_HOST,
)
self.request_type = request_type
self.timeout = timeout if timeout is not None else settings.PROXY_CHECKING_TIMEOUT
self.url = url
async def find_imgs(page_url):
'''
:param page_url:page_url
:return:
'''
async with aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) as session:
async with session.get(page_url, proxy=socks5, ) as response:
bsobj = BeautifulSoup(await response.text(encoding = 'GBK'), 'lxml')
for url in bsobj.find_all('input', {'src': True}):
new_imgae_url = str(url['src'])
await download(new_imgae_url)
async def crawler(urls_q, proxies_q, proxies_q_good):
async with create_engine_async(maxsize=treads, timeout=6000,
**connection) as db_client:
async with aiohttp.ClientSession(
connector=ProxyConnector(),
request_class=ProxyClientRequest) as http_client:
parser_jobs = [
asyncio.ensure_future(
worker(urls_q, proxies_q, proxies_q_good, db_client,
http_client)) for _ in range(treads)
]
await asyncio.gather(*parser_jobs)
async def test_proxy_connect_http(loop):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
loop_mock = mock.Mock()
loop_mock.getaddrinfo = make_mocked_coro([
[0, 0, 0, 0, ['127.0.0.1', 1080]]])
loop_mock.create_connection = make_mocked_coro((tr, proto))
req = ProxyClientRequest(
'GET', URL('http://python.org'), loop=loop,
proxy=URL('http://127.0.0.1'))
connector = ProxyConnector(loop=loop_mock)
await connector.connect(req)
def get_connector(self, socks, limit=400):
try:
return self.socks_connector if socks else self.connector
except AttributeError:
if socks:
self.socks_connector = ProxyConnector(limit=limit,
loop=self.loop,
verify_ssl=False)
return self.socks_connector
self.connector = TCPConnector(limit=limit,
loop=self.loop,
verify_ssl=False)
return self.connector
async def test_proxy_negotiate_fail(loop):
loop_mock = mock.Mock()
loop_mock.getaddrinfo = make_mocked_coro(
[[0, 0, 0, 0, ['127.0.0.1', 1080]]])
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro(raise_exception=aiosocks.SocksError())):
req = ProxyClientRequest(
'GET', URL('http://python.org'), loop=loop,
proxy=URL('socks5://127.0.0.1'))
connector = ProxyConnector(loop=loop_mock)
with pytest.raises(aiosocks.SocksError):
await connector.connect(req)
async def http_fetch(self, url):
with await self.semaphore:
if self.socks_proxy:
connector = ProxyConnector(remote_resolve=True)
request_class = ProxyClientRequest
else:
connector = GoogleDirectConnector(self.google_ip)
request_class = aiohttp.client_reqrep.ClientRequest
with aiohttp.ClientSession(loop=self.loop, connector=connector, request_class=request_class) as session:
async with session.get(url, proxy=self.socks_proxy, headers=self.headers) as resp:
result = await resp.read()
return result
async def worker(ipq):
global counter
async with aiohttp.ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest) as http_client:
while ipq.qsize():
ip = ipq.get()
proxy = f'socks5://{ip}'
counter += 1
# print('Scan:', proxy, counter)
try:
resp = await http_client.get(url, proxy=proxy, timeout=4)
assert resp.status == 200
print(proxy, counter)
except Exception as e:
# print(type(e), e)
pass
async def test_proxy_connect_fail(loop, exc):
loop_mock = mock.Mock()
loop_mock.getaddrinfo = make_mocked_coro(
[[0, 0, 0, 0, ['127.0.0.1', 1080]]])
cc_coro = make_mocked_coro(raise_exception=exc[0]())
with mock.patch('aiosocks.connector.create_connection', cc_coro):
req = ProxyClientRequest('GET',
URL('http://python.org'),
loop=loop,
proxy=URL('socks5://127.0.0.1'))
connector = ProxyConnector(loop=loop_mock)
with pytest.raises(exc[1]):
await connector.connect(req, [], ClientTimeout())
async def worker(urls_q, proxies_q, proxies_q_good):
while urls_q.qsize():
# Get proxy from Queue
if not proxies_q_good.empty():
proxy = await proxies_q_good.get()
else:
if not proxies_q.empty():
proxy = await proxies_q.get()
else:
await asyncio.sleep(1)
continue
# Get data from Queue
key = await urls_q.get()
url = base_link.format(quote(key))
print(proxy.row, '-->', 'url', '| WPQ:', proxies_q_good.qsize(),
'| APQ:', proxies_q.qsize())
# Make http request with SOCKS proxy
try:
async with aiohttp.ClientSession(connector=ProxyConnector(),
request_class=ProxyClientRequest,
conn_timeout=120) as http_client:
async with http_client.get(url,
headers=headers,
proxy=proxy.socks5) as resp:
assert resp.status == 200
code = await resp.text()
assert 'body' in code
except Exception as e:
print(type(e), e, '[worker, http_client.Exception]')
await urls_q.put(key)
continue
# If proxy is working put it into good (working) proxies Queue again
proxies_q_good.put_nowait(proxy)
# Create dictionary data, to save it into database
try:
sites = get_sites(code, key)
if sites:
with open(result_file, 'a', encoding='utf-8') as result:
for site in sites:
result.write('{}\t{}\t{}\n'.format(*site))
# result.write('{}\n'.format(URL(site[1]).host))
except Exception as e:
print(type(e), e, '[data_formatting Exception]')
continue
async def test_http_connect(loop):
async def handler(request):
return web.Response(text='Test message')
async with RawTestServer(handler, host='127.0.0.1', loop=loop) as ws:
async with FakeSocks4Srv(loop) as srv:
conn = ProxyConnector(loop=loop, remote_resolve=False)
async with aiohttp.ClientSession(
connector=conn, loop=loop,
request_class=ProxyClientRequest) as ses:
proxy = 'socks4://127.0.0.1:{}'.format(srv.port)
async with ses.get(ws.make_url('/'), proxy=proxy) as resp:
assert resp.status == 200
assert (await resp.text()) == 'Test message'
async def worker(ipq):
async with aiohttp.ClientSession(
connector=ProxyConnector(),
request_class=ProxyClientRequest) as http_client:
while ipq.qsize():
ip = await ipq.get()
proxy = f'socks5://{ip}'
# print('Scan:', proxy)
try:
resp = await http_client.get(_judge, proxy=proxy, timeout=4)
assert resp.status == 200
# print(ip)
with open('proxies_working.txt', 'a', encoding='utf-8') as f:
f.write(ip + '\n')
except Exception as e:
# print(type(e), e)
pass
async def _request(self, proxy_address, timeout) -> tuple:
checker_result = CheckerResult()
if self.url is None:
raise Exception()
headers = {'User-Agent': async_requests.get_random_user_agent()}
conn = ProxyConnector(remote_resolve=True)
with aiohttp.ClientSession(
connector=conn, request_class=ProxyClientRequest) as session:
async with session.request(
self.request_type, self.url, proxy=proxy_address, timeout=timeout, headers=headers) as \
response:
is_working = await self._check(response, checker_result)
return is_working, checker_result
async def request(method, url, **kwargs):
session_kwargs = {}
if 'proxy' in kwargs and kwargs['proxy'].startswith('socks'):
session_kwargs['connector'] = ProxyConnector(remote_resolve=False)
session_kwargs['request_class'] = ProxyClientRequest
if 'cookies'in kwargs:
session_kwargs['cookies'] = kwargs['cookies']
del kwargs['cookies']
if 'timeout' not in kwargs:
kwargs['timeout'] = 10
async with aiohttp.ClientSession(**session_kwargs) as session:
async with session.request(method, url, **kwargs) as response:
status = response.status
text = await response.text()
return Response(status, text)
async def test_connect_proxy_ip(loop):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
with mock.patch('aiosocks.connector.create_connection',
make_mocked_coro((tr, proto))):
loop.getaddrinfo = make_mocked_coro([[0, 0, 0, 0, ['127.0.0.1',
1080]]])
req = ProxyClientRequest('GET',
URL('http://python.org'),
loop=loop,
proxy=URL('socks5://proxy.org'))
connector = ProxyConnector(loop=loop)
conn = await connector.connect(req, [], ClientTimeout())
assert loop.getaddrinfo.called
assert conn.protocol is proto
conn.close()