def get_tor_connector(string):
ip, port = parse_proxy_address(string)
login, password = generate_credentials()
addr = aiosocks.Socks5Addr(ip, int(port))
auth = aiosocks.Socks5Auth(login, password=password)
conn = SocksConnector(proxy=addr, proxy_auth=auth, remote_resolve=True)
return conn
async def scan_address(self, address, port):
try:
if address.endswith(".onion"):
proxy = aiosocks.Socks5Addr('127.0.0.1', 9050)
else:
proxy = None
conn = BitcoinProtocol(address,
port,
self.network,
self.testnet,
self.log,
proxy=proxy)
try:
await asyncio.wait_for(conn.handshake, timeout=10)
except:
pass
if conn.handshake.result() == True:
try:
await asyncio.wait_for(conn.addresses_received, timeout=10)
except:
pass
for a in conn.addresses:
if a["address"] not in self.not_scanned_addresses:
if a["address"] not in self.scanning_addresses:
if a["address"] not in self.scanned_addresses:
self.not_scanned_addresses[a["address"]] = a
self.online_nodes += 1
# add record to db
net = network_type(address)
if net == "TOR":
geo = {
"country": None,
"city": None,
"geo": None,
"timezone": None,
"asn": None,
"org": None
}
else:
geo = await self.loop.run_in_executor(
None, model.get_geoip, address)
await model.report_online(address, port, net, conn.user_agent,
conn.latency, conn.version,
conn.start_height, conn.services,
geo, int(time.time()), self.db_pool)
else:
await model.report_offline(address, self.db_pool)
conn.__del__()
except:
try:
await model.report_offline(address, self.db_pool)
conn.__del__()
except:
pass
self.scan_threads -= 1
del self.scanning_addresses[address]
self.scanned_addresses.add(address)
def make_socks5(loop,
*,
addr=None,
auth=None,
rr=True,
dst=None,
r=None,
ap_factory=None,
whiter=None):
addr = addr or aiosocks.Socks5Addr('localhost', 1080)
auth = auth or aiosocks.Socks5Auth('user', 'pwd')
dst = dst or ('python.org', 80)
proto = aiosocks.Socks5Protocol(proxy=addr,
proxy_auth=auth,
dst=dst,
remote_resolve=rr,
loop=loop,
app_protocol_factory=ap_factory,
waiter=whiter)
proto._stream_writer = mock.Mock()
proto._stream_writer.drain = fake_coroutine(True)
if not isinstance(r, (list, tuple)):
proto.read_response = mock.Mock(
side_effect=coro(mock.Mock(return_value=r)))
else:
proto.read_response = mock.Mock(
side_effect=coro(mock.Mock(side_effect=r)))
proto._get_dst_addr = mock.Mock(
side_effect=coro(mock.Mock(return_value=(socket.AF_INET,
'127.0.0.1'))))
return proto
def test_socks5_ctor(loop):
addr = aiosocks.Socks5Addr('localhost', 1080)
auth = aiosocks.Socks5Auth('user', 'pwd')
dst = ('python.org', 80)
with pytest.raises(ValueError):
aiosocks.Socks5Protocol(None, None, dst, loop=loop,
waiter=None, app_protocol_factory=None)
with pytest.raises(ValueError):
aiosocks.Socks5Protocol(None, auth, dst, loop=loop,
waiter=None, app_protocol_factory=None)
with pytest.raises(ValueError):
aiosocks.Socks5Protocol(aiosocks.Socks4Addr('host'),
auth, dst, loop=loop,
waiter=None, app_protocol_factory=None)
with pytest.raises(ValueError):
aiosocks.Socks5Protocol(addr, aiosocks.Socks4Auth('l'),
dst, loop=loop,
waiter=None, app_protocol_factory=None)
aiosocks.Socks5Protocol(addr, None, dst, loop=loop,
waiter=None, app_protocol_factory=None)
aiosocks.Socks5Protocol(addr, auth, dst, loop=loop,
waiter=None, app_protocol_factory=None)
async def worker(urls_q, proxies_q, proxies_q_good):
while True:
# Get proxy server from Queue and make proxy row string
if not proxies_q_good.empty():
proxy = await proxies_q_good.get()
else:
proxy = await proxies_q.get()
if proxy is None:
await asyncio.sleep(1)
continue
row = 'http://{host}:{port}'.format(host=proxy.host, port=proxy.port)
# Get url from Queue
if urls_q.empty():
return
else:
keyword = await urls_q.get()
url = base_link.format(keyword.replace(' ', '+'))
print(row, '-->', 'url', '| WPQ:', proxies_q_good.qsize(), '| APQ:',
proxies_q.qsize())
# except Exception as e:
# print(type(e), e, '[worker, print.Exception]')
# continue
# Make http request with SOCKS proxy
try:
addr = aiosocks.Socks5Addr(proxy.host, proxy.port)
conn = SocksConnector(proxy=addr)
with async_timeout.timeout(30):
async with aiohttp.ClientSession(
connector=conn) as http_client:
async with http_client.get(url, headers=headers) as resp:
assert resp.status == 200
code = await resp.text()
assert 'body' in code
except Exception as e:
print(type(e), e, '[worker, http_client.Exception]')
await urls_q.put(keyword)
continue
# If proxy is working put it into good (working) proxies Queue again
proxies_q_good.put_nowait(proxy)
# Create dictionary data, to save it into database
try:
position = get_position(code)
with open('data/google_positions_result.txt',
'a',
encoding='utf-8') as result:
result.write('{}\t{}\n'.format(keyword, position))
except Exception as e:
print(type(e), e, '[data_formatting Exception]')
continue
await asyncio.sleep(1)
async def test_socks5_invalid_address_type(loop):
async with FakeSocksSrv(loop, b'\x05\x02\x01\x00\x05\x00\x00\xFF') as srv:
addr = aiosocks.Socks5Addr('127.0.0.1', srv.port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
with pytest.raises(aiosocks.SocksError) as ct:
await aiosocks.create_connection(None, addr, auth, dst, loop=loop)
assert 'invalid data' in str(ct.value)
async def test_socks5_cmd_not_granted(loop):
async with FakeSocksSrv(loop, b'\x05\x02\x01\x00\x05\x01\x00') as srv:
addr = aiosocks.Socks5Addr('127.0.0.1', srv.port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
with pytest.raises(aiosocks.SocksError) as ct:
await aiosocks.create_connection(None, addr, auth, dst, loop=loop)
assert 'General SOCKS server failure' in str(ct.value)
async def test_socks5_auth_ver_err(loop):
async with FakeSocksSrv(loop, b'\x04\x02') as srv:
addr = aiosocks.Socks5Addr('127.0.0.1', srv.port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
with pytest.raises(aiosocks.SocksError) as ct:
await aiosocks.create_connection(None, addr, auth, dst, loop=loop)
assert 'invalid version' in str(ct.value)
async def test_socks5_auth_failed(loop):
async with FakeSocksSrv(loop, b'\x05\x02\x01\x01') as srv:
addr = aiosocks.Socks5Addr('127.0.0.1', srv.port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
with pytest.raises(aiosocks.SocksError) as ct:
await aiosocks.create_connection(None, addr, auth, dst, loop=loop)
assert 'authentication failed' in str(ct.value)
def test_init(self):
addr = aiosocks.Socks5Addr('localhost')
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
# proxy argument
with self.assertRaises(AssertionError) as ct:
conn = aiosocks.create_connection(None, None, auth, dst)
self.loop.run_until_complete(conn)
self.assertEqual(str(ct.exception),
'proxy must be Socks4Addr() or Socks5Addr() tuple')
with self.assertRaises(AssertionError) as ct:
conn = aiosocks.create_connection(None, auth, auth, dst)
self.loop.run_until_complete(conn)
self.assertEqual(str(ct.exception),
'proxy must be Socks4Addr() or Socks5Addr() tuple')
# proxy_auth
with self.assertRaises(AssertionError) as ct:
conn = aiosocks.create_connection(None, addr, addr, dst)
self.loop.run_until_complete(conn)
self.assertIn('proxy_auth must be None or Socks4Auth()',
str(ct.exception))
# dst
with self.assertRaises(AssertionError) as ct:
conn = aiosocks.create_connection(None, addr, auth, None)
self.loop.run_until_complete(conn)
self.assertIn('invalid dst format, tuple("dst_host", dst_port))',
str(ct.exception))
# addr and auth compatibility
with self.assertRaises(ValueError) as ct:
conn = aiosocks.create_connection(None, addr,
aiosocks.Socks4Auth(''), dst)
self.loop.run_until_complete(conn)
self.assertIn('proxy is Socks5Addr but proxy_auth is not Socks5Auth',
str(ct.exception))
with self.assertRaises(ValueError) as ct:
conn = aiosocks.create_connection(None, aiosocks.Socks4Addr(''),
auth, dst)
self.loop.run_until_complete(conn)
self.assertIn('proxy is Socks4Addr but proxy_auth is not Socks4Auth',
str(ct.exception))
# test ssl, server_hostname
with self.assertRaises(ValueError) as ct:
conn = aiosocks.create_connection(None,
addr,
auth,
dst,
server_hostname='python.org')
self.loop.run_until_complete(conn)
self.assertIn('server_hostname is only meaningful with ssl',
str(ct.exception))
async def main():
response = await aiohttp.get('http://icanhazip.com/')
body = await response.text()
print('ip: {}'.format(body.strip()))
addr = aiosocks.Socks5Addr('127.0.0.1', 9050)
conn = SocksConnector(proxy=addr, remote_resolve=False)
response = await aiohttp.get('http://icanhazip.com/', connector=conn)
body = await response.text()
print('tor ip: {}'.format(body.strip()))
async def test_connection_fail():
addr = aiosocks.Socks5Addr('localhost')
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
loop_mock = mock.Mock()
loop_mock.create_connection = make_mocked_coro(raise_exception=OSError())
with pytest.raises(aiosocks.SocksConnectionError):
await aiosocks.create_connection(None, addr, auth, dst, loop=loop_mock)
async def worker(urls_q, proxies_q, proxies_q_good):
while True:
# Get proxy server from Queue and make proxy row string
if not proxies_q_good.empty():
proxy = await proxies_q_good.get()
else:
proxy = await proxies_q.get()
if proxy is None:
await asyncio.sleep(1)
continue
row = 'http://{host}:{port}'.format(host=proxy.host, port=proxy.port)
# Get url from Queue
if urls_q.empty():
return
else:
page_url, link_text, link_url = await urls_q.get()
url = base_link.format(quote('cache:' + link_url))
print(row, '-->', url, '| WPQ:', proxies_q_good.qsize(), '| APQ:', proxies_q.qsize())
# Make http request with SOCKS proxy
try:
addr = aiosocks.Socks5Addr(proxy.host, proxy.port)
conn = SocksConnector(proxy=addr)
with async_timeout.timeout(30):
async with aiohttp.ClientSession(connector=conn) as http_client:
async with http_client.get(url, headers=headers) as resp:
# assert resp.status == 200
code = await resp.text()
assert (link_url in code or 'Not Found' in code)
except Exception as e:
print(type(e), e, '[worker, http_client.Exception]', link_url)
await urls_q.put(tuple(page_url, link_text, link_url))
continue
# If proxy is working put it into good (working) proxies Queue again
proxies_q_good.put_nowait(proxy)
# Create dictionary data, to save it into database
try:
indexed = get_data(code, page_url)
with open('data/google_cache_result.txt', 'a', encoding='utf-8') as result:
if indexed:
result.write('{}; {}\n'.format(link_url, 'indexed'))
else:
result.write('{}; {}\n'.format(link_url, 'no'))
except Exception as e:
print(type(e), e, '[data_formatting Exception]', link_url)
continue
await asyncio.sleep(1)
def test_proxy_negotiate_fail(self, cr_conn_mock):
loop_mock = mock.Mock()
cr_conn_mock.side_effect = \
fake_coroutine(aiosocks.SocksError()).side_effect
req = ClientRequest('GET', 'http://python.org', loop=self.loop)
connector = SocksConnector(aiosocks.Socks5Addr('127.0.0.1'),
None, loop=loop_mock)
loop_mock.getaddrinfo = fake_coroutine([mock.MagicMock()])
with self.assertRaises(aiosocks.SocksError):
self.loop.run_until_complete(connector.connect(req))
async def test_socks5_atype_domain(loop):
pld = b'\x05\x02\x01\x00\x05\x00\x00\x03\x0apython.org\x04W'
async with FakeSocksSrv(loop, pld) as srv:
addr = aiosocks.Socks5Addr('127.0.0.1', srv.port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
transport, protocol = await aiosocks.create_connection(
None, addr, auth, dst, loop=loop)
assert protocol.proxy_sockname == (b'python.org', 1111)
transport.close()
def test_proxy_connector():
socks4_addr = aiosocks.Socks4Addr('h')
socks5_addr = aiosocks.Socks5Addr('h')
http_addr = aiosocks.HttpProxyAddr('http://proxy')
loop = asyncio.new_event_loop()
assert isinstance(proxy_connector(socks4_addr, loop=loop), SocksConnector)
assert isinstance(proxy_connector(socks5_addr, loop=loop), SocksConnector)
assert isinstance(proxy_connector(http_addr, loop=loop),
aiohttp.ProxyConnector)
with pytest.raises(ValueError):
proxy_connector(None)
def test_auth_failed(self):
with fake_socks_srv(self.loop, b'\x05\x02\x01\x01') as port:
addr = aiosocks.Socks5Addr('127.0.0.1', port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
with self.assertRaises(aiosocks.SocksError) as ct:
coro = aiosocks.create_connection(None,
addr,
auth,
dst,
loop=self.loop)
self.loop.run_until_complete(coro)
self.assertIn('authentication failed', str(ct.exception))
def test_proxy_connector(self):
socks4_addr = aiosocks.Socks4Addr('h')
socks5_addr = aiosocks.Socks5Addr('h')
http_addr = HttpProxyAddr('http://proxy')
self.assertIsInstance(proxy_connector(socks4_addr, loop=self.loop),
SocksConnector)
self.assertIsInstance(proxy_connector(socks5_addr, loop=self.loop),
SocksConnector)
self.assertIsInstance(proxy_connector(http_addr, loop=self.loop),
aiohttp.ProxyConnector)
with self.assertRaises(ValueError):
proxy_connector(None)
def test_init(self):
addr = aiosocks.Socks4Addr('localhost', 1080)
auth = aiosocks.Socks4Auth('user')
dst = ('python.org', 80)
with self.assertRaises(ValueError):
aiosocks.Socks4Protocol(None,
None,
dst,
loop=self.loop,
waiter=None,
app_protocol_factory=None)
with self.assertRaises(ValueError):
aiosocks.Socks4Protocol(None,
auth,
dst,
loop=self.loop,
waiter=None,
app_protocol_factory=None)
with self.assertRaises(ValueError):
aiosocks.Socks4Protocol(aiosocks.Socks5Addr('host'),
auth,
dst,
loop=self.loop,
waiter=None,
app_protocol_factory=None)
with self.assertRaises(ValueError):
aiosocks.Socks4Protocol(addr,
aiosocks.Socks5Auth('l', 'p'),
dst,
loop=self.loop,
waiter=None,
app_protocol_factory=None)
aiosocks.Socks4Protocol(addr,
None,
dst,
loop=self.loop,
waiter=None,
app_protocol_factory=None)
aiosocks.Socks4Protocol(addr,
auth,
dst,
loop=self.loop,
waiter=None,
app_protocol_factory=None)
def test_cmd_not_granted(self):
with fake_socks_srv(self.loop,
b'\x05\x02\x01\x00\x05\x01\x00') as port:
addr = aiosocks.Socks5Addr('127.0.0.1', port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
with self.assertRaises(aiosocks.SocksError) as ct:
coro = aiosocks.create_connection(None,
addr,
auth,
dst,
loop=self.loop)
self.loop.run_until_complete(coro)
self.assertIn('General SOCKS server failure', str(ct.exception))
async def test_socks5_connect_success_usr_pwd(loop):
pld = b'\x05\x02\x01\x00\x05\x00\x00\x01\x01\x01\x01\x01\x04Wtest'
async with FakeSocksSrv(loop, pld) as srv:
addr = aiosocks.Socks5Addr('127.0.0.1', srv.port)
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
transport, protocol = await aiosocks.create_connection(
None, addr, auth, dst, loop=loop)
assert protocol.proxy_sockname == ('1.1.1.1', 1111)
data = await protocol._stream_reader.read(4)
assert data == b'test'
transport.close()
def test_connection_fail(self):
addr = aiosocks.Socks5Addr('localhost')
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
loop_mock = mock.Mock()
loop_mock.create_connection = fake_coroutine(OSError())
with self.assertRaises(aiosocks.SocksConnectionError):
conn = aiosocks.create_connection(None,
addr,
auth,
dst,
loop=loop_mock)
self.loop.run_until_complete(conn)
def __init__(self,
address,
port,
login=None,
password=None,
timeout=10,
loop=None):
super().__init__(timeout)
self.close()
addr = aiosocks.Socks5Addr(address, port)
if login and password:
auth = aiosocks.Socks5Auth(login, password=password)
else:
auth = None
conn = self.connector(proxy=addr, proxy_auth=auth, loop=loop)
self.session = aiohttp.ClientSession(connector=conn)
def test_connect_remote_resolve(self, cr_conn_mock):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
cr_conn_mock.side_effect = \
fake_coroutine((tr, proto)).side_effect
req = ClientRequest('GET', 'http://python.org', loop=self.loop)
connector = SocksConnector(aiosocks.Socks5Addr('127.0.0.1'),
None, loop=self.loop, remote_resolve=True)
connector._resolve_host = fake_coroutine([mock.MagicMock()])
conn = self.loop.run_until_complete(connector.connect(req))
self.assertEqual(connector._resolve_host.call_count, 1)
conn.close()
async def test_create_connection_init():
addr = aiosocks.Socks5Addr('localhost')
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
# proxy argument
with pytest.raises(AssertionError) as ct:
await aiosocks.create_connection(None, None, auth, dst)
assert 'proxy must be Socks4Addr() or Socks5Addr() tuple' in str(ct.value)
with pytest.raises(AssertionError) as ct:
await aiosocks.create_connection(None, auth, auth, dst)
assert 'proxy must be Socks4Addr() or Socks5Addr() tuple' in str(ct.value)
# proxy_auth
with pytest.raises(AssertionError) as ct:
await aiosocks.create_connection(None, addr, addr, dst)
assert 'proxy_auth must be None or Socks4Auth()' in str(ct.value)
# dst
with pytest.raises(AssertionError) as ct:
await aiosocks.create_connection(None, addr, auth, None)
assert 'invalid dst format, tuple("dst_host", dst_port))' in str(ct.value)
# addr and auth compatibility
with pytest.raises(ValueError) as ct:
await aiosocks.create_connection(None, addr, aiosocks.Socks4Auth(''),
dst)
assert 'proxy is Socks5Addr but proxy_auth is not Socks5Auth' \
in str(ct.value)
with pytest.raises(ValueError) as ct:
await aiosocks.create_connection(None, aiosocks.Socks4Addr(''), auth,
dst)
assert 'proxy is Socks4Addr but proxy_auth is not Socks4Auth' \
in str(ct.value)
# test ssl, server_hostname
with pytest.raises(ValueError) as ct:
await aiosocks.create_connection(None,
addr,
auth,
dst,
server_hostname='python.org')
assert 'server_hostname is only meaningful with ssl' in str(ct.value)
async def test_negotiate_fail():
addr = aiosocks.Socks5Addr('localhost')
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
loop_mock = mock.Mock()
loop_mock.create_connection = make_mocked_coro((mock.Mock(), mock.Mock()))
with mock.patch('aiosocks.asyncio.Future') as future_mock:
future_mock.side_effect = make_mocked_coro(
raise_exception=aiosocks.SocksError())
with pytest.raises(aiosocks.SocksError):
await aiosocks.create_connection(None,
addr,
auth,
dst,
loop=loop_mock)
async def proxy_request(self, url, **kwargs):
post = kwargs.get('post')
post = True if post != {} else False
post_data = kwargs.get('post_data')
headers = kwargs.get('headers')
j = kwargs.get('j')
j = True if j != {} else False
proxy_addr = aiosocks.Socks5Addr('', 1080)
proxy_auth = aiosocks.Socks5Auth('', password='')
proxy_connection = aiosocks.connector.SocksConnector(
proxy=proxy_addr, proxy_auth=proxy_auth, remote_resolve=True)
with aiohttp.ClientSession(connector=proxy_connection) as session:
async with session.post(url,
data=post_data if post else None,
headers=headers) as resp:
if j:
return await resp.json()
else:
return await resp.text()
def test_negotiate_fail(self, future_mock):
addr = aiosocks.Socks5Addr('localhost')
auth = aiosocks.Socks5Auth('usr', 'pwd')
dst = ('python.org', 80)
loop_mock = mock.Mock()
loop_mock.create_connection = fake_coroutine(
(mock.Mock(), mock.Mock()))
fut = fake_coroutine(aiosocks.SocksError())
future_mock.side_effect = fut.side_effect
with self.assertRaises(aiosocks.SocksError):
conn = aiosocks.create_connection(None,
addr,
auth,
dst,
loop=loop_mock)
self.loop.run_until_complete(conn)
def test_connect_proxy_ip(self, cr_conn_mock):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
cr_conn_mock.side_effect = \
fake_coroutine((tr, proto)).side_effect
loop_mock = mock.Mock()
req = ClientRequest('GET', 'http://python.org', loop=self.loop)
connector = SocksConnector(aiosocks.Socks5Addr('127.0.0.1'),
None, loop=loop_mock)
loop_mock.getaddrinfo = fake_coroutine([mock.MagicMock()])
conn = self.loop.run_until_complete(connector.connect(req))
self.assertTrue(loop_mock.getaddrinfo.is_called)
self.assertIs(conn._transport, tr)
conn.close()
def test_connect_proxy_domain(self, cr_conn_mock):
tr, proto = mock.Mock(name='transport'), mock.Mock(name='protocol')
cr_conn_mock.side_effect = \
fake_coroutine((tr, proto)).side_effect
loop_mock = mock.Mock()
req = ClientRequest('GET', 'http://python.org', loop=self.loop)
connector = SocksConnector(aiosocks.Socks5Addr('proxy.example'),
None, loop=loop_mock)
connector._resolve_host = fake_coroutine([mock.MagicMock()])
conn = self.loop.run_until_complete(connector.connect(req))
self.assertTrue(connector._resolve_host.is_called)
self.assertEqual(connector._resolve_host.call_count, 1)
self.assertIs(conn._transport, tr)
conn.close()