async def test_proxy_chain():
proxy = ProxyChain([
Proxy.from_url(SOCKS5_IPV4_URL),
Proxy.from_url(SOCKS4_URL),
Proxy.from_url(HTTP_PROXY_URL),
])
# noinspection PyTypeChecker
status_code = await make_request(proxy=proxy, url=TEST_URL)
assert status_code == 200
async def open_connection(proxy_url=None, host=None, port=None, *,
proxy_type=ProxyType.SOCKS5,
proxy_host='127.0.0.1', proxy_port=1080,
username=None, password=None, rdns=True,
loop=None, **kwargs):
if host is None or port is None:
raise ValueError('host and port must be specified') # pragma: no cover
if loop is None:
loop = asyncio.get_event_loop()
if proxy_url is not None:
proxy_type, proxy_host, proxy_port, username, password \
= parse_proxy_url(proxy_url)
proxy = Proxy.create(
proxy_type=proxy_type,
host=proxy_host,
port=proxy_port,
username=username,
password=password,
rdns=rdns,
loop=loop
)
sock = await proxy.connect(host, port)
# noinspection PyTypeChecker
return await asyncio.open_connection(
host=None,
port=None,
sock=sock,
**kwargs
)
async def _wrap_create_connection(self, protocol_factory, host, port,
**kwargs):
proxy = Proxy.create(
proxy_type=self._proxy_type,
host=self._proxy_host,
port=self._proxy_port,
username=self._proxy_username,
password=self._proxy_password,
rdns=self._rdns,
loop=self._loop,
)
connect_timeout = None
timeout = kwargs.get('timeout')
if timeout is not None:
connect_timeout = getattr(timeout, 'sock_connect', None)
sock = await proxy.connect(host, port, timeout=connect_timeout)
return await super()._wrap_create_connection(protocol_factory,
None,
None,
sock=sock,
**kwargs)
async def _wrap_create_connection(self, protocol_factory, host, port,
**kwargs):
proxies = []
for info in self._proxy_infos:
proxy = Proxy.create(proxy_type=info.proxy_type,
host=info.host,
port=info.port,
username=info.username,
password=info.password,
rdns=info.rdns,
loop=self._loop)
proxies.append(proxy)
proxy = ProxyChain(proxies)
connect_timeout = None
timeout = kwargs.get('timeout')
if timeout is not None:
connect_timeout = getattr(timeout, 'sock_connect', None)
sock = await proxy.connect(host, port, timeout=connect_timeout)
return await super()._wrap_create_connection(protocol_factory,
None,
None,
sock=sock,
**kwargs)
async def test_socks5_proxy_ipv4(rdns, resolve_host):
proxy = Proxy.from_url(SOCKS5_IPV4_URL, rdns=rdns)
status_code = await make_request(
proxy=proxy,
url=TEST_URL,
resolve_host=resolve_host
)
assert status_code == 200
async def async_rotate_fetch(
url: str,
protocol: ProxyProto = 'socks5',
return_proxy: Boolean = False) -> Union[Data, Tuple[Data, Data]]:
"""
Rotate proxies and perform a GET request. Returns a Data object of `response.status_code`, `response.text`, and
`response.headers`.
:param url: URL to fetch.
:param protocol: 'socks5' or 'http'; must correspond to your ActProxy proxies' type.
:param return_proxy: Boolean; Return tuple(proxy, ProxyConnector) instead of just ProxyConnector.
:return: A Data object of the response with resp.text, resp.status_code, and resp.headers
"""
url_parts = urlparse(url)
url_port = 443 if url_parts.scheme == 'https' else 80
url_path = url_parts.path
actproxy = one_hot_proxy()
if proxy := AsyncProxy.from_url(
f'{protocol}://{actproxy.username}:{actproxy.password}@{actproxy.host}:{actproxy.port}'
):
sock = await proxy.connect(dest_host=url_parts.hostname,
dest_port=url_port)
reader, writer = await asyncio.open_connection(
host=None,
port=None,
sock=sock,
ssl=ssl.create_default_context(),
server_hostname=url_parts.hostname,
)
request = (b'GET ' + url_path.encode() + b' HTTP/1.1\r\n'
b'Host: ' + url_parts.hostname.encode() + b'\r\n'
b'Connection: close\r\n\r\n')
writer.write(request)
_resp = await reader.read(-1)
body = _resp.decode('UTF-8').split('\r\n\r\n', 1)[-1]
status_code = int(
_resp.decode('UTF-8').split('\r\n', 1)[0].split(' ')[1])
headers = parse_it(_resp)
response_data = Data(text=body,
status_code=status_code,
headers=headers)
if return_proxy:
return response_data, actproxy
else:
return response_data
async def _work():
proxy = Proxy.from_url('socks5://127.0.0.1:1080')
sock = await proxy.connect(dest_host='127.0.0.1',
dest_port=port)
reader, writer = await asyncio.open_connection(host=None,
port=None,
sock=sock)
writer.write(b'Ping\r\n')
await reader.read(-1)
async def test_socks5_proxy_with_invalid_proxy_port(unused_tcp_port):
proxy = Proxy.create(
proxy_type=ProxyType.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=unused_tcp_port,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(ProxyConnectionError):
await make_request(proxy=proxy, url=TEST_URL)
async def test_socks5_proxy_with_connect_timeout():
proxy = Proxy.create(
proxy_type=ProxyType.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=SOCKS5_IPV4_PORT,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(ProxyTimeoutError):
await make_request(proxy=proxy, url=TEST_URL, timeout=0.0001)
async def test_socks5_proxy_with_invalid_credentials():
proxy = Proxy.create(
proxy_type=ProxyType.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=SOCKS5_IPV4_PORT,
username=LOGIN,
password=PASSWORD + 'aaa',
)
with pytest.raises(ProxyError):
await make_request(proxy=proxy, url=TEST_URL)
async def _proxy_connect(self, timeout=None, local_addr=None):
# Use `python-socks` library for newer Python >= 3.6
# Use `PySocks` library for older Python <= 3.5
if sys.version_info >= (3, 6):
import python_socks
# python_socks internal errors are not inherited from
# builtin IOError (just from Exception). Instead of adding those
# in exceptions clauses everywhere through the code, we
# rather monkey-patch them in place.
python_socks._errors.ProxyError = ConnectionError
python_socks._errors.ProxyConnectionError = ConnectionError
python_socks._errors.ProxyTimeoutError = ConnectionError
from python_socks.async_.asyncio import Proxy
# We expect a dict/tuple in the format of `PySocks` (for compatibility)
# I.E.: (proxy_type, addr, port, rdns, username, password).
if isinstance(self._proxy, (tuple, list)):
parsed = self._parse_proxy("python-socks", *self._proxy)
elif isinstance(self._proxy, dict):
parsed = self._parse_proxy("python-socks", **self._proxy)
else:
raise TypeError("Proxy of unknown format!", type(self._proxy))
proxy = Proxy.create(*parsed)
# WARNING: If `local_addr` is set we use manual socket creation, because,
# unfortunately, `Proxy.connect()` does not expose `local_addr`
# argument, so if we want to bind socket locally, we need to manually
# create, bind and connect socket, and then pass to `Proxy.connect()` method.
if local_addr is None:
sock = await proxy.connect(dest_host=self._ip,
dest_port=self._port,
timeout=timeout)
else:
# Here we start manual setup of the socket.
# The `address` represents the proxy ip and proxy port,
# not the destination one (!), because the socket
# connects to the proxy server, not destination server.
# IPv family is also checked on proxy address.
if ':' in proxy.proxy_host:
mode, address = socket.AF_INET6, (proxy.proxy_host,
proxy.proxy_port, 0, 0)
else:
mode, address = socket.AF_INET, (proxy.proxy_host,
proxy.proxy_port)
# Create a non-blocking socket and bind it (if local address is specified).
sock = socket.socket(mode, socket.SOCK_STREAM)
sock.setblocking(False)
sock.bind(local_addr)
# Actual TCP connection is performed here.
await asyncio.wait_for(asyncio.get_event_loop().sock_connect(
sock=sock, address=address),
timeout=timeout)
# As our socket is already created and connected,
# this call sets the destination host/port and
# starts protocol negotiations with the proxy server.
sock = await proxy.connect(dest_host=self._ip,
dest_port=self._port,
timeout=timeout,
_socket=sock)
else:
import socks
# We expect a dict/tuple in the format of `PySocks`.
# I.E.: (proxy_type, addr, port, rdns, username, password).
if isinstance(self._proxy, (tuple, list)):
parsed = self._parse_proxy("pysocks", *self._proxy)
elif isinstance(self._proxy, dict):
parsed = self._parse_proxy("pysocks", **self._proxy)
else:
raise TypeError("Proxy of unknown format!", type(self._proxy))
# Here `address` represents destination address (not proxy), because of
# the `PySocks` implementation of the connection routine.
# IPv family is checked on proxy address, not destination address.
if ':' in parsed[1]:
mode, address = socket.AF_INET6, (self._ip, self._port, 0, 0)
else:
mode, address = socket.AF_INET, (self._ip, self._port)
# Setup socket, proxy, timeout and bind it (if necessary).
sock = socks.socksocket(mode, socket.SOCK_STREAM)
sock.set_proxy(*parsed)
sock.settimeout(timeout)
if local_addr is not None:
sock.bind(local_addr)
# Actual TCP connection and negotiation performed here.
await asyncio.wait_for(asyncio.get_event_loop().sock_connect(
sock=sock, address=address),
timeout=timeout)
sock.setblocking(False)
return sock
async def test_socks5_proxy_ipv4_with_auth_none(rdns):
proxy = Proxy.from_url(SOCKS5_IPV4_URL_WO_AUTH, rdns=rdns)
status_code = await make_request(proxy=proxy, url=TEST_URL)
assert status_code == 200
async def test_http_proxy():
proxy = Proxy.from_url(HTTP_PROXY_URL)
status_code = await make_request(proxy=proxy, url=TEST_URL)
assert status_code == 200
async def test_socks5_proxy_ipv6():
proxy = Proxy.from_url(SOCKS5_IPV6_URL)
status_code = await make_request(proxy=proxy, url=TEST_URL)
assert status_code == 200
async def test_socks5_proxy_hostname_ipv4(url):
proxy = Proxy.from_url(SOCKS5_IPV4_HOSTNAME_URL)
status_code = await make_request(proxy=proxy, url=url)
assert status_code == 200
async def test_socks4_proxy(url, rdns, resolve_host):
proxy = Proxy.from_url(SOCKS4_URL, rdns=rdns)
status_code = await make_request(proxy=proxy,
url=url,
resolve_host=resolve_host)
assert status_code == 200