def session(self):
if not self._session or self._session.closed:
if certifi:
context = ssl.create_default_context(cafile=certifi.where())
connector = ProxyConnector(ssl_context=context)
else:
connector = ProxyConnector()
self._session = aiohttp.ClientSession(
connector=connector,
request_class=ProxyClientRequest,
json_serialize=self.json_serialize)
return self._session
def __init__(self):
self.connector = ProxyConnector()
self.session = aiohttp.ClientSession(
connector=self.connector,
request_class=ProxyClientRequest,
timeout=aiohttp.ClientTimeout(total=4))
self.proxies = None
if config["Proxy"]["PROXY_TYPE"] != "None":
if config["Proxy"]["PROXY_TYPE"].lower() == "http":
if config["Proxy"]["PROXY_USERNAME"] != "" and config["Proxy"][
"PROXY_PASSWORD"] != "":
self.proxies = "http://" + config["Proxy"][
"PROXY_USERNAME"] + ":" + config["Proxy"][
"PROXY_PASSWORD"] + "@" + config["Proxy"][
"PROXY_ADDRESS"] + ":" + config["Proxy"][
"PROXY_PORT"]
else:
self.proxies = "http://" + config["Proxy"][
"PROXY_ADDRESS"] + ":" + config["Proxy"]["PROXY_PORT"]
elif config["Proxy"]["PROXY_TYPE"].lower() == "socks5":
if config["Proxy"]["PROXY_USERNAME"] != "" and config["Proxy"][
"PROXY_PASSWORD"] != "":
self.proxies = "socks5://" + config["Proxy"][
"PROXY_USERNAME"] + ":" + config["Proxy"][
"PROXY_PASSWORD"] + "@" + config["Proxy"][
"PROXY_ADDRESS"] + ":" + config["Proxy"][
"PROXY_PORT"]
else:
self.proxies = "socks5://" + config["Proxy"][
"PROXY_ADDRESS"] + ":" + config["Proxy"]["PROXY_PORT"]
async def async_request_aio_http(arr_handle):
timeout = aiohttp.ClientTimeout(total=30)
auth = Socks5Auth(login='', password='')
connector = ProxyConnector(verify_ssl=False)
socks_api = ""
auth = ""
async with aiohttp.ClientSession(connector=connector,
request_class=ProxyClientRequest,
timeout=timeout) as session:
tasks = []
for item in arr_handle:
url = item['url']
data_post = {}
task = asyncio.ensure_future(
fetch("GET", url, data_post, session, socks_api, auth))
tasks.append(task)
responses = await asyncio.gather(*tasks)
await session.close()
return responses
async def test_https_connect(loop):
async def handler(request):
return web.Response(text='Test message')
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
keyfile = os.path.join(here, 'sample.key')
certfile = os.path.join(here, 'sample.crt')
sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
sslcontext.load_cert_chain(certfile, keyfile)
ws = RawTestServer(handler, scheme='https', host='127.0.0.1', loop=loop)
await ws.start_server(loop=loop, ssl=sslcontext)
v_fp = (b'0\x9a\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd'
b'\xcb~7U\x14D@L'
b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x8b')
inv_fp = (b'0\x9d\xc9D\x83\xdc\x91\'\x88\x91\x11\xa1d\x97\xfd'
b'\xcb~7U\x14D@L'
b'\x11\xab\x99\xa8\xae\xb7\x14\xee\x9e')
async with FakeSocks4Srv(loop) as srv:
v_conn = ProxyConnector(loop=loop,
remote_resolve=False,
fingerprint=v_fp)
inv_conn = ProxyConnector(loop=loop,
remote_resolve=False,
fingerprint=inv_fp)
async with aiohttp.ClientSession(
connector=v_conn, loop=loop,
request_class=ProxyClientRequest) as ses:
proxy = 'socks4://127.0.0.1:{}'.format(srv.port)
async with ses.get(ws.make_url('/'), proxy=proxy) as resp:
assert resp.status == 200
assert (await resp.text()) == 'Test message'
async with aiohttp.ClientSession(
connector=inv_conn, loop=loop,
request_class=ProxyClientRequest) as ses:
proxy = 'socks4://127.0.0.1:{}'.format(srv.port)
with pytest.raises(aiohttp.ServerFingerprintMismatch):
async with ses.get(ws.make_url('/'), proxy=proxy) as resp:
assert resp.status == 200
async def run(self):
# TODO: rework proxy
async with ClientSession(
cookie_jar=DummyCookieJar(),
connector=ProxyConnector(),
request_class=ProxyClientRequest,
) as session:
tasks = self.prepare_tasks(session)
return await asyncio.gather(*tasks)
async def __work():
session = aiohttp.ClientSession(connector=ProxyConnector(),
request_class=ProxyClientRequest)
on_start()
results = await asyncio.gather(self.loop.create_task(
worker(session, *worker_args, **worker_kwargs)),
return_exceptions=True)
on_end(results[0])
await session.close()
async def __aiter__(self):
# TODO: rework proxy
async with ClientSession(
cookie_jar=DummyCookieJar(),
connector=ProxyConnector(),
request_class=ProxyClientRequest,
) as session:
tasks = self.prepare_tasks(session)
for result in asyncio.as_completed(tasks):
yield await result
async def download(url):
connector = ProxyConnector()
socks = None
async with aiohttp.ClientSession(
connector=connector, request_class=ProxyClientRequest) as session:
ret, status = await session_get(session, url, socks)
if 'window.location.href' in ret and len(ret) < 1000:
url = ret.split("window.location.href='")[1].split("'")[0]
ret, status = await session_get(session, url, socks)
return ret, status
def __init__(self):
self.connector = ProxyConnector()
self.session = aiohttp.ClientSession(connector=self.connector, request_class=ProxyClientRequest,
timeout=aiohttp.ClientTimeout(total=4))
self.proxies = None
if config["Proxy"]["PROXY_TYPE"] != "None":
self.proxies = {
"http": config["Proxy"]["PROXY_ADDRESS"],
"https": config["Proxy"]["PROXY_ADDRESS"]
}
async def get_visible_ip(proxy_url: str,
ip_judge: str = 'http://ifconfig.me/ip') -> str:
try:
async with ClientSession(connector=ProxyConnector(),
request_class=ProxyClientRequest) as client:
async with client.get(ip_judge, proxy=proxy_url,
timeout=5) as response:
return await response.text()
except (ClientError, TimeoutError):
raise ServerConnectionError
async def is_available_to_resource(self, resource: str) -> bool:
try:
async with ClientSession(
connector=ProxyConnector(),
request_class=ProxyClientRequest) as client:
async with client.get(resource, proxy=self.url,
timeout=5) as response:
if response.status != 200:
return False
except (ClientError, asyncio.TimeoutError):
return False
return True
async def download(url):
connector = ProxyConnector()
if DEPLOY_MODE == "dev":
socks = None
elif DEPLOY_MODE == "Prod":
socks = random.choice(socks5_address_prod)
async with aiohttp.ClientSession(
connector=connector, request_class=ProxyClientRequest) as session:
ret, status = await session_get(session, url, socks)
if 'window.location.href' in ret and len(ret) < 1000:
url = ret.split("window.location.href='")[1].split("'")[0]
ret, status = await session_get(session, url, socks)
return ret, status
async def init_app(loop=None):
app = web.Application(middlewares=[], loop=loop)
conn = ProxyConnector(remote_resolve=False)
app['client_session'] = aiohttp.ClientSession(
connector=conn,
request_class=ProxyClientRequest,
raise_for_status=True)
app['check_semaphore'] = asyncio.Semaphore(settings.CHECK_SEMAPHORE)
app['parse_semaphore'] = asyncio.Semaphore(settings.PARSE_SEMAPHORE)
app['db'] = await init_db()
app.router.add_get("/", get_proxies)
return app
async def test_http_connect(loop):
async def handler(request):
return web.Response(text='Test message')
async with RawTestServer(handler, host='127.0.0.1', loop=loop) as ws:
async with FakeSocks4Srv(loop) as srv:
conn = ProxyConnector(loop=loop, remote_resolve=False)
async with aiohttp.ClientSession(
connector=conn, loop=loop,
request_class=ProxyClientRequest) as ses:
proxy = 'socks4://127.0.0.1:{}'.format(srv.port)
async with ses.get(ws.make_url('/'), proxy=proxy) as resp:
assert resp.status == 200
assert (await resp.text()) == 'Test message'
async def __work():
semaphore = asyncio.Semaphore(max_concurrent_workers)
session = aiohttp.ClientSession(connector=ProxyConnector(),
request_class=ProxyClientRequest)
async def __task(element: Union[Proxy, Any]):
async with semaphore:
return await worker(session, element, *worker_args,
**worker_kwargs)
on_start()
tasks = [
self.loop.create_task(__task(element)) for element in elements
]
results = await asyncio.gather(*tasks, return_exceptions=True)
await session.close()
on_end(results)
async def get_url(url, headers):
content = False
conn = ProxyConnector()
try:
async with aiohttp.ClientSession(
connector=conn, request_class=ProxyClientRequest) as session:
async with session.get(url,
proxy='socks5://127.0.0.1:1080',
headers=headers) as resp:
if resp.status == 200:
content = await resp.text()
else:
print('failed to get {}, status {}'.format(
url, resp.status))
except Exception as e:
print('failed to get {}, error {}'.format(url, e))
return content
async def get_proxy_ping(proxy_url: str,
ping_target: str = 'https://google.com') -> int:
conn = ProxyConnector()
start_time = time.process_time()
try:
async with ClientSession(connector=conn,
request_class=ProxyClientRequest) as client:
async with client.get(ping_target, proxy=proxy_url,
timeout=5) as response:
end_time = time.process_time()
if response.status != 200:
ping = None
else:
ping = int((end_time - start_time) * 1000)
except (ClientError, TimeoutError, ValueError):
ping = None
return ping
async def run(self):
"""
HTTP session initialization
:return: None
"""
conn = ProxyConnector()
if not self.proxies:
self.proxies = []
if isinstance(self.proxies, str):
self.proxies = [self.proxies]
async with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as self.session:
try:
async for ret in self.startup():
if isinstance(ret, Request):
await self.push_request(ret)
except TypeError:
logger.error('self.startup() returned in the wrong way, please use yield to return data.')
async for req in self._get_request():
await self.request_handler(req)
async def fetch(url_list, proxy, proxy_user, proxy_password, loop):
# 请求及解析
requests_task_list = []
# 构造代理
if proxy_user and proxy_password:
proxy_auth = aiohttp.BasicAuth(proxy_user, proxy_password)
else:
proxy_auth = None
# 创建连接对象，多个连接由同一个对象发出请求，减少创建对象资源
if proxy and 'socks5' in proxy:
# socks5代理需要额外处理
connector, request_class = ProxyConnector(enable_cleanup_closed=True), ProxyClientRequest
client = ClientSession(connector=connector, request_class=request_class)
else:
connector = aiohttp.TCPConnector(enable_cleanup_closed=True)
client = ClientSession(connector=connector)
async with client:
# 请求
for url in url_list:
task = loop.create_task(requests_page(client, url, proxy, proxy_auth))
requests_task_list.append(task)
await asyncio.wait(requests_task_list)
# 解析任务并保存
save_file_task_list = []
requests_failed_url_list = []
for i, task_future in enumerate(requests_task_list):
html, url = task_future.result()
if html:
# save_file_task = loop.create_task(save_html_to_local_file(url, html))
url_md5 = get_token(url)
html = parse_html(html)
redis_help.hset(html_dict_key, url_md5, html)
print(f'requests {url} successful')
else:
requests_failed_url_list.append(url)
print(f'requests {url} error')
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy:
:return:
"""
#auth = Socks5Auth(login=None, password=None),
connector = ProxyConnector()
socks = 'socks5://' + proxy
print("测试代理{0}".format(proxy))
async with aiohttp.ClientSession(
connector=connector,
request_class=ProxyClientRequest) as session:
try:
async with session.get('http://spys.one',
proxy=socks) as resp: # proxy_auth=auth
#print(await resp.text())
if resp.status == 200:
self.useproxy.append(proxy)
print("存入代理{0}".format(proxy))
except: #aiohttp.ProxyConnectionError:
#print('connection problem')
pass
async def check(self, proxy):
possible_exceptions = (
aiohttp.client_exceptions.ClientProxyConnectionError,
concurrent.futures._base.TimeoutError,
aiohttp.client_exceptions.ClientOSError,
aiohttp.client_exceptions.ClientHttpProxyError,
aiohttp.client_exceptions.ServerDisconnectedError,
aiohttp.client_exceptions.ClientResponseError,
aiosocksy.errors.SocksError, aiohttp.client_exceptions.InvalidURL,
aiohttp.client_exceptions.ClientPayloadError)
start_time = time.time()
try:
is_port_open = await check_port_open(proxy.host, proxy.port)
if not is_port_open:
raise concurrent.futures._base.TimeoutError(
'Could not open connection')
async with async_timeout.timeout(self.timeout):
async with aiohttp.ClientSession(
connector=ProxyConnector(verify_ssl=False, limit=0),
request_class=ProxyClientRequest,
conn_timeout=self.timeout,
read_timeout=self.timeout) as session:
async with session.get(
self.url,
proxy=str(proxy),
headers=random_session()['headers']) as response:
content = await response.read()
# self.logger.debug('Got response [{}]: {} bytes'.format(response.status, len(content))) #DELETE_DEBUG
result = response
except possible_exceptions as e:
result = e
delta_time = time.time() - start_time
is_passed = True
is_banned = False
if isinstance(result, aiohttp.client_reqrep.ClientResponse):
if self.check_xpath:
any_xpath_worked = False
try:
doc = lxml.html.fromstring(content)
if settings.SHOW_RESPONSE_BODY:
self.logger.info(content)
for xpath in self.check_xpath:
xpath_result = doc.xpath(xpath)
if xpath_result:
any_xpath_worked = True
if xpath_result and isinstance(
xpath, xpath_check.BanXPathCheck):
is_banned = True
except (lxml.etree.ParserError, lxml.etree.XMLSyntaxError):
pass
except lxml.etree.XPathEvalError as e:
self.logger.warning('XPath is wrong: {}'.format(xpath))
raise e
if not any_xpath_worked:
is_passed = False
# self.logger.debug('No any xpath worked for proxy {} on url {} ({}):'.format(proxy, self.url, ", ".join(self.check_xpath))) #DELETE_DEBUG
if self.status and int(result.status) not in self.status:
self.logger.debug(
'{} not passed status code check on {}. {} got, but {} expected'
.format(proxy, self.url, result.status, self.status))
is_passed = False
else:
is_passed = False
if isinstance(result, aiohttp.client_reqrep.ClientResponse):
status = int(result.status)
else:
status = None
check_result = CheckResult()
check_result.proxy = proxy
check_result.is_passed = is_passed
check_result.is_banned = is_banned
check_result.check = self
check_result.time = delta_time
check_result.done_at = datetime.datetime.utcnow()
check_result.status = status
if isinstance(result, BaseException):
check_result.error = str(result)
get_session().add(check_result)
await proxy.on_check_executed()
error = ''
if isinstance(result, BaseException):
error = ', error: {}'.format(str(result))
self.logger.info(
'Finished check (is passed: {}) for {} on {} by {:0.3f} s{}'.
format(is_passed, proxy, self.url, delta_time, error))
return check_result
def __init__(self,
token: base.String,
loop: Optional[Union[asyncio.BaseEventLoop,
asyncio.AbstractEventLoop]] = None,
connections_limit: Optional[base.Integer] = 10,
proxy: Optional[base.String] = None,
proxy_auth: Optional[aiohttp.BasicAuth] = None,
validate_token: Optional[base.Boolean] = True,
parse_mode=None):
"""
Instructions how to get Bot token is found here: https://core.telegram.org/bots#3-how-do-i-create-a-bot
:param token: token from @BotFather
:type token: :obj:`str`
:param loop: event loop
:type loop: Optional Union :obj:`asyncio.BaseEventLoop`, :obj:`asyncio.AbstractEventLoop`
:param connections_limit: connections limit for aiohttp.ClientSession
:type connections_limit: :obj:`int`
:param proxy: HTTP proxy URL
:type proxy: :obj:`str`
:param proxy_auth: Authentication information
:type proxy_auth: Optional :obj:`aiohttp.BasicAuth`
:param validate_token: Validate token.
:type validate_token: :obj:`bool`
:param parse_mode: You can set default parse mode
:type parse_mode: :obj:`str`
:raise: when token is invalid throw an :obj:`aiogram.utils.exceptions.ValidationError`
"""
# Authentication
if validate_token:
api.check_token(token)
self.__token = token
# Proxy settings
self.proxy = proxy
self.proxy_auth = proxy_auth
# Asyncio loop instance
if loop is None:
loop = asyncio.get_event_loop()
self.loop = loop
# aiohttp main session
ssl_context = ssl.create_default_context(cafile=certifi.where())
if isinstance(proxy, str) and proxy.startswith('socks5://'):
from aiosocksy.connector import ProxyClientRequest, ProxyConnector
connector = ProxyConnector(limit=connections_limit,
ssl_context=ssl_context,
loop=self.loop)
request_class = ProxyClientRequest
else:
connector = aiohttp.TCPConnector(limit=connections_limit,
ssl_context=ssl_context,
loop=self.loop)
request_class = aiohttp.ClientRequest
self.session = aiohttp.ClientSession(connector=connector,
request_class=request_class,
loop=self.loop,
json_serialize=json.dumps)
# Temp sessions
self._temp_sessions = []
# Data stored in bot instance
self._data = {}
self.parse_mode = parse_mode