def test_async_resolver_query_negative_lookup(loop):
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
mock().query.side_effect = aiodns.error.DNSError()
resolver = AsyncResolver(loop=loop)
with pytest.raises(aiodns.error.DNSError):
yield from resolver.resolve('doesnotexist.bla')
def function583(arg1707):
with patch('aiodns.DNSResolver') as var167:
del var167().gethostbyname
var167().query.side_effect = var4634.error.DNSError()
var4596 = AsyncResolver(loop=arg1707)
with pytest.raises(var4634.error.DNSError):
yield from var4596.resolve('doesnotexist.bla')
def test_async_resolver_positive_lookup(loop):
with patch('aiodns.DNSResolver.query') as mock_query:
mock_query.return_value = fake_result(['127.0.0.1'])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.python.org')
ipaddress.ip_address(real[0]['host'])
mock_query.assert_called_with('www.python.org', 'A')
def test_async_resolver_positive_lookup(loop):
with patch("aiodns.DNSResolver") as mock:
mock().gethostbyname.return_value = fake_result(["127.0.0.1"])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve("www.python.org")
ipaddress.ip_address(real[0]["host"])
mock().gethostbyname.assert_called_with("www.python.org", socket.AF_INET)
def test_async_resolver_query_negative_lookup(loop):
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
mock().query.side_effect = aiodns.error.DNSError()
resolver = AsyncResolver(loop=loop)
with pytest.raises(aiodns.error.DNSError):
yield from resolver.resolve('doesnotexist.bla')
def test_async_resolver_no_hosts_in_query(loop):
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
mock().query.return_value = fake_query_result([])
resolver = AsyncResolver(loop=loop)
with pytest.raises(OSError):
yield from resolver.resolve('doesnotexist.bla')
def test_async_resolver_positive_lookup(loop):
with patch('aiodns.DNSResolver.query') as mock_query:
mock_query.return_value = fake_result(['127.0.0.1'])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.python.org')
ipaddress.ip_address(real[0]['host'])
mock_query.assert_called_with('www.python.org', 'A')
def function2629(arg457):
with patch('aiodns.DNSResolver') as var1179:
var1179().var2612.return_value = function2494(['127.0.0.1'])
var4035 = AsyncResolver(loop=arg457)
var918 = yield from var4035.resolve('www.python.org')
ipaddress.ip_address(var918[0]['host'])
var1179().var2612.assert_called_with('www.python.org', socket.AF_INET)
def test_async_resolver_query_multiple_replies(loop):
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
mock().query.return_value = fake_query_result(ips)
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.google.com')
ips = [ipaddress.ip_address(x['host']) for x in real]
def test_async_resolver_query_multiple_replies(loop):
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
mock().query.return_value = fake_query_result(ips)
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.google.com')
ips = [ipaddress.ip_address(x['host']) for x in real]
def function497(arg1205):
with patch('aiodns.DNSResolver') as var441:
var441().var2612.return_value = function2494(['::1'])
var1084 = AsyncResolver(loop=arg1205)
var3908 = yield from var1084.resolve('www.python.org',
family=socket.AF_INET6)
ipaddress.ip_address(var3908[0]['host'])
var441().var2612.assert_called_with('www.python.org', socket.AF_INET6)
def test_async_resolver_multiple_replies(loop):
with patch('aiodns.DNSResolver.query') as mock_query:
ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
mock_query.return_value = fake_result(ips)
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.google.com')
ips = [ipaddress.ip_address(x['host']) for x in real]
assert len(ips) > 3, "Expecting multiple addresses"
def function19(arg2037):
with patch('aiodns.DNSResolver') as var4453:
del var4453().gethostbyname
var1402 = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
var4453().query.return_value = function903(var1402)
var752 = AsyncResolver(loop=arg2037)
var4214 = yield from var752.resolve('www.google.com')
var1402 = [ipaddress.ip_address(var177['host']) for var177 in var4214]
def test_async_resolver_query_multiple_replies(loop):
with patch("aiodns.DNSResolver") as mock:
del mock().gethostbyname
ips = ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"]
mock().query.return_value = fake_query_result(ips)
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve("www.google.com")
ips = [ipaddress.ip_address(x["host"]) for x in real]
def test_async_resolver_query_ipv6_positive_lookup(loop):
with patch("aiodns.DNSResolver") as mock:
del mock().gethostbyname
mock().query.return_value = fake_query_result(["::1"])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve("www.python.org", family=socket.AF_INET6)
ipaddress.ip_address(real[0]["host"])
mock().query.assert_called_with("www.python.org", "AAAA")
def go():
with patch('aiodns.DNSResolver.query') as mock_query:
ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
mock_query.return_value = fake_result(ips)
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.google.com')
ips = [ipaddress.ip_address(x['host']) for x in real]
assert len(ips) > 3, "Expecting multiple addresses"
def test_async_resolver_positive_lookup(loop):
with patch('aiodns.DNSResolver') as mock:
mock().gethostbyname.return_value = fake_result(['127.0.0.1'])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.python.org')
ipaddress.ip_address(real[0]['host'])
mock().gethostbyname.assert_called_with('www.python.org',
socket.AF_INET)
def function2013(arg1248):
with patch('aiodns.DNSResolver') as var4503:
var389 = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
var4503().var2612.return_value = function2494(var389)
var725 = AsyncResolver(loop=arg1248)
var4020 = yield from var725.resolve('www.google.com')
var389 = [ipaddress.ip_address(var4312['host']) for var4312 in var4020]
assert (len(var389) > 3), 'Expecting multiple addresses'
def function2470(arg1399):
with patch('aiodns.DNSResolver') as var1266:
del var1266().gethostbyname
var1266().query.return_value = function903(['127.0.0.1'])
var4374 = AsyncResolver(loop=arg1399)
var573 = yield from var4374.resolve('www.python.org')
ipaddress.ip_address(var573[0]['host'])
var1266().query.assert_called_with('www.python.org', 'A')
def test_async_resolver_query_ipv6_positive_lookup(loop):
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
mock().query.return_value = fake_query_result(['::1'])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.python.org',
family=socket.AF_INET6)
ipaddress.ip_address(real[0]['host'])
mock().query.assert_called_with('www.python.org', 'AAAA')
def go():
with patch('aiodns.DNSResolver.query') as mock_query:
mock_query.side_effect = aiodns.error.DNSError()
resolver = AsyncResolver(loop=loop)
try:
yield from resolver.resolve('doesnotexist.bla')
assert False, "Expecting aiodns.error.DNSError"
except aiodns.error.DNSError:
pass
def go():
with patch('aiodns.DNSResolver.query') as mock_query:
mock_query.side_effect = aiodns.error.DNSError()
resolver = AsyncResolver(loop=loop)
try:
yield from resolver.resolve('doesnotexist.bla')
assert False, "Expecting aiodns.error.DNSError"
except aiodns.error.DNSError:
pass
def test_async_resolver_ipv6_positive_lookup(loop):
with patch('aiodns.DNSResolver') as mock:
mock().gethostbyname.return_value = fake_result(['::1'])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.python.org',
family=socket.AF_INET6)
ipaddress.ip_address(real[0]['host'])
mock().gethostbyname.assert_called_with('www.python.org',
socket.AF_INET6)
def function298(arg2184):
with patch('aiodns.DNSResolver') as var168:
del var168().gethostbyname
var168().query.return_value = function903(['::1'])
var4169 = AsyncResolver(loop=arg2184)
var4092 = yield from var4169.resolve('www.python.org',
family=socket.AF_INET6)
ipaddress.ip_address(var4092[0]['host'])
var168().query.assert_called_with('www.python.org', 'AAAA')
async def init_session(self, session: Optional[ClientSession] = None):
"""
初始化 ClientSession, 使用 get/post/head 方法之前需要调用一次,
ClientSession 内部维护了连接池, 因此不建议每一个请求创建一个 session,
这里默认为每一个类创建一个 persistent session, 或者手动设置一个, 以实现复用,
在 __init__.py 中初始化 session 会出现 warning, 官方在 aiohttp 4.0 之后将只允许在协程中创建 session,
See:
https://github.com/aio-libs/aiohttp/issues/3658
https://github.com/aio-libs/aiohttp/issues/4932
:param session: 用于复用的 ClientSession 对象
"""
if not self.session:
if session:
self.session = session
return
if self._dns_server:
logger.debug(f"Use custom DNS Server: {self._dns_server}")
resolver = AsyncResolver(nameservers=self._dns_server)
con = TCPConnector(ssl=False,
ttl_dns_cache=300,
resolver=resolver)
else:
con = TCPConnector(ssl=False, ttl_dns_cache=300)
jar = CookieJar(unsafe=True)
self.session = ClientSession(connector=con, cookie_jar=jar)
async def bulk_get_request(datas, port):
ports = get_ports(port)
new_datas = gen_new_datas(datas, ports)
logger.log('INFOR', f'正在异步进行子域的GET请求')
limit_open_conn = config.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
limit_open_conn = utils.get_semaphore()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(limit_open_conn)
header = None
if config.fake_header:
header = utils.gen_fake_header()
async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
# 等待所有task完成 错误聚合到结果列表里
results = await asyncio.gather(*tasks, return_exceptions=True)
new_datas = deal_results(new_datas, results)
logger.log('INFOR', f'完成异步进行子域的GET请求')
return new_datas
async def bulk_get_request(datas, port):
logger.log('INFOR', f'正在异步进行子域的GET请求')
ports = get_ports(port)
new_datas = gen_new_datas(datas, ports)
header = None
if config.fake_header:
header = utils.gen_fake_header()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=config.limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(utils.get_semaphore())
async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore))
task.add_done_callback(
functools.partial(request_callback, index=i, datas=new_datas))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
await asyncio.wait(tasks) # 等待所有task完成
logger.log('INFOR', f'完成异步进行子域的GET请求')
return new_datas
async def test_async_resolver_multiple_replies(loop) -> None:
with patch("aiodns.DNSResolver") as mock:
ips = ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"]
mock().gethostbyname.return_value = fake_result(ips)
resolver = AsyncResolver()
real = await resolver.resolve("www.google.com")
ips = [ipaddress.ip_address(x["host"]) for x in real]
assert len(ips) > 3, "Expecting multiple addresses"
async def test_async_resolver_positive_lookup(loop) -> None:
with patch("aiodns.DNSResolver") as mock:
mock().gethostbyname.return_value = fake_result(["127.0.0.1"])
resolver = AsyncResolver()
real = await resolver.resolve("www.python.org")
ipaddress.ip_address(real[0]["host"])
mock().gethostbyname.assert_called_with("www.python.org",
socket.AF_INET)
async def test_async_resolver_multiple_replies(loop) -> None:
with patch('aiodns.DNSResolver') as mock:
ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
mock().gethostbyname.return_value = fake_result(ips)
resolver = AsyncResolver()
real = await resolver.resolve('www.google.com')
ips = [ipaddress.ip_address(x['host']) for x in real]
assert len(ips) > 3, "Expecting multiple addresses"
async def test_async_resolver_ipv6_positive_lookup(loop) -> None:
with patch('aiodns.DNSResolver') as mock:
mock().gethostbyname.return_value = fake_result(['::1'])
resolver = AsyncResolver()
real = await resolver.resolve('www.python.org', family=socket.AF_INET6)
ipaddress.ip_address(real[0]['host'])
mock().gethostbyname.assert_called_with('www.python.org',
socket.AF_INET6)
async def test_async_resolver_query_positive_lookup(loop) -> None:
with patch('aiodns.DNSResolver') as mock:
del mock().gethostbyname
mock().query.return_value = fake_query_result(['127.0.0.1'])
resolver = AsyncResolver()
real = await resolver.resolve('www.python.org')
ipaddress.ip_address(real[0]['host'])
mock().query.assert_called_with('www.python.org', 'A')
def __init__(self,
loop=None,
api_key=None,
proxy_dict=None,
env=None,
json_encoder=None,
adapter=None):
self.loop = loop
if api_key:
self._FCM_API_KEY = api_key
elif os.getenv('FCM_API_KEY', None):
self._FCM_API_KEY = os.getenv('FCM_API_KEY', None)
else:
raise AuthenticationError(
"Please provide the api_key in the google-services.json file")
self.FCM_REQ_PROXIES = None
self.requests_session = requests.Session()
self.aiodns = AsyncResolver(loop=self.loop,
nameservers=["8.8.8.8", "8.8.4.4"])
self.aiohttp_connector = aiohttp.TCPConnector(loop=self.loop,
limit=0,
ttl_dns_cache=600,
resolver=self.aiodns,
family=socket.AF_INET)
self.aiohttp_session = ClientSession(loop=self.loop,
headers=self.request_headers(),
connector=self.aiohttp_connector,
connector_owner=False)
retries = Retry(backoff_factor=1,
status_forcelist=[502, 503, 504],
method_whitelist=(Retry.DEFAULT_METHOD_WHITELIST
| frozenset(['POST'])))
self.requests_session.mount(
'http://', adapter or HTTPAdapter(max_retries=retries))
self.requests_session.mount(
'https://', adapter or HTTPAdapter(max_retries=retries))
self.requests_session.headers.update(self.request_headers())
self.requests_session.mount(self.INFO_END_POINT,
HTTPAdapter(max_retries=self.INFO_RETRIES))
if proxy_dict and isinstance(proxy_dict, dict) and (
('http' in proxy_dict) or ('https' in proxy_dict)):
self.FCM_REQ_PROXIES = proxy_dict
self.requests_session.proxies.update(proxy_dict)
self.send_request_responses = []
if env == 'app_engine':
try:
from requests_toolbelt.adapters import appengine
appengine.monkeypatch()
except ModuleNotFoundError:
pass
self.json_encoder = json_encoder
async def fetch_async(url):
resolver = AsyncResolver(nameservers=["8.8.8.8", "8.8.4.4"])
conn = aiohttp.TCPConnector(resolver=resolver)
async with aiohttp.ClientSession(connector=conn) as session:
try:
async with session.get(url, timeout=10) as resp:
print(resp.headers)
except concurrent.futures._base.TimeoutError as e:
print('chao', e)
async def get_content(url, params):
resolver = AsyncResolver(nameservers=NAMESERVERS)
conn = aiohttp.TCPConnector(resolver=resolver)
async with aiohttp.ClientSession(connector=conn) as session:
async with await session.get(url, params=params) as response:
content = await response.read()
logging.info(
"Response: %d (%d bytes) from %s", response.status, len(content), url,
)
return content
def _init(self):
'''
'''
resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
conn = aiohttp.TCPConnector(resolver=resolver, limit=10)
if self.loop:
self.sess = aiohttp.ClientSession(
connector=conn,
headers={'User-Agent': self.conf['user_agent']},
loop=self.loop)
else:
self.sess = aiohttp.ClientSession(
connector=conn,
headers={'User-Agent': self.conf['user_agent']})
def __init__(self, loop=None):
self.loop = loop
self.conf = getConf('config/dn.yaml')
resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
conn = aiohttp.TCPConnector(resolver=resolver, limit=10)
if self.loop:
self.sess = aiohttp.ClientSession(
connector=conn,
headers={'User-Agent': self.conf['user_agent']},
loop=self.loop)
else:
self.sess = aiohttp.ClientSession(
connector=conn,
headers={'User-Agent': self.conf['user_agent']})
async def bulk_get_request(datas, port):
ports = get_ports(port)
new_datas = gen_new_datas(datas, ports)
logger.log('INFOR', f'正在异步进行子域的GET请求')
limit_open_conn = config.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
limit_open_conn = utils.get_semaphore()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(limit_open_conn)
header = None
if config.fake_header:
header = utils.gen_fake_header()
async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore))
task.add_done_callback(functools.partial(request_callback,
index=i,
datas=new_datas))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
# 等待所有task完成 错误聚合到结果列表里
futures = asyncio.as_completed(tasks)
for future in tqdm.tqdm(futures,
total=len(tasks),
desc='Progress',
smoothing=1.0,
ncols=True):
try:
await future
except:
pass
logger.log('INFOR', f'完成异步进行子域的GET请求')
return new_datas
def __init__(self, loop, ch):
#-# self.APPID = conf['wx_appid'].encode('utf8')
#-# self.APPSECRET = conf['wx_appsecret'].encode('utf8')
#-# self.TOKEN = conf['wx_token'].encode('utf8')
#-# self.ENCODINGAESKEY = conf['wx_encodingaeskey'].encode('utf8')
self.APPID = conf['wx_appid']
self.APPSECRET = conf['wx_appsecret']
self.TOKEN = conf['wx_token']
self.ENCODINGAESKEY = conf['wx_encodingaeskey']
resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
conn = aiohttp.TCPConnector(resolver=resolver, limit=10)
self.loop = loop
if self.loop:
self.sess = aiohttp.ClientSession(
connector=conn,
headers={'User-Agent': conf['user_agent']},
loop=self.loop)
else:
self.sess = aiohttp.ClientSession(
connector=conn, headers={'User-Agent': conf['user_agent']})
self.ch = ch # CommonHandler 实例 提供 db/cache 支持
def go():
with patch('aiodns.DNSResolver.query') as mock_query:
mock_query.return_value = fake_result(['127.0.0.1'])
resolver = AsyncResolver(loop=loop)
real = yield from resolver.resolve('www.python.org')
ipaddress.ip_address(real[0]['host'])
def test_async_negative_lookup(loop):
with patch('aiodns.DNSResolver.query') as mock_query:
mock_query.side_effect = aiodns.error.DNSError()
resolver = AsyncResolver(loop=loop)
with pytest.raises(aiodns.error.DNSError):
yield from resolver.resolve('doesnotexist.bla')
def test_async_resolver_negative_lookup(loop):
with patch("aiodns.DNSResolver") as mock:
mock().gethostbyname.side_effect = aiodns.error.DNSError()
resolver = AsyncResolver(loop=loop)
with pytest.raises(aiodns.error.DNSError):
yield from resolver.resolve("doesnotexist.bla")
def test_close_for_async_resolver(loop):
resolver = AsyncResolver(loop=loop)
yield from resolver.close()
