Exemple #1
0
def test_async_resolver_query_negative_lookup(loop):
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        mock().query.side_effect = aiodns.error.DNSError()
        resolver = AsyncResolver(loop=loop)
        with pytest.raises(aiodns.error.DNSError):
            yield from resolver.resolve('doesnotexist.bla')
def function583(arg1707):
    with patch('aiodns.DNSResolver') as var167:
        del var167().gethostbyname
        var167().query.side_effect = var4634.error.DNSError()
        var4596 = AsyncResolver(loop=arg1707)
        with pytest.raises(var4634.error.DNSError):
            yield from var4596.resolve('doesnotexist.bla')
Exemple #3
0
def test_async_resolver_positive_lookup(loop):
    with patch('aiodns.DNSResolver.query') as mock_query:
        mock_query.return_value = fake_result(['127.0.0.1'])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.python.org')
        ipaddress.ip_address(real[0]['host'])
        mock_query.assert_called_with('www.python.org', 'A')
def test_async_resolver_positive_lookup(loop):
    with patch("aiodns.DNSResolver") as mock:
        mock().gethostbyname.return_value = fake_result(["127.0.0.1"])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve("www.python.org")
        ipaddress.ip_address(real[0]["host"])
        mock().gethostbyname.assert_called_with("www.python.org", socket.AF_INET)
Exemple #5
0
def test_async_resolver_query_negative_lookup(loop):
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        mock().query.side_effect = aiodns.error.DNSError()
        resolver = AsyncResolver(loop=loop)
        with pytest.raises(aiodns.error.DNSError):
            yield from resolver.resolve('doesnotexist.bla')
def test_async_resolver_no_hosts_in_query(loop):
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        mock().query.return_value = fake_query_result([])
        resolver = AsyncResolver(loop=loop)
        with pytest.raises(OSError):
            yield from resolver.resolve('doesnotexist.bla')
Exemple #7
0
def test_async_resolver_positive_lookup(loop):
    with patch('aiodns.DNSResolver.query') as mock_query:
        mock_query.return_value = fake_result(['127.0.0.1'])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.python.org')
        ipaddress.ip_address(real[0]['host'])
        mock_query.assert_called_with('www.python.org', 'A')
def function2629(arg457):
    with patch('aiodns.DNSResolver') as var1179:
        var1179().var2612.return_value = function2494(['127.0.0.1'])
        var4035 = AsyncResolver(loop=arg457)
        var918 = yield from var4035.resolve('www.python.org')
        ipaddress.ip_address(var918[0]['host'])
        var1179().var2612.assert_called_with('www.python.org', socket.AF_INET)
Exemple #9
0
def test_async_resolver_query_multiple_replies(loop):
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
        mock().query.return_value = fake_query_result(ips)
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.google.com')
        ips = [ipaddress.ip_address(x['host']) for x in real]
Exemple #10
0
def test_async_resolver_query_multiple_replies(loop):
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
        mock().query.return_value = fake_query_result(ips)
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.google.com')
        ips = [ipaddress.ip_address(x['host']) for x in real]
def function497(arg1205):
    with patch('aiodns.DNSResolver') as var441:
        var441().var2612.return_value = function2494(['::1'])
        var1084 = AsyncResolver(loop=arg1205)
        var3908 = yield from var1084.resolve('www.python.org',
                                             family=socket.AF_INET6)
        ipaddress.ip_address(var3908[0]['host'])
        var441().var2612.assert_called_with('www.python.org', socket.AF_INET6)
Exemple #12
0
def test_async_resolver_multiple_replies(loop):
    with patch('aiodns.DNSResolver.query') as mock_query:
        ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
        mock_query.return_value = fake_result(ips)
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.google.com')
        ips = [ipaddress.ip_address(x['host']) for x in real]
        assert len(ips) > 3, "Expecting multiple addresses"
def function19(arg2037):
    with patch('aiodns.DNSResolver') as var4453:
        del var4453().gethostbyname
        var1402 = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
        var4453().query.return_value = function903(var1402)
        var752 = AsyncResolver(loop=arg2037)
        var4214 = yield from var752.resolve('www.google.com')
        var1402 = [ipaddress.ip_address(var177['host']) for var177 in var4214]
def test_async_resolver_query_multiple_replies(loop):
    with patch("aiodns.DNSResolver") as mock:
        del mock().gethostbyname
        ips = ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"]
        mock().query.return_value = fake_query_result(ips)
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve("www.google.com")
        ips = [ipaddress.ip_address(x["host"]) for x in real]
def test_async_resolver_query_ipv6_positive_lookup(loop):
    with patch("aiodns.DNSResolver") as mock:
        del mock().gethostbyname
        mock().query.return_value = fake_query_result(["::1"])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve("www.python.org", family=socket.AF_INET6)
        ipaddress.ip_address(real[0]["host"])
        mock().query.assert_called_with("www.python.org", "AAAA")
Exemple #16
0
 def go():
     with patch('aiodns.DNSResolver.query') as mock_query:
         ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
         mock_query.return_value = fake_result(ips)
         resolver = AsyncResolver(loop=loop)
         real = yield from resolver.resolve('www.google.com')
         ips = [ipaddress.ip_address(x['host']) for x in real]
         assert len(ips) > 3, "Expecting multiple addresses"
Exemple #17
0
def test_async_resolver_positive_lookup(loop):
    with patch('aiodns.DNSResolver') as mock:
        mock().gethostbyname.return_value = fake_result(['127.0.0.1'])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.python.org')
        ipaddress.ip_address(real[0]['host'])
        mock().gethostbyname.assert_called_with('www.python.org',
                                                socket.AF_INET)
def function2013(arg1248):
    with patch('aiodns.DNSResolver') as var4503:
        var389 = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
        var4503().var2612.return_value = function2494(var389)
        var725 = AsyncResolver(loop=arg1248)
        var4020 = yield from var725.resolve('www.google.com')
        var389 = [ipaddress.ip_address(var4312['host']) for var4312 in var4020]
        assert (len(var389) > 3), 'Expecting multiple addresses'
def function2470(arg1399):
    with patch('aiodns.DNSResolver') as var1266:
        del var1266().gethostbyname
        var1266().query.return_value = function903(['127.0.0.1'])
        var4374 = AsyncResolver(loop=arg1399)
        var573 = yield from var4374.resolve('www.python.org')
        ipaddress.ip_address(var573[0]['host'])
        var1266().query.assert_called_with('www.python.org', 'A')
Exemple #20
0
def test_async_resolver_query_ipv6_positive_lookup(loop):
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        mock().query.return_value = fake_query_result(['::1'])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.python.org',
                                           family=socket.AF_INET6)
        ipaddress.ip_address(real[0]['host'])
        mock().query.assert_called_with('www.python.org', 'AAAA')
Exemple #21
0
 def go():
     with patch('aiodns.DNSResolver.query') as mock_query:
         mock_query.side_effect = aiodns.error.DNSError()
         resolver = AsyncResolver(loop=loop)
         try:
             yield from resolver.resolve('doesnotexist.bla')
             assert False, "Expecting aiodns.error.DNSError"
         except aiodns.error.DNSError:
             pass
Exemple #22
0
 def go():
     with patch('aiodns.DNSResolver.query') as mock_query:
         mock_query.side_effect = aiodns.error.DNSError()
         resolver = AsyncResolver(loop=loop)
         try:
             yield from resolver.resolve('doesnotexist.bla')
             assert False, "Expecting aiodns.error.DNSError"
         except aiodns.error.DNSError:
             pass
Exemple #23
0
def test_async_resolver_ipv6_positive_lookup(loop):
    with patch('aiodns.DNSResolver') as mock:
        mock().gethostbyname.return_value = fake_result(['::1'])
        resolver = AsyncResolver(loop=loop)
        real = yield from resolver.resolve('www.python.org',
                                           family=socket.AF_INET6)
        ipaddress.ip_address(real[0]['host'])
        mock().gethostbyname.assert_called_with('www.python.org',
                                                socket.AF_INET6)
def function298(arg2184):
    with patch('aiodns.DNSResolver') as var168:
        del var168().gethostbyname
        var168().query.return_value = function903(['::1'])
        var4169 = AsyncResolver(loop=arg2184)
        var4092 = yield from var4169.resolve('www.python.org',
                                             family=socket.AF_INET6)
        ipaddress.ip_address(var4092[0]['host'])
        var168().query.assert_called_with('www.python.org', 'AAAA')
Exemple #25
0
    async def init_session(self, session: Optional[ClientSession] = None):
        """
        初始化 ClientSession, 使用 get/post/head 方法之前需要调用一次,
        ClientSession 内部维护了连接池, 因此不建议每一个请求创建一个 session,
        这里默认为每一个类创建一个 persistent session, 或者手动设置一个, 以实现复用,
        在 __init__.py 中初始化 session 会出现 warning, 官方在 aiohttp 4.0 之后将只允许在协程中创建 session,
        See:

            https://github.com/aio-libs/aiohttp/issues/3658
            https://github.com/aio-libs/aiohttp/issues/4932

        :param session: 用于复用的 ClientSession 对象
        """
        if not self.session:
            if session:
                self.session = session
                return

            if self._dns_server:
                logger.debug(f"Use custom DNS Server: {self._dns_server}")
                resolver = AsyncResolver(nameservers=self._dns_server)
                con = TCPConnector(ssl=False,
                                   ttl_dns_cache=300,
                                   resolver=resolver)
            else:
                con = TCPConnector(ssl=False, ttl_dns_cache=300)

            jar = CookieJar(unsafe=True)
            self.session = ClientSession(connector=con, cookie_jar=jar)
Exemple #26
0
async def bulk_get_request(datas, port):
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
    logger.log('INFOR', f'正在异步进行子域的GET请求')

    limit_open_conn = config.limit_open_conn
    if limit_open_conn is None:  # 默认情况
        limit_open_conn = utils.get_semaphore()
    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
        limit_open_conn = utils.get_semaphore()
    # 使用异步域名解析器 自定义域名服务器
    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
                                limit=limit_open_conn,
                                limit_per_host=config.limit_per_host,
                                resolver=resolver)

    semaphore = asyncio.Semaphore(limit_open_conn)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
            # 等待所有task完成 错误聚合到结果列表里
            results = await asyncio.gather(*tasks, return_exceptions=True)
            new_datas = deal_results(new_datas, results)

    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
Exemple #27
0
async def bulk_get_request(datas, port):
    logger.log('INFOR', f'正在异步进行子域的GET请求')
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
    # 使用异步域名解析器 自定义域名服务器
    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
                                limit=config.limit_open_conn,
                                limit_per_host=config.limit_per_host,
                                resolver=resolver)
    semaphore = asyncio.Semaphore(utils.get_semaphore())
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
            task.add_done_callback(
                functools.partial(request_callback, index=i, datas=new_datas))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
            await asyncio.wait(tasks)  # 等待所有task完成
    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
Exemple #28
0
async def test_async_resolver_multiple_replies(loop) -> None:
    with patch("aiodns.DNSResolver") as mock:
        ips = ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"]
        mock().gethostbyname.return_value = fake_result(ips)
        resolver = AsyncResolver()
        real = await resolver.resolve("www.google.com")
        ips = [ipaddress.ip_address(x["host"]) for x in real]
        assert len(ips) > 3, "Expecting multiple addresses"
Exemple #29
0
async def test_async_resolver_positive_lookup(loop) -> None:
    with patch("aiodns.DNSResolver") as mock:
        mock().gethostbyname.return_value = fake_result(["127.0.0.1"])
        resolver = AsyncResolver()
        real = await resolver.resolve("www.python.org")
        ipaddress.ip_address(real[0]["host"])
        mock().gethostbyname.assert_called_with("www.python.org",
                                                socket.AF_INET)
Exemple #30
0
async def test_async_resolver_multiple_replies(loop) -> None:
    with patch('aiodns.DNSResolver') as mock:
        ips = ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']
        mock().gethostbyname.return_value = fake_result(ips)
        resolver = AsyncResolver()
        real = await resolver.resolve('www.google.com')
        ips = [ipaddress.ip_address(x['host']) for x in real]
        assert len(ips) > 3, "Expecting multiple addresses"
Exemple #31
0
async def test_async_resolver_ipv6_positive_lookup(loop) -> None:
    with patch('aiodns.DNSResolver') as mock:
        mock().gethostbyname.return_value = fake_result(['::1'])
        resolver = AsyncResolver()
        real = await resolver.resolve('www.python.org', family=socket.AF_INET6)
        ipaddress.ip_address(real[0]['host'])
        mock().gethostbyname.assert_called_with('www.python.org',
                                                socket.AF_INET6)
Exemple #32
0
async def test_async_resolver_query_positive_lookup(loop) -> None:
    with patch('aiodns.DNSResolver') as mock:
        del mock().gethostbyname
        mock().query.return_value = fake_query_result(['127.0.0.1'])
        resolver = AsyncResolver()
        real = await resolver.resolve('www.python.org')
        ipaddress.ip_address(real[0]['host'])
        mock().query.assert_called_with('www.python.org', 'A')
Exemple #33
0
    def __init__(self,
                 loop=None,
                 api_key=None,
                 proxy_dict=None,
                 env=None,
                 json_encoder=None,
                 adapter=None):
        self.loop = loop

        if api_key:
            self._FCM_API_KEY = api_key
        elif os.getenv('FCM_API_KEY', None):
            self._FCM_API_KEY = os.getenv('FCM_API_KEY', None)
        else:
            raise AuthenticationError(
                "Please provide the api_key in the google-services.json file")

        self.FCM_REQ_PROXIES = None
        self.requests_session = requests.Session()

        self.aiodns = AsyncResolver(loop=self.loop,
                                    nameservers=["8.8.8.8", "8.8.4.4"])
        self.aiohttp_connector = aiohttp.TCPConnector(loop=self.loop,
                                                      limit=0,
                                                      ttl_dns_cache=600,
                                                      resolver=self.aiodns,
                                                      family=socket.AF_INET)
        self.aiohttp_session = ClientSession(loop=self.loop,
                                             headers=self.request_headers(),
                                             connector=self.aiohttp_connector,
                                             connector_owner=False)

        retries = Retry(backoff_factor=1,
                        status_forcelist=[502, 503, 504],
                        method_whitelist=(Retry.DEFAULT_METHOD_WHITELIST
                                          | frozenset(['POST'])))
        self.requests_session.mount(
            'http://', adapter or HTTPAdapter(max_retries=retries))
        self.requests_session.mount(
            'https://', adapter or HTTPAdapter(max_retries=retries))
        self.requests_session.headers.update(self.request_headers())
        self.requests_session.mount(self.INFO_END_POINT,
                                    HTTPAdapter(max_retries=self.INFO_RETRIES))

        if proxy_dict and isinstance(proxy_dict, dict) and (
            ('http' in proxy_dict) or ('https' in proxy_dict)):
            self.FCM_REQ_PROXIES = proxy_dict
            self.requests_session.proxies.update(proxy_dict)
        self.send_request_responses = []

        if env == 'app_engine':
            try:
                from requests_toolbelt.adapters import appengine
                appengine.monkeypatch()
            except ModuleNotFoundError:
                pass

        self.json_encoder = json_encoder
Exemple #34
0
async def fetch_async(url):
    resolver = AsyncResolver(nameservers=["8.8.8.8", "8.8.4.4"])
    conn = aiohttp.TCPConnector(resolver=resolver)
    async with aiohttp.ClientSession(connector=conn) as session:
        try:
            async with session.get(url, timeout=10) as resp:
                print(resp.headers)
        except concurrent.futures._base.TimeoutError as e:
            print('chao', e)
Exemple #35
0
async def get_content(url, params):
    resolver = AsyncResolver(nameservers=NAMESERVERS)
    conn = aiohttp.TCPConnector(resolver=resolver)
    async with aiohttp.ClientSession(connector=conn) as session:
        async with await session.get(url, params=params) as response:
            content = await response.read()
            logging.info(
                "Response: %d (%d bytes) from %s", response.status, len(content), url,
            )
            return content
Exemple #36
0
 def _init(self):
     '''
     '''
     resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
     conn = aiohttp.TCPConnector(resolver=resolver, limit=10)
     if self.loop:
         self.sess = aiohttp.ClientSession(
             connector=conn,
             headers={'User-Agent': self.conf['user_agent']},
             loop=self.loop)
     else:
         self.sess = aiohttp.ClientSession(
             connector=conn,
             headers={'User-Agent': self.conf['user_agent']})
Exemple #37
0
    def __init__(self, loop=None):
        self.loop = loop
        self.conf = getConf('config/dn.yaml')

        resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
        conn = aiohttp.TCPConnector(resolver=resolver, limit=10)
        if self.loop:
            self.sess = aiohttp.ClientSession(
                connector=conn,
                headers={'User-Agent': self.conf['user_agent']},
                loop=self.loop)
        else:
            self.sess = aiohttp.ClientSession(
                connector=conn,
                headers={'User-Agent': self.conf['user_agent']})
async def bulk_get_request(datas, port):
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
    logger.log('INFOR', f'正在异步进行子域的GET请求')

    limit_open_conn = config.limit_open_conn
    if limit_open_conn is None:  # 默认情况
        limit_open_conn = utils.get_semaphore()
    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
        limit_open_conn = utils.get_semaphore()
    # 使用异步域名解析器 自定义域名服务器
    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
                                limit=limit_open_conn,
                                limit_per_host=config.limit_per_host,
                                resolver=resolver)

    semaphore = asyncio.Semaphore(limit_open_conn)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
            task.add_done_callback(functools.partial(request_callback,
                                                     index=i,
                                                     datas=new_datas))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
            # 等待所有task完成 错误聚合到结果列表里
            futures = asyncio.as_completed(tasks)
            for future in tqdm.tqdm(futures,
                                    total=len(tasks),
                                    desc='Progress',
                                    smoothing=1.0,
                                    ncols=True):
                try:
                    await future
                except:
                    pass

    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
Exemple #39
0
 def __init__(self, loop, ch):
     #-#        self.APPID = conf['wx_appid'].encode('utf8')
     #-#        self.APPSECRET = conf['wx_appsecret'].encode('utf8')
     #-#        self.TOKEN = conf['wx_token'].encode('utf8')
     #-#        self.ENCODINGAESKEY = conf['wx_encodingaeskey'].encode('utf8')
     self.APPID = conf['wx_appid']
     self.APPSECRET = conf['wx_appsecret']
     self.TOKEN = conf['wx_token']
     self.ENCODINGAESKEY = conf['wx_encodingaeskey']
     resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
     conn = aiohttp.TCPConnector(resolver=resolver, limit=10)
     self.loop = loop
     if self.loop:
         self.sess = aiohttp.ClientSession(
             connector=conn,
             headers={'User-Agent': conf['user_agent']},
             loop=self.loop)
     else:
         self.sess = aiohttp.ClientSession(
             connector=conn, headers={'User-Agent': conf['user_agent']})
     self.ch = ch  # CommonHandler 实例 提供 db/cache 支持
Exemple #40
0
 def go():
     with patch('aiodns.DNSResolver.query') as mock_query:
         mock_query.return_value = fake_result(['127.0.0.1'])
         resolver = AsyncResolver(loop=loop)
         real = yield from resolver.resolve('www.python.org')
         ipaddress.ip_address(real[0]['host'])
Exemple #41
0
def test_async_negative_lookup(loop):
    with patch('aiodns.DNSResolver.query') as mock_query:
        mock_query.side_effect = aiodns.error.DNSError()
        resolver = AsyncResolver(loop=loop)
        with pytest.raises(aiodns.error.DNSError):
            yield from resolver.resolve('doesnotexist.bla')
def test_async_resolver_negative_lookup(loop):
    with patch("aiodns.DNSResolver") as mock:
        mock().gethostbyname.side_effect = aiodns.error.DNSError()
        resolver = AsyncResolver(loop=loop)
        with pytest.raises(aiodns.error.DNSError):
            yield from resolver.resolve("doesnotexist.bla")
Exemple #43
0
def test_close_for_async_resolver(loop):
    resolver = AsyncResolver(loop=loop)
    yield from resolver.close()