def set_proxy(url, auth=None): global _proxy global _proxy_auth if not url: _proxy = None else: type = url.split(':')[0] _proxy = (type, url, auth) if auth else ( type, url, ) _proxy_auth = aiohttp.BasicAuth if type != "http": if not auth: socks_conn = SocksConnector.from_url(url, rdns=True, limit=10) else: if len(auth) < 2: auth += ('', ) socks_conn = SocksConnector.from_url(url, rnds=True, username=auth[0], password=auth[1], limit=10) _pools['default'] = aiohttp.ClientSession(connector=socks_conn, loop=_loop) _proxy_auth = { "socks4": None, # as socks imped by aiohttp_socks and it handles auth internally "socks5": None, }.get(type, None)
async def RequestUrl(config, init): #loggin.info("[<] " + str(datetime.now()) + ':: get+requestURL') _connector = None if config.Proxy_host is not None: if config.Proxy_host.lower() == "tor": connector = SocksConnector(socks_ver=SocksVer.SOCKS5, host='127.0.0.1', port=9050, rdns=True) elif config.Proxy_port and config.Proxy_type: if config.Proxy_type.lower() == "socks5": _type = SocksVer.SOCKS5 elif config.Proxy_type.lower() == "socks4": _type = SocksVer.SOCKS4 elif config.Proxy_type.lower() == 'http': _type = 'Hello' else: print("Error: Proxy types allowed are: socks5 and socks4.") sys.exit(1) _connector = SocksConnector(socks_ver=_type, host=config.Proxy_host, port=config.Proxy_port, rdns=True) else: print( "Error: Please specify --proxy-host, --proxy-port, and --proxy-type" ) sys.exit(1) else: if config.Proxy_port or config.Proxy_type: print( "Error: Please specify --proxy-host, --proxy-port, and --proxy-type" ) sys.exit(1) if config.Profile: if config.Profile_full: _url = await url.MobileProfile(config.Username, init) response = await MobileRequest(_url, connector=_connector) else: _url = await url.Profile(config.Username, init) response = await Request(_url, connector=_connector) elif config.TwitterSearch: _url = await url.Search(config, init) response = await Request(_url, options=_connector, proxy=config.True_proxy) else: if config.Following: _url = await url.Following(config.Username, init) elif config.Followers: _url = await url.Followers(config.Username, init) else: _url = await url.Favorites(config.Username, init) response = await MobileRequest(_url, connector=_connector) if config.Debug: print(_url, file=open("twint-request_urls.log", "a", encoding="utf-8")) return response
def get_connector(config): _connector = None if config.Proxy_host is not None: if config.Proxy_host.lower() == "tor": _connector = SocksConnector(socks_ver=SocksVer.SOCKS5, host='127.0.0.1', port=9050, rdns=True) elif config.Proxy_port and config.Proxy_type: if config.Proxy_type.lower() == "socks5": _type = SocksVer.SOCKS5 elif config.Proxy_type.lower() == "socks4": _type = SocksVer.SOCKS4 else: print("Error: Proxy types allowed are: socks5 and socks4.") sys.exit(1) _connector = SocksConnector(socks_ver=_type, host=config.Proxy_host, port=config.Proxy_port, rdns=True) else: print( "Error: Please specify --proxy-host, --proxy-port, and --proxy-type" ) sys.exit(1) else: if config.Proxy_port or config.Proxy_type: print( "Error: Please specify --proxy-host, --proxy-port, and --proxy-type" ) sys.exit(1) return _connector
def make_aiohttp_session(proxy: Optional[dict], headers=None, timeout=None): if headers is None: headers = {'User-Agent': 'Electrum'} if timeout is None: timeout = aiohttp.ClientTimeout(total=30) elif isinstance(timeout, (int, float)): timeout = aiohttp.ClientTimeout(total=timeout) ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=ca_path) if proxy: connector = SocksConnector( socks_ver=SocksVer.SOCKS5 if proxy['mode'] == 'socks5' else SocksVer.SOCKS4, host=proxy['host'], port=int(proxy['port']), username=proxy.get('user', None), password=proxy.get('password', None), rdns=True, ssl=ssl_context, ) else: connector = aiohttp.TCPConnector(ssl=ssl_context) return aiohttp.ClientSession(headers=headers, timeout=timeout, connector=connector)
async def fetch(domain): """Fetch some data """ rows = getD_db(domain.split("/")[2]) _t_list = [] for row in rows: _t_list.append(row[0]) try: _t_list.remove("http://" + domain.split("/")[2]) except ValueError: pass # the domain is not in list, cannot be removed if domain not in _t_list: try: if domain.split('/')[2] in lists: if not (domain.endswith("gz") or domain.endswith("tar") or domain.endswith("rpm") or domain.endswith("jpg") or domain.endswith("bz2") or domain.endswith("drpm") or domain.endswith("zip") or domain.endswith("filez") or domain.endswith("dirtree")): connector = SocksConnector(socks_ver=SocksVer.SOCKS5, host='127.0.0.1', port=9050, rdns=True) ua = { 'User-Agent': 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12' } timeout = aiohttp.ClientTimeout(total=60 * 3) async with aiohttp.ClientSession(connector=connector, headers=ua) as session: session = RateLimiter(session) try: async with await session.get( domain, timeout=timeout) as response: if response.status == 200: soup = BeautifulSoup( await response.read(), "html.parser") for link in soup.find_all('a'): try: _l = urllib.parse.urljoin( domain, link.get('href')) rows = getU_db(_l) _t_list_url = [] for row in rows: _t_list_url.append(row[0]) if _l not in _t_list_url: await getContent( domain, soup, link.get('href')) except Exception as e: print(e) except Exception as e: #_obj = { # "domain": domain, # "timestamp": str(datetime.now()), # "error": str(e) # } pass except Exception as e: print(e)
def get_random_proxy_params(self, use_proxy=None): if not use_proxy: connector = None proxy_url = None elif use_proxy.startswith('HTTP'): connector = None proxy = next(PROXIES_ITER) if len(proxy) == 2: host, port = proxy proxy_url = f'http://{host}:{port}' elif len(proxy) == 4: host, port, username, password = proxy proxy_url = f'http://{username}:{password}@{host}:{port}' elif use_proxy.startswith('SOCKS'): proxy = next(PROXIES_ITER) if len(proxy) == 2: host, port = proxy username = password = None elif len(proxy) == 4: host, port, username, password = proxy connector = SocksConnector(SocksVer.SOCKS5, host, port, username=username, password=password) proxy_url = None return connector, proxy_url
def __init__(self, bot): self.bot = bot connector = None socks5_url = self.bot.config.get('socks5_proxy_url') if socks5_url: from aiohttp_socks import SocksConnector connector = SocksConnector.from_url(socks5_url, rdns=True) self.http = aiohttp.ClientSession( loop=self.bot.loop, read_timeout=self.bot.config.get('http_read_timeout', 60), connector=connector if self.bot.config.get('use_socks5_for_all_connections') else None, headers={ 'User-Agent': self.bot.config['user_agent'] + ' ' + self.bot.http.user_agent }) self.aioec = aioec.Client( loop=self.bot.loop, connector=connector, base_url=self.bot.config.get('ec_api_base_url')) # keep track of paginators so we can end them when the cog is unloaded self.paginators = weakref.WeakSet()
async def get_idex_depth(symbol, cnt): global p_count proxy = proxys[cnt] if cnt > 39: cnt -= 20 if cnt > 19: cnt -= 20 header = { 'IDEX-API-KEY': gv.idex_apis[cnt], } url = f"https://api.idex.io/v1/orderbook?market={symbol}&level=2&limit=20" socks_url = 'socks5://' + proxy[2] + ':' + proxy[3] + '@' + proxy[0] + ':' + proxy[1] connector = SocksConnector.from_url(socks_url) try: async with aiohttp.ClientSession(connector=connector, headers=header) as session: async with session.get(url) as response: html = await response.text() jhtml = json.loads(html) p_count += 1 if 'sequence' in html: return jhtml elif 'code' in html: if jhtml['code'] != 'MARKET_NOT_FOUND': print('IDEX', symbol, jhtml['code']) return None except Exception as exc: print(exc, proxy) return None
async def open_session(self, proxy: str = None) -> aiohttp.ClientSession: if proxy is None: self.session = aiohttp.ClientSession() return self.session connector = SocksConnector.from_url(proxy) self.session = aiohttp.ClientSession(connector=connector) return self.session
def __init__(self, bot): self.bot = bot connector = None socks5_url = self.bot.config.get('socks5_proxy_url') if socks5_url: from aiohttp_socks import SocksConnector connector = SocksConnector.from_url(socks5_url, rdns=True) self.http = aiohttp.ClientSession( loop=self.bot.loop, read_timeout=self.bot.config.get('http_read_timeout', 60), connector=connector if self.bot.config.get('use_socks5_for_all_connections') else None, headers={ 'User-Agent': self.bot.config['user_agent'] + ' ' + self.bot.http.user_agent }) self.emote_client = EmoteClient(self.bot) with open('data/ec-emotes-final.json') as f: self.ec_emotes = json.load(f) # keep track of paginators so we can end them when the cog is unloaded self.paginators = weakref.WeakSet()
async def prove(self): # socks proxy proxy = 'socks5://{}:{}'.format(self.target_host, str(self.target_port)) connector = SocksConnector.from_url(proxy) url = 'http://example.com/' async with ClientSession(connector=connector) as session: async with session.get(url=url) as res1: if res1: text1 = await res1.text() if 'More information...' in text1: self.flag = 1 self.res.append({"info": proxy, "key": "proxy unauth"}) return # http proxy proxy = 'http://{}:{}'.format(self.target_host, str(self.target_port)) async with ClientSession() as session: async with session.get(url=url, proxy=proxy) as res2: if res2: text2 = await res2.text() if 'More information...' in text2: self.flag = 1 self.res.append({"info": proxy, "key": "proxy unauth"}) return
async def load_content( url: str, proxy_address: str = None, timeout: int = None): timeout_ = DEFAULT_TIMEOUT if not timeout else timeout if proxy_address is None: async with aiohttp.ClientSession() as session: return { 'response': await fetch(session, url=url), 'proxy': proxy_address } session = aiohttp.ClientSession( connector=SocksConnector.from_url(f'socks5://{proxy_address}'), timeout=aiohttp.ClientTimeout(total=timeout_) ) try: async with session as session: return { 'response': await fetch(session, url=url), 'proxy': proxy_address } except Exception: print(f'Bad proxy: {proxy_address}') return { 'response': None, 'proxy': proxy_address }
def get_random_proxy_params(use_proxy): if not use_proxy: connector = None proxy_url = None elif use_proxy == 'HTTP': connector = None proxy = next(PROXY_ITER['HTTP']) if len(proxy) == 2: host, port = proxy proxy_url = f'http://{host}:{port}' elif len(proxy) == 4: host, port, username, password = proxy proxy_url = f'http://{username}:{password}@{host}:{port}' elif use_proxy == 'SOCKS': proxy = next(PROXY_ITER['SOCKS']) if len(proxy) == 2: host, port = proxy username = password = None elif len(proxy) == 4: host, port, username, password = proxy connector = SocksConnector(SocksVer.SOCKS5, host, int(port), username=username, password=password) proxy_url = None return connector, proxy_url
async def async_session(): with Controller.from_port(port=9051) as controller: controller.authenticate(password='') controller.signal(Signal.NEWNYM) connector = SocksConnector.from_url('socks5://127.0.0.1:9050') return await aiohttp.ClientSession(connector=connector ).__aenter__(), connector
async def prove(self): # socks proxy proxy = 'socks5://{}:{}'.format(self.target_host, str(self.target_port)) connector = SocksConnector.from_url(proxy) url = self.ceye_dns_api(t='url') async with ClientSession(connector=connector) as session: async with session.options(url=url) as res1: if res1: text1 = await res1.text() if 'HTTP Record Insert Success' in text1: self.flag = 1 self.res.append({"info": proxy, "key": "proxy unauth"}) return # http proxy proxy = 'http://{}:{}'.format(self.target_host, str(self.target_port)) async with ClientSession() as session: async with session.get(url=url, proxy=proxy) as res2: if res2: text2 = await res2.text() if 'HTTP Record Insert Success' in text2: self.flag = 1 self.res.append({"info": proxy, "key": "proxy unauth"}) return
async def test_socks4_connector(url, rdns): connector = SocksConnector.from_url( SOCKS4_URL, rdns=rdns, ) async with aiohttp.ClientSession(connector=connector) as session: async with session.get(url) as resp: assert resp.status == 200
def async_session(): with Controller.from_port(port=9051) as controller: controller.authenticate( password='******') controller.signal(Signal.NEWNYM) connector = SocksConnector.from_url('socks5://127.0.0.1:9050') return asyncio.get_event_loop().run_until_complete( aiohttp.ClientSession(connector=connector).__aenter__()), connector
async def proxy_session(self, proxy_url: str) -> aiohttp.ClientSession: """ proxy_url ="socks5://user:[email protected]:1080" or "socks5://127.0.0.1:1080" :param proxy_url: :return: """ connector = SocksConnector.from_url(proxy_url) self.session = aiohttp.ClientSession(connector=connector) return self.session
def get_connector(config): logme.debug(__name__ + ':get_connector') _connector = None if config.Proxy_host is not None: if config.Proxy_host.lower() == "tor": _connector = SocksConnector(socks_ver=SocksVer.SOCKS5, host='127.0.0.1', port=9050, rdns=True) elif config.Proxy_port and config.Proxy_type: if config.Proxy_type.lower() == "socks5": _type = SocksVer.SOCKS5 elif config.Proxy_type.lower() == "socks4": _type = SocksVer.SOCKS4 elif config.Proxy_type.lower() == "http": global httpproxy httpproxy = "http://" + config.Proxy_host + ":" + str( config.Proxy_port) return _connector else: logme.critical("get_connector:proxy-type-error") print( "Error: Proxy types allowed are: http, socks5 and socks4. No https." ) sys.exit(1) _connector = SocksConnector(socks_ver=_type, host=config.Proxy_host, port=config.Proxy_port, rdns=True) else: logme.critical(__name__ + ':get_connector:proxy-port-type-error') print( "Error: Please specify --proxy-host, --proxy-port, and --proxy-type" ) sys.exit(1) else: if config.Proxy_port or config.Proxy_type: logme.critical(__name__ + ':get_connector:proxy-host-arg-error') print( "Error: Please specify --proxy-host, --proxy-port, and --proxy-type" ) sys.exit(1) return _connector
async def get(url, proxy=PROXY): connector = SocksConnector.from_url(PROXY, verify_ssl=False) if proxy: async with aiohttp.ClientSession(connector=connector) as session: async with session.get(url, headers={'User-agent':HEADER}) as response: return await response.text() else: async with aiohttp.ClientSession(connector=TCPConnector(verify_ssl=False)) as session: async with session.get(url,headers={'User-agent':HEADER}) as response: return await response.text()
async def _connect(self): try: socks_port = await self.app.tor.socks_port() conn = SocksConnector.from_url(f"socks5://127.0.0.1:{socks_port}", rdns=True) async with aiohttp.ClientSession(connector=conn) as session: self.session = session await self.ping() except Exception as e: logger.exception(e) raise e
async def test_socks5_connector_with_timeout(): connector = SocksConnector( socks_ver=SocksVer.SOCKS5, host=SOCKS5_IPV4_HOST, port=SOCKS5_IPV4_PORT, username=LOGIN, password=PASSWORD, ) with pytest.raises(asyncio.TimeoutError): async with aiohttp.ClientSession(connector=connector) as session: async with session.get(HTTP_URL_DELAY_3_SEC, timeout=1) as resp: await resp.text()
async def test_socks5_connector_with_invalid_credentials(): connector = SocksConnector( socks_ver=SocksVer.SOCKS5, host=SOCKS5_IPV4_HOST, port=SOCKS5_IPV4_PORT, username=LOGIN, password=PASSWORD + 'aaa', ) with pytest.raises(SocksError): async with aiohttp.ClientSession(connector=connector) as session: async with session.get(HTTP_TEST_URL) as resp: await resp.text()
def _create_onetime_pool(): if _proxy and _proxy[0] != "http": url = _proxy[1] if len(_proxy) > 2: # auth auth = _proxy[2] socks_conn = SocksConnector.from_url(url, rdns=True, username=auth[0], password=auth[1], limit=1, force_close=True) else: socks_conn = SocksConnector.from_url(url, rdns=True, limit=1, force_close=True) return aiohttp.ClientSession(connector=socks_conn, loop=_loop) else: return aiohttp.ClientSession(connector=aiohttp.TCPConnector( limit=1, force_close=True), loop=_loop)
async def test_socks5_connector_with_invalid_proxy_port(unused_tcp_port): connector = SocksConnector( socks_ver=SocksVer.SOCKS5, host=SOCKS5_IPV4_HOST, port=unused_tcp_port, username=LOGIN, password=PASSWORD, ) with pytest.raises(SocksConnectionError): async with aiohttp.ClientSession(connector=connector) as session: async with session.get(HTTP_TEST_URL) as resp: await resp.text()
async def open_session(self, proxy: typing.Optional[str] = None ) -> aiohttp.ClientSession: """ @type proxy: object """ if proxy is None: self.session = aiohttp.ClientSession() return self.session connector = SocksConnector.from_url(proxy) self.session = aiohttp.ClientSession(connector=connector) return self.session
def make_aiohttp_session(proxy): if proxy: connector = SocksConnector( socks_ver=SocksVer.SOCKS5 if proxy['mode'] == 'socks5' else SocksVer.SOCKS4, host=proxy['host'], port=int(proxy['port']), username=proxy.get('user', None), password=proxy.get('password', None), rdns=True ) return aiohttp.ClientSession(headers={'User-Agent' : 'Qtum Electrum'}, timeout=aiohttp.ClientTimeout(total=10), connector=connector) else: return aiohttp.ClientSession(headers={'User-Agent' : 'Qtum Electrum'}, timeout=aiohttp.ClientTimeout(total=10))
def __init__(self, token): self.token = token if MODE == MODE_LOCAL_POLLING: self.conn = SocksConnector(socks_ver=SocksVer.SOCKS5, host='orbtl.s5.opennetwork.cc', port=999, username='******', password='******') # self.conn = None else: self.conn = None self.BASE_URL = 'https://api.telegram.org/bot{}/'.format(token) self.session = None
def get_proxy_params(self, proxy_url): schema, username, password, host, port = re.match( '(.*?)://(?:(.*?):(.*?)@)?(.*?):(\d+)', proxy_url).groups() port = int(port) if schema.startswith('http'): connector = None proxy_url = proxy_url elif schema.startswith('socks'): connector = SocksConnector(SocksVer.SOCKS5, host, port, username=username, password=password) proxy_url = None return connector, proxy_url
async def download(self, request: Request) -> Union[Response, Exception]: session = self._default_session new_session = False proxy = None if request.proxy: proxy_url = URL(request.proxy) if proxy_url.scheme in ('sock4', 'sock5'): connector = SocksConnector.from_url(request.url) session = ClientSession(cookie_jar=CookieJar(unsafe=True), connector=connector) new_session = True elif proxy_url.scheme == 'https' and URL(request.url).scheme == 'https': return await self.download_by_requests(request) else: proxy = request.proxy try: if request.cookies: session.cookie_jar.update_cookies(request.cookies) resp = await session.request(method=request.method, url=request.url, params=request.params, data=request.data, proxy=proxy, headers=request.headers, timeout=request.timeout) status = resp.status text = await resp.text(encoding=request.encoding) cookies = resp.cookies response = Response(text=text, status=status, cookies=cookies) if request.cookies: response.cookies.update(request.cookies) return response except Exception as e: self.logger.error(traceback.format_exc(limit=10)) return e finally: if new_session: await session.close()