def set_proxy(url, auth=None):
global _proxy
global _proxy_auth
if not url:
_proxy = None
else:
type = url.split(':')[0]
_proxy = (type, url, auth) if auth else (
type,
url,
)
_proxy_auth = aiohttp.BasicAuth
if type != "http":
if not auth:
socks_conn = SocksConnector.from_url(url, rdns=True, limit=10)
else:
if len(auth) < 2:
auth += ('', )
socks_conn = SocksConnector.from_url(url,
rnds=True,
username=auth[0],
password=auth[1],
limit=10)
_pools['default'] = aiohttp.ClientSession(connector=socks_conn,
loop=_loop)
_proxy_auth = {
"socks4":
None, # as socks imped by aiohttp_socks and it handles auth internally
"socks5": None,
}.get(type, None)
async def RequestUrl(config, init):
#loggin.info("[<] " + str(datetime.now()) + ':: get+requestURL')
_connector = None
if config.Proxy_host is not None:
if config.Proxy_host.lower() == "tor":
connector = SocksConnector(socks_ver=SocksVer.SOCKS5,
host='127.0.0.1',
port=9050,
rdns=True)
elif config.Proxy_port and config.Proxy_type:
if config.Proxy_type.lower() == "socks5":
_type = SocksVer.SOCKS5
elif config.Proxy_type.lower() == "socks4":
_type = SocksVer.SOCKS4
elif config.Proxy_type.lower() == 'http':
_type = 'Hello'
else:
print("Error: Proxy types allowed are: socks5 and socks4.")
sys.exit(1)
_connector = SocksConnector(socks_ver=_type,
host=config.Proxy_host,
port=config.Proxy_port,
rdns=True)
else:
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
else:
if config.Proxy_port or config.Proxy_type:
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
if config.Profile:
if config.Profile_full:
_url = await url.MobileProfile(config.Username, init)
response = await MobileRequest(_url, connector=_connector)
else:
_url = await url.Profile(config.Username, init)
response = await Request(_url, connector=_connector)
elif config.TwitterSearch:
_url = await url.Search(config, init)
response = await Request(_url,
options=_connector,
proxy=config.True_proxy)
else:
if config.Following:
_url = await url.Following(config.Username, init)
elif config.Followers:
_url = await url.Followers(config.Username, init)
else:
_url = await url.Favorites(config.Username, init)
response = await MobileRequest(_url, connector=_connector)
if config.Debug:
print(_url, file=open("twint-request_urls.log", "a", encoding="utf-8"))
return response
def get_connector(config):
_connector = None
if config.Proxy_host is not None:
if config.Proxy_host.lower() == "tor":
_connector = SocksConnector(socks_ver=SocksVer.SOCKS5,
host='127.0.0.1',
port=9050,
rdns=True)
elif config.Proxy_port and config.Proxy_type:
if config.Proxy_type.lower() == "socks5":
_type = SocksVer.SOCKS5
elif config.Proxy_type.lower() == "socks4":
_type = SocksVer.SOCKS4
else:
print("Error: Proxy types allowed are: socks5 and socks4.")
sys.exit(1)
_connector = SocksConnector(socks_ver=_type,
host=config.Proxy_host,
port=config.Proxy_port,
rdns=True)
else:
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
else:
if config.Proxy_port or config.Proxy_type:
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
return _connector
def make_aiohttp_session(proxy: Optional[dict], headers=None, timeout=None):
if headers is None:
headers = {'User-Agent': 'Electrum'}
if timeout is None:
timeout = aiohttp.ClientTimeout(total=30)
elif isinstance(timeout, (int, float)):
timeout = aiohttp.ClientTimeout(total=timeout)
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH,
cafile=ca_path)
if proxy:
connector = SocksConnector(
socks_ver=SocksVer.SOCKS5
if proxy['mode'] == 'socks5' else SocksVer.SOCKS4,
host=proxy['host'],
port=int(proxy['port']),
username=proxy.get('user', None),
password=proxy.get('password', None),
rdns=True,
ssl=ssl_context,
)
else:
connector = aiohttp.TCPConnector(ssl=ssl_context)
return aiohttp.ClientSession(headers=headers,
timeout=timeout,
connector=connector)
async def fetch(domain):
"""Fetch some data
"""
rows = getD_db(domain.split("/")[2])
_t_list = []
for row in rows:
_t_list.append(row[0])
try:
_t_list.remove("http://" + domain.split("/")[2])
except ValueError:
pass # the domain is not in list, cannot be removed
if domain not in _t_list:
try:
if domain.split('/')[2] in lists:
if not (domain.endswith("gz") or domain.endswith("tar")
or domain.endswith("rpm") or domain.endswith("jpg")
or domain.endswith("bz2") or domain.endswith("drpm")
or domain.endswith("zip") or domain.endswith("filez")
or domain.endswith("dirtree")):
connector = SocksConnector(socks_ver=SocksVer.SOCKS5,
host='127.0.0.1',
port=9050,
rdns=True)
ua = {
'User-Agent':
'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12'
}
timeout = aiohttp.ClientTimeout(total=60 * 3)
async with aiohttp.ClientSession(connector=connector,
headers=ua) as session:
session = RateLimiter(session)
try:
async with await session.get(
domain, timeout=timeout) as response:
if response.status == 200:
soup = BeautifulSoup(
await response.read(), "html.parser")
for link in soup.find_all('a'):
try:
_l = urllib.parse.urljoin(
domain, link.get('href'))
rows = getU_db(_l)
_t_list_url = []
for row in rows:
_t_list_url.append(row[0])
if _l not in _t_list_url:
await getContent(
domain, soup,
link.get('href'))
except Exception as e:
print(e)
except Exception as e:
#_obj = {
# "domain": domain,
# "timestamp": str(datetime.now()),
# "error": str(e)
# }
pass
except Exception as e:
print(e)
def get_random_proxy_params(self, use_proxy=None):
if not use_proxy:
connector = None
proxy_url = None
elif use_proxy.startswith('HTTP'):
connector = None
proxy = next(PROXIES_ITER)
if len(proxy) == 2:
host, port = proxy
proxy_url = f'http://{host}:{port}'
elif len(proxy) == 4:
host, port, username, password = proxy
proxy_url = f'http://{username}:{password}@{host}:{port}'
elif use_proxy.startswith('SOCKS'):
proxy = next(PROXIES_ITER)
if len(proxy) == 2:
host, port = proxy
username = password = None
elif len(proxy) == 4:
host, port, username, password = proxy
connector = SocksConnector(SocksVer.SOCKS5,
host,
port,
username=username,
password=password)
proxy_url = None
return connector, proxy_url
def __init__(self, bot):
self.bot = bot
connector = None
socks5_url = self.bot.config.get('socks5_proxy_url')
if socks5_url:
from aiohttp_socks import SocksConnector
connector = SocksConnector.from_url(socks5_url, rdns=True)
self.http = aiohttp.ClientSession(
loop=self.bot.loop,
read_timeout=self.bot.config.get('http_read_timeout', 60),
connector=connector if self.bot.config.get('use_socks5_for_all_connections') else None,
headers={
'User-Agent':
self.bot.config['user_agent'] + ' '
+ self.bot.http.user_agent
})
self.aioec = aioec.Client(
loop=self.bot.loop,
connector=connector,
base_url=self.bot.config.get('ec_api_base_url'))
# keep track of paginators so we can end them when the cog is unloaded
self.paginators = weakref.WeakSet()
async def get_idex_depth(symbol, cnt):
global p_count
proxy = proxys[cnt]
if cnt > 39:
cnt -= 20
if cnt > 19:
cnt -= 20
header = {
'IDEX-API-KEY': gv.idex_apis[cnt],
}
url = f"https://api.idex.io/v1/orderbook?market={symbol}&level=2&limit=20"
socks_url = 'socks5://' + proxy[2] + ':' + proxy[3] + '@' + proxy[0] + ':' + proxy[1]
connector = SocksConnector.from_url(socks_url)
try:
async with aiohttp.ClientSession(connector=connector, headers=header) as session:
async with session.get(url) as response:
html = await response.text()
jhtml = json.loads(html)
p_count += 1
if 'sequence' in html:
return jhtml
elif 'code' in html:
if jhtml['code'] != 'MARKET_NOT_FOUND':
print('IDEX', symbol, jhtml['code'])
return None
except Exception as exc:
print(exc, proxy)
return None
async def open_session(self, proxy: str = None) -> aiohttp.ClientSession:
if proxy is None:
self.session = aiohttp.ClientSession()
return self.session
connector = SocksConnector.from_url(proxy)
self.session = aiohttp.ClientSession(connector=connector)
return self.session
def __init__(self, bot):
self.bot = bot
connector = None
socks5_url = self.bot.config.get('socks5_proxy_url')
if socks5_url:
from aiohttp_socks import SocksConnector
connector = SocksConnector.from_url(socks5_url, rdns=True)
self.http = aiohttp.ClientSession(
loop=self.bot.loop,
read_timeout=self.bot.config.get('http_read_timeout', 60),
connector=connector
if self.bot.config.get('use_socks5_for_all_connections') else None,
headers={
'User-Agent':
self.bot.config['user_agent'] + ' ' + self.bot.http.user_agent
})
self.emote_client = EmoteClient(self.bot)
with open('data/ec-emotes-final.json') as f:
self.ec_emotes = json.load(f)
# keep track of paginators so we can end them when the cog is unloaded
self.paginators = weakref.WeakSet()
async def prove(self):
# socks proxy
proxy = 'socks5://{}:{}'.format(self.target_host, str(self.target_port))
connector = SocksConnector.from_url(proxy)
url = 'http://example.com/'
async with ClientSession(connector=connector) as session:
async with session.get(url=url) as res1:
if res1:
text1 = await res1.text()
if 'More information...' in text1:
self.flag = 1
self.res.append({"info": proxy, "key": "proxy unauth"})
return
# http proxy
proxy = 'http://{}:{}'.format(self.target_host, str(self.target_port))
async with ClientSession() as session:
async with session.get(url=url, proxy=proxy) as res2:
if res2:
text2 = await res2.text()
if 'More information...' in text2:
self.flag = 1
self.res.append({"info": proxy, "key": "proxy unauth"})
return
async def load_content(
url: str,
proxy_address: str = None,
timeout: int = None):
timeout_ = DEFAULT_TIMEOUT if not timeout else timeout
if proxy_address is None:
async with aiohttp.ClientSession() as session:
return {
'response': await fetch(session, url=url),
'proxy': proxy_address
}
session = aiohttp.ClientSession(
connector=SocksConnector.from_url(f'socks5://{proxy_address}'),
timeout=aiohttp.ClientTimeout(total=timeout_)
)
try:
async with session as session:
return {
'response': await fetch(session, url=url),
'proxy': proxy_address
}
except Exception:
print(f'Bad proxy: {proxy_address}')
return {
'response': None,
'proxy': proxy_address
}
def get_random_proxy_params(use_proxy):
if not use_proxy:
connector = None
proxy_url = None
elif use_proxy == 'HTTP':
connector = None
proxy = next(PROXY_ITER['HTTP'])
if len(proxy) == 2:
host, port = proxy
proxy_url = f'http://{host}:{port}'
elif len(proxy) == 4:
host, port, username, password = proxy
proxy_url = f'http://{username}:{password}@{host}:{port}'
elif use_proxy == 'SOCKS':
proxy = next(PROXY_ITER['SOCKS'])
if len(proxy) == 2:
host, port = proxy
username = password = None
elif len(proxy) == 4:
host, port, username, password = proxy
connector = SocksConnector(SocksVer.SOCKS5,
host,
int(port),
username=username,
password=password)
proxy_url = None
return connector, proxy_url
async def async_session():
with Controller.from_port(port=9051) as controller:
controller.authenticate(password='')
controller.signal(Signal.NEWNYM)
connector = SocksConnector.from_url('socks5://127.0.0.1:9050')
return await aiohttp.ClientSession(connector=connector
).__aenter__(), connector
async def prove(self):
# socks proxy
proxy = 'socks5://{}:{}'.format(self.target_host,
str(self.target_port))
connector = SocksConnector.from_url(proxy)
url = self.ceye_dns_api(t='url')
async with ClientSession(connector=connector) as session:
async with session.options(url=url) as res1:
if res1:
text1 = await res1.text()
if 'HTTP Record Insert Success' in text1:
self.flag = 1
self.res.append({"info": proxy, "key": "proxy unauth"})
return
# http proxy
proxy = 'http://{}:{}'.format(self.target_host, str(self.target_port))
async with ClientSession() as session:
async with session.get(url=url, proxy=proxy) as res2:
if res2:
text2 = await res2.text()
if 'HTTP Record Insert Success' in text2:
self.flag = 1
self.res.append({"info": proxy, "key": "proxy unauth"})
return
async def test_socks4_connector(url, rdns):
connector = SocksConnector.from_url(
SOCKS4_URL,
rdns=rdns,
)
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(url) as resp:
assert resp.status == 200
def async_session():
with Controller.from_port(port=9051) as controller:
controller.authenticate(
password='******')
controller.signal(Signal.NEWNYM)
connector = SocksConnector.from_url('socks5://127.0.0.1:9050')
return asyncio.get_event_loop().run_until_complete(
aiohttp.ClientSession(connector=connector).__aenter__()), connector
async def proxy_session(self, proxy_url: str) -> aiohttp.ClientSession:
"""
proxy_url ="socks5://user:[email protected]:1080" or "socks5://127.0.0.1:1080"
:param proxy_url:
:return:
"""
connector = SocksConnector.from_url(proxy_url)
self.session = aiohttp.ClientSession(connector=connector)
return self.session
def get_connector(config):
logme.debug(__name__ + ':get_connector')
_connector = None
if config.Proxy_host is not None:
if config.Proxy_host.lower() == "tor":
_connector = SocksConnector(socks_ver=SocksVer.SOCKS5,
host='127.0.0.1',
port=9050,
rdns=True)
elif config.Proxy_port and config.Proxy_type:
if config.Proxy_type.lower() == "socks5":
_type = SocksVer.SOCKS5
elif config.Proxy_type.lower() == "socks4":
_type = SocksVer.SOCKS4
elif config.Proxy_type.lower() == "http":
global httpproxy
httpproxy = "http://" + config.Proxy_host + ":" + str(
config.Proxy_port)
return _connector
else:
logme.critical("get_connector:proxy-type-error")
print(
"Error: Proxy types allowed are: http, socks5 and socks4. No https."
)
sys.exit(1)
_connector = SocksConnector(socks_ver=_type,
host=config.Proxy_host,
port=config.Proxy_port,
rdns=True)
else:
logme.critical(__name__ + ':get_connector:proxy-port-type-error')
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
else:
if config.Proxy_port or config.Proxy_type:
logme.critical(__name__ + ':get_connector:proxy-host-arg-error')
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
return _connector
async def get(url, proxy=PROXY):
connector = SocksConnector.from_url(PROXY, verify_ssl=False)
if proxy:
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(url, headers={'User-agent':HEADER}) as response:
return await response.text()
else:
async with aiohttp.ClientSession(connector=TCPConnector(verify_ssl=False)) as session:
async with session.get(url,headers={'User-agent':HEADER}) as response:
return await response.text()
async def _connect(self):
try:
socks_port = await self.app.tor.socks_port()
conn = SocksConnector.from_url(f"socks5://127.0.0.1:{socks_port}",
rdns=True)
async with aiohttp.ClientSession(connector=conn) as session:
self.session = session
await self.ping()
except Exception as e:
logger.exception(e)
raise e
async def test_socks5_connector_with_timeout():
connector = SocksConnector(
socks_ver=SocksVer.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=SOCKS5_IPV4_PORT,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(asyncio.TimeoutError):
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(HTTP_URL_DELAY_3_SEC, timeout=1) as resp:
await resp.text()
async def test_socks5_connector_with_invalid_credentials():
connector = SocksConnector(
socks_ver=SocksVer.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=SOCKS5_IPV4_PORT,
username=LOGIN,
password=PASSWORD + 'aaa',
)
with pytest.raises(SocksError):
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(HTTP_TEST_URL) as resp:
await resp.text()
def _create_onetime_pool():
if _proxy and _proxy[0] != "http":
url = _proxy[1]
if len(_proxy) > 2: # auth
auth = _proxy[2]
socks_conn = SocksConnector.from_url(url,
rdns=True,
username=auth[0],
password=auth[1],
limit=1,
force_close=True)
else:
socks_conn = SocksConnector.from_url(url,
rdns=True,
limit=1,
force_close=True)
return aiohttp.ClientSession(connector=socks_conn, loop=_loop)
else:
return aiohttp.ClientSession(connector=aiohttp.TCPConnector(
limit=1, force_close=True),
loop=_loop)
async def test_socks5_connector_with_invalid_proxy_port(unused_tcp_port):
connector = SocksConnector(
socks_ver=SocksVer.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=unused_tcp_port,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(SocksConnectionError):
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(HTTP_TEST_URL) as resp:
await resp.text()
async def open_session(self,
proxy: typing.Optional[str] = None
) -> aiohttp.ClientSession:
"""
@type proxy: object
"""
if proxy is None:
self.session = aiohttp.ClientSession()
return self.session
connector = SocksConnector.from_url(proxy)
self.session = aiohttp.ClientSession(connector=connector)
return self.session
def make_aiohttp_session(proxy):
if proxy:
connector = SocksConnector(
socks_ver=SocksVer.SOCKS5 if proxy['mode'] == 'socks5' else SocksVer.SOCKS4,
host=proxy['host'],
port=int(proxy['port']),
username=proxy.get('user', None),
password=proxy.get('password', None),
rdns=True
)
return aiohttp.ClientSession(headers={'User-Agent' : 'Qtum Electrum'}, timeout=aiohttp.ClientTimeout(total=10), connector=connector)
else:
return aiohttp.ClientSession(headers={'User-Agent' : 'Qtum Electrum'}, timeout=aiohttp.ClientTimeout(total=10))
def __init__(self, token):
self.token = token
if MODE == MODE_LOCAL_POLLING:
self.conn = SocksConnector(socks_ver=SocksVer.SOCKS5,
host='orbtl.s5.opennetwork.cc',
port=999,
username='******',
password='******')
# self.conn = None
else:
self.conn = None
self.BASE_URL = 'https://api.telegram.org/bot{}/'.format(token)
self.session = None
def get_proxy_params(self, proxy_url):
schema, username, password, host, port = re.match(
'(.*?)://(?:(.*?):(.*?)@)?(.*?):(\d+)', proxy_url).groups()
port = int(port)
if schema.startswith('http'):
connector = None
proxy_url = proxy_url
elif schema.startswith('socks'):
connector = SocksConnector(SocksVer.SOCKS5,
host,
port,
username=username,
password=password)
proxy_url = None
return connector, proxy_url
async def download(self, request: Request) -> Union[Response, Exception]:
session = self._default_session
new_session = False
proxy = None
if request.proxy:
proxy_url = URL(request.proxy)
if proxy_url.scheme in ('sock4', 'sock5'):
connector = SocksConnector.from_url(request.url)
session = ClientSession(cookie_jar=CookieJar(unsafe=True), connector=connector)
new_session = True
elif proxy_url.scheme == 'https' and URL(request.url).scheme == 'https':
return await self.download_by_requests(request)
else:
proxy = request.proxy
try:
if request.cookies:
session.cookie_jar.update_cookies(request.cookies)
resp = await session.request(method=request.method,
url=request.url,
params=request.params,
data=request.data,
proxy=proxy,
headers=request.headers,
timeout=request.timeout)
status = resp.status
text = await resp.text(encoding=request.encoding)
cookies = resp.cookies
response = Response(text=text, status=status, cookies=cookies)
if request.cookies:
response.cookies.update(request.cookies)
return response
except Exception as e:
self.logger.error(traceback.format_exc(limit=10))
return e
finally:
if new_session:
await session.close()
