def get_connector(config):
logme.debug(__name__ + ':get_connector')
_connector = None
global proxyauth
if config.Proxy_Username:
if config.Proxy_Password:
proxyauth = aiohttp.BasicAuth(config.Proxy_Username,
config.Proxy_Password)
else:
print("Error: Proxy username requires a password.")
else:
proxyauth = None
if config.Proxy_host:
if config.Proxy_host.lower() == "tor":
_connector = ProxyConnector(host='127.0.0.1', port=9050, rdns=True)
elif config.Proxy_port and config.Proxy_type:
if config.Proxy_type.lower() == "socks5":
_type = ProxyType.SOCKS5
elif config.Proxy_type.lower() == "socks4":
_type = ProxyType.SOCKS4
elif config.Proxy_type.lower() == "http":
global httpproxy
httpproxy = "http://" + config.Proxy_host + ":" + str(
config.Proxy_port)
return _connector
else:
logme.critical("get_connector:proxy-type-error")
print(
"Error: Proxy types allowed are: http, socks5 and socks4. No https."
)
sys.exit(1)
_connector = ProxyConnector(proxy_type=_type,
host=config.Proxy_host,
port=config.Proxy_port,
rdns=True)
else:
logme.critical(__name__ + ':get_connector:proxy-port-type-error')
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
else:
if config.Proxy_port or config.Proxy_type:
logme.critical(__name__ + ':get_connector:proxy-host-arg-error')
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
return _connector
def make_aiohttp_session(proxy: Optional[dict], headers=None, timeout=None):
if headers is None:
headers = {'User-Agent': 'Electrum'}
if timeout is None:
# The default timeout is high intentionally.
# DNS on some systems can be really slow, see e.g. #5337
timeout = aiohttp.ClientTimeout(total=45)
elif isinstance(timeout, (int, float)):
timeout = aiohttp.ClientTimeout(total=timeout)
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH,
cafile=ca_path)
if proxy:
connector = ProxyConnector(
proxy_type=ProxyType.SOCKS5
if proxy['mode'] == 'socks5' else ProxyType.SOCKS4,
host=proxy['host'],
port=int(proxy['port']),
username=proxy.get('user', None),
password=proxy.get('password', None),
rdns=True,
ssl=ssl_context,
)
else:
connector = aiohttp.TCPConnector(ssl=ssl_context)
return aiohttp.ClientSession(headers=headers,
timeout=timeout,
connector=connector)
async def get(self, check_size: bool = False, size="sample") -> bytes:
try:
headers = {}
connector: ProxyConnector = ProxyConnector()
proxy = ymConfig.getConfig('setting').get('proxy')
if proxy:
connector = ProxyConnector.from_url(proxy)
if size == "sample":
self.url = self.sample_url
elif size == "file":
self.url = self.file_url
elif size == "jpeg":
self.url = self.jpeg_url
async with aiohttp.request('GET', self.url, headers=headers, connector=connector,
timeout=aiohttp.ClientTimeout(600)) as resp:
img_bytes: bytes = await resp.read()
if connector:
await connector.close()
if check_size:
pass
'''img: PIL.Image.Image = PIL.Image.open(BytesIO(initial_bytes=img_bytes))
if img.size != (self.width, self.height):
raise ValueError(f'Image Size Error: expected {(self.width, self.height)} but got {img.size}')'''
except (asyncio.TimeoutError, ValueError) as e:
raise e
except PIL.UnidentifiedImageError:
raise ValueError(f'Image load fail {str(img_bytes[:20])}...')
return img_bytes
async def get(self, offest=0, limit=40) -> 'list':
await self.__build()
# params = {}
connector: ProxyConnector = ProxyConnector()
proxy = ymConfig.getConfig('setting').get('proxy')
if proxy:
connector = ProxyConnector.from_url(proxy)
data: list = []
for k, t in self.rips.items():
async with aiohttp.request('GET', k,
connector=connector) as response:
raw = await response.read()
logger.debug(k + f"\n[{t[0]}, {t[1]}]")
data = data + json.loads(raw)['posts'][t[0]:t[1]]
result: [AnipicData] = []
for one in data:
anipic: AnipicData = AnipicData()
one['file_url'] = f"https://images.anime-pictures.net/{one['md5'][0:3]}/{one['md5'] + one['ext']}"
anipic.__dict__.update(one)
async with aiohttp.request(
'GET',
f'https://anime-pictures.net/pictures/view_post/{anipic.id}?type=json',
connector=connector,
timeout=aiohttp.ClientTimeout(20)) as oneInfo:
meta = json.loads(await oneInfo.read())
anipic.tags = meta['tags']
result.append(anipic)
if connector:
await connector.close()
return result
async def replace_session(self):
await self.session.close()
proxy_connector = ProxyConnector(**env.TELEGRAPH_PROXY_DICT, loop=self.loop) \
if env.TELEGRAPH_PROXY_DICT else None
self.session = RetryClient(connector=proxy_connector,
timeout=ClientTimeout(total=10),
loop=self.loop,
json_serialize=self._json_serialize)
async def test_socks5_proxy_with_invalid_proxy_port(unused_tcp_port):
connector = ProxyConnector(
proxy_type=ProxyType.SOCKS5,
host=PROXY_HOST_IPV4,
port=unused_tcp_port,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(ProxyConnectionError):
await fetch(connector=connector, url=TEST_URL_IPV4)
async def test_socks5_proxy_with_timeout():
connector = ProxyConnector(
proxy_type=ProxyType.SOCKS5,
host=PROXY_HOST_IPV4,
port=SOCKS5_PROXY_PORT,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(asyncio.TimeoutError):
await fetch(connector=connector, url=TEST_URL_IPV4_DELAY, timeout=1)
def get_connector(config):
logme.debug(__name__ + ":get_connector")
_connector = None
if config.Proxy_host:
if config.Proxy_host.lower() == "tor":
_connector = ProxyConnector(host="127.0.0.1", port=9050, rdns=True)
elif config.Proxy_port and config.Proxy_type:
if config.Proxy_type.lower() == "socks5":
_type = ProxyType.SOCKS5
elif config.Proxy_type.lower() == "socks4":
_type = ProxyType.SOCKS4
elif config.Proxy_type.lower() == "http":
global httpproxy
httpproxy = "http://" + config.Proxy_host + ":" + str(
config.Proxy_port)
return _connector
else:
logme.critical("get_connector:proxy-type-error")
print(
"Error: Proxy types allowed are: http, socks5 and socks4. No https."
)
sys.exit(1)
_connector = ProxyConnector(
proxy_type=_type,
host=config.Proxy_host,
port=config.Proxy_port,
rdns=True,
)
else:
logme.critical(__name__ + ":get_connector:proxy-port-type-error")
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
else:
if config.Proxy_port or config.Proxy_type:
logme.critical(__name__ + ":get_connector:proxy-host-arg-error")
print(
"Error: Please specify --proxy-host, --proxy-port, and --proxy-type"
)
sys.exit(1)
return _connector
async def test_socks5_proxy_with_invalid_credentials():
connector = ProxyConnector(
proxy_type=ProxyType.SOCKS5,
host=PROXY_HOST_IPV4,
port=SOCKS5_PROXY_PORT,
username=LOGIN,
password=PASSWORD + 'aaa',
)
with pytest.raises(ProxyError):
await fetch(connector=connector, url=TEST_URL_IPV4)
async def test_socks5_proxy_with_invalid_proxy_port(unused_tcp_port):
connector = ProxyConnector(
proxy_type=ProxyType.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=unused_tcp_port,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(ProxyConnectionError):
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(HTTP_TEST_URL) as resp:
await resp.text()
async def test_socks5_proxy_with_timeout():
connector = ProxyConnector(
proxy_type=ProxyType.SOCKS5,
host=SOCKS5_IPV4_HOST,
port=SOCKS5_IPV4_PORT,
username=LOGIN,
password=PASSWORD,
)
with pytest.raises(asyncio.TimeoutError):
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get(HTTP_URL_DELAY_3_SEC, timeout=1) as resp:
await resp.text()
async def _request(self, method, url, **kw):
while not self._current:
await self.change_server()
server = self._current
logging.getLogger(__name__).warning('using {}'.format(server.hostname))
try:
connector = ProxyConnector(proxy_type=ProxyType.HTTP,
host=server.hostname,
password=self.password,
username=self.username,
port=80,
verify_ssl=False)
self._connector = connector
return await super()._request(method, url, **kw)
except Exception as e:
logging.getLogger(__name__).exception(e, exc_info=True)
def _create_session(self):
proxy_kwargs = {}
if self.proxy_url:
proxy_kwargs['proxy_type'],\
proxy_kwargs['host'],\
proxy_kwargs['port'],\
proxy_kwargs['username'],\
proxy_kwargs['password'] = parse_proxy_url(self.proxy_url)
connector = ProxyConnector(
ttl_dns_cache=self.ttl_dns_cache,
verify_ssl=False,
**proxy_kwargs,
)
else:
connector = aiohttp.TCPConnector(
ttl_dns_cache=self.ttl_dns_cache,
verify_ssl=False,
)
return aiohttp.ClientSession(connector=connector)
async def get(self) -> 'list':
# params = {}
self.__build()
connector: ProxyConnector = ProxyConnector()
proxy = ymConfig.getConfig('setting').get('proxy')
if proxy:
connector = ProxyConnector.from_url(proxy)
data: list = []
for k, t in self.rips.items():
async with aiohttp.request('GET', k, connector=connector) as response:
raw = await response.read()
logger.debug(k + f"\n[{t[0]}, {t[1]}]")
data = data + json.loads(raw)[t[0]:t[1]]
result: list = self._formatData(data)
await connector.close()
self.hasParm = 0
self.hasAction = ''
self.rips.clear()
return result
async def fetToken(self) -> 'str':
connector: ProxyConnector = ProxyConnector()
proxy = ymConfig.getConfig('setting').get('proxy')
if proxy:
connector = ProxyConnector.from_url(proxy)
user = ymConfig.getConfig('anipic').get('user')
password = ymConfig.getConfig('anipic').get('password')
async with aiohttp.request('POST',
'https://anime-pictures.net/login/submit',
params={
'login': user,
'password': password
},
connector=connector) as response:
raw = await response.read()
if connector:
await connector.close()
result = json.loads(raw)
if result['success']:
return result['token']
else:
return ''
import asyncio
import aiohttp
from aiohttp_socks import ProxyConnector, ProxyType
# connector = ProxyConnector.from_url('socks5://127.0.0.1:7891')
connector = ProxyConnector(
proxy_type=ProxyType.HTTP,
host='127.0.0.1',
port=7890,
# username='******',
# password='******',
# rdns=True
)
async def main():
async with aiohttp.ClientSession(connector=connector) as session:
async with session.get('https://httpbin.org/get') as response:
print(await response.text())
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(main())
async def get(self) -> 'list':
# params = {}
self.__build()
connector: ProxyConnector = ProxyConnector()
if proxy := ymConfig.getConfig('setting').get('proxy'):
connector = ProxyConnector.from_url(proxy)
async def get(self) -> 'list':
self.__build()
cookies = {'nw': '1'}
connector: ProxyConnector = ProxyConnector()
proxy = ymConfig.getConfig('setting').get('proxy')
if proxy:
connector = ProxyConnector.from_url(proxy)
if self.hasAction != RipperConst.DETAIL:
find = ['table', 'itg gltc']
else:
find = ['div', 'gdtm']
result: [EhentaiData] = []
for k, t in self.rips.items():
logger.debug(k + f"\n[{t[0]}, {t[1]}]")
async with aiohttp.request('GET',
k,
cookies=cookies,
connector=connector) as response:
soup = BeautifulSoup((await response.read()).decode("utf-8"),
"lxml")
sear = soup.find_all(find[0], class_=find[1])
if self.hasAction == RipperConst.DETAIL:
sear = sear[t[0]:t[1]] # no ads
for one in sear:
link = one.contents[0].contents[0]['href']
async with aiohttp.request('GET',
link,
connector=connector) as oneInfo:
url = BeautifulSoup(await oneInfo.read(),
"lxml").find_all(
'img', id='img')[0]['src']
ehentai: EhentaiData = EhentaiData()
ehentai.__dict__.update({'preview': url})
result.append(ehentai)
else:
rm = self.perPages[self.hasAction]
tables = sear[0].contents[1:rm + 1]
tables += sear[0].contents[rm + 2:]
tables = tables[t[0]:t[1]]
for ga in tables:
la = ga.contents[2].contents[0]['href'].split('/')
body = {
"method": "gdata",
"gidlist": [[int(la[-3]), la[-2]]],
"namespace": 1
}
json.dumps(body, indent=1)
async with aiohttp.request(
'POST',
self.api,
json=body,
connector=connector,
timeout=aiohttp.ClientTimeout(20)) as oneInfo:
one = json.loads(await oneInfo.read())
ehentai: EhentaiData = EhentaiData()
ehentai.__dict__.update(one['gmetadata'][0])
ehentai.gr = self.gr
ehentai.id = ehentai.gid
ehentai.preview = f"https://ehgt.org/m/{str(1000000000 + int(la[-3]))[1:7]}/{la[-3]}-00.jpg"
result.append(ehentai)
await connector.close()
self.hasParm = 0
self.hasAction = ''
self.rips.clear()
return result
async def getConnector() -> ProxyConnector:
connector: ProxyConnector = ProxyConnector()
proxy = vrConfig.getConfig('setting').get('proxy')
if proxy:
connector = ProxyConnector.from_url(proxy)
return connector