async def request_handler(self, request: Request): """ This method can be rewritten to handle the request exception, or do more. :param request: base.http.Request :return: """ auth = None proxy_url = None if self.proxies: proxy_str = random.choice(self.proxies) proxy = Url(proxy_str) if proxy.scheme == 'socks5' or proxy.scheme == 'socks4': auth = Socks5Auth(proxy.username, proxy.password) proxy_url = proxy.url_index() elif proxy.scheme == 'http' or proxy.scheme == 'https': auth = aiohttp.BasicAuth(proxy.username, proxy.password) proxy_url = proxy.url_index() logger.info(f'Open {request.url.url_full()}') logger.debug(f'Current proxy: {proxy_url}') async with self.session.request(request.method, request.url.url_full(), headers=request.headers, data=request.data, timeout=request.timeout, json=request._json, proxy=proxy_url, proxy_auth=auth) as response: async for ret in self.process(response, request): if isinstance(ret, Request): await self.push_request(ret) else: await self.storage(ret)
async def async_request_aio_http(arr_handle): timeout = aiohttp.ClientTimeout(total=30) auth = Socks5Auth(login='', password='') connector = ProxyConnector(verify_ssl=False) socks_api = "" auth = "" async with aiohttp.ClientSession(connector=connector, request_class=ProxyClientRequest, timeout=timeout) as session: tasks = [] for item in arr_handle: url = item['url'] data_post = {} task = asyncio.ensure_future( fetch("GET", url, data_post, session, socks_api, auth)) tasks.append(task) responses = await asyncio.gather(*tasks) await session.close() return responses
async def session_get(session, url, socks): auth = Socks5Auth(login='******', password='******') headers = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'} timeout = aiohttp.ClientTimeout(total=20) response = await session.get(url, proxy=socks, proxy_auth=auth, timeout=timeout, headers=headers, ssl=ssl.SSLContext()) return await response.text(), response.status
async def session_get(session, url, socks): auth = Socks5Auth(login='******', password='******') headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } timeout = aiohttp.ClientTimeout(total=20) response = await session.get(url, proxy=socks, proxy_auth=auth, headers=headers, timeout=timeout, ssl=ssl.SSLContext()) return await response.text(), response.status
# определение файла для хранения БД для двух режимов db_filename_test = "test_database.db" db_filename_prom = "prom_database.db" # определение файла для хранения лога для двух режимов log_filename_test = "test.log" log_filename_prom = "prom.log" # определение списка администраторов бота, массивом идентификаторов пользователей в Телеграм ADMIN_IDS = [109099327] # Настройка прокси Socks5 proxy_url = 'socks5://ip:port' proxy_login = '******' proxy_password = '******' LOG_DIRECTORY = "./logs" if use_proxy: PROXY_URL = proxy_url PROXY_AUTH = Socks5Auth(login=proxy_login, password=proxy_password) else: PROXY_URL = None PROXY_AUTH = None if mode == "prom": TOKEN = telegrambot_prom DB_FILENAME = db_filename_prom LOG_FILENAME = os.path.join(LOG_DIRECTORY, log_filename_prom) else: TOKEN = telegrambot_test DB_FILENAME = db_filename_test LOG_FILENAME = os.path.join(LOG_DIRECTORY, log_filename_test)