async def init_session(self, session: Optional[ClientSession] = None): """ 初始化 ClientSession, 使用 get/post/head 方法之前需要调用一次, ClientSession 内部维护了连接池, 因此不建议每一个请求创建一个 session, 这里默认为每一个类创建一个 persistent session, 或者手动设置一个, 以实现复用, 在 __init__.py 中初始化 session 会出现 warning, 官方在 aiohttp 4.0 之后将只允许在协程中创建 session, See: https://github.com/aio-libs/aiohttp/issues/3658 https://github.com/aio-libs/aiohttp/issues/4932 :param session: 用于复用的 ClientSession 对象 """ if not self.session: if session: self.session = session return if self._dns_server: logger.debug(f"Use custom DNS Server: {self._dns_server}") resolver = AsyncResolver(nameservers=self._dns_server) con = TCPConnector(ssl=False, ttl_dns_cache=300, resolver=resolver) else: con = TCPConnector(ssl=False, ttl_dns_cache=300) jar = CookieJar(unsafe=True) self.session = ClientSession(connector=con, cookie_jar=jar)
async def run(sites, retesting=False): if IPV4_ONLY: connector = TCPConnector(verify_ssl=False, family=socket.AF_INET) else: connector = TCPConnector(verify_ssl=False) sem = asyncio.Semaphore(100) tasks = [] if not retesting: read_timeout = READ_TIMEOUT conn_timeout = CONN_TIMEOUT else: read_timeout = READ_TIMEOUT + READ_TIMEOUT / 2 conn_timeout = CONN_TIMEOUT + CONN_TIMEOUT / 2 async with ClientSession(connector=connector, read_timeout=read_timeout, conn_timeout=conn_timeout, headers=client_headers) as session: for site_ in sites: task = asyncio.ensure_future( bound_gethead(sem, site_, session, retesting=retesting)) tasks.append(task) responses = await asyncio.gather(*tasks)
async def build_list(cls): headers = {"X-API-KEY": Keys.site_api} # Set up the session if DEBUG_MODE: http_session = ClientSession(connector=TCPConnector( resolver=AsyncResolver(), family=socket.AF_INET, verify_ssl=False, )) else: http_session = ClientSession(connector=TCPConnector( resolver=AsyncResolver())) # Get all the snakes if cls.snakes is None: response = await http_session.get(URLs.site_names_api, params={"get_all": "true"}, headers=headers) cls.snakes = await response.json() # Get the special cases if cls.special_cases is None: response = await http_session.get(URLs.site_special_api, headers=headers) special_cases = await response.json() cls.special_cases = { snake['name'].lower(): snake for snake in special_cases } # Close the session http_session.close()
async def get(url): connector = TCPConnector(verify_ssl=False) session = aiohttp.ClientSession(connector=TCPConnector(verify_ssl=False)) response = await session.get(url) await response.text() await session.close() return response
async def __aenter__(self): conn = TCPConnector(limit=100) self._session = ClientSession(connector=conn, headers=self.auth.header) conn_remote = TCPConnector(limit=100) self._session_remote = ClientSession(connector=conn_remote) return self
def __init__(self, **kwargs): self.commands = [] self.tasks = [] self.components = [] self.modals = [] self.public_key = VerifyKey(bytes.fromhex(kwargs["public_key"])) self.token = kwargs["token"] self._loop = kwargs.get("loop") bind_to = env.get("BIND_INTERFACE") if bind_to is not None: connector = TCPConnector(local_addr=(bind_to, 0)) else: connector = TCPConnector() self.session = kwargs.get("session", ClientSession(loop=self.loop, connector=connector)) self.guild_id = kwargs.get("guild_id") # Can be used during development to avoid the 1 hour cache self.app_id = None # Filled by setup() self.http = None self.redis = None self.listeners = {} self.modules = set()
def aiohttp_session(*, auth=None, **kwargs): headers = dict() if auth: headers['Authorization'] = auth.encode() ssl_context = create_default_context(cafile=certifi.where()) try: connector = TCPConnector(ssl=ssl_context) except TypeError: connector = TCPConnector(ssl_context=ssl_context) return ClientSession(headers=headers, connector=connector, **kwargs)
def custom_session(path_to_cert=None, custom_headers=None): if path_to_cert: try: # with open | no need for importing `os` to do path.exists with open(path_to_cert) as cert: cert.read() sslcontext = create_default_context(cafile=path_to_cert) conn = TCPConnector(ssl_context=sslcontext) except: return 'Cert file not Found!' else: conn = TCPConnector(verify_ssl=False) session = ClientSession(connector=conn, headers=custom_headers) return session
def fetch_all(urls): stage = aio.map(_fetch,urls, workers = MAX_WORKERS, on_start = lambda: ClientSession(connector=TCPConnector(limit=None)), on_done = lambda _status, session: session.close()) return stage
def get_client_session(self, config: SeekerConfig) -> ClientSession: async def _on_request_start( session: ClientSession, trace_config_ctx: SimpleNamespace, params: TraceRequestStartParams ) -> None: current_attempt = \ trace_config_ctx.trace_request_ctx['current_attempt'] if(current_attempt > 1): logger.warning( f'::warn ::Retry Attempt #{current_attempt} ' + f'of {config.max_tries}: {params.url}') trace_config = TraceConfig() trace_config.on_request_start.append(_on_request_start) limit_per_host = max(0, config.connect_limit_per_host) connector = TCPConnector( limit_per_host=limit_per_host, ttl_dns_cache=600 # 10-minute DNS cache ) retry_options = ExponentialRetry( attempts=config.max_tries, max_timeout=config.max_time, exceptions=[ aiohttp.ClientError, asyncio.TimeoutError ]) return RetryClient( raise_for_status=True, connector=connector, timeout=ClientTimeout(total=config.timeout), headers={'User-Agent': config.agent}, retry_options=retry_options, trace_configs=[trace_config])
async def get_google_weather(future, location): start = time.time() info = None if location is not None: url = google_weather_url + '+' + location else: url = google_weather_url + '+' + default_location LOGGER.info(url) try: async with ClientSession(connector=TCPConnector(ssl=False)) as session: html = await fetch(session, url) LOGGER.info( f'g-weather content fetch in {time.time() - start} secs.') parse_start = time.time() info = await parse_google_weather(html) LOGGER.info( f'g-weather parsing took {time.time() - parse_start} secs.') except BaseException as ex: LOGGER.error(f'Unable to connect Weather API : {repr(ex)}') info = WeatherInfo.WeatherInfo('0', "0", "0", "00:00", "", "", "") info.set_error(ex) LOGGER.info(f'Weather Time Taken {time.time() - start}') future.set_result(info)
async def run(num): tasks = [] # Semaphore, 相当于基于服务器的处理速度和测试客户端的硬件条件,一批批的发 # 直至发送完全部(下面定义的number/num) sem = asyncio.Semaphore(3000) # 创建session,且对本地的TCP连接不做限制limit=0 # 超时时间指定 # total:全部请求最终完成时间 # connect: aiohttp从本机连接池里取出一个将要进行的请求的时间 # sock_connect:单个请求连接到服务器的时间 # sock_read:单个请求从服务器返回的时间 timeout = ClientTimeout(total=300, connect=60, sock_connect=60, sock_read=60) async with ClientSession(connector=TCPConnector(limit=0), timeout=timeout) as session: for i in range(0, num): # 如果是分批的发,就使用并传递Semaphore task = asyncio.ensure_future( bound_register(sem=sem, firstname=str(18300000000 + i), lastname=str(18300000000 + i), nickname=str(18300000000 + i), phone=str(18300000000 + i), password="******", session=session, i=i)) tasks.append(task) responses = asyncio.gather(*tasks) await responses
def __init__(self, **kwargs): super().__init__(**kwargs) self.http_session = ClientSession(connector=TCPConnector( resolver=AsyncResolver(), family=socket.AF_INET)) self._guild_available = asyncio.Event() self.loop.create_task(self.send_log("SeasonalBot", "Connected!"))
async def __run_(self, task, proxy): """ # 运行查询 :param task: 每次任务查询所需参数 元组 :param proxy:代理 :return: """ # 生成HTTP头 USER AGENT headers = {'User-Agent': UserAgent().chrome} # 获取COOKIES cookies = await self.__cookies_get() # 设置异步参数 connect = TCPConnector(ssl=False, limit=30) # 创建会话 async with ClientSession(connector=connect, headers=headers, cookies=cookies) as session: # 获取数据 message = await self.__fetch_(session, task, proxy) if self.__debug and message: print(message) # 异步等待 await asyncio.sleep(random.randint(1, 10))
def __init__(self, name: str, details: dict): self.name = name self.task = None super().__init__(platform=fortnitepy.Platform.MAC, connector=TCPConnector(limit=None), auth=fortnitepy.AdvancedAuth(**details))
def __init__( self, pool: int = 8, proxy: str = None, proxy_auth: BasicAuth = None, auto_proxy: bool = None, timeout: int = 2333, ssl: bool = False ) -> None: """ :param pool: 同时发起线程数 :param proxy: 每个连接的代理 :param proxy_auth: 每个连接的代理的身份 :param auto_proxy: 自动设置系统代理 :param timeout: 该客户端会话超时时间 :param ssl: 验证SSL """ super().__init__( trust_env=auto_proxy, connector=TCPConnector(limit=pool), timeout=ClientTimeout(total=timeout), headers=self.request_headers ) self.proxy = proxy self.proxy_auth = proxy_auth self.verify_ssl = ssl
async def get_sitemaps(self): if isinstance(self.robots_url, str): sitemaps = await self._find_sitemaps_from_robots(self.robots_url) else: sitemaps = self.robots_url while sitemaps: sitemap = sitemaps.pop() conn = TCPConnector(verify_ssl=False) async with ClientSession(connector=conn) as client: async with client.get(sitemap) as response: if response.status == 200: try: if 'gzip' in response.content_type: buf = BytesIO(await response.read()) f = gzip.GzipFile(fileobj=buf) xml = xmltodict.parse(f.read()) else: xml = xmltodict.parse(await response.read()) sitemaps.extend(self._search_sitemaps(xml)) self.output_sitemap(xml, sitemap.split('/')[-1]) except Exception as e: print(str(e)) print( "Error parsing site map: {0} | content-type: {1}" .format(sitemap, response.content_type))
async def run(cod_stops, fetch_conf): """Async function that generates a aiohttp ClientSession and fetches the given stops. Arguments: cod_stops (list): List of stop codes in CRTM's format (e.g. 8_17491). fetch_conf (dict): Dictionary with configuration parameters for fetching the content. Returns: list: The responses. """ tasks = [] # Fetch all responses within one Client session, # keep connection alive for all requests. connector = TCPConnector(limit=fetch_conf['max_connections']) timeout = ClientTimeout(total=fetch_conf['timeout']) async with ClientSession(connector=connector, timeout=timeout) as session: for cod_stop in cod_stops: task = asyncio.ensure_future(fetch(cod_stop, session, fetch_conf)) tasks.append(task) responses = await asyncio.gather(*tasks) # you now have all response bodies in this variable return responses
def create_session(connection_pool_size=100, **kwargs) -> aiohttp.ClientSession: """ Create a session with parameters suited for AutoExtract """ kwargs.setdefault('timeout', AIO_API_TIMEOUT) if "connector" not in kwargs: kwargs["connector"] = TCPConnector(limit=connection_pool_size) return aiohttp.ClientSession(**kwargs)
async def get_content_async(data): """ Asynchronously hit each hacker news article item for download. :param data: tuple of url and article type :return: list of json responses """ parent_tasks = [] child_tasks = [] timeout = ClientTimeout(total=7 * 60) connector = TCPConnector(ssl=False, family=socket.AF_INET) async with ClientSession(loop=asyncio.get_event_loop(), timeout=timeout, connector=connector) as session: for url, article_type in data: parent_tasks.append(asyncio.create_task(fetch(url, session))) parent_results = await asyncio.gather(*parent_tasks) for index, section in enumerate(parent_results): _, article_type = data[index] for article_id in section: child_tasks.append( asyncio.create_task( fetch( f'https://hacker-news.firebaseio.com/v0/item/{article_id}.json', session, article_type))) return await asyncio.gather(*child_tasks)
async def main(): global session session = aiohttp.ClientSession(connector=TCPConnector(ssl=False)) scrape_index_tasks = [ asyncio.ensure_future(scrape_api()) for _ in range(10000) ] await asyncio.gather(*scrape_index_tasks) # 启动协程
async def main(): connector = TCPConnector(ssl=False) async with ClientSession(connector=connector) as session: file_links = await get_file_links(session, ROOT_URL) tasks = [download_file(session, file_link) for file_link in file_links] await asyncio.gather(*tasks)
async def run(self, req_type, req_url, req_count=100, per_req_path=False, **req_params): """ run a req_count number of requests with the specified method and url. per_req_path allows for appending a count number at the end of url which is useful for eg putting objects in a seq. Note that every request is fired off as a coroutine so ordering is not guaranteed """ reqs = [] timeout = ClientTimeout(total=600) # TODO: figure out why we get 104s at connection counts > 4000 async with ClientSession( connector=TCPConnector(keepalive_timeout=3000), timeout=timeout) as s: for i in range(int(req_count)): url = req_url if per_req_path: url += str(i) self.logger.debug('making request: %d' % i) reqs.append( asyncio.ensure_future( self.make_request(req_type, url, s, req_count=i, **req_params))) responses = await asyncio.gather(*reqs)
def __init__(self, *args, **kwargs): self.logger = logging.getLogger(__name__).getChild( self.__class__.__name__) config = kwargs.pop("config", {}) authtoken = kwargs.pop("authtoken", None) context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH, cafile=config.get("ca_file"), capath=config.get("ca_path"), ) cert = config.get("cert") if cert is not None: context.load_cert_chain(cert, config.get("key")) if not config.get("verify", True): self.logger.warning("Certificate validation disabled") context.check_hostname = False context.verify_mode = ssl.VerifyMode.CERT_NONE # pylint: disable=no-member ciphers = config.get("ciphers") if ciphers is not None: context.set_ciphers(ciphers) self.proxy = config.get("proxy") conn = TCPConnector(ssl_context=context) self.bob_authclient = AuthClientHelper(config=config, connector=conn, authtoken=authtoken) kwargs["connector"] = conn if "headers" not in kwargs: kwargs["headers"] = DEFAULT_HEADERS super().__init__(*args, **kwargs)
async def get_weather_forecast(future, location): start = time.time() if location is not None: url = weather_url + location else: url = weather_url + DEFAULT_LOC_UUID info = None LOGGER.info(url) try: async with ClientSession(connector=TCPConnector(ssl=False)) as session: html = await fetch(session, url) LOGGER.info( f'weather content fetch in {time.time() - start} secs.') parse_start = time.time() info = await parse_weather_forecast(html) LOGGER.info( f'weather parsing took {time.time() - parse_start} secs.') except ClientConnectorError as ex: LOGGER.error(f'Unable to connect Weather API : {repr(ex)}') info = [] except BaseException as ex: LOGGER.error(f'Unable to connect Weather API : {repr(ex)}') info = [] LOGGER.info(f'Weather Time Taken {time.time() - start}') future.set_result(info)
async def fetcher(database): """Fetch all the feeds""" # disable certificate validation to cope with self-signed certificates in some feed back-ends client = ClientSession(connector=TCPConnector(verify_ssl=False)) sem = Semaphore(MAX_CONCURRENT_REQUESTS) queue = await connect_redis() while True: log.info("Beginning run.") tasks = [] threshold = datetime.now() - timedelta(seconds=FETCH_INTERVAL) async with ClientSession() as session: while True: try: job = await dequeue(queue, 'fetcher') feed = await database.feeds.find_one({'_id': job['_id']}) last_fetched = feed.get('last_fetched', threshold) if last_fetched <= threshold: task = ensure_future( throttle(sem, session, feed, client, database, queue)) tasks.append(task) except Exception: log.error(format_exc()) break responses = gather(*tasks) await responses log.info("Run complete, sleeping %ds...", CHECK_INTERVAL) await sleep(CHECK_INTERVAL) queue.close() await queue.wait_closed()
async def search(self, keyword: str): """ 关键词搜索 """ api = global_url + "comics/search?page={0}" + "&q={0}".format( parse.quote(keyword)) url = api.format(1) temp_header = self.update_signature(url, "GET") async with aiohttp.ClientSession(connector=TCPConnector( ssl=False)) as session: async with session.get(url=url, headers=temp_header, proxy=proxy) as resp: __pages = (await resp.json())["data"]["comics"]["pages"] _return = [] for _ in range(1, 3): url = api.format(_) temp_header = self.update_signature(url, "GET") async with session.get(url=url, headers=temp_header, proxy=proxy) as resp: __res = (await resp.json())["data"]["comics"]["docs"] for __ in __res: if __["likesCount"] < 200: continue if __["pagesCount"] / __["epsCount"] > 60 or __[ "epsCount"] > 10: continue _return.append({"name": __["title"], "id": __["_id"]}) return _return
async def main(calls_data): try: async with ClientSession( connector=TCPConnector(ssl=False, limit=25), headers=voxpredict.data.headers, ) as sess: tasks = [scrape(sess, row) for row in calls_data] audio_data = [ await audio for audio in tqdm(asyncio.as_completed(tasks), total=len(calls_data)) ] with open("audio_data.pickle", "wb") as f: pickle.dump(audio_data, f) audio_data = [ pd.DataFrame(data_dict, index=[0]) for data_dict in audio_data ] audio_df = pd.concat(audio_data) with sqlite3.connect( voxpredict.data.earnings_db_path) as sql_connection: audio_df.to_sql( "call_references", con=sql_connection, index=False, if_exists="append", ) except: asyncio.run(main(calls_data))
def http_session(self) -> ClientSession: session: Optional[ClientSession] = self.lookup.get("http_session") if not session: connector = TCPConnector(limit=0, ssl=False, ttl_dns_cache=300) session = ClientSession(connector=connector) self.lookup["http_session"] = session return session
async def run(num): tasks = [] # Semaphore, 相当于基于服务器的处理速度和测试客户端的硬件条件,一批批的发 # 直至发送完全部(下面定义的number/num) sem = asyncio.Semaphore(2000) # 创建session,且对本地的TCP连接不做限制limit=0 # 超时时间指定 # total:全部请求最终完成时间 # connect: aiohttp从本机连接池里取出一个将要进行的请求的时间 # sock_connect:单个请求连接到服务器的时间 # sock_read:单个请求从服务器返回的时间 import aiohttp # timeout = aiohttp.ClientTimeout(total=300, connect=60, sock_connect=60, sock_read=60) async with ClientSession(connector=TCPConnector(limit=0)) as session: while True: cookie_jar = PushTool.get_cookies() for i in range(0, num): # 如果是分批的发,就使用并传递Semaphore task = asyncio.ensure_future( bound_register(sem=sem, session=session, cookie_jia=cookie_jar)) tasks.append(task) responses = asyncio.gather(*tasks) await responses