async def init_session(self, session: Optional[ClientSession] = None):
"""
初始化 ClientSession, 使用 get/post/head 方法之前需要调用一次,
ClientSession 内部维护了连接池, 因此不建议每一个请求创建一个 session,
这里默认为每一个类创建一个 persistent session, 或者手动设置一个, 以实现复用,
在 __init__.py 中初始化 session 会出现 warning, 官方在 aiohttp 4.0 之后将只允许在协程中创建 session,
See:
https://github.com/aio-libs/aiohttp/issues/3658
https://github.com/aio-libs/aiohttp/issues/4932
:param session: 用于复用的 ClientSession 对象
"""
if not self.session:
if session:
self.session = session
return
if self._dns_server:
logger.debug(f"Use custom DNS Server: {self._dns_server}")
resolver = AsyncResolver(nameservers=self._dns_server)
con = TCPConnector(ssl=False,
ttl_dns_cache=300,
resolver=resolver)
else:
con = TCPConnector(ssl=False, ttl_dns_cache=300)
jar = CookieJar(unsafe=True)
self.session = ClientSession(connector=con, cookie_jar=jar)
async def run(sites, retesting=False):
if IPV4_ONLY:
connector = TCPConnector(verify_ssl=False, family=socket.AF_INET)
else:
connector = TCPConnector(verify_ssl=False)
sem = asyncio.Semaphore(100)
tasks = []
if not retesting:
read_timeout = READ_TIMEOUT
conn_timeout = CONN_TIMEOUT
else:
read_timeout = READ_TIMEOUT + READ_TIMEOUT / 2
conn_timeout = CONN_TIMEOUT + CONN_TIMEOUT / 2
async with ClientSession(connector=connector,
read_timeout=read_timeout,
conn_timeout=conn_timeout,
headers=client_headers) as session:
for site_ in sites:
task = asyncio.ensure_future(
bound_gethead(sem, site_, session, retesting=retesting))
tasks.append(task)
responses = await asyncio.gather(*tasks)
async def build_list(cls):
headers = {"X-API-KEY": Keys.site_api}
# Set up the session
if DEBUG_MODE:
http_session = ClientSession(connector=TCPConnector(
resolver=AsyncResolver(),
family=socket.AF_INET,
verify_ssl=False,
))
else:
http_session = ClientSession(connector=TCPConnector(
resolver=AsyncResolver()))
# Get all the snakes
if cls.snakes is None:
response = await http_session.get(URLs.site_names_api,
params={"get_all": "true"},
headers=headers)
cls.snakes = await response.json()
# Get the special cases
if cls.special_cases is None:
response = await http_session.get(URLs.site_special_api,
headers=headers)
special_cases = await response.json()
cls.special_cases = {
snake['name'].lower(): snake
for snake in special_cases
}
# Close the session
http_session.close()
async def get(url):
connector = TCPConnector(verify_ssl=False)
session = aiohttp.ClientSession(connector=TCPConnector(verify_ssl=False))
response = await session.get(url)
await response.text()
await session.close()
return response
async def __aenter__(self):
conn = TCPConnector(limit=100)
self._session = ClientSession(connector=conn, headers=self.auth.header)
conn_remote = TCPConnector(limit=100)
self._session_remote = ClientSession(connector=conn_remote)
return self
def __init__(self, **kwargs):
self.commands = []
self.tasks = []
self.components = []
self.modals = []
self.public_key = VerifyKey(bytes.fromhex(kwargs["public_key"]))
self.token = kwargs["token"]
self._loop = kwargs.get("loop")
bind_to = env.get("BIND_INTERFACE")
if bind_to is not None:
connector = TCPConnector(local_addr=(bind_to, 0))
else:
connector = TCPConnector()
self.session = kwargs.get("session", ClientSession(loop=self.loop, connector=connector))
self.guild_id = kwargs.get("guild_id") # Can be used during development to avoid the 1 hour cache
self.app_id = None
# Filled by setup()
self.http = None
self.redis = None
self.listeners = {}
self.modules = set()
def aiohttp_session(*, auth=None, **kwargs):
headers = dict()
if auth:
headers['Authorization'] = auth.encode()
ssl_context = create_default_context(cafile=certifi.where())
try:
connector = TCPConnector(ssl=ssl_context)
except TypeError:
connector = TCPConnector(ssl_context=ssl_context)
return ClientSession(headers=headers, connector=connector, **kwargs)
def custom_session(path_to_cert=None, custom_headers=None):
if path_to_cert:
try:
# with open | no need for importing `os` to do path.exists
with open(path_to_cert) as cert:
cert.read()
sslcontext = create_default_context(cafile=path_to_cert)
conn = TCPConnector(ssl_context=sslcontext)
except:
return 'Cert file not Found!'
else:
conn = TCPConnector(verify_ssl=False)
session = ClientSession(connector=conn, headers=custom_headers)
return session
def fetch_all(urls):
stage = aio.map(_fetch,urls,
workers = MAX_WORKERS,
on_start = lambda: ClientSession(connector=TCPConnector(limit=None)),
on_done = lambda _status, session: session.close())
return stage
def get_client_session(self, config: SeekerConfig) -> ClientSession:
async def _on_request_start(
session: ClientSession,
trace_config_ctx: SimpleNamespace,
params: TraceRequestStartParams
) -> None:
current_attempt = \
trace_config_ctx.trace_request_ctx['current_attempt']
if(current_attempt > 1):
logger.warning(
f'::warn ::Retry Attempt #{current_attempt} ' +
f'of {config.max_tries}: {params.url}')
trace_config = TraceConfig()
trace_config.on_request_start.append(_on_request_start)
limit_per_host = max(0, config.connect_limit_per_host)
connector = TCPConnector(
limit_per_host=limit_per_host,
ttl_dns_cache=600 # 10-minute DNS cache
)
retry_options = ExponentialRetry(
attempts=config.max_tries,
max_timeout=config.max_time,
exceptions=[
aiohttp.ClientError,
asyncio.TimeoutError
])
return RetryClient(
raise_for_status=True,
connector=connector,
timeout=ClientTimeout(total=config.timeout),
headers={'User-Agent': config.agent},
retry_options=retry_options,
trace_configs=[trace_config])
async def get_google_weather(future, location):
start = time.time()
info = None
if location is not None:
url = google_weather_url + '+' + location
else:
url = google_weather_url + '+' + default_location
LOGGER.info(url)
try:
async with ClientSession(connector=TCPConnector(ssl=False)) as session:
html = await fetch(session, url)
LOGGER.info(
f'g-weather content fetch in {time.time() - start} secs.')
parse_start = time.time()
info = await parse_google_weather(html)
LOGGER.info(
f'g-weather parsing took {time.time() - parse_start} secs.')
except BaseException as ex:
LOGGER.error(f'Unable to connect Weather API : {repr(ex)}')
info = WeatherInfo.WeatherInfo('0', "0", "0", "00:00", "", "", "")
info.set_error(ex)
LOGGER.info(f'Weather Time Taken {time.time() - start}')
future.set_result(info)
async def run(num):
tasks = []
# Semaphore， 相当于基于服务器的处理速度和测试客户端的硬件条件，一批批的发
# 直至发送完全部（下面定义的number/num）
sem = asyncio.Semaphore(3000)
# 创建session，且对本地的TCP连接不做限制limit=0
# 超时时间指定
# total:全部请求最终完成时间
# connect: aiohttp从本机连接池里取出一个将要进行的请求的时间
# sock_connect：单个请求连接到服务器的时间
# sock_read：单个请求从服务器返回的时间
timeout = ClientTimeout(total=300,
connect=60,
sock_connect=60,
sock_read=60)
async with ClientSession(connector=TCPConnector(limit=0),
timeout=timeout) as session:
for i in range(0, num):
# 如果是分批的发，就使用并传递Semaphore
task = asyncio.ensure_future(
bound_register(sem=sem,
firstname=str(18300000000 + i),
lastname=str(18300000000 + i),
nickname=str(18300000000 + i),
phone=str(18300000000 + i),
password="******",
session=session,
i=i))
tasks.append(task)
responses = asyncio.gather(*tasks)
await responses
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.http_session = ClientSession(connector=TCPConnector(
resolver=AsyncResolver(), family=socket.AF_INET))
self._guild_available = asyncio.Event()
self.loop.create_task(self.send_log("SeasonalBot", "Connected!"))
async def __run_(self, task, proxy):
"""
# 运行查询
:param task: 每次任务查询所需参数 元组
:param proxy:代理
:return:
"""
# 生成HTTP头 USER AGENT
headers = {'User-Agent': UserAgent().chrome}
# 获取COOKIES
cookies = await self.__cookies_get()
# 设置异步参数
connect = TCPConnector(ssl=False, limit=30)
# 创建会话
async with ClientSession(connector=connect,
headers=headers,
cookies=cookies) as session:
# 获取数据
message = await self.__fetch_(session, task, proxy)
if self.__debug and message: print(message)
# 异步等待
await asyncio.sleep(random.randint(1, 10))
def __init__(self, name: str, details: dict):
self.name = name
self.task = None
super().__init__(platform=fortnitepy.Platform.MAC,
connector=TCPConnector(limit=None),
auth=fortnitepy.AdvancedAuth(**details))
def __init__(
self,
pool: int = 8,
proxy: str = None,
proxy_auth: BasicAuth = None,
auto_proxy: bool = None,
timeout: int = 2333,
ssl: bool = False
) -> None:
"""
:param pool: 同时发起线程数
:param proxy: 每个连接的代理
:param proxy_auth: 每个连接的代理的身份
:param auto_proxy: 自动设置系统代理
:param timeout: 该客户端会话超时时间
:param ssl: 验证SSL
"""
super().__init__(
trust_env=auto_proxy,
connector=TCPConnector(limit=pool),
timeout=ClientTimeout(total=timeout),
headers=self.request_headers
)
self.proxy = proxy
self.proxy_auth = proxy_auth
self.verify_ssl = ssl
async def get_sitemaps(self):
if isinstance(self.robots_url, str):
sitemaps = await self._find_sitemaps_from_robots(self.robots_url)
else:
sitemaps = self.robots_url
while sitemaps:
sitemap = sitemaps.pop()
conn = TCPConnector(verify_ssl=False)
async with ClientSession(connector=conn) as client:
async with client.get(sitemap) as response:
if response.status == 200:
try:
if 'gzip' in response.content_type:
buf = BytesIO(await response.read())
f = gzip.GzipFile(fileobj=buf)
xml = xmltodict.parse(f.read())
else:
xml = xmltodict.parse(await response.read())
sitemaps.extend(self._search_sitemaps(xml))
self.output_sitemap(xml, sitemap.split('/')[-1])
except Exception as e:
print(str(e))
print(
"Error parsing site map: {0} | content-type: {1}"
.format(sitemap, response.content_type))
async def run(cod_stops, fetch_conf):
"""Async function that generates a aiohttp ClientSession and fetches the
given stops.
Arguments:
cod_stops (list): List of stop codes in CRTM's format (e.g. 8_17491).
fetch_conf (dict): Dictionary with configuration parameters for
fetching the content.
Returns:
list: The responses.
"""
tasks = []
# Fetch all responses within one Client session,
# keep connection alive for all requests.
connector = TCPConnector(limit=fetch_conf['max_connections'])
timeout = ClientTimeout(total=fetch_conf['timeout'])
async with ClientSession(connector=connector, timeout=timeout) as session:
for cod_stop in cod_stops:
task = asyncio.ensure_future(fetch(cod_stop, session, fetch_conf))
tasks.append(task)
responses = await asyncio.gather(*tasks)
# you now have all response bodies in this variable
return responses
def create_session(connection_pool_size=100,
**kwargs) -> aiohttp.ClientSession:
""" Create a session with parameters suited for AutoExtract """
kwargs.setdefault('timeout', AIO_API_TIMEOUT)
if "connector" not in kwargs:
kwargs["connector"] = TCPConnector(limit=connection_pool_size)
return aiohttp.ClientSession(**kwargs)
async def get_content_async(data):
"""
Asynchronously hit each hacker news article item for download.
:param data: tuple of url and article type
:return: list of json responses
"""
parent_tasks = []
child_tasks = []
timeout = ClientTimeout(total=7 * 60)
connector = TCPConnector(ssl=False, family=socket.AF_INET)
async with ClientSession(loop=asyncio.get_event_loop(),
timeout=timeout,
connector=connector) as session:
for url, article_type in data:
parent_tasks.append(asyncio.create_task(fetch(url, session)))
parent_results = await asyncio.gather(*parent_tasks)
for index, section in enumerate(parent_results):
_, article_type = data[index]
for article_id in section:
child_tasks.append(
asyncio.create_task(
fetch(
f'https://hacker-news.firebaseio.com/v0/item/{article_id}.json',
session, article_type)))
return await asyncio.gather(*child_tasks)
async def main():
global session
session = aiohttp.ClientSession(connector=TCPConnector(ssl=False))
scrape_index_tasks = [
asyncio.ensure_future(scrape_api()) for _ in range(10000)
]
await asyncio.gather(*scrape_index_tasks) # 启动协程
async def main():
connector = TCPConnector(ssl=False)
async with ClientSession(connector=connector) as session:
file_links = await get_file_links(session, ROOT_URL)
tasks = [download_file(session, file_link) for file_link in file_links]
await asyncio.gather(*tasks)
async def run(self,
req_type,
req_url,
req_count=100,
per_req_path=False,
**req_params):
"""
run a req_count number of requests with the specified method and url.
per_req_path allows for appending a count number at the end of url
which is useful for eg putting objects in a seq. Note that every
request is fired off as a coroutine so ordering is not guaranteed
"""
reqs = []
timeout = ClientTimeout(total=600)
# TODO: figure out why we get 104s at connection counts > 4000
async with ClientSession(
connector=TCPConnector(keepalive_timeout=3000),
timeout=timeout) as s:
for i in range(int(req_count)):
url = req_url
if per_req_path:
url += str(i)
self.logger.debug('making request: %d' % i)
reqs.append(
asyncio.ensure_future(
self.make_request(req_type,
url,
s,
req_count=i,
**req_params)))
responses = await asyncio.gather(*reqs)
def __init__(self, *args, **kwargs):
self.logger = logging.getLogger(__name__).getChild(
self.__class__.__name__)
config = kwargs.pop("config", {})
authtoken = kwargs.pop("authtoken", None)
context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH,
cafile=config.get("ca_file"),
capath=config.get("ca_path"),
)
cert = config.get("cert")
if cert is not None:
context.load_cert_chain(cert, config.get("key"))
if not config.get("verify", True):
self.logger.warning("Certificate validation disabled")
context.check_hostname = False
context.verify_mode = ssl.VerifyMode.CERT_NONE # pylint: disable=no-member
ciphers = config.get("ciphers")
if ciphers is not None:
context.set_ciphers(ciphers)
self.proxy = config.get("proxy")
conn = TCPConnector(ssl_context=context)
self.bob_authclient = AuthClientHelper(config=config,
connector=conn,
authtoken=authtoken)
kwargs["connector"] = conn
if "headers" not in kwargs:
kwargs["headers"] = DEFAULT_HEADERS
super().__init__(*args, **kwargs)
async def get_weather_forecast(future, location):
start = time.time()
if location is not None:
url = weather_url + location
else:
url = weather_url + DEFAULT_LOC_UUID
info = None
LOGGER.info(url)
try:
async with ClientSession(connector=TCPConnector(ssl=False)) as session:
html = await fetch(session, url)
LOGGER.info(
f'weather content fetch in {time.time() - start} secs.')
parse_start = time.time()
info = await parse_weather_forecast(html)
LOGGER.info(
f'weather parsing took {time.time() - parse_start} secs.')
except ClientConnectorError as ex:
LOGGER.error(f'Unable to connect Weather API : {repr(ex)}')
info = []
except BaseException as ex:
LOGGER.error(f'Unable to connect Weather API : {repr(ex)}')
info = []
LOGGER.info(f'Weather Time Taken {time.time() - start}')
future.set_result(info)
async def fetcher(database):
"""Fetch all the feeds"""
# disable certificate validation to cope with self-signed certificates in some feed back-ends
client = ClientSession(connector=TCPConnector(verify_ssl=False))
sem = Semaphore(MAX_CONCURRENT_REQUESTS)
queue = await connect_redis()
while True:
log.info("Beginning run.")
tasks = []
threshold = datetime.now() - timedelta(seconds=FETCH_INTERVAL)
async with ClientSession() as session:
while True:
try:
job = await dequeue(queue, 'fetcher')
feed = await database.feeds.find_one({'_id': job['_id']})
last_fetched = feed.get('last_fetched', threshold)
if last_fetched <= threshold:
task = ensure_future(
throttle(sem, session, feed, client, database,
queue))
tasks.append(task)
except Exception:
log.error(format_exc())
break
responses = gather(*tasks)
await responses
log.info("Run complete, sleeping %ds...", CHECK_INTERVAL)
await sleep(CHECK_INTERVAL)
queue.close()
await queue.wait_closed()
async def search(self, keyword: str):
""" 关键词搜索 """
api = global_url + "comics/search?page={0}" + "&q={0}".format(
parse.quote(keyword))
url = api.format(1)
temp_header = self.update_signature(url, "GET")
async with aiohttp.ClientSession(connector=TCPConnector(
ssl=False)) as session:
async with session.get(url=url, headers=temp_header,
proxy=proxy) as resp:
__pages = (await resp.json())["data"]["comics"]["pages"]
_return = []
for _ in range(1, 3):
url = api.format(_)
temp_header = self.update_signature(url, "GET")
async with session.get(url=url,
headers=temp_header,
proxy=proxy) as resp:
__res = (await resp.json())["data"]["comics"]["docs"]
for __ in __res:
if __["likesCount"] < 200:
continue
if __["pagesCount"] / __["epsCount"] > 60 or __[
"epsCount"] > 10:
continue
_return.append({"name": __["title"], "id": __["_id"]})
return _return
async def main(calls_data):
try:
async with ClientSession(
connector=TCPConnector(ssl=False, limit=25),
headers=voxpredict.data.headers,
) as sess:
tasks = [scrape(sess, row) for row in calls_data]
audio_data = [
await audio for audio in tqdm(asyncio.as_completed(tasks),
total=len(calls_data))
]
with open("audio_data.pickle", "wb") as f:
pickle.dump(audio_data, f)
audio_data = [
pd.DataFrame(data_dict, index=[0]) for data_dict in audio_data
]
audio_df = pd.concat(audio_data)
with sqlite3.connect(
voxpredict.data.earnings_db_path) as sql_connection:
audio_df.to_sql(
"call_references",
con=sql_connection,
index=False,
if_exists="append",
)
except:
asyncio.run(main(calls_data))
def http_session(self) -> ClientSession:
session: Optional[ClientSession] = self.lookup.get("http_session")
if not session:
connector = TCPConnector(limit=0, ssl=False, ttl_dns_cache=300)
session = ClientSession(connector=connector)
self.lookup["http_session"] = session
return session
async def run(num):
tasks = []
# Semaphore， 相当于基于服务器的处理速度和测试客户端的硬件条件，一批批的发
# 直至发送完全部（下面定义的number/num）
sem = asyncio.Semaphore(2000)
# 创建session，且对本地的TCP连接不做限制limit=0
# 超时时间指定
# total:全部请求最终完成时间
# connect: aiohttp从本机连接池里取出一个将要进行的请求的时间
# sock_connect：单个请求连接到服务器的时间
# sock_read：单个请求从服务器返回的时间
import aiohttp
# timeout = aiohttp.ClientTimeout(total=300, connect=60, sock_connect=60, sock_read=60)
async with ClientSession(connector=TCPConnector(limit=0)) as session:
while True:
cookie_jar = PushTool.get_cookies()
for i in range(0, num):
# 如果是分批的发，就使用并传递Semaphore
task = asyncio.ensure_future(
bound_register(sem=sem,
session=session,
cookie_jia=cookie_jar))
tasks.append(task)
responses = asyncio.gather(*tasks)
await responses
