async def user_tasks(username):
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
async with aiohttp.ClientSession(
trace_configs=[trace_config],
connector=aiohttp.TCPConnector(ssl=False)) as session:
# exercise_id = str(username) + '-1'
# question_list = ['0101010003001004', '0101010003001014', '0101010003001020',
# '0101010003001021', '0101010003001025', '0101010003001059',
# '0101010003001050', '0101010003001046',
# '0101010005001009', '0101010005001011']
token = await login(session, username)
print("token:", token)
headers = {"token": token}
select_grade_res = await test_select_grade(session, headers)
print("res:", select_grade_res)
await knowlege_test(session,
headers=headers,
select_grade_res=select_grade_res,
username=username)
# await get_exercise(session, headers=headers)
# await get_question(session, headers=headers, exercise_id=exercise_id)
# await submit_exercise(headers, exercise_id, question_list)
await logout(session, headers)
async def get_museum_session() -> aiohttp.ClientSession:
"""
Return an aiohttp session to use for requests to the MuseumPlus service
This ensures that credentials are used and amount of active connections
is limited.
Generating a museum session should be postponed until absolutely necessary;
this allows passari to accurately determine when the previous
session key has expired and a new one should be generated.
"""
user = CONFIG["museumplus"]["username"]
session_key = await get_museum_session_key()
# Refresh the session key file after every successful request to indicate
# that the session key still works
trace_config = aiohttp.TraceConfig()
trace_config.on_response_chunk_received.append(refresh_session_key)
return aiohttp.ClientSession(
connector=aiohttp.TCPConnector(limit=3),
timeout=aiohttp.ClientTimeout(
# Allow up to a 10 minute gap between data segments.
# Without the parameter set the session will wait forever
# for data that will never arrive if MuseumPlus hangs for
# whatever reason
sock_connect=600,
sock_read=600),
trace_configs=[trace_config],
auth=aiohttp.BasicAuth(login=f"user[{user}]",
password=f"session[{session_key}]"),
headers={"User-Agent": USER_AGENT})
async def test_request_tracing_interpose_headers(loop):
class MyClientRequest(ClientRequest):
headers = None
def __init__(self, *args, **kwargs):
super(MyClientRequest, self).__init__(*args, **kwargs)
MyClientRequest.headers = self.headers
async def new_headers(session,
trace_config_ctx,
method,
url,
headers,
trace_request_ctx=None):
headers['foo'] = 'bar'
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(new_headers)
session = aiohttp.ClientSession(loop=loop,
request_class=MyClientRequest,
trace_configs=[trace_config])
await session.get('http://example.com')
assert MyClientRequest.headers['foo'] == 'bar'
def request_tracer(results_collector):
async def on_request_start(session, context, params):
context.on_request_start = session.loop.time()
context.is_redirect = False
async def on_request_end(session, context, params):
total = session.loop.time() - context.on_request_start
context.on_request_end = total
results_collector['total'] = round(total * 1000, 2)
async def on_request_exception(session, context, params):
total = session.loop.time() - context.on_request_start
context.on_request_end = total
results_collector['total'] = round(total * 1000, 2)
results_collector['details'] = {
'method': params.method,
'url': str(params.url),
'error': type(params.exception).__name__
}
# exception=ClientConnectorError(ConnectionKey(host='webhssssssook.site', port=443, is_ssl=True, ssl=None,
# proxy=None, proxy_auth=None, proxy_headers_hash=-1738310210140354566),
# gaierror(-2, 'Name or service not known')))
# Might have strerror
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_exception.append(on_request_exception)
return trace_config
async def test_request_tracing_exception() -> None:
on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
on_request_exception = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
trace_config = aiohttp.TraceConfig()
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_exception.append(on_request_exception)
with mock.patch("aiohttp.client.TCPConnector.connect") as connect_patched:
error = Exception()
if sys.version_info >= (3, 8, 1):
connect_patched.side_effect = error
else:
loop = asyncio.get_event_loop()
f = loop.create_future()
f.set_exception(error)
connect_patched.return_value = f
session = aiohttp.ClientSession(trace_configs=[trace_config])
try:
await session.get("http://example.com")
except Exception:
pass
on_request_exception.assert_called_once_with(
session,
mock.ANY,
aiohttp.TraceRequestExceptionParams(
hdrs.METH_GET, URL("http://example.com"), CIMultiDict(), error
),
)
assert not on_request_end.called
await session.close()
def _session(self, loop=None, auth=None, timeout=None):
async def on_request_start(session, trace_config_ctx, params):
log = logging.getLogger(__name__).log
log(DEBUG2, '%s %s', params.method, params.url)
for k, v in params.headers.items():
log(DEBUG3, '%s: %s', k, v)
async def on_request_end(session, trace_config_ctx, params):
log = logging.getLogger(__name__).log
log(DEBUG1, '%s %s %s %s %s', params.method, params.url,
params.response.status, params.response.reason,
params.response.headers.get('content-length'))
for k, v in params.response.headers.items():
log(DEBUG3, '%s: %s', k, v)
kwargs = {
'loop': loop,
}
if auth is not None:
kwargs['auth'] = auth
if timeout is not None:
kwargs['timeout'] = timeout
if (logging.getLogger(__name__).getEffectiveLevel()
in [DEBUG1, DEBUG2, DEBUG3]):
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
kwargs['trace_configs'] = [trace_config]
return aiohttp.ClientSession(**kwargs)
async def test_request_tracing_exception(loop) -> None:
on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
on_request_exception = mock.Mock(
side_effect=asyncio.coroutine(mock.Mock()))
trace_config = aiohttp.TraceConfig()
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_exception.append(on_request_exception)
with mock.patch("aiohttp.client.TCPConnector.connect") as connect_patched:
error = Exception()
f = loop.create_future()
f.set_exception(error)
connect_patched.return_value = f
session = aiohttp.ClientSession(loop=loop,
trace_configs=[trace_config])
try:
await session.get('http://example.com')
except Exception:
pass
on_request_exception.assert_called_once_with(
session, mock.ANY,
aiohttp.TraceRequestExceptionParams(hdrs.METH_GET,
URL("http://example.com"),
CIMultiDict(), error))
assert not on_request_end.called
def __init__(self, loop, base_url, api_key):
""" Set attributes and initialize logging handlers. """
# Set attributes
self.base_url = base_url
self.api_key = api_key
self.logger = logging.getLogger('csgoleague.api')
# Check API URL
if not self.base_url.startswith('https') and self.base_url.startswith(
'http'):
self.logger.warning(
f'API url "{self.base_url}" should start with "https" instead of "http"'
)
# Register trace config handlers
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(start_request_log)
trace_config.on_request_end.append(end_request_log)
# Start session
self.logger.info('Starting API helper client session')
self.session = aiohttp.ClientSession(
loop=loop,
json_serialize=lambda x: json.dumps(x, ensure_ascii=False),
raise_for_status=True,
trace_configs=[trace_config])
async def test_request_tracing(loop):
trace_config_ctx = mock.Mock()
trace_request_ctx = {}
on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
trace_config = aiohttp.TraceConfig(trace_config_ctx_factory=mock.Mock(
return_value=trace_config_ctx))
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_redirect.append(on_request_redirect)
async with aiohttp.ClientSession(loop=loop,
trace_configs=[trace_config]) as session:
async with await session.get(
'http://example.com',
trace_request_ctx=trace_request_ctx) as resp:
on_request_start.assert_called_once_with(session, trace_config_ctx,
hdrs.METH_GET,
URL("http://example.com"),
CIMultiDict())
on_request_end.assert_called_once_with(session, trace_config_ctx,
hdrs.METH_GET,
URL("http://example.com"),
CIMultiDict(), resp)
assert not on_request_redirect.called
async def test_request_tracing_interpose_headers(loop, aiohttp_client) -> None:
async def handler(request):
return web.Response()
app = web.Application()
app.router.add_get('/', handler)
class MyClientRequest(ClientRequest):
headers = None
def __init__(self, *args, **kwargs):
super(MyClientRequest, self).__init__(*args, **kwargs)
MyClientRequest.headers = self.headers
async def new_headers(session, trace_config_ctx, data):
data.headers['foo'] = 'bar'
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(new_headers)
session = await aiohttp_client(app,
request_class=MyClientRequest,
trace_configs=[trace_config])
await session.get('/')
assert MyClientRequest.headers['foo'] == 'bar'
def make_trace_config(tracer: Tracer) -> aiohttp.TraceConfig:
trace_config = aiohttp.TraceConfig()
zipkin = ZipkingClientSignals(tracer)
trace_config.on_request_start.append(zipkin.on_request_start)
trace_config.on_request_end.append(zipkin.on_request_end)
trace_config.on_request_exception.append(zipkin.on_request_exception)
return trace_config
def _faust_trace_configs(self,
configs: List[TraceConfig] = None
) -> List[TraceConfig]:
if configs is None:
configs = []
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(_on_request_start)
configs.append(trace_config)
return configs
async def start(self):
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(self.on_http_request_start)
trace_config.on_request_end.append(self.on_http_request_end)
self.http._HTTPClient__session = aiohttp.ClientSession(
connector=self.http.connector,
ws_response_class=DiscordClientWebSocketResponse,
trace_configs=[trace_config],
)
self.http._token(self.config.token, bot=True)
self.pool = await asyncpg.create_pool(**self.config.database, max_size=10, command_timeout=60)
self._redis = await aioredis.create_redis_pool(
(self.config.redis["host"], self.config.redis["port"]),
password=self.config.redis["password"],
minsize=5,
maxsize=10,
loop=self.loop,
)
self._amqp = await aio_pika.connect_robust(
login=self.config.rabbitmq["username"],
password=self.config.rabbitmq["password"],
host=self.config.rabbitmq["host"],
port=self.config.rabbitmq["port"],
)
self._amqp_channel = await self._amqp.channel()
self._amqp_queue = await self._amqp_channel.get_queue("gateway.recv")
self.prom = Prometheus(self)
await self.prom.start()
self._connection = await self._get_state()
self._connection._get_client = lambda: self
self.ws = DiscordWebSocket(socket=None, loop=self.loop)
self.ws.token = self.http.token
self.ws._connection = self._connection
self.ws._discord_parsers = self._connection.parsers
self.ws._dispatch = self.dispatch
self.ws.call_hooks = self._connection.call_hooks
for extension in self._cogs:
try:
self.load_extension("cogs." + extension)
except Exception:
log.error(f"Failed to load extension {extension}.", file=sys.stderr)
log.error(traceback.print_exc())
log.info("Running...")
async with self._amqp_queue.iterator() as queue_iter:
async for message in queue_iter:
async with message.process(ignore_processed=True):
await self.receive_message(message.body)
message.ack()
def __init__(self, aiohttp_client_session: aiohttp.ClientSession):
if ENABLE_CONNECTION_TRACING:
request_tracing = aiohttp.TraceConfig()
request_tracing.on_request_start.append(self.on_request_start)
request_tracing.on_request_end.append(self.on_request_end)
request_tracing.on_request_chunk_sent.append(self.chunk_sent)
aiohttp_client_session = aiohttp.ClientSession(
trace_configs=[request_tracing])
self._client_session = aiohttp_client_session
async def main():
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
async with aiohttp.ClientSession(trace_configs=[trace_config]) as session:
tasks = [get(str(i), session) for i in range(1, 11)]
res = await asyncio.gather(*tasks)
print([ele['id'] for ele in res])
def _tracing_session(urls: List[yarl.URL]) -> aiohttp.ClientSession:
async def record_urls(session: aiohttp.ClientSession,
trace_config_ctx: types.SimpleNamespace,
params: aiohttp.TraceRequestStartParams) -> None:
urls.append(params.url)
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(record_urls) # type: ignore
return aiohttp.ClientSession(trace_configs=[trace_config])
def __init__(self,
trace_configs: Optional[List[TraceConfig]] = None,
**kwargs: Any) -> None:
if trace_configs is None:
trace_configs = []
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(_on_request_start)
trace_configs.append(trace_config)
super().__init__(trace_configs=trace_configs, **kwargs)
async def canon(url: str, timeout=5): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_redirect.append(on_request_redirect) trace_config.on_request_end.append(on_request_end) trace_config.on_connection_reuseconn.append(on_connection_reuseconn) async with aiohttp.ClientSession(trace_configs=[trace_config]) as session: async with session.get(url, allow_redirects=True, timeout=timeout) as res: return res.url, (await res.text())
def make_trace_config(tracer: Tracer) -> aiohttp.TraceConfig:
"""Creates aiohttp.TraceConfig with enabled aiojaeger instrumentation
for aiohttp client.
"""
tc = aiohttp.TraceConfig()
jaeger = JaegerClientSignals(tracer)
tc.on_request_start.append(jaeger.on_request_start) # type: ignore
tc.on_request_end.append(jaeger.on_request_end) # type: ignore
tc.on_request_exception.append(jaeger.on_request_exception) # type: ignore
return tc
def get_trace_config():
async def on_request_start(session, trace_config_ctx, params):
trace_config_ctx.trace_request_ctx["start"] = asyncio.get_running_loop().time()
async def on_request_end(session, trace_config_ctx, params):
trace_config_ctx.trace_request_ctx["end"] = asyncio.get_running_loop().time()
trace_config = aiohttp.TraceConfig()
trace_config.on_connection_create_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
return trace_config
async def test_request_tracing(loop, aiohttp_client) -> None:
async def handler(request):
return web.json_response({'ok': True})
app = web.Application()
app.router.add_post('/', handler)
trace_config_ctx = mock.Mock()
trace_request_ctx = {}
body = 'This is request body'
gathered_req_body = BytesIO()
gathered_res_body = BytesIO()
on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
async def on_request_chunk_sent(session, context, params):
gathered_req_body.write(params.chunk)
async def on_response_chunk_received(session, context, params):
gathered_res_body.write(params.chunk)
trace_config = aiohttp.TraceConfig(trace_config_ctx_factory=mock.Mock(
return_value=trace_config_ctx))
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_chunk_sent.append(on_request_chunk_sent)
trace_config.on_response_chunk_received.append(on_response_chunk_received)
trace_config.on_request_redirect.append(on_request_redirect)
session = await aiohttp_client(app, trace_configs=[trace_config])
async with session.post('/',
data=body,
trace_request_ctx=trace_request_ctx) as resp:
await resp.json()
on_request_start.assert_called_once_with(
session.session, trace_config_ctx,
aiohttp.TraceRequestStartParams(hdrs.METH_POST,
session.make_url('/'),
CIMultiDict()))
on_request_end.assert_called_once_with(
session.session, trace_config_ctx,
aiohttp.TraceRequestEndParams(hdrs.METH_POST,
session.make_url('/'), CIMultiDict(),
resp))
assert not on_request_redirect.called
assert gathered_req_body.getvalue() == body.encode('utf8')
assert gathered_res_body.getvalue() == json.dumps({
'ok': True
}).encode('utf8')
async def get_session(self) -> aiohttp.ClientSession:
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append((self._on_request_start))
trace_config.on_request_end.append(self._on_request_end)
if "session" in self.kwargs:
session = self.kwargs['session']
if not session._trace_configs:
trace_config.freeze()
session._trace_configs = [trace_config]
return session
return aiohttp.ClientSession(trace_configs=[trace_config])
def _create_trace_config(self) -> aiohttp.TraceConfig:
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(self._on_request_start) # type: ignore
trace_config.on_request_chunk_sent.append(
self._on_request_chunk_sent # type: ignore
)
trace_config.on_request_end.append(self._on_request_end) # type: ignore
trace_config.on_response_chunk_received.append(
self._on_response_chunk_received # type: ignore
)
return trace_config
def make_trace_config(tracer: Tracer) -> aiohttp.TraceConfig:
"""Creates aiohttp.TraceConfig with enabled aiozipking instrumentation
for aiohttp client.
"""
trace_config = aiohttp.TraceConfig()
zipkin = ZipkinClientSignals(tracer)
trace_config.on_request_start.append(zipkin.on_request_start)
trace_config.on_request_end.append(zipkin.on_request_end)
trace_config.on_request_exception.append(zipkin.on_request_exception)
return trace_config
def init_with_instana(wrapped, instance, argv, kwargs):
instana_trace_config = aiohttp.TraceConfig()
instana_trace_config.on_request_start.append(stan_request_start)
instana_trace_config.on_request_end.append(stan_request_end)
instana_trace_config.on_request_exception.append(stan_request_exception)
if 'trace_configs' in kwargs:
kwargs['trace_configs'].append(instana_trace_config)
else:
kwargs['trace_configs'] = [instana_trace_config]
return wrapped(*argv, **kwargs)
async def fictionhunt(story_id):
fictionhunt = "http://fictionhunt.com"
story_url = f"{fictionhunt}/read/{story_id}"
trace_config = aiohttp.TraceConfig()
trace_config.on_request_end.append(on_request_end)
async with aiohttp.ClientSession(trace_configs=[trace_config]) as session:
title, author, total = await getmetadata(session, story_url)
logger.info(f"Got metadata")
chapters = (get_chapter(session, chapter, story_url)
for chapter in range(1, total + 1))
return title, author, (await asyncio.gather(*chapters))
def __init__(self):
self.start_time = None
self.time_lat = None
self.trace_config = aiohttp.TraceConfig()
# трекер начала запросы
self.trace_config.on_connection_create_start.append(
self.on_request_start)
# трекер конца запросы
self.trace_config.on_connection_create_end.append(self.on_request_end)
self.trace_config.on_request_exception.append(self.on_request_end)
async def get_session(self) -> aiohttp.ClientSession:
"""Create or inject aiohttp session with trace config"""
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append((self._on_request_start))
trace_config.on_request_end.append(self._on_request_end)
if "session" in self.kwargs:
session = self.kwargs["session"]
if not session._trace_configs:
trace_config.freeze()
session._trace_configs = [trace_config]
return session
return aiohttp.ClientSession(trace_configs=[trace_config])
def add_trace_config():
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_dns_resolvehost_start.append(on_dns_resolvehost_start)
trace_config.on_dns_cache_hit.append(on_dns_cache_hit)
trace_config.on_dns_cache_miss.append(on_dns_cache_miss)
trace_config.on_dns_resolvehost_end.append(on_dns_resolvehost_end)
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_redirect.append(on_request_redirect)
trace_config.on_connection_create_start.append(on_connection_create_start)
trace_config.on_connection_create_end.append(on_connection_create_end)
return trace_config
def create_async(self):
# pylint: disable=import-outside-toplevel
import io
import aiohttp
import elasticsearch
from elasticsearch.serializer import JSONSerializer
import esrally.async_connection
class LazyJSONSerializer(JSONSerializer):
def loads(self, s):
meta = RallyAsyncElasticsearch.request_context.get()
if "raw_response" in meta:
return io.BytesIO(s)
else:
return super().loads(s)
async def on_request_start(session, trace_config_ctx, params):
RallyAsyncElasticsearch.on_request_start()
async def on_request_end(session, trace_config_ctx, params):
RallyAsyncElasticsearch.on_request_end()
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
# ensure that we also stop the timer when a request "ends" with an exception (e.g. a timeout)
trace_config.on_request_exception.append(on_request_end)
# override the builtin JSON serializer
self.client_options["serializer"] = LazyJSONSerializer()
self.client_options["trace_config"] = trace_config
class VerifiedAsyncTransport(elasticsearch.AsyncTransport):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# skip verification at this point; we've already verified this earlier with the synchronous client.
# The async client is used in the hot code path and we use customized overrides (such as that we don't
# parse response bodies in some cases for performance reasons, e.g. when using the bulk API).
self._verified_elasticsearch = True
class RallyAsyncElasticsearch(elasticsearch.AsyncElasticsearch,
RequestContextHolder):
pass
return RallyAsyncElasticsearch(
hosts=self.hosts,
transport_class=VerifiedAsyncTransport,
connection_class=esrally.async_connection.AIOHttpConnection,
ssl_context=self.ssl_context,
**self.client_options,
)
