async def user_tasks(username): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) async with aiohttp.ClientSession( trace_configs=[trace_config], connector=aiohttp.TCPConnector(ssl=False)) as session: # exercise_id = str(username) + '-1' # question_list = ['0101010003001004', '0101010003001014', '0101010003001020', # '0101010003001021', '0101010003001025', '0101010003001059', # '0101010003001050', '0101010003001046', # '0101010005001009', '0101010005001011'] token = await login(session, username) print("token:", token) headers = {"token": token} select_grade_res = await test_select_grade(session, headers) print("res:", select_grade_res) await knowlege_test(session, headers=headers, select_grade_res=select_grade_res, username=username) # await get_exercise(session, headers=headers) # await get_question(session, headers=headers, exercise_id=exercise_id) # await submit_exercise(headers, exercise_id, question_list) await logout(session, headers)
async def get_museum_session() -> aiohttp.ClientSession: """ Return an aiohttp session to use for requests to the MuseumPlus service This ensures that credentials are used and amount of active connections is limited. Generating a museum session should be postponed until absolutely necessary; this allows passari to accurately determine when the previous session key has expired and a new one should be generated. """ user = CONFIG["museumplus"]["username"] session_key = await get_museum_session_key() # Refresh the session key file after every successful request to indicate # that the session key still works trace_config = aiohttp.TraceConfig() trace_config.on_response_chunk_received.append(refresh_session_key) return aiohttp.ClientSession( connector=aiohttp.TCPConnector(limit=3), timeout=aiohttp.ClientTimeout( # Allow up to a 10 minute gap between data segments. # Without the parameter set the session will wait forever # for data that will never arrive if MuseumPlus hangs for # whatever reason sock_connect=600, sock_read=600), trace_configs=[trace_config], auth=aiohttp.BasicAuth(login=f"user[{user}]", password=f"session[{session_key}]"), headers={"User-Agent": USER_AGENT})
async def test_request_tracing_interpose_headers(loop): class MyClientRequest(ClientRequest): headers = None def __init__(self, *args, **kwargs): super(MyClientRequest, self).__init__(*args, **kwargs) MyClientRequest.headers = self.headers async def new_headers(session, trace_config_ctx, method, url, headers, trace_request_ctx=None): headers['foo'] = 'bar' trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(new_headers) session = aiohttp.ClientSession(loop=loop, request_class=MyClientRequest, trace_configs=[trace_config]) await session.get('http://example.com') assert MyClientRequest.headers['foo'] == 'bar'
def request_tracer(results_collector): async def on_request_start(session, context, params): context.on_request_start = session.loop.time() context.is_redirect = False async def on_request_end(session, context, params): total = session.loop.time() - context.on_request_start context.on_request_end = total results_collector['total'] = round(total * 1000, 2) async def on_request_exception(session, context, params): total = session.loop.time() - context.on_request_start context.on_request_end = total results_collector['total'] = round(total * 1000, 2) results_collector['details'] = { 'method': params.method, 'url': str(params.url), 'error': type(params.exception).__name__ } # exception=ClientConnectorError(ConnectionKey(host='webhssssssook.site', port=443, is_ssl=True, ssl=None, # proxy=None, proxy_auth=None, proxy_headers_hash=-1738310210140354566), # gaierror(-2, 'Name or service not known'))) # Might have strerror trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) trace_config.on_request_exception.append(on_request_exception) return trace_config
async def test_request_tracing_exception() -> None: on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) on_request_exception = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) trace_config = aiohttp.TraceConfig() trace_config.on_request_end.append(on_request_end) trace_config.on_request_exception.append(on_request_exception) with mock.patch("aiohttp.client.TCPConnector.connect") as connect_patched: error = Exception() if sys.version_info >= (3, 8, 1): connect_patched.side_effect = error else: loop = asyncio.get_event_loop() f = loop.create_future() f.set_exception(error) connect_patched.return_value = f session = aiohttp.ClientSession(trace_configs=[trace_config]) try: await session.get("http://example.com") except Exception: pass on_request_exception.assert_called_once_with( session, mock.ANY, aiohttp.TraceRequestExceptionParams( hdrs.METH_GET, URL("http://example.com"), CIMultiDict(), error ), ) assert not on_request_end.called await session.close()
def _session(self, loop=None, auth=None, timeout=None): async def on_request_start(session, trace_config_ctx, params): log = logging.getLogger(__name__).log log(DEBUG2, '%s %s', params.method, params.url) for k, v in params.headers.items(): log(DEBUG3, '%s: %s', k, v) async def on_request_end(session, trace_config_ctx, params): log = logging.getLogger(__name__).log log(DEBUG1, '%s %s %s %s %s', params.method, params.url, params.response.status, params.response.reason, params.response.headers.get('content-length')) for k, v in params.response.headers.items(): log(DEBUG3, '%s: %s', k, v) kwargs = { 'loop': loop, } if auth is not None: kwargs['auth'] = auth if timeout is not None: kwargs['timeout'] = timeout if (logging.getLogger(__name__).getEffectiveLevel() in [DEBUG1, DEBUG2, DEBUG3]): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) kwargs['trace_configs'] = [trace_config] return aiohttp.ClientSession(**kwargs)
async def test_request_tracing_exception(loop) -> None: on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) on_request_exception = mock.Mock( side_effect=asyncio.coroutine(mock.Mock())) trace_config = aiohttp.TraceConfig() trace_config.on_request_end.append(on_request_end) trace_config.on_request_exception.append(on_request_exception) with mock.patch("aiohttp.client.TCPConnector.connect") as connect_patched: error = Exception() f = loop.create_future() f.set_exception(error) connect_patched.return_value = f session = aiohttp.ClientSession(loop=loop, trace_configs=[trace_config]) try: await session.get('http://example.com') except Exception: pass on_request_exception.assert_called_once_with( session, mock.ANY, aiohttp.TraceRequestExceptionParams(hdrs.METH_GET, URL("http://example.com"), CIMultiDict(), error)) assert not on_request_end.called
def __init__(self, loop, base_url, api_key): """ Set attributes and initialize logging handlers. """ # Set attributes self.base_url = base_url self.api_key = api_key self.logger = logging.getLogger('csgoleague.api') # Check API URL if not self.base_url.startswith('https') and self.base_url.startswith( 'http'): self.logger.warning( f'API url "{self.base_url}" should start with "https" instead of "http"' ) # Register trace config handlers trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(start_request_log) trace_config.on_request_end.append(end_request_log) # Start session self.logger.info('Starting API helper client session') self.session = aiohttp.ClientSession( loop=loop, json_serialize=lambda x: json.dumps(x, ensure_ascii=False), raise_for_status=True, trace_configs=[trace_config])
async def test_request_tracing(loop): trace_config_ctx = mock.Mock() trace_request_ctx = {} on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) trace_config = aiohttp.TraceConfig(trace_config_ctx_factory=mock.Mock( return_value=trace_config_ctx)) trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) trace_config.on_request_redirect.append(on_request_redirect) async with aiohttp.ClientSession(loop=loop, trace_configs=[trace_config]) as session: async with await session.get( 'http://example.com', trace_request_ctx=trace_request_ctx) as resp: on_request_start.assert_called_once_with(session, trace_config_ctx, hdrs.METH_GET, URL("http://example.com"), CIMultiDict()) on_request_end.assert_called_once_with(session, trace_config_ctx, hdrs.METH_GET, URL("http://example.com"), CIMultiDict(), resp) assert not on_request_redirect.called
async def test_request_tracing_interpose_headers(loop, aiohttp_client) -> None: async def handler(request): return web.Response() app = web.Application() app.router.add_get('/', handler) class MyClientRequest(ClientRequest): headers = None def __init__(self, *args, **kwargs): super(MyClientRequest, self).__init__(*args, **kwargs) MyClientRequest.headers = self.headers async def new_headers(session, trace_config_ctx, data): data.headers['foo'] = 'bar' trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(new_headers) session = await aiohttp_client(app, request_class=MyClientRequest, trace_configs=[trace_config]) await session.get('/') assert MyClientRequest.headers['foo'] == 'bar'
def make_trace_config(tracer: Tracer) -> aiohttp.TraceConfig: trace_config = aiohttp.TraceConfig() zipkin = ZipkingClientSignals(tracer) trace_config.on_request_start.append(zipkin.on_request_start) trace_config.on_request_end.append(zipkin.on_request_end) trace_config.on_request_exception.append(zipkin.on_request_exception) return trace_config
def _faust_trace_configs(self, configs: List[TraceConfig] = None ) -> List[TraceConfig]: if configs is None: configs = [] trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(_on_request_start) configs.append(trace_config) return configs
async def start(self): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(self.on_http_request_start) trace_config.on_request_end.append(self.on_http_request_end) self.http._HTTPClient__session = aiohttp.ClientSession( connector=self.http.connector, ws_response_class=DiscordClientWebSocketResponse, trace_configs=[trace_config], ) self.http._token(self.config.token, bot=True) self.pool = await asyncpg.create_pool(**self.config.database, max_size=10, command_timeout=60) self._redis = await aioredis.create_redis_pool( (self.config.redis["host"], self.config.redis["port"]), password=self.config.redis["password"], minsize=5, maxsize=10, loop=self.loop, ) self._amqp = await aio_pika.connect_robust( login=self.config.rabbitmq["username"], password=self.config.rabbitmq["password"], host=self.config.rabbitmq["host"], port=self.config.rabbitmq["port"], ) self._amqp_channel = await self._amqp.channel() self._amqp_queue = await self._amqp_channel.get_queue("gateway.recv") self.prom = Prometheus(self) await self.prom.start() self._connection = await self._get_state() self._connection._get_client = lambda: self self.ws = DiscordWebSocket(socket=None, loop=self.loop) self.ws.token = self.http.token self.ws._connection = self._connection self.ws._discord_parsers = self._connection.parsers self.ws._dispatch = self.dispatch self.ws.call_hooks = self._connection.call_hooks for extension in self._cogs: try: self.load_extension("cogs." + extension) except Exception: log.error(f"Failed to load extension {extension}.", file=sys.stderr) log.error(traceback.print_exc()) log.info("Running...") async with self._amqp_queue.iterator() as queue_iter: async for message in queue_iter: async with message.process(ignore_processed=True): await self.receive_message(message.body) message.ack()
def __init__(self, aiohttp_client_session: aiohttp.ClientSession): if ENABLE_CONNECTION_TRACING: request_tracing = aiohttp.TraceConfig() request_tracing.on_request_start.append(self.on_request_start) request_tracing.on_request_end.append(self.on_request_end) request_tracing.on_request_chunk_sent.append(self.chunk_sent) aiohttp_client_session = aiohttp.ClientSession( trace_configs=[request_tracing]) self._client_session = aiohttp_client_session
async def main(): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) async with aiohttp.ClientSession(trace_configs=[trace_config]) as session: tasks = [get(str(i), session) for i in range(1, 11)] res = await asyncio.gather(*tasks) print([ele['id'] for ele in res])
def _tracing_session(urls: List[yarl.URL]) -> aiohttp.ClientSession: async def record_urls(session: aiohttp.ClientSession, trace_config_ctx: types.SimpleNamespace, params: aiohttp.TraceRequestStartParams) -> None: urls.append(params.url) trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(record_urls) # type: ignore return aiohttp.ClientSession(trace_configs=[trace_config])
def __init__(self, trace_configs: Optional[List[TraceConfig]] = None, **kwargs: Any) -> None: if trace_configs is None: trace_configs = [] trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(_on_request_start) trace_configs.append(trace_config) super().__init__(trace_configs=trace_configs, **kwargs)
async def canon(url: str, timeout=5): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_redirect.append(on_request_redirect) trace_config.on_request_end.append(on_request_end) trace_config.on_connection_reuseconn.append(on_connection_reuseconn) async with aiohttp.ClientSession(trace_configs=[trace_config]) as session: async with session.get(url, allow_redirects=True, timeout=timeout) as res: return res.url, (await res.text())
def make_trace_config(tracer: Tracer) -> aiohttp.TraceConfig: """Creates aiohttp.TraceConfig with enabled aiojaeger instrumentation for aiohttp client. """ tc = aiohttp.TraceConfig() jaeger = JaegerClientSignals(tracer) tc.on_request_start.append(jaeger.on_request_start) # type: ignore tc.on_request_end.append(jaeger.on_request_end) # type: ignore tc.on_request_exception.append(jaeger.on_request_exception) # type: ignore return tc
def get_trace_config(): async def on_request_start(session, trace_config_ctx, params): trace_config_ctx.trace_request_ctx["start"] = asyncio.get_running_loop().time() async def on_request_end(session, trace_config_ctx, params): trace_config_ctx.trace_request_ctx["end"] = asyncio.get_running_loop().time() trace_config = aiohttp.TraceConfig() trace_config.on_connection_create_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) return trace_config
async def test_request_tracing(loop, aiohttp_client) -> None: async def handler(request): return web.json_response({'ok': True}) app = web.Application() app.router.add_post('/', handler) trace_config_ctx = mock.Mock() trace_request_ctx = {} body = 'This is request body' gathered_req_body = BytesIO() gathered_res_body = BytesIO() on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) async def on_request_chunk_sent(session, context, params): gathered_req_body.write(params.chunk) async def on_response_chunk_received(session, context, params): gathered_res_body.write(params.chunk) trace_config = aiohttp.TraceConfig(trace_config_ctx_factory=mock.Mock( return_value=trace_config_ctx)) trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) trace_config.on_request_chunk_sent.append(on_request_chunk_sent) trace_config.on_response_chunk_received.append(on_response_chunk_received) trace_config.on_request_redirect.append(on_request_redirect) session = await aiohttp_client(app, trace_configs=[trace_config]) async with session.post('/', data=body, trace_request_ctx=trace_request_ctx) as resp: await resp.json() on_request_start.assert_called_once_with( session.session, trace_config_ctx, aiohttp.TraceRequestStartParams(hdrs.METH_POST, session.make_url('/'), CIMultiDict())) on_request_end.assert_called_once_with( session.session, trace_config_ctx, aiohttp.TraceRequestEndParams(hdrs.METH_POST, session.make_url('/'), CIMultiDict(), resp)) assert not on_request_redirect.called assert gathered_req_body.getvalue() == body.encode('utf8') assert gathered_res_body.getvalue() == json.dumps({ 'ok': True }).encode('utf8')
async def get_session(self) -> aiohttp.ClientSession: trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append((self._on_request_start)) trace_config.on_request_end.append(self._on_request_end) if "session" in self.kwargs: session = self.kwargs['session'] if not session._trace_configs: trace_config.freeze() session._trace_configs = [trace_config] return session return aiohttp.ClientSession(trace_configs=[trace_config])
def _create_trace_config(self) -> aiohttp.TraceConfig: trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(self._on_request_start) # type: ignore trace_config.on_request_chunk_sent.append( self._on_request_chunk_sent # type: ignore ) trace_config.on_request_end.append(self._on_request_end) # type: ignore trace_config.on_response_chunk_received.append( self._on_response_chunk_received # type: ignore ) return trace_config
def make_trace_config(tracer: Tracer) -> aiohttp.TraceConfig: """Creates aiohttp.TraceConfig with enabled aiozipking instrumentation for aiohttp client. """ trace_config = aiohttp.TraceConfig() zipkin = ZipkinClientSignals(tracer) trace_config.on_request_start.append(zipkin.on_request_start) trace_config.on_request_end.append(zipkin.on_request_end) trace_config.on_request_exception.append(zipkin.on_request_exception) return trace_config
def init_with_instana(wrapped, instance, argv, kwargs): instana_trace_config = aiohttp.TraceConfig() instana_trace_config.on_request_start.append(stan_request_start) instana_trace_config.on_request_end.append(stan_request_end) instana_trace_config.on_request_exception.append(stan_request_exception) if 'trace_configs' in kwargs: kwargs['trace_configs'].append(instana_trace_config) else: kwargs['trace_configs'] = [instana_trace_config] return wrapped(*argv, **kwargs)
async def fictionhunt(story_id): fictionhunt = "http://fictionhunt.com" story_url = f"{fictionhunt}/read/{story_id}" trace_config = aiohttp.TraceConfig() trace_config.on_request_end.append(on_request_end) async with aiohttp.ClientSession(trace_configs=[trace_config]) as session: title, author, total = await getmetadata(session, story_url) logger.info(f"Got metadata") chapters = (get_chapter(session, chapter, story_url) for chapter in range(1, total + 1)) return title, author, (await asyncio.gather(*chapters))
def __init__(self): self.start_time = None self.time_lat = None self.trace_config = aiohttp.TraceConfig() # трекер начала запросы self.trace_config.on_connection_create_start.append( self.on_request_start) # трекер конца запросы self.trace_config.on_connection_create_end.append(self.on_request_end) self.trace_config.on_request_exception.append(self.on_request_end)
async def get_session(self) -> aiohttp.ClientSession: """Create or inject aiohttp session with trace config""" trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append((self._on_request_start)) trace_config.on_request_end.append(self._on_request_end) if "session" in self.kwargs: session = self.kwargs["session"] if not session._trace_configs: trace_config.freeze() session._trace_configs = [trace_config] return session return aiohttp.ClientSession(trace_configs=[trace_config])
def add_trace_config(): trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_dns_resolvehost_start.append(on_dns_resolvehost_start) trace_config.on_dns_cache_hit.append(on_dns_cache_hit) trace_config.on_dns_cache_miss.append(on_dns_cache_miss) trace_config.on_dns_resolvehost_end.append(on_dns_resolvehost_end) trace_config.on_request_end.append(on_request_end) trace_config.on_request_redirect.append(on_request_redirect) trace_config.on_connection_create_start.append(on_connection_create_start) trace_config.on_connection_create_end.append(on_connection_create_end) return trace_config
def create_async(self): # pylint: disable=import-outside-toplevel import io import aiohttp import elasticsearch from elasticsearch.serializer import JSONSerializer import esrally.async_connection class LazyJSONSerializer(JSONSerializer): def loads(self, s): meta = RallyAsyncElasticsearch.request_context.get() if "raw_response" in meta: return io.BytesIO(s) else: return super().loads(s) async def on_request_start(session, trace_config_ctx, params): RallyAsyncElasticsearch.on_request_start() async def on_request_end(session, trace_config_ctx, params): RallyAsyncElasticsearch.on_request_end() trace_config = aiohttp.TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) # ensure that we also stop the timer when a request "ends" with an exception (e.g. a timeout) trace_config.on_request_exception.append(on_request_end) # override the builtin JSON serializer self.client_options["serializer"] = LazyJSONSerializer() self.client_options["trace_config"] = trace_config class VerifiedAsyncTransport(elasticsearch.AsyncTransport): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # skip verification at this point; we've already verified this earlier with the synchronous client. # The async client is used in the hot code path and we use customized overrides (such as that we don't # parse response bodies in some cases for performance reasons, e.g. when using the bulk API). self._verified_elasticsearch = True class RallyAsyncElasticsearch(elasticsearch.AsyncElasticsearch, RequestContextHolder): pass return RallyAsyncElasticsearch( hosts=self.hosts, transport_class=VerifiedAsyncTransport, connection_class=esrally.async_connection.AIOHttpConnection, ssl_context=self.ssl_context, **self.client_options, )