async def _batch_handler_template(self, requests, api_name): ''' batch request handler params: * requests: list of aiohttp request * api_name: called API name raise: * RemoteException: known exceptions from model server * Exception: other exceptions ''' headers = {self._MARSHAL_FLAG: "true"} api_url = f"http://{self.outbound_host}:{self.outbound_port}/{api_name}" with async_trace( ZIPKIN_API_URL, service_name=self.__class__.__name__, span_name=f"[2]merged {api_name}", ) as trace_ctx: headers.update(make_http_headers(trace_ctx)) reqs_s = DataLoader.merge_requests(requests) try: async with aiohttp.ClientSession( auto_decompress=False) as client: async with client.post(api_url, data=reqs_s, headers=headers) as resp: raw = await resp.read() except aiohttp.client_exceptions.ClientConnectionError as e: raise RemoteException(e, payload=HTTPResponse( status=503, body=b"Service Unavailable")) if resp.status != 200: raise RemoteException( f"Bad response status from model server:\n{resp.status}\n{raw}", payload=HTTPResponse( status=resp.status, headers=tuple(resp.headers.items()), body=raw, ), ) merged = DataLoader.split_responses(raw) return tuple( aiohttp.web.Response( body=i.body, headers=i.headers, status=i.status or 500) for i in merged)
async def _batch_handler_template(self, requests, api_route): ''' batch request handler params: * requests: list of aiohttp request * api_route: called API name raise: * RemoteException: known exceptions from model server * Exception: other exceptions ''' from aiohttp.client_exceptions import ClientConnectionError from aiohttp.web import Response headers = {MARSHAL_REQUEST_HEADER: "true"} api_url = f"http://{self.outbound_host}:{self.outbound_port}/{api_route}" with get_tracer().async_span( service_name=self.__class__.__name__, span_name=f"[2]merged {api_route}", request_headers=headers, ): reqs_s = DataLoader.merge_requests(requests) try: client = self.get_client() async with client.post(api_url, data=reqs_s, headers=headers) as resp: raw = await resp.read() except ClientConnectionError as e: raise RemoteException(e, payload=HTTPResponse( status=503, body=b"Service Unavailable")) if resp.status != 200: raise RemoteException( f"Bad response status from model server:\n{resp.status}\n{raw}", payload=HTTPResponse( status=resp.status, headers=tuple(resp.headers.items()), body=raw, ), ) merged = DataLoader.split_responses(raw) return tuple( Response( body=i.body, headers=i.headers, status=i.status or 500) for i in merged)
def to_http_response( self, results: Iterable[InferenceResult], ) -> Iterator[HTTPResponse]: return ( HTTPResponse( r.context.http_status, tuple(r.context.http_headers.items()), r.context.err_msg or r.data, ) for r in results )
def to_http_response(self, result: InferenceResult) -> HTTPResponse: return HTTPResponse.new( status=result.http_status, headers=result.http_headers, body=result.err_msg or result.data, )
def to_http_response(self, result: InferenceResult) -> HTTPResponse: return HTTPResponse( status=result.http_status, headers=tuple(result.http_headers.items()), body=result.err_msg or result.data, )