def to_http_response(self, result: InferenceResult) -> HTTPResponse:
return HTTPResponse.new(
status=result.http_status,
headers=result.http_headers,
body=result.err_msg or result.data,
)
async def _batch_handler_template(self, requests, api_route, max_latency):
'''
batch request handler
params:
* requests: list of aiohttp request
* api_route: called API name
raise:
* RemoteException: known exceptions from model server
* Exception: other exceptions
'''
from aiohttp.client_exceptions import ClientConnectionError
from aiohttp import ClientTimeout
from aiohttp.web import Response
headers = {MARSHAL_REQUEST_HEADER: "true"}
api_url = f"http://{self.outbound_host}:{self.outbound_port}/{api_route}"
with self.tracer.async_span(
service_name=self.__class__.__name__,
span_name=f"[2]merged {api_route}",
request_headers=headers,
):
reqs_s = DataLoader.merge_requests(requests)
try:
client = self.get_client()
timeout = ClientTimeout(
total=(self.mb_max_latency or max_latency) // 1000
)
async with client.post(
api_url, data=reqs_s, headers=headers, timeout=timeout
) as resp:
raw = await resp.read()
except ClientConnectionError as e:
raise RemoteException(
e, payload=HTTPResponse.new(status=503, body=b"Service Unavailable")
)
except asyncio.CancelledError as e:
raise RemoteException(
e,
payload=HTTPResponse(
status=500, body=b"Cancelled before upstream responses"
),
)
except asyncio.TimeoutError as e:
raise RemoteException(
e, payload=HTTPResponse(status=408, body=b"Request timeout"),
)
if resp.status != 200:
raise RemoteException(
f"Bad response status from model server:\n{resp.status}\n{raw}",
payload=HTTPResponse.new(
status=resp.status,
headers=tuple(resp.headers.items()),
body=raw,
),
)
merged = DataLoader.split_responses(raw)
return tuple(
Response(body=i.body, headers=i.headers, status=i.status or 500)
for i in merged
)