async def request_dispatcher(self, request):
with async_trace(
ZIPKIN_API_URL,
service_name=self.__class__.__name__,
span_name="[1]http request",
is_root=True,
standalone=True,
sample_rate=0.001,
):
api_name = request.match_info.get("name")
if api_name in self.batch_handlers:
req = SimpleRequest(request.raw_headers, await request.read())
try:
resp = await self.batch_handlers[api_name](req)
except RemoteException as e:
# known remote exception
logger.error(traceback.format_exc())
resp = aiohttp.web.Response(
status=e.payload.status,
headers=e.payload.headers,
body=e.payload.data,
)
except Exception: # pylint: disable=broad-except
logger.error(traceback.format_exc())
resp = aiohttp.web.InternalServerError()
else:
resp = await self.relay_handler(request)
return resp
def handle_request(self, request: flask.Request, func):
if request.content_type != "application/json":
raise BadInput(
"Request content-type must be 'application/json' for this "
"BentoService API")
resps = self.handle_batch_request(
[SimpleRequest.from_flask_request(request)], func)
return resps[0].to_flask_response()
def to_response(self, result, request: flask.Request) -> flask.Response:
"""Converts corresponding data into an HTTP response
:param result: result of user API function
:param request: request object
"""
simple_req = SimpleRequest.from_flask_request(request)
simple_resp = self.to_batch_response((result, ),
requests=(simple_req, ))[0]
return simple_resp.to_flask_response()
def handle_request(self, request, func):
"""Handle http request that has jsonlized tensorflow tensor. It will convert it
into a tf tensor for the function to consume.
Args:
request: incoming request object.
func: function that will take ndarray as its arg.
Return:
response object
"""
req = SimpleRequest.from_flask_request(request)
res = self.handle_batch_request([req], func)[0]
return res.to_flask_response()
def test_anno_image_input_batch_request_skip_bad(img_file, json_file):
adapter = AnnotatedImageInput(is_batch_input=True)
multipart_data, headers = generate_multipart_body(img_file, json_file)
empty_request = SimpleRequest(headers=headers, data=None)
request = SimpleRequest.from_flask_request(
Request.from_values(
data=multipart_data,
content_type=headers['Content-Type'],
content_length=headers['Content-Length'],
))
image = ("image.jpg", open(img_file, "rb").read())
json = ("annotations.jso", open(json_file, "rb").read())
files = {"image.invalid": image, "annotations.invalid": json}
bad_data, content_type = encode_multipart_formdata(files)
bad_request = SimpleRequest.from_flask_request(
Request.from_values(
data=bad_data,
content_type=content_type,
content_length=len(bad_data),
))
responses = adapter.handle_batch_request(
[empty_request, request, bad_request], predict_image_and_json)
assert len(responses) == 3
assert responses[0] is None
assert responses[1].status == 200 and responses[
1].data == '[[10, 10, 3], "kaith"]'
assert responses[2] is None
bad_responses = adapter.handle_batch_request([empty_request],
predict_image_and_json)
assert len(bad_responses) == 1
assert bad_responses[0] is None
def test_bad_multi_image_batch_input(img_file):
adapter = MultiImageInput(("imageX", "imageY"), is_batch_input=True)
multipart_data, headers = generate_multipart_body(img_file)
request = SimpleRequest.from_flask_request(
Request.from_values(
data=multipart_data,
content_type=headers['Content-Type'],
content_length=headers['Content-Length'],
))
responses = adapter.handle_batch_request(
[request] * 5 + [
SimpleRequest.from_flask_request(
Request.from_values(
data=multipart_data,
content_type='application/octet-stream',
content_length=headers['Content-Length'],
))
],
predict,
)
print(responses[-1])
assert isinstance(responses[-1], SimpleRequest)
def test_multi_image_batch_input(img_file):
adapter = MultiImageInput(("imageX", "imageY"), is_batch_input=True)
multipart_data, headers = generate_multipart_body(img_file)
request = SimpleRequest.from_flask_request(
Request.from_values(
data=multipart_data,
content_type=headers['Content-Type'],
content_length=headers['Content-Length'],
))
responses = adapter.handle_batch_request([request] * 5, predict)
for response in responses:
assert response.status == 200
assert response.data == '[[10, 10, 3], [10, 10, 3]]'
def to_response(self, result, request: flask.Request) -> flask.Response:
"""Converts corresponding data into an HTTP response
:param result: result of user API function
:param request: request object
"""
simple_req = SimpleRequest(headers=request.headers,
data=request.get_data())
simple_resp = self.to_batch_response((result, ),
requests=(simple_req, ))[0]
return flask.Response(
response=simple_resp.data,
status=simple_resp.status,
headers=simple_resp.headers,
)
async def request_dispatcher(self, request):
with async_trace(
ZIPKIN_API_URL,
service_name=self.__class__.__name__,
span_name=f"[1]http request",
is_root=True,
standalone=True,
sample_rate=0.001,
):
api_name = request.match_info.get("name")
if api_name in self.batch_handlers:
req = SimpleRequest(request.raw_headers, await request.read())
resp = await self.batch_handlers[api_name](req)
else:
resp = await self.relay_handler(request)
return resp
def test_anno_image_input_batch_request(img_file, json_file):
adapter = AnnotatedImageInput(is_batch_input=True)
multipart_data, headers = generate_multipart_body(img_file, json_file)
request = SimpleRequest.from_flask_request(
Request.from_values(
data=multipart_data,
content_type=headers['Content-Type'],
content_length=headers['Content-Length'],
))
responses = adapter.handle_batch_request([request] * 5,
predict_image_and_json)
for response in responses:
assert response.status == 200
assert response.data == '[[10, 10, 3], "kaith"]'
def test_file_input_http_request_post_binary(bin_file):
test_file_input = FileInput()
request = mock.MagicMock(spec=flask.Request)
request.method = "POST"
request.files = {}
request.headers = {}
request.get_data.return_value = open(str(bin_file), 'rb').read()
response = test_file_input.handle_request(request, predict)
assert response.status_code == 200
assert b'{"b64": "gTCJOQ=="}' in response.data
simple_request = SimpleRequest.from_flask_request(request)
responses = test_file_input.handle_batch_request([simple_request], predict)
assert responses[0].status == 200
assert '{"b64": "gTCJOQ=="}' == responses[0].data
def test_image_input_http_request_post_binary(img_file):
test_image_input = ImageInput()
request = mock.MagicMock(spec=flask.Request)
request.method = "POST"
request.files = {}
request.headers = {}
request.get_data.return_value = open(str(img_file), 'rb').read()
response = test_image_input.handle_request(request, predict)
assert response.status_code == 200
assert "[10, 10, 3]" in str(response.response)
simple_request = SimpleRequest.from_flask_request(request)
responses = test_image_input.handle_batch_request([simple_request],
predict)
assert responses[0].status == 200
assert "[10, 10, 3]" in str(responses[0].data)
def test_tf_tensor_handle_batch_request(test_cases):
'''
ref: https://www.tensorflow.org/tfx/serving/api_rest#request_format_2
'''
from bentoml.adapters import TfTensorInput
from bentoml.marshal.utils import SimpleRequest
input_adapter = TfTensorInput()
request = MagicMock(spec=flask.Request)
input_data, headers, except_result = test_cases
request.get_data.return_value = json.dumps(input_data).encode('utf-8')
request.headers = headers
responses = input_adapter.handle_batch_request(
[SimpleRequest.from_flask_request(request)] * 3, lambda i: i)
for response in responses:
prediction = json.loads(response.data)
assert_eq_or_both_nan(except_result, prediction)