def test_json_from_http(input_adapter, raw_jsons):
requests = [HTTPRequest(body=r) for r in raw_jsons]
requests += [
HTTPRequest(body=gzip.compress(r),
headers=(("Content-Encoding", "gzip"), ))
for r in raw_jsons
]
requests += [
HTTPRequest(body=r, headers=(("Content-Encoding", "gzip"), ))
for r in raw_jsons
]
requests += [
HTTPRequest(body=r, headers=(("Content-Encoding", "compress"), ))
for r in raw_jsons
]
tasks = map(input_adapter.from_http_request, requests)
iter_tasks = iter(tasks)
for b, t in zip(raw_jsons, iter_tasks):
assert t.data == b
for b, t in zip(raw_jsons, iter_tasks):
assert t.data == b
for b, t in zip(raw_jsons, iter_tasks):
assert t.is_discarded
for b, t in zip(raw_jsons, iter_tasks):
assert t.is_discarded
def from_http_request(self, req: HTTPRequest) -> MultiImgTask:
if len(self.input_names) == 1:
# broad parsing while single input
if req.parsed_headers.content_type == 'multipart/form-data':
_, _, files = HTTPRequest.parse_form_data(req)
if not any(files):
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=
f"BentoML#{self.__class__.__name__} requires inputs"
f"fields {self.input_names}",
)
else:
f = next(iter(files.values()))
task = InferenceTask(
context=InferenceContext(
http_headers=req.parsed_headers),
data=(f, ),
)
else:
# for images/*
task = InferenceTask(
context=InferenceContext(http_headers=req.parsed_headers),
data=(io.BytesIO(req.body), ),
)
elif req.parsed_headers.content_type == 'multipart/form-data':
_, _, files = HTTPRequest.parse_form_data(req)
files = tuple(files.get(k) for k in self.input_names)
if not any(files):
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=f"BentoML#{self.__class__.__name__} requires inputs "
f"fields {self.input_names}",
)
elif not all(files) and not self.allow_none:
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=f"BentoML#{self.__class__.__name__} requires inputs "
f"fields {self.input_names}",
)
else:
task = InferenceTask(
context=InferenceContext(http_headers=req.parsed_headers),
data=files,
)
else:
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=
f"BentoML#{self.__class__.__name__} with multiple inputs "
"accepts requests with Content-Type: multipart/form-data only",
)
return task
def from_http_request(self, req: HTTPRequest) -> InferenceTask[str]:
if req.headers.content_type == 'multipart/form-data':
_, _, files = HTTPRequest.parse_form_data(req)
if len(files) != 1:
return InferenceTask().discard(
http_status=400,
err_msg=
f"BentoML#{self.__class__.__name__} accepts one text file "
"at a time",
)
input_file = next(iter(files.values()))
bytes_ = input_file.read()
charset = chardet.detect(bytes_)['encoding'] or "utf-8"
else:
bytes_ = req.body
charset = req.headers.charset or "utf-8"
try:
return InferenceTask(
http_headers=req.headers,
data=bytes_.decode(charset),
)
except UnicodeDecodeError:
return InferenceTask().discard(
http_status=400,
err_msg=
f"{self.__class__.__name__}: UnicodeDecodeError for {req.body}",
)
except LookupError:
return InferenceTask().discard(
http_status=400,
err_msg=
f"{self.__class__.__name__}: Unsupported charset {req.charset}",
)
def from_http_request(self, req: HTTPRequest) -> InferenceTask[BinaryIO]:
if req.parsed_headers.content_type == 'multipart/form-data':
_, _, files = HTTPRequest.parse_form_data(req)
if len(files) != 1:
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=
f"BentoML#{self.__class__.__name__} requires one and at"
" least one file at a time, if you just upgraded from"
" bentoml 0.7, you may need to use MultiFileAdapter instead",
)
else:
input_file = next(iter(files.values()))
task = InferenceTask(
context=InferenceContext(http_headers=req.parsed_headers),
data=input_file,
)
elif req.body:
task = InferenceTask(
context=InferenceContext(http_headers=req.parsed_headers),
data=io.BytesIO(req.body),
)
else:
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=
f'BentoML#{self.__class__.__name__} unexpected HTTP request'
' format',
)
return task
def from_http_request(self, req: HTTPRequest) -> MultiFileTask:
if req.headers.content_type != 'multipart/form-data':
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=
f"BentoML#{self.__class__.__name__} only accepts requests "
"with Content-Type: multipart/form-data",
)
else:
_, _, files = HTTPRequest.parse_form_data(req)
files = tuple(files.get(k) for k in self.input_names)
if not any(files):
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=f"BentoML#{self.__class__.__name__} requires inputs "
f"fields {self.input_names}",
)
elif not all(files) and not self.allow_none:
task = InferenceTask(data=None)
task.discard(
http_status=400,
err_msg=f"BentoML#{self.__class__.__name__} requires inputs "
f"fields {self.input_names}",
)
else:
task = InferenceTask(
http_headers=req.headers,
data=files,
)
return task
async def request_dispatcher(self, request):
with async_trace(
ZIPKIN_API_URL,
service_name=self.__class__.__name__,
span_name="[1]http request",
is_root=True,
standalone=True,
sample_rate=0.001,
):
api_name = request.match_info.get("name")
if api_name in self.batch_handlers:
req = HTTPRequest(
tuple((k.decode(), v.decode())
for k, v in request.raw_headers),
await request.read(),
)
try:
resp = await self.batch_handlers[api_name](req)
except RemoteException as e:
# known remote exception
logger.error(traceback.format_exc())
resp = aiohttp.web.Response(
status=e.payload.status,
headers=e.payload.headers,
body=e.payload.body,
)
except Exception: # pylint: disable=broad-except
logger.error(traceback.format_exc())
resp = aiohttp.web.HTTPInternalServerError()
else:
resp = await self.relay_handler(request)
return resp
def handle_request(self, request: flask.Request):
req = HTTPRequest.from_flask_request(request)
inf_task = self.input_adapter.from_http_request(req)
results = self.infer((inf_task, ))
result = next(iter(results))
response = self.output_adapter.to_http_response(result)
return response.to_flask_response()
def test_file_input_http_request_post_binary(input_adapter, img_bytes_list):
img_bytes = img_bytes_list[0]
# post binary
request = HTTPRequest(body=img_bytes)
task = input_adapter.from_http_request(request)
assert img_bytes == task.data.read()
# post as multipart/form-data
headers = (("Content-Type", "multipart/form-data; boundary=123456"), )
body = (
b'--123456\n' +
b'Content-Disposition: form-data; name="file"; filename="text.jpg"\n' +
b'Content-Type: application/octet-stream\n\n' + img_bytes +
b'\n--123456--\n')
request = HTTPRequest(headers=headers, body=body)
task = input_adapter.from_http_request(request)
assert img_bytes == task.data.read()
def test_dataframe_handle_request_csv(make_api):
def test_func(df):
return df["name"]
input_adapter = DataframeInput()
api = make_api(input_adapter, test_func)
csv_data = b'name,game,city\njohn,mario,sf'
request = HTTPRequest(headers={'Content-Type': 'text/csv'}, body=csv_data)
result = api.handle_request(request)
assert result.body == '[{"name":"john"}]'
def test_anno_image_input_http_request_multipart_form_image_only(
input_adapter, img_file):
body, content_type = encode_multipart_formdata(
dict(image=("test.jpg", read_bin(img_file)), ))
headers = (("Content-Type", content_type), )
request = HTTPRequest(headers=headers, body=body)
task = input_adapter.from_http_request(request)
assert task.data[0].read() == read_bin(img_file)
assert task.data[1] is None
def test_file_input_http_request_malformatted_input_missing_file(
input_adapter, bin_file):
file_bytes = open(str(bin_file), 'rb').read()
requests = []
body = b''
headers = (("Content-Type", "multipart/form-data; boundary=123456"), )
requests.append(HTTPRequest(headers=headers, body=body))
body = file_bytes
headers = (("Content-Type", "images/jpeg"), )
requests.append(HTTPRequest(headers=headers, body=body))
body, content_type = encode_multipart_formdata(
dict(x=("test.bin", file_bytes), ))
headers = (("Content-Type", content_type), )
requests.append(HTTPRequest(headers=headers, body=body))
for task in map(input_adapter.from_http_request, requests):
assert task.is_discarded
def test_anno_image_input_http_request_malformatted_input_wrong_input_name(
input_adapter, img_file, json_file):
body, content_type = encode_multipart_formdata(
dict(
wrong_image=("test.jpg", read_bin(img_file)),
wrong_annotations=("test.json", read_bin(json_file)),
))
headers = (("Content-Type", content_type), )
request = HTTPRequest(headers=headers, body=body)
task = input_adapter.from_http_request(request)
assert task.is_discarded
def _method(self, req: HTTPRequest) -> InferenceTask:
if req.headers.content_encoding in {"gzip", "x-gzip"}:
# https://tools.ietf.org/html/rfc7230#section-4.2.3
try:
req.body = gzip.decompress(req.body)
except OSError:
return InferenceTask().discard(
http_status=400, err_msg="Gzip decompression error")
req.headers.pop("content-encoding")
return method(self, req)
return method(self, req)
def test_file_input_http_request_multipart_form(input_adapter, bin_file):
file_bytes = open(str(bin_file), 'rb').read()
headers = (("Content-Type", "multipart/form-data; boundary=123456"), )
body = (
b'--123456\n' +
b'Content-Disposition: form-data; name="file"; filename="text.png"\n' +
b'Content-Type: application/octet-stream\n\n' + file_bytes +
b'\n--123456--\n')
request = HTTPRequest(headers=headers, body=body)
task = input_adapter.from_http_request(request)
assert b'\x810\x899' == task.data.read()
def test_file_input_http_request_multipart_form(input_adapter, bin_file):
file_bytes = open(str(bin_file), 'rb').read()
body, content_type = encode_multipart_formdata(
dict(
x=("test.bin", file_bytes),
y=("test.bin", file_bytes),
))
headers = (("Content-Type", content_type), )
request = HTTPRequest(headers=headers, body=body)
task = input_adapter.from_http_request(request)
assert b'\x810\x899' == task.data[0].read()
assert b'\x810\x899' == task.data[1].read()
def test_file_input_http_request_none_file(bin_file):
file_bytes = open(str(bin_file), 'rb').read()
allow_none_input_adapter = MultiFileInput(input_names=["x", "y"],
allow_none=True)
body, content_type = encode_multipart_formdata(
dict(x=("test.bin", file_bytes), ))
headers = (("Content-Type", content_type), )
request = HTTPRequest(headers=headers, body=body)
task = allow_none_input_adapter.from_http_request(request)
assert not task.is_discarded
assert b'\x810\x899' == task.data[0].read()
assert task.data[1] is None
def test_anno_image_input_http_request_too_many_files(input_adapter, img_file,
json_file):
body, content_type = encode_multipart_formdata(
dict(
image=("test.jpg", read_bin(img_file)),
image2=("test.jpg", read_bin(img_file)),
annotations=("test.json", read_bin(json_file)),
))
headers = (("Content-Type", content_type), )
request = HTTPRequest(headers=headers, body=body)
task = input_adapter.from_http_request(request)
assert task.data[0].read() == read_bin(img_file)
assert task.data[1].read() == read_bin(json_file)
def test_tf_tensor_handle_request(make_api, test_cases):
'''
ref: https://www.tensorflow.org/tfx/serving/api_rest#request_format_2
'''
from bentoml.adapters import TfTensorInput
api = make_api(input_adapter=TfTensorInput(), user_func=lambda i: i)
input_data, headers, except_result = test_cases
body = json.dumps(input_data).encode('utf-8')
request = HTTPRequest(headers=headers, body=body)
response = tuple(api.handle_batch_request([request]))[0]
prediction = json.loads(response.body)
assert_eq_or_both_nan(except_result, prediction)
def test_string_input(make_api):
from bentoml.adapters import StringInput, JsonOutput
api = make_api(
input_adapter=StringInput(), output_adapter=JsonOutput(), user_func=lambda i: i,
)
body = b'{"a": 1}'
request = HTTPRequest(body=body)
response = api.handle_request(request)
assert json.loads(response.body) == body.decode()
responses = api.handle_batch_request([request] * 3)
for response in responses:
assert json.loads(response.body) == body.decode()
def api_func():
# handle_request may raise 4xx or 5xx exception.
try:
if request.headers.get(MARSHAL_REQUEST_HEADER):
reqs = DataLoader.split_requests(request.get_data())
responses = api.handle_batch_request(reqs)
response_body = DataLoader.merge_responses(responses)
response = make_response(response_body)
else:
req = HTTPRequest.from_flask_request(request)
resp = api.handle_request(req)
response = resp.to_flask_response()
except BentoMLException as e:
log_exception(sys.exc_info())
if 400 <= e.status_code < 500 and e.status_code not in (401,
403):
response = make_response(
jsonify(message=
"BentoService error handling API request: %s" %
str(e)),
e.status_code,
)
else:
response = make_response('', e.status_code)
except Exception: # pylint: disable=broad-except
# For all unexpected error, return 500 by default. For example,
# if users' model raises an error of division by zero.
log_exception(sys.exc_info())
response = make_response(
'An error has occurred in BentoML user code when handling this '
'request, find the error details in server logs',
500,
)
return response
def test_file_input_http_request_post_binary(input_adapter, bin_file):
with open(str(bin_file), 'rb') as f:
request = HTTPRequest(body=f.read())
task = input_adapter.from_http_request(request)
assert b'\x810\x899' == task.data.read()
def test_file_input_http_request_malformatted_input_missing_file(input_adapter):
request = HTTPRequest(body=None)
task = input_adapter.from_http_request(request)
assert task.is_discarded
def from_aws_lambda_event(self, event: AwsLambdaEvent) -> MultiFileTask:
request = HTTPRequest(
headers=tuple((k, v) for k, v in event.get('headers', {}).items()),
body=event['body'],
)
return self.from_http_request(request)
def view_function():
req = HTTPRequest.from_flask_request(request)
response = api.handle_request(req)
return response.to_flask_response()
