def test_file_input_cli(capsys, bin_file):
test_file_input = FileInput()
test_args = ["--input", bin_file]
test_file_input.handle_cli(test_args, predict)
out, _ = capsys.readouterr()
assert '\\x810\\x899' in out
class ExampleBentoServiceSingle(ExampleBentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(
input=MultiImageInput(input_names=('original', 'compared')), batch=False
)
def predict_legacy_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original], [compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(
input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False
)
def predict_with_sklearn(self, json):
return self.artifacts.sk_model.predict([json])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
def test_file_input_cli_list(capsys, bin_files):
test_file_input = FileInput()
test_args = ["--input"] + glob.glob(bin_files)
test_file_input.handle_cli(test_args, predict)
out, _ = capsys.readouterr()
lines = out.strip().split('\n')
for line in lines[-10:]:
assert '\\x810\\x899' in line
class NonBatchExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original],
[compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=False)
def predict_with_sklearn(self, json_value):
return self.artifacts.sk_model.predict([json_value])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_strict_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
task.discard(http_status=400, err_msg="application/json only")
return
result = self.artifacts.model.predict_json([input_data])[0]
return result
@bentoml.api(input=JsonInput(), batch=False)
def predict_direct_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
return InferenceError(http_status=400,
err_msg="application/json only")
result = self.artifacts.model.predict_json([input_data])[0]
return InferenceResult(http_status=200, data=json.dumps(result))
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_batch_size(self, input_datas=10):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
batch_size = len(input_datas)
return [batch_size] * batch_size
class MnistTensorflow(bentoml.BentoService):
@bentoml.api(input=FileInput(), batch=True)
def predict(self, inputs):
loaded_func = self.artifacts.trackable.signatures[
tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
inputs = [i.read() for i in inputs]
inputs = tf.constant(inputs, dtype=tf.string)
pred = loaded_func(raw_inputs=inputs)
output = pred['outputs']
return output
class ExampleBentoService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True)
def predict_with_sklearn(self, jsons):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=LegacyImageInput(input_names=('original', 'compared')))
def predict_legacy_images(self, original, compared):
return self.artifacts.model.predict_legacy_images(original, compared)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
def test_file_input_http_request_malformatted_input_wrong_input_name():
test_file_input = FileInput()
request = mock.MagicMock(spec=flask.Request)
request.method = "POST"
request.files = {"abc": None}
request.headers = {}
request.get_data.return_value = None
with pytest.raises(BadInput) as e:
test_file_input.handle_request(request, predict)
assert "unexpected HTTP request format" in str(e.value)
def test_file_input_http_request_post_binary(bin_file):
test_file_input = FileInput()
request = mock.MagicMock(spec=flask.Request)
request.method = "POST"
request.files = {}
request.headers = {}
request.get_data.return_value = open(str(bin_file), 'rb').read()
response = test_file_input.handle_request(request, predict)
assert response.status_code == 200
assert b'{"b64": "gTCJOQ=="}' in response.data
simple_request = SimpleRequest.from_flask_request(request)
responses = test_file_input.handle_batch_request([simple_request], predict)
assert responses[0].status == 200
assert '{"b64": "gTCJOQ=="}' == responses[0].data
class ExampleBentoServiceSingle(ExampleBentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original],
[compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=False)
def predict_with_sklearn(self, json):
return self.artifacts.sk_model.predict([json])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_strict_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
task.discard(http_status=400, err_msg="application/json only")
return
result = self.artifacts.model.predict_json([input_data])[0]
return result
@bentoml.api(input=JsonInput(), batch=False)
def predict_direct_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
return InferenceError(http_status=400,
err_msg="application/json only")
result = self.artifacts.model.predict_json([input_data])[0]
return InferenceResult(http_status=200, data=json.dumps(result))
class PytorchImageClassifier(bentoml.BentoService):
@bentoml.utils.cached_property
def transform(self):
return transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
@bentoml.api(input=FileInput(), batch=True)
def predict(self, file_streams: List[BinaryIO]) -> List[str]:
input_datas = []
for fs in file_streams:
img = Image.open(fs).resize((32, 32))
input_datas.append(self.transform(img))
outputs = self.artifacts.net(Variable(torch.stack(input_datas)))
_, output_classes = outputs.max(dim=1)
return [classes[output_class] for output_class in output_classes]
def test_file_input_aws_lambda_event(bin_file):
test_file_input = FileInput()
with open(str(bin_file), "rb") as file_file:
content = file_file.read()
try:
file_bytes_encoded = base64.encodebytes(content)
except AttributeError:
file_bytes_encoded = base64.encodebytes(str(bin_file))
aws_lambda_event = {
"body": file_bytes_encoded,
"headers": {
"Content-Type": "images/png"
},
}
aws_result = test_file_input.handle_aws_lambda_event(
aws_lambda_event, predict)
assert aws_result["statusCode"] == 200
assert aws_result["body"] == '{"b64": "gTCJOQ=="}'
def test_file_input_http_request_multipart_form(bin_file):
test_file_input = FileInput()
request = mock.MagicMock(spec=flask.Request)
file_bytes = open(str(bin_file), 'rb').read()
file_attr = {
'filename': 'test_img.png',
'read.return_value': file_bytes,
'stream': io.BytesIO(file_bytes),
}
file = mock.Mock(**file_attr)
request.method = "POST"
request.files = {"file_file": file}
request.headers = {}
request.get_data.return_value = None
response = test_file_input.handle_request(request, predict)
assert response.status_code == 200
assert b'{"b64": "gTCJOQ=="}' in response.data
class DocparserService(bentoml.BentoService):
def __init__(self):
super().__init__()
model_config = InferenceConfig()
self.preprocess_obj = ForwardModel(model_config)
def preprocess(self, image):
images = np.expand_dims(image, axis=0)[:,:,:,:3]
molded_images, image_metas, windows = self.preprocess_obj.mold_inputs(images)
molded_images = molded_images.astype(np.float32)
image_shape = molded_images[0].shape
for g in molded_images[1:]:
assert g.shape == image_shape, \
"After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes."
anchors = self.preprocess_obj.get_anchors(image_shape)
anchors = np.broadcast_to(anchors, (images.shape[0],) + anchors.shape)
self.images = images
self.molded_images = molded_images
self.windows = windows
# response body format row wise.
return {
INPUT_IMAGE: molded_images[0].tolist(),
INPUT_IMAGE_META: image_metas[0].tolist(),
INPUT_ANCHORS: anchors[0].tolist()
}
@bentoml.api(input=FileInput(), batch=True)
def predict(self, inputs):
# print(inputs)
inputs = [self.preprocess(Image.open(io.BytesIO(i.read()))) for i in inputs]
inputs = {
INPUT_ANCHORS: [i[INPUT_ANCHORS] for i in inputs],
INPUT_IMAGE: [i[INPUT_IMAGE] for i in inputs],
INPUT_IMAGE_META: [i[INPUT_IMAGE_META] for i in inputs],
}
loaded_func = self.artifacts.trackable.signatures[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
input_anchors = tf.constant(inputs[INPUT_ANCHORS], dtype=tf.float32)
input_image = tf.constant(inputs[INPUT_IMAGE], dtype=tf.float32)
input_image_meta = tf.constant(inputs[INPUT_IMAGE_META], dtype=tf.float32)
pred = loaded_func(
input_anchors=input_anchors,
input_image=input_image,
input_image_meta=input_image_meta,
)
results = self.preprocess_obj.result_to_dict(self.images, self.molded_images, self.windows, pred)
outputs = []
structure_parser = stage2_structure_parser.StructureParser()
for idx, r in enumerate(results):
pred_bboxes = r['rois']
pred_class_ids = r['class_ids']
pred_scores = r['scores']
classes_with_background = ['background'] + DocsDataset.ALL_CLASSES
orig_shape = list(self.images[idx].shape)
num_preds = len(pred_bboxes)
prediction_list = []
for pred_nr in range(num_preds):
class_name = classes_with_background[pred_class_ids[pred_nr]]
pred_bbox = pred_bboxes[pred_nr]
pred_score = pred_scores[pred_nr]
prediction_list.append({'pred_nr': pred_nr, 'class_name': class_name, 'pred_score': pred_score,
'bbox_orig_coords': pred_bbox, 'orig_img_shape': orig_shape})
img_relations_dict = structure_parser.create_structure_for_doc_from_list({'prediction_list': prediction_list, 'orig_img_shape': orig_shape}, do_postprocessing=True)
predictions_dict = convert_bbox_list_to_save_format(img_relations_dict['all_bboxes'])
try:
del img_relations_dict['all_bboxes']
except KeyError:
pass
predictions_dict["relations"] = img_relations_dict
outputs.append(predictions_dict)
return outputs
# bento_svc = DocparserService()
# bento_svc.pack("trackable", "serving_model/1")
# saved_path = bento_svc.save()
class ExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated
def predict_dataframe_v1(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(
input=MultiImageInput(input_names=('original', 'compared')), batch=True
)
def predict_multi_images(self, originals, compareds):
return self.artifacts.model.predict_multi_images(originals, compareds)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(
input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True,
)
def predict_with_sklearn(self, jsons):
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict"
@bentoml.api(
route=CUSTOM_ROUTE, input=JsonInput(), batch=True,
)
def customezed_route(self, input_datas):
return input_datas
CUSTOM_SCHEMA = {
"application/json": {
"schema": {
"type": "object",
"required": ["field1", "field2"],
"properties": {
"field1": {"type": "string"},
"field2": {"type": "uuid"},
},
},
}
}
@bentoml.api(input=JsonInput(request_schema=CUSTOM_SCHEMA), batch=True)
def customezed_schema(self, input_datas):
return input_datas
@bentoml.api(input=JsonInput(), batch=True)
def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
return self.artifacts.model.predict_json(filtered_jsons)
@bentoml.api(input=JsonInput(), batch=True)
def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
rets = self.artifacts.model.predict_json(filtered_jsons)
return [
InferenceResult(http_status=200, data=json.dumps(result)) for result in rets
]
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_batch_size(self, input_datas=10):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
batch_size = len(input_datas)
return [batch_size] * batch_size
@bentoml.api(input=JsonInput())
def echo_json(self, input_data):
return input_data
if version.parse(BENTOML_VERSION) > version.parse("0.12.1+0"):
@bentoml.api(input=JsonInput(), output=JsonOutput(ensure_ascii=True))
def echo_json_ensure_ascii(self, input_data):
return input_data
class TextDetectionService(BentoService):
@api(input=FileInput())
def predict(self, image):
result = self.artifacts.model.detect_text(image[0])
return result
class ExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=True)
def predict_multi_images(self, originals, compareds):
return self.artifacts.model.predict_multi_images(originals, compareds)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(
input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_with_sklearn(self, jsons):
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict"
@bentoml.api(
route=CUSTOM_ROUTE,
input=JsonInput(),
batch=True,
)
def customezed_route(self, input_datas):
return input_datas
@bentoml.api(input=JsonInput(), batch=True)
def predict_strict_json(self,
input_datas,
tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
return self.artifacts.model.predict_json(filtered_jsons)
@bentoml.api(input=JsonInput(), batch=True)
def predict_direct_json(self,
input_datas,
tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
rets = self.artifacts.model.predict_json(filtered_jsons)
return [
InferenceResult(http_status=200, data=json.dumps(result))
for result in rets
]
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
def input_adapter():
return FileInput()