def test_image_input_cli(capsys, img_file):
test_image_input = ImageInput()
test_args = ["--input", img_file]
test_image_input.handle_cli(test_args, predict)
out, _ = capsys.readouterr()
assert out.strip().endswith("(10, 10, 3)")
def test_image_input_cli_list(capsys, img_files):
test_image_input = ImageInput()
test_args = ["--input"] + glob.glob(img_files)
test_image_input.handle_cli(test_args, predict)
out, _ = capsys.readouterr()
lines = out.strip().split('\n')
for line in lines[-10:]:
assert line.strip().endswith("(10, 10, 3)")
def test_image_input_http_request_malformatted_input_wrong_input_name():
test_image_input = ImageInput()
request = mock.MagicMock(spec=flask.Request)
request.method = "POST"
request.files = {"abc": None}
request.headers = {}
request.get_data.return_value = None
with pytest.raises(BadInput) as e:
test_image_input.handle_request(request, predict)
assert "unexpected HTTP request format" in str(e.value)
class CocoDetectronClassifier(bentoml.BentoService):
@bentoml.api(input=ImageInput(), batch=False)
def predict(self, img: np.ndarray) -> Dict:
_aug = T.ResizeShortestEdge(
[800, 800], 1333
)
boxes = None
scores = None
pred_classes = None
pred_masks= None
try:
original_image = img[:, :, ::-1]
height, width = original_image.shape[:2]
image = _aug.get_transform(original_image).apply_image(original_image)
image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
inputs = {"image": image, "height": height, "width": width}
predictions = self.artifacts.model([inputs])[0]
pred_instances = predictions["instances"]
boxes = (pred_instances.pred_boxes).to("cpu").tensor.detach().numpy()
scores = (pred_instances.scores).to("cpu").detach().numpy()
pred_classes = (pred_instances.pred_classes).to("cpu").detach().numpy()
pred_masks = (pred_instances.pred_masks).to("cpu").detach().numpy()
except Exception as error:
print(f"Caught error whilst processing {error}")
for line in get_traceback_list():
print(line[:-1])
result = {
"boxes" : boxes,
"scores" : scores,
"classes" : pred_classes,
"masks" : pred_masks
}
return result
class EasyOCRService(bentoml.BentoService):
@bentoml.api(input=ImageInput(), batch=False)
def predict(self, image):
reader = self.artifacts.chinese_small
raw_results = reader.readtext(np.array(image))
text_instances = [x[1] for x in raw_results]
return {"text": text_instances}
class EmotionRecognitionService(bentoml.BentoService):
@bentoml.api(input=ImageInput(), output=JsonOutput(), batch=False)
def predict(self, image):
print(image)
return DeepFace.analyze(np.array(image),
actions=['emotion'],
enforce_detection=False)
class ExampleBentoServiceSingle(ExampleBentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(
input=MultiImageInput(input_names=('original', 'compared')), batch=False
)
def predict_legacy_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original], [compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(
input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False
)
def predict_with_sklearn(self, json):
return self.artifacts.sk_model.predict([json])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
class NonBatchExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original],
[compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=False)
def predict_with_sklearn(self, json_value):
return self.artifacts.sk_model.predict([json_value])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_strict_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
task.discard(http_status=400, err_msg="application/json only")
return
result = self.artifacts.model.predict_json([input_data])[0]
return result
@bentoml.api(input=JsonInput(), batch=False)
def predict_direct_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
return InferenceError(http_status=400,
err_msg="application/json only")
result = self.artifacts.model.predict_json([input_data])[0]
return InferenceResult(http_status=200, data=json.dumps(result))
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_batch_size(self, input_datas=10):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
batch_size = len(input_datas)
return [batch_size] * batch_size
def test_image_input_http_request_post_binary(img_file):
test_image_input = ImageInput()
request = mock.MagicMock(spec=flask.Request)
request.method = "POST"
request.files = {}
request.headers = {}
request.get_data.return_value = open(str(img_file), 'rb').read()
response = test_image_input.handle_request(request, predict)
assert response.status_code == 200
assert "[10, 10, 3]" in str(response.response)
simple_request = SimpleRequest.from_flask_request(request)
responses = test_image_input.handle_batch_request([simple_request],
predict)
assert responses[0].status == 200
assert "[10, 10, 3]" in str(responses[0].data)
def test_image_input_http_request_multipart_form(img_file):
test_image_input = ImageInput()
request = mock.MagicMock(spec=flask.Request)
file_bytes = open(str(img_file), 'rb').read()
file_attr = {
'filename': 'test_img.png',
'read.return_value': file_bytes,
'stream': io.BytesIO(file_bytes),
}
file = mock.Mock(**file_attr)
request.method = "POST"
request.files = {"image_file": file}
request.headers = {}
request.get_data.return_value = None
response = test_image_input.handle_request(request, predict)
assert response.status_code == 200
assert "[10, 10, 3]" in str(response.response)
def test_image_input_aws_lambda_event(img_file):
test_image_input = ImageInput()
with open(str(img_file), "rb") as image_file:
content = image_file.read()
try:
image_bytes_encoded = base64.encodebytes(content)
except AttributeError:
image_bytes_encoded = base64.encodebytes(str(img_file))
aws_lambda_event = {
"body": image_bytes_encoded,
"headers": {
"Content-Type": "images/png"
},
}
aws_result = test_image_input.handle_aws_lambda_event(
aws_lambda_event, predict)
assert aws_result["statusCode"] == 200
assert aws_result["body"] == "[10, 10, 3]"
class KerasFashionMnistService(BentoService):
@api(input=ImageInput(pilmode='L'))
def predict(self, imgs):
inputs = []
for img in imgs:
img = Image.fromarray(img).resize((28, 28))
img /= 255.0
img = np.array(img.getdata()).reshape((28, 28, 1))
inputs.append(img)
inputs = np.stack(inputs)
class_idxs = self.artifacts.classifier.predict_classes(inputs)
return [class_names[class_idx] for class_idx in class_idxs]
class ExampleBentoService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True)
def predict_with_sklearn(self, jsons):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=LegacyImageInput(input_names=('original', 'compared')))
def predict_legacy_images(self, original, compared):
return self.artifacts.model.predict_legacy_images(original, compared)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
class ExampleBentoService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=DataframeInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True)
def predict(self, df):
"""An API for testing simple bento model service
"""
return self.artifacts.model.predict(df)
@bentoml.api(input=DataframeInput(dtype={"col1": "int"}), batch=True)
def predict_dataframe(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images(original, compared)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_data):
return self.artifacts.model.predict_json(input_data)
CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict"
@bentoml.api(
route=CUSTOM_ROUTE,
input=JsonInput(),
batch=True,
)
def customize_route(self, input_data):
return input_data
def test_image_input_http_request_single_image_different_name(img_file):
test_image_input = ImageInput(input_names=("my_image",))
request = mock.MagicMock(spec=flask.Request)
file_attr = {
'filename': 'test_img.png',
'read.return_value': open(str(img_file), 'rb').read(),
}
file = mock.Mock(**file_attr)
request.method = "POST"
request.files = {"a_differnt_name_used": file}
request.headers = {}
request.get_data.return_value = None
response = test_image_input.handle_request(request, predict)
assert response.status_code == 200
assert "[10, 10, 3]" in str(response.response)
class ExampleBentoServiceSingle(ExampleBentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original],
[compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=False)
def predict_with_sklearn(self, json):
return self.artifacts.sk_model.predict([json])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_strict_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
task.discard(http_status=400, err_msg="application/json only")
return
result = self.artifacts.model.predict_json([input_data])[0]
return result
@bentoml.api(input=JsonInput(), batch=False)
def predict_direct_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
return InferenceError(http_status=400,
err_msg="application/json only")
result = self.artifacts.model.predict_json([input_data])[0]
return InferenceResult(http_status=200, data=json.dumps(result))
class ExampleBentoService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=DataframeInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True)
def predict(self, df):
"""An API for testing simple bento model service
"""
return self.artifacts.model.predict(df)
@bentoml.api(input=DataframeInput(dtype={"col1": "int"}), batch=True)
def predict_dataframe(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(input=LegacyImageInput(input_names=('original', 'compared')),
batch=False)
def predict_legacy_images(self, original, compared):
return self.artifacts.model.predict_legacy_images(original, compared)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_data):
return self.artifacts.model.predict_json(input_data)
@bentoml.api(input=LegacyJsonInput(), batch=False)
def predict_legacy_json(self, input_data):
return self.artifacts.model.predict_legacy_json(input_data)
class FaceDetectionService(BentoService):
def prepare_input(self, img: np.ndarray) -> np.ndarray:
img = np.transpose(img[:, :, :3], (2, 0, 1))
return np.expand_dims(img, axis=0).astype(np.float32)
def to_json(self, results: np.ndarray) -> Dict:
# results: (N, 6) as x1,y1,x2,y2,score,batch_idx
return {
"boxes": results[:, :4].astype(np.int32).tolist(),
"scores": results[:, 4].astype(np.float32).tolist()
}
@api(input=ImageInput(), batch=True, mb_max_batch_size=8, mb_max_latency=1000)
def detect(self, imgs: List[np.ndarray]):
input_name = self.artifacts.model.get_inputs()[0].name
preds = []
for img in imgs:
results = self.artifacts.model.run(None, {input_name: self.prepare_input(img) })[0]
preds.append(
self.to_json(results)
)
return preds
class DetectronClassifier(bentoml.BentoService):
@bentoml.api(input=ImageInput(), batch=False)
def predict(self, original_image: np.ndarray) -> Dict:
_aug = T.ResizeShortestEdge([800, 800], 1333)
height, width = original_image.shape[:2]
image = _aug.get_transform(original_image).apply_image(original_image)
image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
inputs = {"image": image, "height": height, "width": width}
predictions = self.artifacts.model([inputs])[0]
pred_instances = predictions["instances"]
boxes = (pred_instances.pred_boxes).to("cpu").tensor.detach().numpy()
scores = (pred_instances.scores).to("cpu").detach().numpy()
pred_classes = (pred_instances.pred_classes).to("cpu").detach().numpy()
pred_masks = (pred_instances.pred_masks).to("cpu").detach().numpy()
result = {
"boxes": boxes,
"scores": scores,
"classes": pred_classes,
"masks": pred_masks,
}
return result
class TestImageService(bentoml.BentoService):
@bentoml.api(input=ImageInput())
def test(self, images):
return images
def input_adapter():
return ImageInput(pilmode="L")
class ExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=True)
def predict_multi_images(self, originals, compareds):
return self.artifacts.model.predict_multi_images(originals, compareds)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(
input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_with_sklearn(self, jsons):
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict"
@bentoml.api(
route=CUSTOM_ROUTE,
input=JsonInput(),
batch=True,
)
def customezed_route(self, input_datas):
return input_datas
@bentoml.api(input=JsonInput(), batch=True)
def predict_strict_json(self,
input_datas,
tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
return self.artifacts.model.predict_json(filtered_jsons)
@bentoml.api(input=JsonInput(), batch=True)
def predict_direct_json(self,
input_datas,
tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
rets = self.artifacts.model.predict_json(filtered_jsons)
return [
InferenceResult(http_status=200, data=json.dumps(result))
for result in rets
]
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
class ExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated
def predict_dataframe_v1(self, df):
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(
input=MultiImageInput(input_names=('original', 'compared')), batch=True
)
def predict_multi_images(self, originals, compareds):
return self.artifacts.model.predict_multi_images(originals, compareds)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(
input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True,
)
def predict_with_sklearn(self, jsons):
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict"
@bentoml.api(
route=CUSTOM_ROUTE, input=JsonInput(), batch=True,
)
def customezed_route(self, input_datas):
return input_datas
CUSTOM_SCHEMA = {
"application/json": {
"schema": {
"type": "object",
"required": ["field1", "field2"],
"properties": {
"field1": {"type": "string"},
"field2": {"type": "uuid"},
},
},
}
}
@bentoml.api(input=JsonInput(request_schema=CUSTOM_SCHEMA), batch=True)
def customezed_schema(self, input_datas):
return input_datas
@bentoml.api(input=JsonInput(), batch=True)
def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
return self.artifacts.model.predict_json(filtered_jsons)
@bentoml.api(input=JsonInput(), batch=True)
def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None):
filtered_jsons = []
for j, t in zip(input_datas, tasks):
if t.http_headers.content_type != "application/json":
t.discard(http_status=400, err_msg="application/json only")
else:
filtered_jsons.append(j)
rets = self.artifacts.model.predict_json(filtered_jsons)
return [
InferenceResult(http_status=200, data=json.dumps(result)) for result in rets
]
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_batch_size(self, input_datas=10):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
batch_size = len(input_datas)
return [batch_size] * batch_size
@bentoml.api(input=JsonInput())
def echo_json(self, input_data):
return input_data
if version.parse(BENTOML_VERSION) > version.parse("0.12.1+0"):
@bentoml.api(input=JsonInput(), output=JsonOutput(ensure_ascii=True))
def echo_json_ensure_ascii(self, input_data):
return input_data
