def test_image_input_cli(capsys, img_file): test_image_input = ImageInput() test_args = ["--input", img_file] test_image_input.handle_cli(test_args, predict) out, _ = capsys.readouterr() assert out.strip().endswith("(10, 10, 3)")
def test_image_input_cli_list(capsys, img_files): test_image_input = ImageInput() test_args = ["--input"] + glob.glob(img_files) test_image_input.handle_cli(test_args, predict) out, _ = capsys.readouterr() lines = out.strip().split('\n') for line in lines[-10:]: assert line.strip().endswith("(10, 10, 3)")
def test_image_input_http_request_malformatted_input_wrong_input_name(): test_image_input = ImageInput() request = mock.MagicMock(spec=flask.Request) request.method = "POST" request.files = {"abc": None} request.headers = {} request.get_data.return_value = None with pytest.raises(BadInput) as e: test_image_input.handle_request(request, predict) assert "unexpected HTTP request format" in str(e.value)
class CocoDetectronClassifier(bentoml.BentoService): @bentoml.api(input=ImageInput(), batch=False) def predict(self, img: np.ndarray) -> Dict: _aug = T.ResizeShortestEdge( [800, 800], 1333 ) boxes = None scores = None pred_classes = None pred_masks= None try: original_image = img[:, :, ::-1] height, width = original_image.shape[:2] image = _aug.get_transform(original_image).apply_image(original_image) image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) inputs = {"image": image, "height": height, "width": width} predictions = self.artifacts.model([inputs])[0] pred_instances = predictions["instances"] boxes = (pred_instances.pred_boxes).to("cpu").tensor.detach().numpy() scores = (pred_instances.scores).to("cpu").detach().numpy() pred_classes = (pred_instances.pred_classes).to("cpu").detach().numpy() pred_masks = (pred_instances.pred_masks).to("cpu").detach().numpy() except Exception as error: print(f"Caught error whilst processing {error}") for line in get_traceback_list(): print(line[:-1]) result = { "boxes" : boxes, "scores" : scores, "classes" : pred_classes, "masks" : pred_masks } return result
class EasyOCRService(bentoml.BentoService): @bentoml.api(input=ImageInput(), batch=False) def predict(self, image): reader = self.artifacts.chinese_small raw_results = reader.readtext(np.array(image)) text_instances = [x[1] for x in raw_results] return {"text": text_instances}
class EmotionRecognitionService(bentoml.BentoService): @bentoml.api(input=ImageInput(), output=JsonOutput(), batch=False) def predict(self, image): print(image) return DeepFace.analyze(np.array(image), actions=['emotion'], enforce_detection=False)
class ExampleBentoServiceSingle(ExampleBentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=MultiImageInput(input_names=('original', 'compared')), batch=False ) def predict_legacy_images(self, original, compared): return self.artifacts.model.predict_multi_images([original], [compared])[0] @bentoml.api(input=ImageInput(), batch=False) def predict_image(self, image): return self.artifacts.model.predict_image([image])[0] @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False ) def predict_with_sklearn(self, json): return self.artifacts.sk_model.predict([json])[0] @bentoml.api(input=FileInput(), batch=False) def predict_file(self, file_): return self.artifacts.model.predict_file([file_])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_json(self, input_data): return self.artifacts.model.predict_json([input_data])[0]
class NonBatchExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=False) def predict_multi_images(self, original, compared): return self.artifacts.model.predict_multi_images([original], [compared])[0] @bentoml.api(input=ImageInput(), batch=False) def predict_image(self, image): return self.artifacts.model.predict_image([image])[0] @bentoml.api(input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False) def predict_with_sklearn(self, json_value): return self.artifacts.sk_model.predict([json_value])[0] @bentoml.api(input=FileInput(), batch=False) def predict_file(self, file_): return self.artifacts.model.predict_file([file_])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_json(self, input_data): return self.artifacts.model.predict_json([input_data])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_strict_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": task.discard(http_status=400, err_msg="application/json only") return result = self.artifacts.model.predict_json([input_data])[0] return result @bentoml.api(input=JsonInput(), batch=False) def predict_direct_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": return InferenceError(http_status=400, err_msg="application/json only") result = self.artifacts.model.predict_json([input_data])[0] return InferenceResult(http_status=200, data=json.dumps(result)) @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_batch_size(self, input_datas=10): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) batch_size = len(input_datas) return [batch_size] * batch_size
def test_image_input_http_request_post_binary(img_file): test_image_input = ImageInput() request = mock.MagicMock(spec=flask.Request) request.method = "POST" request.files = {} request.headers = {} request.get_data.return_value = open(str(img_file), 'rb').read() response = test_image_input.handle_request(request, predict) assert response.status_code == 200 assert "[10, 10, 3]" in str(response.response) simple_request = SimpleRequest.from_flask_request(request) responses = test_image_input.handle_batch_request([simple_request], predict) assert responses[0].status == 200 assert "[10, 10, 3]" in str(responses[0].data)
def test_image_input_http_request_multipart_form(img_file): test_image_input = ImageInput() request = mock.MagicMock(spec=flask.Request) file_bytes = open(str(img_file), 'rb').read() file_attr = { 'filename': 'test_img.png', 'read.return_value': file_bytes, 'stream': io.BytesIO(file_bytes), } file = mock.Mock(**file_attr) request.method = "POST" request.files = {"image_file": file} request.headers = {} request.get_data.return_value = None response = test_image_input.handle_request(request, predict) assert response.status_code == 200 assert "[10, 10, 3]" in str(response.response)
def test_image_input_aws_lambda_event(img_file): test_image_input = ImageInput() with open(str(img_file), "rb") as image_file: content = image_file.read() try: image_bytes_encoded = base64.encodebytes(content) except AttributeError: image_bytes_encoded = base64.encodebytes(str(img_file)) aws_lambda_event = { "body": image_bytes_encoded, "headers": { "Content-Type": "images/png" }, } aws_result = test_image_input.handle_aws_lambda_event( aws_lambda_event, predict) assert aws_result["statusCode"] == 200 assert aws_result["body"] == "[10, 10, 3]"
class KerasFashionMnistService(BentoService): @api(input=ImageInput(pilmode='L')) def predict(self, imgs): inputs = [] for img in imgs: img = Image.fromarray(img).resize((28, 28)) img /= 255.0 img = np.array(img.getdata()).reshape((28, 28, 1)) inputs.append(img) inputs = np.stack(inputs) class_idxs = self.artifacts.classifier.predict_classes(inputs) return [class_names[class_idx] for class_idx in class_idxs]
class ExampleBentoService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True) def predict_with_sklearn(self, jsons): """predict_dataframe expects dataframe as input """ return self.artifacts.sk_model.predict(jsons) @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=LegacyImageInput(input_names=('original', 'compared'))) def predict_legacy_images(self, original, compared): return self.artifacts.model.predict_legacy_images(original, compared) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas
class ExampleBentoService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=DataframeInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True) def predict(self, df): """An API for testing simple bento model service """ return self.artifacts.model.predict(df) @bentoml.api(input=DataframeInput(dtype={"col1": "int"}), batch=True) def predict_dataframe(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=False) def predict_multi_images(self, original, compared): return self.artifacts.model.predict_multi_images(original, compared) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_data): return self.artifacts.model.predict_json(input_data) CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict" @bentoml.api( route=CUSTOM_ROUTE, input=JsonInput(), batch=True, ) def customize_route(self, input_data): return input_data
def test_image_input_http_request_single_image_different_name(img_file): test_image_input = ImageInput(input_names=("my_image",)) request = mock.MagicMock(spec=flask.Request) file_attr = { 'filename': 'test_img.png', 'read.return_value': open(str(img_file), 'rb').read(), } file = mock.Mock(**file_attr) request.method = "POST" request.files = {"a_differnt_name_used": file} request.headers = {} request.get_data.return_value = None response = test_image_input.handle_request(request, predict) assert response.status_code == 200 assert "[10, 10, 3]" in str(response.response)
class ExampleBentoServiceSingle(ExampleBentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=False) def predict_multi_images(self, original, compared): return self.artifacts.model.predict_multi_images([original], [compared])[0] @bentoml.api(input=ImageInput(), batch=False) def predict_image(self, image): return self.artifacts.model.predict_image([image])[0] @bentoml.api(input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False) def predict_with_sklearn(self, json): return self.artifacts.sk_model.predict([json])[0] @bentoml.api(input=FileInput(), batch=False) def predict_file(self, file_): return self.artifacts.model.predict_file([file_])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_json(self, input_data): return self.artifacts.model.predict_json([input_data])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_strict_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": task.discard(http_status=400, err_msg="application/json only") return result = self.artifacts.model.predict_json([input_data])[0] return result @bentoml.api(input=JsonInput(), batch=False) def predict_direct_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": return InferenceError(http_status=400, err_msg="application/json only") result = self.artifacts.model.predict_json([input_data])[0] return InferenceResult(http_status=200, data=json.dumps(result))
class ExampleBentoService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=DataframeInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True) def predict(self, df): """An API for testing simple bento model service """ return self.artifacts.model.predict(df) @bentoml.api(input=DataframeInput(dtype={"col1": "int"}), batch=True) def predict_dataframe(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api(input=LegacyImageInput(input_names=('original', 'compared')), batch=False) def predict_legacy_images(self, original, compared): return self.artifacts.model.predict_legacy_images(original, compared) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_data): return self.artifacts.model.predict_json(input_data) @bentoml.api(input=LegacyJsonInput(), batch=False) def predict_legacy_json(self, input_data): return self.artifacts.model.predict_legacy_json(input_data)
class FaceDetectionService(BentoService): def prepare_input(self, img: np.ndarray) -> np.ndarray: img = np.transpose(img[:, :, :3], (2, 0, 1)) return np.expand_dims(img, axis=0).astype(np.float32) def to_json(self, results: np.ndarray) -> Dict: # results: (N, 6) as x1,y1,x2,y2,score,batch_idx return { "boxes": results[:, :4].astype(np.int32).tolist(), "scores": results[:, 4].astype(np.float32).tolist() } @api(input=ImageInput(), batch=True, mb_max_batch_size=8, mb_max_latency=1000) def detect(self, imgs: List[np.ndarray]): input_name = self.artifacts.model.get_inputs()[0].name preds = [] for img in imgs: results = self.artifacts.model.run(None, {input_name: self.prepare_input(img) })[0] preds.append( self.to_json(results) ) return preds
class DetectronClassifier(bentoml.BentoService): @bentoml.api(input=ImageInput(), batch=False) def predict(self, original_image: np.ndarray) -> Dict: _aug = T.ResizeShortestEdge([800, 800], 1333) height, width = original_image.shape[:2] image = _aug.get_transform(original_image).apply_image(original_image) image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) inputs = {"image": image, "height": height, "width": width} predictions = self.artifacts.model([inputs])[0] pred_instances = predictions["instances"] boxes = (pred_instances.pred_boxes).to("cpu").tensor.detach().numpy() scores = (pred_instances.scores).to("cpu").detach().numpy() pred_classes = (pred_instances.pred_classes).to("cpu").detach().numpy() pred_masks = (pred_instances.pred_masks).to("cpu").detach().numpy() result = { "boxes": boxes, "scores": scores, "classes": pred_classes, "masks": pred_masks, } return result
class TestImageService(bentoml.BentoService): @bentoml.api(input=ImageInput()) def test(self, images): return images
def input_adapter(): return ImageInput(pilmode="L")
class ExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=True) def predict_multi_images(self, originals, compareds): return self.artifacts.model.predict_multi_images(originals, compareds) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_with_sklearn(self, jsons): return self.artifacts.sk_model.predict(jsons) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict" @bentoml.api( route=CUSTOM_ROUTE, input=JsonInput(), batch=True, ) def customezed_route(self, input_datas): return input_datas @bentoml.api(input=JsonInput(), batch=True) def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) return self.artifacts.model.predict_json(filtered_jsons) @bentoml.api(input=JsonInput(), batch=True) def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) rets = self.artifacts.model.predict_json(filtered_jsons) return [ InferenceResult(http_status=200, data=json.dumps(result)) for result in rets ] @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas
class ExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api( input=MultiImageInput(input_names=('original', 'compared')), batch=True ) def predict_multi_images(self, originals, compareds): return self.artifacts.model.predict_multi_images(originals, compareds) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_with_sklearn(self, jsons): return self.artifacts.sk_model.predict(jsons) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict" @bentoml.api( route=CUSTOM_ROUTE, input=JsonInput(), batch=True, ) def customezed_route(self, input_datas): return input_datas CUSTOM_SCHEMA = { "application/json": { "schema": { "type": "object", "required": ["field1", "field2"], "properties": { "field1": {"type": "string"}, "field2": {"type": "uuid"}, }, }, } } @bentoml.api(input=JsonInput(request_schema=CUSTOM_SCHEMA), batch=True) def customezed_schema(self, input_datas): return input_datas @bentoml.api(input=JsonInput(), batch=True) def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) return self.artifacts.model.predict_json(filtered_jsons) @bentoml.api(input=JsonInput(), batch=True) def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) rets = self.artifacts.model.predict_json(filtered_jsons) return [ InferenceResult(http_status=200, data=json.dumps(result)) for result in rets ] @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_batch_size(self, input_datas=10): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) batch_size = len(input_datas) return [batch_size] * batch_size @bentoml.api(input=JsonInput()) def echo_json(self, input_data): return input_data if version.parse(BENTOML_VERSION) > version.parse("0.12.1+0"): @bentoml.api(input=JsonInput(), output=JsonOutput(ensure_ascii=True)) def echo_json_ensure_ascii(self, input_data): return input_data