def test_file_input_cli(capsys, bin_file): test_file_input = FileInput() test_args = ["--input", bin_file] test_file_input.handle_cli(test_args, predict) out, _ = capsys.readouterr() assert '\\x810\\x899' in out
class ExampleBentoServiceSingle(ExampleBentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=MultiImageInput(input_names=('original', 'compared')), batch=False ) def predict_legacy_images(self, original, compared): return self.artifacts.model.predict_multi_images([original], [compared])[0] @bentoml.api(input=ImageInput(), batch=False) def predict_image(self, image): return self.artifacts.model.predict_image([image])[0] @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False ) def predict_with_sklearn(self, json): return self.artifacts.sk_model.predict([json])[0] @bentoml.api(input=FileInput(), batch=False) def predict_file(self, file_): return self.artifacts.model.predict_file([file_])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_json(self, input_data): return self.artifacts.model.predict_json([input_data])[0]
def test_file_input_cli_list(capsys, bin_files): test_file_input = FileInput() test_args = ["--input"] + glob.glob(bin_files) test_file_input.handle_cli(test_args, predict) out, _ = capsys.readouterr() lines = out.strip().split('\n') for line in lines[-10:]: assert '\\x810\\x899' in line
class NonBatchExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=False) def predict_multi_images(self, original, compared): return self.artifacts.model.predict_multi_images([original], [compared])[0] @bentoml.api(input=ImageInput(), batch=False) def predict_image(self, image): return self.artifacts.model.predict_image([image])[0] @bentoml.api(input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False) def predict_with_sklearn(self, json_value): return self.artifacts.sk_model.predict([json_value])[0] @bentoml.api(input=FileInput(), batch=False) def predict_file(self, file_): return self.artifacts.model.predict_file([file_])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_json(self, input_data): return self.artifacts.model.predict_json([input_data])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_strict_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": task.discard(http_status=400, err_msg="application/json only") return result = self.artifacts.model.predict_json([input_data])[0] return result @bentoml.api(input=JsonInput(), batch=False) def predict_direct_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": return InferenceError(http_status=400, err_msg="application/json only") result = self.artifacts.model.predict_json([input_data])[0] return InferenceResult(http_status=200, data=json.dumps(result)) @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_batch_size(self, input_datas=10): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) batch_size = len(input_datas) return [batch_size] * batch_size
class MnistTensorflow(bentoml.BentoService): @bentoml.api(input=FileInput(), batch=True) def predict(self, inputs): loaded_func = self.artifacts.trackable.signatures[ tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] inputs = [i.read() for i in inputs] inputs = tf.constant(inputs, dtype=tf.string) pred = loaded_func(raw_inputs=inputs) output = pred['outputs'] return output
class ExampleBentoService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True) def predict_with_sklearn(self, jsons): """predict_dataframe expects dataframe as input """ return self.artifacts.sk_model.predict(jsons) @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): """predict_dataframe expects dataframe as input """ return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=LegacyImageInput(input_names=('original', 'compared'))) def predict_legacy_images(self, original, compared): return self.artifacts.model.predict_legacy_images(original, compared) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas
def test_file_input_http_request_malformatted_input_wrong_input_name(): test_file_input = FileInput() request = mock.MagicMock(spec=flask.Request) request.method = "POST" request.files = {"abc": None} request.headers = {} request.get_data.return_value = None with pytest.raises(BadInput) as e: test_file_input.handle_request(request, predict) assert "unexpected HTTP request format" in str(e.value)
def test_file_input_http_request_post_binary(bin_file): test_file_input = FileInput() request = mock.MagicMock(spec=flask.Request) request.method = "POST" request.files = {} request.headers = {} request.get_data.return_value = open(str(bin_file), 'rb').read() response = test_file_input.handle_request(request, predict) assert response.status_code == 200 assert b'{"b64": "gTCJOQ=="}' in response.data simple_request = SimpleRequest.from_flask_request(request) responses = test_file_input.handle_batch_request([simple_request], predict) assert responses[0].status == 200 assert '{"b64": "gTCJOQ=="}' == responses[0].data
class ExampleBentoServiceSingle(ExampleBentoService): """ Example BentoService class made for testing purpose """ @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=False) def predict_multi_images(self, original, compared): return self.artifacts.model.predict_multi_images([original], [compared])[0] @bentoml.api(input=ImageInput(), batch=False) def predict_image(self, image): return self.artifacts.model.predict_image([image])[0] @bentoml.api(input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=False) def predict_with_sklearn(self, json): return self.artifacts.sk_model.predict([json])[0] @bentoml.api(input=FileInput(), batch=False) def predict_file(self, file_): return self.artifacts.model.predict_file([file_])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_json(self, input_data): return self.artifacts.model.predict_json([input_data])[0] @bentoml.api(input=JsonInput(), batch=False) def predict_strict_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": task.discard(http_status=400, err_msg="application/json only") return result = self.artifacts.model.predict_json([input_data])[0] return result @bentoml.api(input=JsonInput(), batch=False) def predict_direct_json(self, input_data, task: InferenceTask = None): if task.http_headers.content_type != "application/json": return InferenceError(http_status=400, err_msg="application/json only") result = self.artifacts.model.predict_json([input_data])[0] return InferenceResult(http_status=200, data=json.dumps(result))
class PytorchImageClassifier(bentoml.BentoService): @bentoml.utils.cached_property def transform(self): return transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]) @bentoml.api(input=FileInput(), batch=True) def predict(self, file_streams: List[BinaryIO]) -> List[str]: input_datas = [] for fs in file_streams: img = Image.open(fs).resize((32, 32)) input_datas.append(self.transform(img)) outputs = self.artifacts.net(Variable(torch.stack(input_datas))) _, output_classes = outputs.max(dim=1) return [classes[output_class] for output_class in output_classes]
def test_file_input_aws_lambda_event(bin_file): test_file_input = FileInput() with open(str(bin_file), "rb") as file_file: content = file_file.read() try: file_bytes_encoded = base64.encodebytes(content) except AttributeError: file_bytes_encoded = base64.encodebytes(str(bin_file)) aws_lambda_event = { "body": file_bytes_encoded, "headers": { "Content-Type": "images/png" }, } aws_result = test_file_input.handle_aws_lambda_event( aws_lambda_event, predict) assert aws_result["statusCode"] == 200 assert aws_result["body"] == '{"b64": "gTCJOQ=="}'
def test_file_input_http_request_multipart_form(bin_file): test_file_input = FileInput() request = mock.MagicMock(spec=flask.Request) file_bytes = open(str(bin_file), 'rb').read() file_attr = { 'filename': 'test_img.png', 'read.return_value': file_bytes, 'stream': io.BytesIO(file_bytes), } file = mock.Mock(**file_attr) request.method = "POST" request.files = {"file_file": file} request.headers = {} request.get_data.return_value = None response = test_file_input.handle_request(request, predict) assert response.status_code == 200 assert b'{"b64": "gTCJOQ=="}' in response.data
class DocparserService(bentoml.BentoService): def __init__(self): super().__init__() model_config = InferenceConfig() self.preprocess_obj = ForwardModel(model_config) def preprocess(self, image): images = np.expand_dims(image, axis=0)[:,:,:,:3] molded_images, image_metas, windows = self.preprocess_obj.mold_inputs(images) molded_images = molded_images.astype(np.float32) image_shape = molded_images[0].shape for g in molded_images[1:]: assert g.shape == image_shape, \ "After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes." anchors = self.preprocess_obj.get_anchors(image_shape) anchors = np.broadcast_to(anchors, (images.shape[0],) + anchors.shape) self.images = images self.molded_images = molded_images self.windows = windows # response body format row wise. return { INPUT_IMAGE: molded_images[0].tolist(), INPUT_IMAGE_META: image_metas[0].tolist(), INPUT_ANCHORS: anchors[0].tolist() } @bentoml.api(input=FileInput(), batch=True) def predict(self, inputs): # print(inputs) inputs = [self.preprocess(Image.open(io.BytesIO(i.read()))) for i in inputs] inputs = { INPUT_ANCHORS: [i[INPUT_ANCHORS] for i in inputs], INPUT_IMAGE: [i[INPUT_IMAGE] for i in inputs], INPUT_IMAGE_META: [i[INPUT_IMAGE_META] for i in inputs], } loaded_func = self.artifacts.trackable.signatures[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] input_anchors = tf.constant(inputs[INPUT_ANCHORS], dtype=tf.float32) input_image = tf.constant(inputs[INPUT_IMAGE], dtype=tf.float32) input_image_meta = tf.constant(inputs[INPUT_IMAGE_META], dtype=tf.float32) pred = loaded_func( input_anchors=input_anchors, input_image=input_image, input_image_meta=input_image_meta, ) results = self.preprocess_obj.result_to_dict(self.images, self.molded_images, self.windows, pred) outputs = [] structure_parser = stage2_structure_parser.StructureParser() for idx, r in enumerate(results): pred_bboxes = r['rois'] pred_class_ids = r['class_ids'] pred_scores = r['scores'] classes_with_background = ['background'] + DocsDataset.ALL_CLASSES orig_shape = list(self.images[idx].shape) num_preds = len(pred_bboxes) prediction_list = [] for pred_nr in range(num_preds): class_name = classes_with_background[pred_class_ids[pred_nr]] pred_bbox = pred_bboxes[pred_nr] pred_score = pred_scores[pred_nr] prediction_list.append({'pred_nr': pred_nr, 'class_name': class_name, 'pred_score': pred_score, 'bbox_orig_coords': pred_bbox, 'orig_img_shape': orig_shape}) img_relations_dict = structure_parser.create_structure_for_doc_from_list({'prediction_list': prediction_list, 'orig_img_shape': orig_shape}, do_postprocessing=True) predictions_dict = convert_bbox_list_to_save_format(img_relations_dict['all_bboxes']) try: del img_relations_dict['all_bboxes'] except KeyError: pass predictions_dict["relations"] = img_relations_dict outputs.append(predictions_dict) return outputs # bento_svc = DocparserService() # bento_svc.pack("trackable", "serving_model/1") # saved_path = bento_svc.save()
class ExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api( input=MultiImageInput(input_names=('original', 'compared')), batch=True ) def predict_multi_images(self, originals, compareds): return self.artifacts.model.predict_multi_images(originals, compareds) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_with_sklearn(self, jsons): return self.artifacts.sk_model.predict(jsons) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict" @bentoml.api( route=CUSTOM_ROUTE, input=JsonInput(), batch=True, ) def customezed_route(self, input_datas): return input_datas CUSTOM_SCHEMA = { "application/json": { "schema": { "type": "object", "required": ["field1", "field2"], "properties": { "field1": {"type": "string"}, "field2": {"type": "uuid"}, }, }, } } @bentoml.api(input=JsonInput(request_schema=CUSTOM_SCHEMA), batch=True) def customezed_schema(self, input_datas): return input_datas @bentoml.api(input=JsonInput(), batch=True) def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) return self.artifacts.model.predict_json(filtered_jsons) @bentoml.api(input=JsonInput(), batch=True) def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) rets = self.artifacts.model.predict_json(filtered_jsons) return [ InferenceResult(http_status=200, data=json.dumps(result)) for result in rets ] @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_batch_size(self, input_datas=10): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) batch_size = len(input_datas) return [batch_size] * batch_size @bentoml.api(input=JsonInput()) def echo_json(self, input_data): return input_data if version.parse(BENTOML_VERSION) > version.parse("0.12.1+0"): @bentoml.api(input=JsonInput(), output=JsonOutput(ensure_ascii=True)) def echo_json_ensure_ascii(self, input_data): return input_data
class TextDetectionService(BentoService): @api(input=FileInput()) def predict(self, image): result = self.artifacts.model.detect_text(image[0]) return result
class ExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=True) def predict_multi_images(self, originals, compareds): return self.artifacts.model.predict_multi_images(originals, compareds) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_with_sklearn(self, jsons): return self.artifacts.sk_model.predict(jsons) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict" @bentoml.api( route=CUSTOM_ROUTE, input=JsonInput(), batch=True, ) def customezed_route(self, input_datas): return input_datas @bentoml.api(input=JsonInput(), batch=True) def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) return self.artifacts.model.predict_json(filtered_jsons) @bentoml.api(input=JsonInput(), batch=True) def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) rets = self.artifacts.model.predict_json(filtered_jsons) return [ InferenceResult(http_status=200, data=json.dumps(result)) for result in rets ] @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas
def input_adapter(): return FileInput()