class OnnxService(BentoService): def __init__(self): super().__init__() self.news_label = { 1: 'World', 2: 'Sports', 3: 'Business', 4: 'Sci/Tec' } def classify_categories(self, sentence): text_pipeline, _ = get_pipeline(self.artifacts.tokenizer, self.artifacts.vocab) text = to_numpy(torch.tensor(text_pipeline(sentence)).to(device)) tensor_name = self.artifacts.model.get_inputs()[0].name output_name = self.artifacts.model.get_outputs()[0].name onnx_inputs = {tensor_name: text} print(f'providers: {self.artifacts.model.get_providers()}') try: r = self.artifacts.model.run([output_name], onnx_inputs)[0] return r.argmax(1).item() + 1 except (RuntimeError, InvalidArgument) as e: print(f"ERROR with shape: {onnx_inputs[tensor_name].shape} - {e}") @api(input=JsonInput(), output=JsonOutput()) def predict(self, parsed_json): sentence = parsed_json.get('text') return { 'categories': self.news_label[self.classify_categories(sentence)] }
class EmotionRecognitionService(bentoml.BentoService): @bentoml.api(input=ImageInput(), output=JsonOutput(), batch=False) def predict(self, image): print(image) return DeepFace.analyze(np.array(image), actions=['emotion'], enforce_detection=False)
class PyTorchModel(bentoml.BentoService): ''' @bentoml.utils.cached_property # reuse transformer def transform(self): return transforms.Compose([transforms.CenterCrop((29, 29)), transforms.ToTensor()]) ''' @bentoml.api(input=JsonInput(), output=JsonOutput(), batch=True) def predict(self, input_json) -> List[str]: sentence = input_json[0]['sentence'] tokenized_sentence = config.TOKENIZER.encode(sentence) sentence = sentence.split() #print(sentence) #print(tokenized_sentence) test_dataset = EntityDataset(texts=[sentence], pos=[[0] * len(sentence)], tags=[[0] * len(sentence)]) device = torch.device("cpu") with torch.no_grad(): data = test_dataset[0] for k, v in data.items(): data[k] = v.to(device).unsqueeze(0) tag, pos, _ = self.artifacts.ner(**data) tags = enc_tag.inverse_transform( tag.argmax(2).cpu().numpy().reshape( -1))[:len(tokenized_sentence)] i = 0 names = [] while i < len(tags): item = tags[i] indices = [] if (item == "B-per"): while (item == "B-per"): indices.append(i) i += 1 item = tags[i] tokenized_name = tokenized_sentence[ indices[0]:indices[-1] + 1] name = config.TOKENIZER.decode(tokenized_name) names.append(name) indices = [] i += 1 resp = ','.join(names) return [resp]
class ProfanityFilterService(BentoService): def model_pred(self, sentence): self.artifacts.model.eval() tokens = self.artifacts.tokenizer.tokenize(sentence) length = torch.LongTensor([len(tokens)]).to(device) idx = [vocab.stoi[token] for token in tokens] tensor = torch.LongTensor(idx).unsqueeze(-1).to(device) prediction = self.artifacts.model(tensor, length) probabilities = nn.functional.softmax(prediction, dim=-1) return probabilities.squeeze()[-1].item() @api(input=JsonInput(), output=JsonOutput()) def predict(self, parsed_json): return self.model_pred(parsed_json["text"])
class ToxicCommentClassification(BentoService): def tokenize_df(self, df): comments = df['comment_text'].values tokenized = self.artifacts.x_tokenizer.texts_to_sequences(comments) input_data = sequence.pad_sequences(tokenized, maxlen=max_text_length) return input_data @api(input=DataframeInput(), output=JsonOutput(), batch=True) def predict(self, df: pd.DataFrame) -> List[str]: input_data = self.tokenize_df(df) prediction = self.artifacts.model.predict(input_data) result = [] for i in prediction: result.append(list_of_classes[np.argmax(i)]) return result
def test_string_input(make_api): from bentoml.adapters import StringInput, JsonOutput api = make_api( input_adapter=StringInput(), output_adapter=JsonOutput(), user_func=lambda i: i, ) body = b'{"a": 1}' request = HTTPRequest(body=body) response = api.handle_request(request) assert json.loads(response.body) == body.decode() responses = api.handle_batch_request([request] * 3) for response in responses: assert json.loads(response.body) == body.decode()
class PytorchService(BentoService): def __init__(self): super().__init__() self.news_label = {1: 'World', 2: 'Sports', 3: 'Business', 4: 'Sci/Tec'} def classify_categories(self, sentence): text_pipeline, _ = get_pipeline(self.artifacts.tokenizer, self.artifacts.vocab) with torch.no_grad(): text = torch.tensor(text_pipeline(sentence)).to(device) offsets = torch.tensor([0]).to(device) output = self.artifacts.model(text, offsets=offsets) return output.argmax(1).item() + 1 @api(input=JsonInput(), output=JsonOutput()) def predict(self, parsed_json): label = self.classify_categories(parsed_json.get("text")) return {'categories': self.news_label[label]}
class ExampleService(bentoml.BentoService): """ Example BentoService class made for testing purpose """ @bentoml.api( input=DataframeInput(dtype={"col1": "int"}), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_dataframe(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(DataframeHandler, dtype={"col1": "int"}, batch=True) # deprecated def predict_dataframe_v1(self, df): return self.artifacts.model.predict_dataframe(df) @bentoml.api(input=MultiImageInput(input_names=('original', 'compared')), batch=True) def predict_multi_images(self, originals, compareds): return self.artifacts.model.predict_multi_images(originals, compareds) @bentoml.api(input=ImageInput(), batch=True) def predict_image(self, images): return self.artifacts.model.predict_image(images) @bentoml.api( input=JsonInput(), mb_max_latency=1000, mb_max_batch_size=2000, batch=True, ) def predict_with_sklearn(self, jsons): return self.artifacts.sk_model.predict(jsons) @bentoml.api(input=FileInput(), batch=True) def predict_file(self, files): return self.artifacts.model.predict_file(files) @bentoml.api(input=JsonInput(), batch=True) def predict_json(self, input_datas): return self.artifacts.model.predict_json(input_datas) CUSTOM_ROUTE = "$~!@%^&*()_-+=[]\\|;:,./predict" @bentoml.api( route=CUSTOM_ROUTE, input=JsonInput(), batch=True, ) def customezed_route(self, input_datas): return input_datas CUSTOM_SCHEMA = { "application/json": { "schema": { "type": "object", "required": ["field1", "field2"], "properties": { "field1": { "type": "string" }, "field2": { "type": "uuid" }, }, }, } } @bentoml.api(input=JsonInput(request_schema=CUSTOM_SCHEMA), batch=True) def customezed_schema(self, input_datas): return input_datas @bentoml.api(input=JsonInput(), batch=True) def predict_strict_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) return self.artifacts.model.predict_json(filtered_jsons) @bentoml.api(input=JsonInput(), batch=True) def predict_direct_json(self, input_datas, tasks: Sequence[InferenceTask] = None): filtered_jsons = [] for j, t in zip(input_datas, tasks): if t.http_headers.content_type != "application/json": t.discard(http_status=400, err_msg="application/json only") else: filtered_jsons.append(j) rets = self.artifacts.model.predict_json(filtered_jsons) return [ InferenceResult(http_status=200, data=json.dumps(result)) for result in rets ] @bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True) def echo_with_delay(self, input_datas): data = input_datas[0] time.sleep(data['b'] + data['a'] * len(input_datas)) return input_datas @bentoml.api(input=JsonInput()) def echo_json(self, input_data): return input_data if version.parse(BENTOML_VERSION) > version.parse("0.12.1+0"): @bentoml.api(input=JsonInput(), output=JsonOutput(ensure_ascii=True)) def echo_json_ensure_ascii(self, input_data): return input_data
class Service(BentoService): @api(input=StringInput(), output=JsonOutput()) def say_hi(self, x): return f"Hello {x}!"