class NonBatchExampleService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original],
[compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=False)
def predict_with_sklearn(self, json_value):
return self.artifacts.sk_model.predict([json_value])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_strict_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
task.discard(http_status=400, err_msg="application/json only")
return
result = self.artifacts.model.predict_json([input_data])[0]
return result
@bentoml.api(input=JsonInput(), batch=False)
def predict_direct_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
return InferenceError(http_status=400,
err_msg="application/json only")
result = self.artifacts.model.predict_json([input_data])[0]
return InferenceResult(http_status=200, data=json.dumps(result))
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_batch_size(self, input_datas=10):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
batch_size = len(input_datas)
return [batch_size] * batch_size
class MnistTensorflow(bentoml.BentoService):
@bentoml.api(input=FileInput(), batch=True)
def predict(self, inputs):
loaded_func = self.artifacts.trackable.signatures[
tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
inputs = [i.read() for i in inputs]
inputs = tf.constant(inputs, dtype=tf.string)
pred = loaded_func(raw_inputs=inputs)
output = pred['outputs']
return output
class ExampleBentoService(bentoml.BentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True)
def predict_with_sklearn(self, jsons):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.sk_model.predict(jsons)
@bentoml.api(
input=DataframeInput(dtype={"col1": "int"}),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=True,
)
def predict_dataframe(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(DataframeHandler, dtype={"col1": "int"},
batch=True) # deprecated
def predict_dataframe_v1(self, df):
"""predict_dataframe expects dataframe as input
"""
return self.artifacts.model.predict_dataframe(df)
@bentoml.api(input=ImageInput(), batch=True)
def predict_image(self, images):
return self.artifacts.model.predict_image(images)
@bentoml.api(input=FileInput(), batch=True)
def predict_file(self, files):
return self.artifacts.model.predict_file(files)
@bentoml.api(input=LegacyImageInput(input_names=('original', 'compared')))
def predict_legacy_images(self, original, compared):
return self.artifacts.model.predict_legacy_images(original, compared)
@bentoml.api(input=JsonInput(), batch=True)
def predict_json(self, input_datas):
return self.artifacts.model.predict_json(input_datas)
@bentoml.api(input=JsonInput(), mb_max_latency=10000 * 1000, batch=True)
def echo_with_delay(self, input_datas):
data = input_datas[0]
time.sleep(data['b'] + data['a'] * len(input_datas))
return input_datas
class ExampleBentoServiceSingle(ExampleBentoService):
"""
Example BentoService class made for testing purpose
"""
@bentoml.api(input=MultiImageInput(input_names=('original', 'compared')),
batch=False)
def predict_multi_images(self, original, compared):
return self.artifacts.model.predict_multi_images([original],
[compared])[0]
@bentoml.api(input=ImageInput(), batch=False)
def predict_image(self, image):
return self.artifacts.model.predict_image([image])[0]
@bentoml.api(input=JsonInput(),
mb_max_latency=1000,
mb_max_batch_size=2000,
batch=False)
def predict_with_sklearn(self, json):
return self.artifacts.sk_model.predict([json])[0]
@bentoml.api(input=FileInput(), batch=False)
def predict_file(self, file_):
return self.artifacts.model.predict_file([file_])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_json(self, input_data):
return self.artifacts.model.predict_json([input_data])[0]
@bentoml.api(input=JsonInput(), batch=False)
def predict_strict_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
task.discard(http_status=400, err_msg="application/json only")
return
result = self.artifacts.model.predict_json([input_data])[0]
return result
@bentoml.api(input=JsonInput(), batch=False)
def predict_direct_json(self, input_data, task: InferenceTask = None):
if task.http_headers.content_type != "application/json":
return InferenceError(http_status=400,
err_msg="application/json only")
result = self.artifacts.model.predict_json([input_data])[0]
return InferenceResult(http_status=200, data=json.dumps(result))
class PytorchImageClassifier(bentoml.BentoService):
@bentoml.utils.cached_property
def transform(self):
return transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
@bentoml.api(input=FileInput(), batch=True)
def predict(self, file_streams: List[BinaryIO]) -> List[str]:
input_datas = []
for fs in file_streams:
img = Image.open(fs).resize((32, 32))
input_datas.append(self.transform(img))
outputs = self.artifacts.net(Variable(torch.stack(input_datas)))
_, output_classes = outputs.max(dim=1)
return [classes[output_class] for output_class in output_classes]
class TextDetectionService(BentoService):
@api(input=FileInput())
def predict(self, image):
result = self.artifacts.model.detect_text(image[0])
return result