def wrapped_func(*args, **kwargs):
with trace(
service_name=self.__class__.__name__,
span_name="user defined inference api callback function",
):
if append_arg and append_arg in kwargs:
tasks = kwargs.pop(append_arg)
elif append_arg in kwargs:
tasks = kwargs[append_arg]
else:
tasks = []
try:
return self._user_func(*args, **kwargs)
except Exception as e: # pylint: disable=broad-except
if self.batch:
for task in tasks:
if not task.is_discarded:
task.discard(
http_status=500,
err_msg=
f"Exception happened in API function: {e}",
)
return [None] * sum(1 if t.batch is None else t.batch
for t in tasks)
else:
task = tasks
if not task.is_discarded:
task.discard(
http_status=500,
err_msg=
f"Exception happened in API function: {e}",
)
return [None
] * (1 if task.batch is None else task.batch)
def handle_batch_request(self, requests: Sequence[HTTPRequest]):
with trace(
service_name=self.__class__.__name__,
span_name=f"call `{self.input_adapter.__class__.__name__}`",
):
inf_tasks = map(self.input_adapter.from_http_request, requests)
results = self.infer(inf_tasks)
return map(self.output_adapter.to_http_response, results)
def handle_batch_request(self, requests: Sequence[HTTPRequest]):
with trace(
service_name=self.__class__.__name__,
span_name=f"call `{self.input_adapter.__class__.__name__}`",
):
inf_tasks = map(self.input_adapter.from_http_request, requests)
results = self.infer(inf_tasks)
responses = tuple(map(self.output_adapter.to_http_response, results))
for inf_task, response in zip(inf_tasks, responses):
response.headers['X-Request-Id'] = inf_task.task_id
return responses
def handle_batch_request(self, request: flask.Request):
from bentoml.marshal.utils import DataLoader
requests = DataLoader.split_requests(request.get_data())
with trace(
service_name=self.__class__.__name__,
span_name=f"call `{self.handler.__class__.__name__}`",
):
responses = self.handler.handle_batch_request(requests, self.func)
return DataLoader.merge_responses(responses)
def func_with_tracing(*args, **kwargs):
with trace(
service_name=self.__class__.__name__,
span_name="user defined inference api callback function",
):
return self._func(*args, **kwargs)
def api_func_with_tracing():
with trace(
request.headers,
service_name=self.__class__.__name__,
):
return api_func()
