def predict(
request: Any = Body(..., media_type="application/json"), debug=False):
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
debug_obj("payload", request, debug)
prediction = predictor_impl.predict(request)
debug_obj("prediction", prediction, debug)
try:
json_string = json.dumps(prediction)
except:
json_string = util.json_tricks_encoder().encode(prediction)
response = Response(content=json_string, media_type="application/json")
if api.tracker is not None:
try:
predicted_value = api.tracker.extract_predicted_value(prediction)
api.post_tracker_metrics(predicted_value)
if (api.tracker.model_type == "classification"
and predicted_value not in local_cache["class_set"]):
tasks = BackgroundTasks()
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
response.background = tasks
except:
cx_logger().warn("unable to record prediction metric",
exc_info=True)
return response
def predict(
request: Any = Body(..., media_type="application/json"), debug=False):
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
debug_obj("payload", request, debug)
prediction = predictor_impl.predict(request)
try:
json_string = json.dumps(prediction)
except Exception as e:
raise UserRuntimeException(
f"the return value of predict() or one of its nested values is not JSON serializable",
str(e),
) from e
debug_obj("prediction", json_string, debug)
response = Response(content=json_string, media_type="application/json")
if api.tracker is not None:
try:
predicted_value = api.tracker.extract_predicted_value(prediction)
api.post_tracker_metrics(predicted_value)
if (api.tracker.model_type == "classification"
and predicted_value not in local_cache["class_set"]):
tasks = BackgroundTasks()
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
response.background = tasks
except:
cx_logger().warn("unable to record prediction metric",
exc_info=True)
return response
def predict(request: Request):
tasks = BackgroundTasks()
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
dynamic_batcher = local_cache["dynamic_batcher"]
kwargs = build_predict_kwargs(request)
if dynamic_batcher:
prediction = dynamic_batcher.predict(**kwargs)
else:
prediction = predictor_impl.predict(**kwargs)
if isinstance(prediction, bytes):
response = Response(content=prediction,
media_type="application/octet-stream")
elif isinstance(prediction, str):
response = Response(content=prediction, media_type="text/plain")
elif isinstance(prediction, Response):
response = prediction
else:
try:
json_string = json.dumps(prediction)
except Exception as e:
raise UserRuntimeException(
str(e),
"please return an object that is JSON serializable (including its nested fields), a bytes object, "
"a string, or a starlette.response.Response object",
) from e
response = Response(content=json_string, media_type="application/json")
if local_cache["provider"] != "local" and api.monitoring is not None:
try:
predicted_value = api.monitoring.extract_predicted_value(
prediction)
api.post_monitoring_metrics(predicted_value)
if (api.monitoring.model_type == "classification"
and predicted_value not in local_cache["class_set"]):
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
except:
logger().warn("unable to record prediction metric", exc_info=True)
if util.has_method(predictor_impl, "post_predict"):
kwargs = build_post_predict_kwargs(prediction, request)
request_thread_pool.submit(predictor_impl.post_predict, **kwargs)
if len(tasks.tasks) > 0:
response.background = tasks
return response
def predict(request: Any = Body(..., media_type="application/json"), debug=False):
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
debug_obj("payload", request, debug)
prediction = predictor_impl.predict(request)
if isinstance(prediction, bytes):
response = Response(content=prediction, media_type="application/octet-stream")
elif isinstance(prediction, str):
response = Response(content=prediction, media_type="text/plain")
debug_obj("prediction", prediction, debug)
elif isinstance(prediction, Response):
response = prediction
else:
try:
json_string = json.dumps(prediction)
debug_obj("prediction", prediction, debug)
except Exception as e:
raise UserRuntimeException(
str(e),
"please return an object that is JSON serializable (including its nested fields), a bytes object, a string, or a starlette.response.Response object",
) from e
response = Response(content=json_string, media_type="application/json")
if local_cache["provider"] != "local" and api.tracker is not None:
try:
predicted_value = api.tracker.extract_predicted_value(prediction)
api.post_tracker_metrics(predicted_value)
if (
api.tracker.model_type == "classification"
and predicted_value not in local_cache["class_set"]
):
tasks = BackgroundTasks()
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
response.background = tasks
except:
cx_logger().warn("unable to record prediction metric", exc_info=True)
return response
