def predict():
debug = request.args.get("debug", "false").lower() == "true"
try:
payload = request.get_json()
except:
return "malformed json", status.HTTP_400_BAD_REQUEST
api = local_cache["api"]
predictor = local_cache["predictor"]
try:
try:
debug_obj("payload", payload, debug)
output = predictor.predict(payload, api["predictor"]["metadata"])
debug_obj("prediction", output, debug)
except Exception as e:
raise UserRuntimeException(api["predictor"]["path"], "predict",
str(e)) from e
except Exception as e:
cx_logger().exception("prediction failed")
return prediction_failed(str(e))
g.prediction = output
return jsonify(output)
def predict(
request: Any = Body(..., media_type="application/json"), debug=False):
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
debug_obj("payload", request, debug)
prediction = predictor_impl.predict(request)
debug_obj("prediction", prediction, debug)
try:
json_string = json.dumps(prediction)
except:
json_string = util.json_tricks_encoder().encode(prediction)
response = Response(content=json_string, media_type="application/json")
if api.tracker is not None:
try:
predicted_value = api.tracker.extract_predicted_value(prediction)
api.post_tracker_metrics(predicted_value)
if (api.tracker.model_type == "classification"
and predicted_value not in local_cache["class_set"]):
tasks = BackgroundTasks()
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
response.background = tasks
except:
cx_logger().warn("unable to record prediction metric",
exc_info=True)
return response
def predict(
request: Any = Body(..., media_type="application/json"), debug=False):
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
debug_obj("payload", request, debug)
prediction = predictor_impl.predict(request)
try:
json_string = json.dumps(prediction)
except Exception as e:
raise UserRuntimeException(
f"the return value of predict() or one of its nested values is not JSON serializable",
str(e),
) from e
debug_obj("prediction", json_string, debug)
response = Response(content=json_string, media_type="application/json")
if api.tracker is not None:
try:
predicted_value = api.tracker.extract_predicted_value(prediction)
api.post_tracker_metrics(predicted_value)
if (api.tracker.model_type == "classification"
and predicted_value not in local_cache["class_set"]):
tasks = BackgroundTasks()
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
response.background = tasks
except:
cx_logger().warn("unable to record prediction metric",
exc_info=True)
return response
def predict(app_name, api_name):
debug = request.args.get("debug", "false").lower() == "true"
try:
sample = request.get_json()
except Exception as e:
return "malformed json", status.HTTP_400_BAD_REQUEST
sess = local_cache["sess"]
api = local_cache["api"]
ctx = local_cache["ctx"]
request_handler = local_cache.get("request_handler")
input_metadata = local_cache["input_metadata"]
output_metadata = local_cache["output_metadata"]
try:
debug_obj("sample", sample, debug)
prepared_sample = sample
if request_handler is not None and util.has_function(
request_handler, "pre_inference"):
try:
prepared_sample = request_handler.pre_inference(
sample, input_metadata)
debug_obj("pre_inference", prepared_sample, debug)
except Exception as e:
raise UserRuntimeException(api["request_handler"],
"pre_inference request handler",
str(e)) from e
inference_input = convert_to_onnx_input(prepared_sample,
input_metadata)
model_outputs = sess.run([], inference_input)
result = []
for model_output in model_outputs:
if type(model_output) is np.ndarray:
result.append(model_output.tolist())
else:
result.append(model_output)
debug_obj("inference", result, debug)
if request_handler is not None and util.has_function(
request_handler, "post_inference"):
try:
result = request_handler.post_inference(
result, output_metadata)
except Exception as e:
raise UserRuntimeException(api["request_handler"],
"post_inference request handler",
str(e)) from e
debug_obj("post_inference", result, debug)
except Exception as e:
logger.exception("prediction failed")
return prediction_failed(str(e))
g.prediction = result
return jsonify(result)
def predict():
debug = request.args.get("debug", "false").lower() == "true"
try:
payload = request.get_json()
except Exception as e:
return "malformed json", status.HTTP_400_BAD_REQUEST
sess = local_cache["sess"]
api = local_cache["api"]
ctx = local_cache["ctx"]
request_handler = local_cache.get("request_handler")
input_metadata = local_cache["input_metadata"]
output_metadata = local_cache["output_metadata"]
try:
debug_obj("payload", payload, debug)
prepared_payload = payload
if request_handler is not None and util.has_function(
request_handler, "pre_inference"):
try:
prepared_payload = request_handler.pre_inference(
payload, input_metadata, api["onnx"]["metadata"])
debug_obj("pre_inference", prepared_payload, debug)
except Exception as e:
raise UserRuntimeException(api["onnx"]["request_handler"],
"pre_inference request handler",
str(e)) from e
inference_input = convert_to_onnx_input(prepared_payload,
input_metadata)
model_output = sess.run([], inference_input)
debug_obj("inference", model_output, debug)
result = model_output
if request_handler is not None and util.has_function(
request_handler, "post_inference"):
try:
result = request_handler.post_inference(
model_output, output_metadata, api["onnx"]["metadata"])
except Exception as e:
raise UserRuntimeException(api["onnx"]["request_handler"],
"post_inference request handler",
str(e)) from e
debug_obj("post_inference", result, debug)
except Exception as e:
cx_logger().exception("prediction failed")
return prediction_failed(str(e))
g.prediction = result
return jsonify(result)
def predict(
request: Any = Body(..., media_type="application/json"), debug=False):
api = local_cache["api"]
predictor_impl = local_cache["predictor_impl"]
debug_obj("payload", request, debug)
prediction = predictor_impl.predict(request)
debug_obj("prediction", prediction, debug)
if isinstance(prediction, bytes):
response = Response(content=prediction,
media_type="application/octet-stream")
elif isinstance(prediction, str):
response = Response(content=prediction, media_type="text/plain")
elif isinstance(prediction, Response):
response = prediction
else:
try:
json_string = json.dumps(prediction)
except Exception as e:
raise UserRuntimeException(
str(e),
"please return an object that is JSON serializable (including its nested fields), a bytes object, a string, or a starlette.response.Response object",
) from e
response = Response(content=json_string, media_type="application/json")
if api.tracker is not None:
try:
predicted_value = api.tracker.extract_predicted_value(prediction)
api.post_tracker_metrics(predicted_value)
if (api.tracker.model_type == "classification"
and predicted_value not in local_cache["class_set"]):
tasks = BackgroundTasks()
tasks.add_task(api.upload_class, class_name=predicted_value)
local_cache["class_set"].add(predicted_value)
response.background = tasks
except:
cx_logger().warn("unable to record prediction metric",
exc_info=True)
return response
def run_predict(payload, debug=False):
ctx = local_cache["ctx"]
api = local_cache["api"]
request_handler = local_cache.get("request_handler")
prepared_payload = payload
debug_obj("payload", payload, debug)
if request_handler is not None and util.has_function(
request_handler, "pre_inference"):
try:
prepared_payload = request_handler.pre_inference(
payload,
local_cache["model_metadata"]["signatureDef"],
api["tensorflow"]["metadata"],
)
debug_obj("pre_inference", prepared_payload, debug)
except Exception as e:
raise UserRuntimeException(api["tensorflow"]["request_handler"],
"pre_inference request handler",
str(e)) from e
validate_payload(prepared_payload)
prediction_request = create_prediction_request(prepared_payload)
response_proto = local_cache["stub"].Predict(prediction_request,
timeout=300.0)
result = parse_response_proto(response_proto)
debug_obj("inference", result, debug)
if request_handler is not None and util.has_function(
request_handler, "post_inference"):
try:
result = request_handler.post_inference(
result, local_cache["model_metadata"]["signatureDef"],
api["tensorflow"]["metadata"])
debug_obj("post_inference", result, debug)
except Exception as e:
raise UserRuntimeException(api["tensorflow"]["request_handler"],
"post_inference request handler",
str(e)) from e
return result
