def predict(self, batch):
"""
PREDICT COMMAND = {
"command": "predict",
"batch": [ REQUEST_INPUT ]
}
:param batch: list of request
:return:
"""
headers, input_batch, req_id_map = Service.retrieve_data_for_inference(
batch)
self.context.request_ids = req_id_map
self.context.request_processor = headers
metrics = MetricsStore(req_id_map, self.context.model_name)
self.context.metrics = metrics
start_time = time.time()
# noinspection PyBroadException
try:
ret = self._entry_point(input_batch, self.context)
except MemoryError:
logger.error("System out of memory", exc_info=True)
return create_predict_response(None, req_id_map,
"Out of resources", 507)
except Exception as e: # pylint: disable=broad-except
logger.warning("Invoking custom service failed.", exc_info=True)
message = "Prediction failed: " + str(e) + " \n " + str(
traceback.format_exc())
return create_predict_response(None, req_id_map, message, 503)
if not isinstance(ret, list):
logger.warning("model: %s, Invalid return type: %s.",
self.context.model_name, type(ret))
return create_predict_response(None, req_id_map,
"Invalid model predict output", 503)
if len(ret) != len(input_batch):
logger.warning(
"model: %s, number of batch response mismatched, expect: %d, got: %d.",
self.context.model_name, len(input_batch), len(ret))
return create_predict_response(
None, req_id_map, "number of batch response mismatched", 503)
duration = round((time.time() - start_time) * 1000, 2)
metrics.add_time(PREDICTION_METRIC, duration)
return create_predict_response(ret,
req_id_map,
"Prediction success",
200,
context=self.context)
def test_create_predict_response_with_error(self):
msg = codec.create_predict_response(None, {0: "request_id"}, "failed",
200)
assert msg == b'\x00\x00\x00\xc8\x00\x00\x00\x06failed\x00\x00\x00\nrequest_id\x00\x00\x00\x00\x00\x00\x00' \
b'\xc8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05error\xff\xff\xff\xff'
def test_create_predict_response(self):
msg = codec.create_predict_response(["OK"], {0: "request_id"},
"success", 200)
assert msg == b'\x00\x00\x00\xc8\x00\x00\x00\x07success\x00\x00\x00\nrequest_id\x00\x00\x00\x00\x00\x00' \
b'\x00\xc8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02OK\xff\xff\xff\xff'