def health_status( user_model: Any, seldon_metrics: SeldonMetrics ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Call the user model to check the health of the model Parameters ---------- user_model User defined class instance seldon_metrics A SeldonMetrics instance Returns ------- Health check output """ if hasattr(user_model, "health_status_raw"): try: return user_model.health_status_raw() except SeldonNotImplementedError: pass client_response = client_health_status(user_model) metrics = client_custom_metrics(user_model, seldon_metrics, HEALTH_METRIC_METHOD_TAG) return construct_response_json(user_model, False, {}, client_response, None, metrics)
def construct_response(user_model: SeldonComponent, is_request: bool, client_request: prediction_pb2.SeldonMessage, client_raw_response: Union[np.ndarray, str, bytes, dict]) -> prediction_pb2.SeldonMessage: """ Parameters ---------- user_model Client user class is_request Whether this is part of the request flow as opposed to the response flow client_request The request received client_raw_response The raw client response from their model Returns ------- A SeldonMessage proto response """ data_type = client_request.WhichOneof("data_oneof") meta = prediction_pb2.Meta() meta_json: Dict = {} tags = client_custom_tags(user_model) if tags: meta_json["tags"] = tags metrics = client_custom_metrics(user_model) if metrics: meta_json["metrics"] = metrics if client_request.meta: if client_request.meta.puid: meta_json["puid"] = client_request.meta.puid json_format.ParseDict(meta_json, meta) if isinstance(client_raw_response, np.ndarray) or isinstance(client_raw_response, list): client_raw_response = np.array(client_raw_response) if is_request: names = client_feature_names(user_model, client_request.data.names) else: names = client_class_names(user_model, client_raw_response) if data_type == "data": # If request is using defaultdata then return what was sent if is numeric response else ndarray if np.issubdtype(client_raw_response.dtype, np.number): default_data_type = client_request.data.WhichOneof("data_oneof") else: default_data_type = "ndarray" else: # If numeric response return as tensor else return as ndarray if np.issubdtype(client_raw_response.dtype, np.number): default_data_type = "tensor" else: default_data_type = "ndarray" data = array_to_grpc_datadef(default_data_type, client_raw_response, names) return prediction_pb2.SeldonMessage(data=data, meta=meta) elif isinstance(client_raw_response, str): return prediction_pb2.SeldonMessage(strData=client_raw_response, meta=meta) elif isinstance(client_raw_response, dict): jsonDataResponse = ParseDict(client_raw_response, prediction_pb2.SeldonMessage().jsonData) return prediction_pb2.SeldonMessage(jsonData=jsonDataResponse, meta=meta) elif isinstance(client_raw_response, (bytes, bytearray)): return prediction_pb2.SeldonMessage(binData=client_raw_response, meta=meta) else: raise SeldonMicroserviceException("Unknown data type returned as payload:" + client_raw_response)
def test_component_bad(): with pytest.raises(SeldonMicroserviceException): c = Component(False) client_custom_metrics(c, SeldonMetrics(), TEST_METRIC_METHOD_TAG)
def test_component_ok(): c = Component(True) print(client_custom_metrics(c, SeldonMetrics(), TEST_METRIC_METHOD_TAG)) assert client_custom_metrics( c, SeldonMetrics(), TEST_METRIC_METHOD_TAG) == [EXPECTED_COUNTER_METRIC]
def predict( user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict, bytes], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict, bytes]: """ Call the user model to get a prediction and package the response Parameters ---------- user_model User defined class instance request The incoming request Returns ------- The prediction """ # TODO: Find a way to choose predict_rest or predict_grpc when payload is # not decoded is_proto = isinstance(request, prediction_pb2.SeldonMessage) if hasattr(user_model, "predict_rest") and not is_proto: logger.warning("predict_rest is deprecated. Please use predict_raw") return user_model.predict_rest(request) elif hasattr(user_model, "predict_grpc") and is_proto: logger.warning("predict_grpc is deprecated. Please use predict_raw") return user_model.predict_grpc(request) else: if hasattr(user_model, "predict_raw"): try: response = user_model.predict_raw(request) handle_raw_custom_metrics(response, seldon_metrics, is_proto, PREDICT_METRIC_METHOD_TAG) return response except SeldonNotImplementedError: pass if is_proto: (features, meta, datadef, data_type) = extract_request_parts(request) client_response = client_predict(user_model, features, datadef.names, meta=meta) metrics = client_custom_metrics( user_model, seldon_metrics, PREDICT_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response( user_model, False, request, client_response.data, meta, metrics, client_response.tags, ) else: (features, meta, datadef, data_type) = extract_request_parts_json(request) class_names = datadef[ "names"] if datadef and "names" in datadef else [] client_response = client_predict(user_model, features, class_names, meta=meta) metrics = client_custom_metrics( user_model, seldon_metrics, PREDICT_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response_json( user_model, False, request, client_response.data, meta, metrics, client_response.tags, )
def aggregate( user_model: Any, request: Union[prediction_pb2.SeldonMessageList, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Aggregate a list of payloads Parameters ---------- user_model A Seldon user model request SeldonMessage proto seldon_metrics A SeldonMetrics instance Returns ------- Aggregated SeldonMessage proto """ def merge_meta(meta_list): tags = {} for meta in meta_list: if meta: tags.update(meta.get("tags", {})) return {"tags": tags} def merge_metrics(meta_list, custom_metrics): metrics = [] for meta in meta_list: if meta: metrics.extend(meta.get("metrics", [])) metrics.extend(custom_metrics) return metrics is_proto = isinstance(request, prediction_pb2.SeldonMessageList) if hasattr(user_model, "aggregate_rest"): logger.warning( "aggregate_rest is deprecated. Please use aggregate_raw") return user_model.aggregate_rest(request) elif hasattr(user_model, "aggregate_grpc"): logger.warning( "aggregate_grpc is deprecated. Please use aggregate_raw") return user_model.aggregate_grpc(request) else: if hasattr(user_model, "aggregate_raw"): try: response = user_model.aggregate_raw(request) handle_raw_custom_metrics(response, seldon_metrics, is_proto, AGGREGATE_METRIC_METHOD_TAG) return response except SeldonNotImplementedError: pass if is_proto: features_list = [] names_list = [] meta_list = [] for msg in request.seldonMessages: (features, meta, datadef, data_type) = extract_request_parts(msg) features_list.append(features) names_list.append(datadef.names) meta_list.append(meta) client_response = client_aggregate(user_model, features_list, names_list) metrics = client_custom_metrics( user_model, seldon_metrics, AGGREGATE_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response( user_model, False, request.seldonMessages[0], client_response.data, merge_meta(meta_list), merge_metrics(meta_list, metrics), client_response.tags, ) else: features_list = [] names_list = [] if isinstance(request, list): msgs = request elif "seldonMessages" in request and isinstance( request["seldonMessages"], list): msgs = request["seldonMessages"] else: raise SeldonMicroserviceException( f"Invalid request data type: {request}") meta_list = [] for msg in msgs: (features, meta, datadef, data_type) = extract_request_parts_json(msg) class_names = datadef[ "names"] if datadef and "names" in datadef else [] features_list.append(features) names_list.append(class_names) meta_list.append(meta) client_response = client_aggregate(user_model, features_list, names_list) metrics = client_custom_metrics( user_model, seldon_metrics, AGGREGATE_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response_json( user_model, False, msgs[0], client_response.data, merge_meta(meta_list), merge_metrics(meta_list, metrics), client_response.tags, )
def route( user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Parameters ---------- user_model A Seldon user model request A SelodonMessage proto seldon_metrics A SeldonMetrics instance Returns ------- """ is_proto = isinstance(request, prediction_pb2.SeldonMessage) if hasattr(user_model, "route_rest"): logger.warning("route_rest is deprecated. Please use route_raw") return user_model.route_rest(request) elif hasattr(user_model, "route_grpc"): logger.warning("route_grpc is deprecated. Please use route_raw") return user_model.route_grpc(request) else: if hasattr(user_model, "route_raw"): try: response = user_model.route_raw(request) handle_raw_custom_metrics(response, seldon_metrics, is_proto, ROUTER_METRIC_METHOD_TAG) return response except SeldonNotImplementedError: pass if is_proto: (features, meta, datadef, data_type) = extract_request_parts(request) client_response = client_route(user_model, features, datadef.names, meta=meta) if not isinstance(client_response.data, int): raise SeldonMicroserviceException( "Routing response must be int but got " + str(client_response.data)) client_response_arr = np.array([[client_response.data]]) metrics = client_custom_metrics( user_model, seldon_metrics, ROUTER_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response( user_model, False, request, client_response_arr, None, metrics, client_response.tags, ) else: (features, meta, datadef, data_type) = extract_request_parts_json(request) class_names = datadef[ "names"] if datadef and "names" in datadef else [] client_response = client_route(user_model, features, class_names, meta=meta) if not isinstance(client_response.data, int): raise SeldonMicroserviceException( "Routing response must be int but got " + str(client_response.data)) client_response_arr = np.array([[client_response.data]]) metrics = client_custom_metrics( user_model, seldon_metrics, ROUTER_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response_json( user_model, False, request, client_response_arr, None, metrics, client_response.tags, )
def transform_output( user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Parameters ---------- user_model User defined class to handle transform input request The incoming request Returns ------- The transformed request """ is_proto = isinstance(request, prediction_pb2.SeldonMessage) if hasattr(user_model, "transform_output_rest"): logger.warning( "transform_input_rest is deprecated. Please use transform_input_raw" ) return user_model.transform_output_rest(request) elif hasattr(user_model, "transform_output_grpc"): logger.warning( "transform_input_grpc is deprecated. Please use transform_input_raw" ) return user_model.transform_output_grpc(request) else: if hasattr(user_model, "transform_output_raw"): try: response = user_model.transform_output_raw(request) handle_raw_custom_metrics( response, seldon_metrics, is_proto, OUTPUT_TRANSFORM_METRIC_METHOD_TAG, ) return response except SeldonNotImplementedError: pass if is_proto: (features, meta, datadef, data_type) = extract_request_parts(request) client_response = client_transform_output(user_model, features, datadef.names, meta=meta) metrics = client_custom_metrics( user_model, seldon_metrics, OUTPUT_TRANSFORM_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response( user_model, False, request, client_response.data, meta, metrics, client_response.tags, ) else: (features, meta, datadef, data_type) = extract_request_parts_json(request) class_names = datadef[ "names"] if datadef and "names" in datadef else [] client_response = client_transform_output(user_model, features, class_names, meta=meta) metrics = client_custom_metrics( user_model, seldon_metrics, OUTPUT_TRANSFORM_METRIC_METHOD_TAG, client_response.metrics, ) return construct_response_json( user_model, False, request, client_response.data, meta, metrics, client_response.tags, )
def send_feedback( user_model: Any, request: prediction_pb2.Feedback, predictive_unit_id: str, seldon_metrics: SeldonMetrics, ) -> prediction_pb2.SeldonMessage: """ Parameters ---------- user_model A Seldon user model request SeldonMesage proto predictive_unit_id The ID of the enclosing container predictive unit. Will be taken from environment. Returns ------- """ seldon_metrics.update_reward(request.reward) if hasattr(user_model, "send_feedback_rest"): logger.warning( "send_feedback_rest is deprecated. Please use send_feedback_raw") request_json = json_format.MessageToJson(request) response_json = user_model.send_feedback_rest(request_json) return json_to_seldon_message(response_json) elif hasattr(user_model, "send_feedback_grpc"): logger.warning( "send_feedback_grpc is deprecated. Please use send_feedback_raw") response_json = user_model.send_feedback_grpc(request) return json_to_seldon_message(response_json) else: if hasattr(user_model, "send_feedback_raw"): try: response = user_model.send_feedback_raw(request) handle_raw_custom_metrics(response, seldon_metrics, True, FEEDBACK_METRIC_METHOD_TAG) return response except SeldonNotImplementedError: pass (datadef_request, features, truth, reward) = extract_feedback_request_parts(request) routing = request.response.meta.routing.get(predictive_unit_id) client_response = client_send_feedback(user_model, features, datadef_request.names, reward, truth, routing) metrics = client_custom_metrics( user_model, seldon_metrics, FEEDBACK_METRIC_METHOD_TAG, client_response.metrics, ) if client_response.data is None: client_response.data = np.array([]) return construct_response( user_model, False, request.request, client_response.data, None, metrics, client_response.tags, )
def construct_response_json( user_model: SeldonComponent, is_request: bool, client_request_raw: Union[List, Dict], client_raw_response: Union[np.ndarray, str, bytes, dict], meta: dict = None, ) -> Union[List, Dict]: """ This class converts a raw REST response into a JSON object that has the same structure as the SeldonMessage proto. This is necessary as the conversion using the SeldonMessage proto changes the Numeric types of all ints in a JSON into Floats. Parameters ---------- user_model Client user class is_request Whether this is part of the request flow as opposed to the response flow client_request_raw The request received in JSON format client_raw_response The raw client response from their model Returns ------- A SeldonMessage JSON response """ response = {} if isinstance(client_raw_response, dict): response["jsonData"] = client_raw_response elif isinstance(client_raw_response, (bytes, bytearray)): base64_data = base64.b64encode(client_raw_response) response["binData"] = base64_data.decode("utf-8") elif isinstance(client_raw_response, str): response["strData"] = client_raw_response else: is_np = isinstance(client_raw_response, np.ndarray) is_list = isinstance(client_raw_response, list) if not (is_np or is_list): raise SeldonMicroserviceException( "Unknown data type returned as payload (must be list or np array):" + str(client_raw_response)) if is_np: np_client_raw_response = client_raw_response list_client_raw_response = client_raw_response.tolist() else: np_client_raw_response = np.array(client_raw_response) list_client_raw_response = client_raw_response response["data"] = {} if "data" in client_request_raw: if np.issubdtype(np_client_raw_response.dtype, np.number): if "tensor" in client_request_raw["data"]: default_data_type = "tensor" result_client_response = { "values": np_client_raw_response.ravel().tolist(), "shape": np_client_raw_response.shape, } elif "tftensor" in client_request_raw["data"]: default_data_type = "tftensor" tf_json_str = json_format.MessageToJson( tf.make_tensor_proto(np_client_raw_response)) result_client_response = json.loads(tf_json_str) else: default_data_type = "ndarray" result_client_response = list_client_raw_response else: default_data_type = "ndarray" result_client_response = list_client_raw_response else: if np.issubdtype(np_client_raw_response.dtype, np.number): default_data_type = "tensor" result_client_response = { "values": np_client_raw_response.ravel().tolist(), "shape": np_client_raw_response.shape, } else: default_data_type = "ndarray" result_client_response = list_client_raw_response response["data"][default_data_type] = result_client_response if is_request: req_names = client_request_raw.get("data", {}).get("names", []) names = client_feature_names(user_model, req_names) else: names = client_class_names(user_model, np_client_raw_response) response["data"]["names"] = names response["meta"] = {} if meta: tags = meta.get("tags", {}) else: tags = {} custom_tags = client_custom_tags(user_model) if custom_tags: tags.update(custom_tags) if tags: response["meta"]["tags"] = tags metrics = client_custom_metrics(user_model) if metrics: response["meta"]["metrics"] = metrics puid = client_request_raw.get("meta", {}).get("puid", None) if puid: response["meta"]["puid"] = puid return response
def test_component_ok(): c = Component(True) assert client_custom_metrics(c, SeldonMetrics()) == c.metrics()
def predict( user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Call the user model to get a prediction and package the response Parameters ---------- user_model User defined class instance request The incoming request Returns ------- The prediction """ is_proto = isinstance(request, prediction_pb2.SeldonMessage) if hasattr(user_model, "predict_rest") and not is_proto: logger.warning("predict_rest is deprecated. Please use predict_raw") return user_model.predict_rest(request) elif hasattr(user_model, "predict_grpc") and is_proto: logger.warning("predict_grpc is deprecated. Please use predict_raw") return user_model.predict_grpc(request) else: if hasattr(user_model, "predict_raw"): try: response = user_model.predict_raw(request) if is_proto: metrics = seldon_message_to_json(response.meta).get( "metrics", []) else: metrics = response.get("meta", {}).get("metrics", []) seldon_metrics.update(metrics) return response except SeldonNotImplementedError: pass if is_proto: (features, meta, datadef, data_type) = extract_request_parts(request) client_response = client_predict(user_model, features, datadef.names, meta=meta) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response(user_model, False, request, client_response, meta, metrics) else: (features, meta, datadef, data_type) = extract_request_parts_json(request) class_names = datadef[ "names"] if datadef and "names" in datadef else [] client_response = client_predict(user_model, features, class_names, meta=meta) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response_json(user_model, False, request, client_response, meta, metrics)
def test_component_bad(): with pytest.raises(SeldonMicroserviceException): c = Component(False) client_custom_metrics(c)