def health_status( user_model: Any, seldon_metrics: SeldonMetrics ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Call the user model to check the health of the model Parameters ---------- user_model User defined class instance Returns ------- Health check output """ if hasattr(user_model, "health_status_raw"): try: return user_model.health_status_raw() except SeldonNotImplementedError: pass client_response = client_health_status(user_model) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response_json(user_model, False, {}, client_response, None, metrics)
def test_metrics_clear(): metrics = SeldonMetrics() metrics.update([RAW_COUNTER_METRIC], method="predict") assert len(metrics.data) > 0 metrics.clear() assert len(metrics.data) == 0
def client_custom_metrics( user_model: SeldonComponent, seldon_metrics: SeldonMetrics, method: str, runtime_metrics: List[Dict] = [], ) -> List[Dict]: """ Get custom metrics for client and update SeldonMetrics. This function will return empty list if INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental variable is NOT set to "true" or "True". Parameters ---------- user_model A Seldon user model seldon_metrics A SeldonMetrics instance method: tag of a method that collected the metrics runtime_metrics: metrics that were defined on runtime Returns ------- A list of custom metrics """ if not validate_metrics(runtime_metrics): raise SeldonMicroserviceException( f"Bad metric created during request: {json.dumps(runtime_metrics)}", status_code=500, reason="MICROSERVICE_BAD_METRIC", ) seldon_metrics.update(runtime_metrics, method) if hasattr(user_model, "metrics"): try: metrics = user_model.metrics() if not validate_metrics(metrics): raise SeldonMicroserviceException( f"Bad metric created during request: {json.dumps(metrics)}", status_code=500, reason="MICROSERVICE_BAD_METRIC", ) seldon_metrics.update(metrics, method) if INCLUDE_METRICS_IN_CLIENT_RESPONSE: return metrics + runtime_metrics else: return [] except SeldonNotImplementedError: pass logger.debug("custom_metrics is not implemented") if INCLUDE_METRICS_IN_CLIENT_RESPONSE: return runtime_metrics else: return []
def handle_raw_custom_metrics( msg: Union[prediction_pb2.SeldonMessage, Dict], seldon_metrics: SeldonMetrics, is_proto: bool, ): """ Update SeldonMetrics object with custom metrics from raw methods. If INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental variable is set to "true" metrics will be dropped from msg. """ if is_proto: metrics = seldon_message_to_json(msg.meta).get("metrics", []) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] else: metrics = msg.get("meta", {}).get("metrics", []) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg["meta"]["metrics"] seldon_metrics.update(metrics)
def client_custom_metrics(user_model: SeldonComponent, seldon_metrics: SeldonMetrics) -> List[Dict]: """ Get custom metrics for client and update SeldonMetrics. This function will return empty list if INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental variable is NOT set to "true" or "True". Parameters ---------- user_model A Seldon user model seldon_metrics A SeldonMetrics instance Returns ------- A list of custom metrics """ if hasattr(user_model, "metrics"): try: metrics = user_model.metrics() if not validate_metrics(metrics): j_str = json.dumps(metrics) raise SeldonMicroserviceException( "Bad metric created during request: " + j_str, reason="MICROSERVICE_BAD_METRIC", ) seldon_metrics.update(metrics) if INCLUDE_METRICS_IN_CLIENT_RESPONSE: return metrics else: return [] except SeldonNotImplementedError: pass logger.debug("custom_metrics is not implemented") return []
def aggregate( user_model: Any, request: Union[prediction_pb2.SeldonMessageList, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Aggregate a list of payloads Parameters ---------- user_model A Seldon user model request SeldonMessage proto Returns ------- Aggregated SeldonMessage proto """ def merge_meta(meta_list): tags = {} for meta in meta_list: if meta: tags.update(meta.get("tags", {})) return {"tags": tags} def merge_metrics(meta_list, custom_metrics): metrics = [] for meta in meta_list: if meta: metrics.extend(meta.get("metrics", [])) metrics.extend(custom_metrics) return metrics is_proto = isinstance(request, prediction_pb2.SeldonMessageList) if hasattr(user_model, "aggregate_rest"): logger.warning( "aggregate_rest is deprecated. Please use aggregate_raw") return user_model.aggregate_rest(request) elif hasattr(user_model, "aggregate_grpc"): logger.warning( "aggregate_grpc is deprecated. Please use aggregate_raw") return user_model.aggregate_grpc(request) else: if hasattr(user_model, "aggregate_raw"): try: response = user_model.aggregate_raw(request) if is_proto: metrics = seldon_message_to_json(response.meta).get( "metrics", []) else: metrics = response.get("meta", {}).get("metrics", []) seldon_metrics.update(metrics) return response except SeldonNotImplementedError: pass if is_proto: features_list = [] names_list = [] meta_list = [] for msg in request.seldonMessages: (features, meta, datadef, data_type) = extract_request_parts(msg) features_list.append(features) names_list.append(datadef.names) meta_list.append(meta) client_response = client_aggregate(user_model, features_list, names_list) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response( user_model, False, request.seldonMessages[0], client_response, merge_meta(meta_list), merge_metrics(meta_list, metrics), ) else: features_list = [] names_list = [] if isinstance(request, list): msgs = request elif "seldonMessages" in request and isinstance( request["seldonMessages"], list): msgs = request["seldonMessages"] else: raise SeldonMicroserviceException( f"Invalid request data type: {request}") meta_list = [] for msg in msgs: (features, meta, datadef, data_type) = extract_request_parts_json(msg) class_names = datadef[ "names"] if datadef and "names" in datadef else [] features_list.append(features) names_list.append(class_names) meta_list.append(meta) client_response = client_aggregate(user_model, features_list, names_list) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response_json( user_model, False, msgs[0], client_response, merge_meta(meta_list), merge_metrics(meta_list, metrics), )
def predict( user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Call the user model to get a prediction and package the response Parameters ---------- user_model User defined class instance request The incoming request Returns ------- The prediction """ is_proto = isinstance(request, prediction_pb2.SeldonMessage) if hasattr(user_model, "predict_rest") and not is_proto: logger.warning("predict_rest is deprecated. Please use predict_raw") return user_model.predict_rest(request) elif hasattr(user_model, "predict_grpc") and is_proto: logger.warning("predict_grpc is deprecated. Please use predict_raw") return user_model.predict_grpc(request) else: if hasattr(user_model, "predict_raw"): try: response = user_model.predict_raw(request) if is_proto: metrics = seldon_message_to_json(response.meta).get( "metrics", []) else: metrics = response.get("meta", {}).get("metrics", []) seldon_metrics.update(metrics) return response except SeldonNotImplementedError: pass if is_proto: (features, meta, datadef, data_type) = extract_request_parts(request) client_response = client_predict(user_model, features, datadef.names, meta=meta) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response(user_model, False, request, client_response, meta, metrics) else: (features, meta, datadef, data_type) = extract_request_parts_json(request) class_names = datadef[ "names"] if datadef and "names" in datadef else [] client_response = client_predict(user_model, features, class_names, meta=meta) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response_json(user_model, False, request, client_response, meta, metrics)
def route( user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict], seldon_metrics: SeldonMetrics, ) -> Union[prediction_pb2.SeldonMessage, List, Dict]: """ Parameters ---------- user_model A Seldon user model request A SelodonMessage proto Returns ------- """ is_proto = isinstance(request, prediction_pb2.SeldonMessage) if hasattr(user_model, "route_rest"): logger.warning("route_rest is deprecated. Please use route_raw") return user_model.route_rest(request) elif hasattr(user_model, "route_grpc"): logger.warning("route_grpc is deprecated. Please use route_raw") return user_model.route_grpc(request) else: if hasattr(user_model, "route_raw"): try: response = user_model.route_raw(request) if is_proto: metrics = seldon_message_to_json(response.meta).get( "metrics", []) else: metrics = response.get("meta", {}).get("metrics", []) seldon_metrics.update(metrics) return response except SeldonNotImplementedError: pass if is_proto: (features, meta, datadef, data_type) = extract_request_parts(request) client_response = client_route(user_model, features, datadef.names, meta=meta) if not isinstance(client_response, int): raise SeldonMicroserviceException( "Routing response must be int but got " + str(client_response)) client_response_arr = np.array([[client_response]]) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response(user_model, False, request, client_response_arr, None, metrics) else: (features, meta, datadef, data_type) = extract_request_parts_json(request) class_names = datadef[ "names"] if datadef and "names" in datadef else [] client_response = client_route(user_model, features, class_names, meta=meta) if not isinstance(client_response, int): raise SeldonMicroserviceException( "Routing response must be int but got " + str(client_response)) client_response_arr = np.array([[client_response]]) metrics = client_custom_metrics(user_model) if seldon_metrics is not None: seldon_metrics.update(metrics) return construct_response_json(user_model, False, request, client_response_arr, None, metrics)