def client_custom_metrics( user_model: SeldonComponent, seldon_metrics: SeldonMetrics, method: str, runtime_metrics: List[Dict] = [], ) -> List[Dict]: """ Get custom metrics for client and update SeldonMetrics. This function will return empty list if INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental variable is NOT set to "true" or "True". Parameters ---------- user_model A Seldon user model seldon_metrics A SeldonMetrics instance method: tag of a method that collected the metrics runtime_metrics: metrics that were defined on runtime Returns ------- A list of custom metrics """ if not validate_metrics(runtime_metrics): raise SeldonMicroserviceException( f"Bad metric created during request: {json.dumps(runtime_metrics)}", status_code=500, reason="MICROSERVICE_BAD_METRIC", ) seldon_metrics.update(runtime_metrics, method) if hasattr(user_model, "metrics"): try: metrics = user_model.metrics() if not validate_metrics(metrics): raise SeldonMicroserviceException( f"Bad metric created during request: {json.dumps(metrics)}", status_code=500, reason="MICROSERVICE_BAD_METRIC", ) seldon_metrics.update(metrics, method) if INCLUDE_METRICS_IN_CLIENT_RESPONSE: return metrics + runtime_metrics else: return [] except SeldonNotImplementedError: pass logger.debug("custom_metrics is not implemented") if INCLUDE_METRICS_IN_CLIENT_RESPONSE: return runtime_metrics else: return []
def client_custom_metrics(user_model: SeldonComponent) -> List[Dict]: """ Get custom metrics Parameters ---------- user_model A Seldon user model Returns ------- A list of custom metrics """ if hasattr(user_model, "metrics"): try: metrics = user_model.metrics() if not validate_metrics(metrics): j_str = json.dumps(metrics) raise SeldonMicroserviceException( "Bad metric created during request: " + j_str, reason="MICROSERVICE_BAD_METRIC", ) return metrics except SeldonNotImplementedError: pass logger.info("custom_metrics is not implemented") return []
def client_custom_metrics(user_model: SeldonComponent, seldon_metrics: SeldonMetrics) -> List[Dict]: """ Get custom metrics for client and update SeldonMetrics. This function will return empty list if INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental variable is NOT set to "true" or "True". Parameters ---------- user_model A Seldon user model seldon_metrics A SeldonMetrics instance Returns ------- A list of custom metrics """ if hasattr(user_model, "metrics"): try: metrics = user_model.metrics() if not validate_metrics(metrics): j_str = json.dumps(metrics) raise SeldonMicroserviceException( "Bad metric created during request: " + j_str, reason="MICROSERVICE_BAD_METRIC", ) seldon_metrics.update(metrics) if INCLUDE_METRICS_IN_CLIENT_RESPONSE: return metrics else: return [] except SeldonNotImplementedError: pass logger.debug("custom_metrics is not implemented") return []
def client_custom_metrics(user_model: SeldonComponent) -> List[Dict]: """ Get custom metrics Parameters ---------- user_model A Seldon user model Returns ------- A list of custom metrics """ try: metrics = user_model.metrics() if not validate_metrics(metrics): j_str = json.dumps(metrics) raise SeldonMicroserviceException( "Bad metric created during request: " + j_str, reason="MICROSERVICE_BAD_METRIC") return metrics except (NotImplementedError, AttributeError): return []
def test_validate_no_type(): assert validate_metrics([{"key": "a", "value": 1}]) == False
def test_validate_bad_type(): assert validate_metrics([{"type": "ABC", "key": "a", "value": 1}]) == False
def test_validate_ok(): assert validate_metrics([{ "type": COUNTER, "key": "a", "value": 1 }]) == True
def test_validate_no_list(): assert validate_metrics({"type": COUNTER, "key": "a", "value": 1}) == False
def test_validate_bad_value(): assert validate_metrics([{ "type": COUNTER, "key": "a", "value": "1" }]) == False
def test_validate_no_value(): assert validate_metrics([{"type": COUNTER, "key": "a"}]) == False
def test_validate_no_key(): assert validate_metrics([{"type": COUNTER, "value": 1}]) == False
def post(self): """ Handle post request. Extract data. Call event handler and optionally send a reply event. """ if not self.model.ready: self.model.load() try: body = json.loads(self.request.body) except json.decoder.JSONDecodeError as e: raise tornado.web.HTTPError( status_code=HTTPStatus.BAD_REQUEST, reason="Unrecognized request format: %s" % e, ) # Extract payload from request request_handler: RequestHandler = get_request_handler( self.protocol, body) request_handler.validate() request = request_handler.extract_request() # Create event from request body event = v1.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = http_marshaller.FromRequest(event, self.request.headers, self.request.body, json.loads) logging.debug(json.dumps(event.Properties())) # Extract any desired request headers headers = {} for (key, val) in self.request.headers.get_all(): headers[key] = val response: Optional[ModelResponse] = self.model.process_event( request, headers) if response is None: return runtime_metrics = response.metrics if runtime_metrics is not None: if validate_metrics(runtime_metrics): self.seldon_metrics.update(runtime_metrics, self.event_type) else: logging.error("Metrics returned are invalid: " + str(runtime_metrics)) if response.data is not None: responseStr = json.dumps(response.data) # Create event from response if reply_url is active if not self.reply_url == "": if event.EventID() is None or event.EventID() == "": resp_event_id = uuid.uuid1().hex else: resp_event_id = event.EventID() revent = ( v1.Event().SetContentType("application/json").SetData( responseStr).SetEventID(resp_event_id).SetSource( self.event_source).SetEventType( self.event_type).SetExtensions( event.Extensions())) logging.debug(json.dumps(revent.Properties())) sendCloudEvent(revent, self.reply_url) self.write(json.dumps(response.data))