def client_custom_metrics(
user_model: SeldonComponent,
seldon_metrics: SeldonMetrics,
method: str,
runtime_metrics: List[Dict] = [],
) -> List[Dict]:
"""
Get custom metrics for client and update SeldonMetrics.
This function will return empty list if INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental
variable is NOT set to "true" or "True".
Parameters
----------
user_model
A Seldon user model
seldon_metrics
A SeldonMetrics instance
method:
tag of a method that collected the metrics
runtime_metrics:
metrics that were defined on runtime
Returns
-------
A list of custom metrics
"""
if not validate_metrics(runtime_metrics):
raise SeldonMicroserviceException(
f"Bad metric created during request: {json.dumps(runtime_metrics)}",
status_code=500,
reason="MICROSERVICE_BAD_METRIC",
)
seldon_metrics.update(runtime_metrics, method)
if hasattr(user_model, "metrics"):
try:
metrics = user_model.metrics()
if not validate_metrics(metrics):
raise SeldonMicroserviceException(
f"Bad metric created during request: {json.dumps(metrics)}",
status_code=500,
reason="MICROSERVICE_BAD_METRIC",
)
seldon_metrics.update(metrics, method)
if INCLUDE_METRICS_IN_CLIENT_RESPONSE:
return metrics + runtime_metrics
else:
return []
except SeldonNotImplementedError:
pass
logger.debug("custom_metrics is not implemented")
if INCLUDE_METRICS_IN_CLIENT_RESPONSE:
return runtime_metrics
else:
return []
def client_custom_metrics(user_model: SeldonComponent) -> List[Dict]:
"""
Get custom metrics
Parameters
----------
user_model
A Seldon user model
Returns
-------
A list of custom metrics
"""
if hasattr(user_model, "metrics"):
try:
metrics = user_model.metrics()
if not validate_metrics(metrics):
j_str = json.dumps(metrics)
raise SeldonMicroserviceException(
"Bad metric created during request: " + j_str,
reason="MICROSERVICE_BAD_METRIC",
)
return metrics
except SeldonNotImplementedError:
pass
logger.info("custom_metrics is not implemented")
return []
def client_custom_metrics(user_model: SeldonComponent,
seldon_metrics: SeldonMetrics) -> List[Dict]:
"""
Get custom metrics for client and update SeldonMetrics.
This function will return empty list if INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental
variable is NOT set to "true" or "True".
Parameters
----------
user_model
A Seldon user model
seldon_metrics
A SeldonMetrics instance
Returns
-------
A list of custom metrics
"""
if hasattr(user_model, "metrics"):
try:
metrics = user_model.metrics()
if not validate_metrics(metrics):
j_str = json.dumps(metrics)
raise SeldonMicroserviceException(
"Bad metric created during request: " + j_str,
reason="MICROSERVICE_BAD_METRIC",
)
seldon_metrics.update(metrics)
if INCLUDE_METRICS_IN_CLIENT_RESPONSE:
return metrics
else:
return []
except SeldonNotImplementedError:
pass
logger.debug("custom_metrics is not implemented")
return []
def client_custom_metrics(user_model: SeldonComponent) -> List[Dict]:
"""
Get custom metrics
Parameters
----------
user_model
A Seldon user model
Returns
-------
A list of custom metrics
"""
try:
metrics = user_model.metrics()
if not validate_metrics(metrics):
j_str = json.dumps(metrics)
raise SeldonMicroserviceException(
"Bad metric created during request: " + j_str, reason="MICROSERVICE_BAD_METRIC")
return metrics
except (NotImplementedError, AttributeError):
return []
def test_validate_no_type():
assert validate_metrics([{"key": "a", "value": 1}]) == False
def test_validate_bad_type():
assert validate_metrics([{"type": "ABC", "key": "a", "value": 1}]) == False
def test_validate_ok():
assert validate_metrics([{
"type": COUNTER,
"key": "a",
"value": 1
}]) == True
def test_validate_no_list():
assert validate_metrics({"type": COUNTER, "key": "a", "value": 1}) == False
def test_validate_bad_value():
assert validate_metrics([{
"type": COUNTER,
"key": "a",
"value": "1"
}]) == False
def test_validate_no_value():
assert validate_metrics([{"type": COUNTER, "key": "a"}]) == False
def test_validate_no_key():
assert validate_metrics([{"type": COUNTER, "value": 1}]) == False
def post(self):
"""
Handle post request. Extract data. Call event handler and optionally send a reply event.
"""
if not self.model.ready:
self.model.load()
try:
body = json.loads(self.request.body)
except json.decoder.JSONDecodeError as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Unrecognized request format: %s" % e,
)
# Extract payload from request
request_handler: RequestHandler = get_request_handler(
self.protocol, body)
request_handler.validate()
request = request_handler.extract_request()
# Create event from request body
event = v1.Event()
http_marshaller = marshaller.NewDefaultHTTPMarshaller()
event = http_marshaller.FromRequest(event, self.request.headers,
self.request.body, json.loads)
logging.debug(json.dumps(event.Properties()))
# Extract any desired request headers
headers = {}
for (key, val) in self.request.headers.get_all():
headers[key] = val
response: Optional[ModelResponse] = self.model.process_event(
request, headers)
if response is None:
return
runtime_metrics = response.metrics
if runtime_metrics is not None:
if validate_metrics(runtime_metrics):
self.seldon_metrics.update(runtime_metrics, self.event_type)
else:
logging.error("Metrics returned are invalid: " +
str(runtime_metrics))
if response.data is not None:
responseStr = json.dumps(response.data)
# Create event from response if reply_url is active
if not self.reply_url == "":
if event.EventID() is None or event.EventID() == "":
resp_event_id = uuid.uuid1().hex
else:
resp_event_id = event.EventID()
revent = (
v1.Event().SetContentType("application/json").SetData(
responseStr).SetEventID(resp_event_id).SetSource(
self.event_source).SetEventType(
self.event_type).SetExtensions(
event.Extensions()))
logging.debug(json.dumps(revent.Properties()))
sendCloudEvent(revent, self.reply_url)
self.write(json.dumps(response.data))
