def process_event(self, inputs: Union[List, Dict],
headers: Dict) -> ModelResponse:
"""
Process the event and return Alibi Detect score
Parameters
----------
inputs
Input data
headers
Header options
Returns
-------
Alibi Detect response
"""
logging.info("PROCESSING EVENT.")
logging.info(str(headers))
logging.info("----")
try:
X = read_inputs_as_numpy(inputs)
except Exception as e:
raise Exception(
"Failed to initialize NumPy array from inputs: %s, %s" %
(e, inputs))
ret_instance_score = True
if (HEADER_RETURN_INSTANCE_SCORE in headers
and headers[HEADER_RETURN_INSTANCE_SCORE] == "false"):
ret_instance_score = False
ad_preds = self.model.predict(X,
return_instance_score=ret_instance_score)
data = json.loads(json.dumps(ad_preds, cls=NumpyEncoder))
return ModelResponse(data=data, metrics=None)
def process_event(self, inputs: Union[List, Dict],
headers: Dict) -> Optional[ModelResponse]:
"""
Process the event and return Alibi Detect score
Parameters
----------
inputs
Input data
headers
Header options
Returns
-------
Alibi Detect response
"""
logging.info("PROCESSING EVENT.")
logging.info(str(headers))
logging.info("----")
try:
X = read_inputs_as_numpy(inputs)
except Exception as e:
raise Exception(
"Failed to initialize NumPy array from inputs: %s, %s" %
(e, inputs))
ret_instance_score = False
if (HEADER_RETURN_INSTANCE_SCORE in headers
and headers[HEADER_RETURN_INSTANCE_SCORE]
== "true") or RETURN_INSTANCE_SCORE:
ret_instance_score = True
outlier_type = "instance"
if HEADER_OUTLIER_TYPE in headers and headers[HEADER_OUTLIER_TYPE]:
outlier_type = headers[HEADER_OUTLIER_TYPE]
ret_feature_score = False
if (HEADER_RETURN_FEATURE_SCORE in headers
and headers[HEADER_RETURN_FEATURE_SCORE]
== "true") or RETURN_FEATURE_SCORE:
ret_feature_score = True
od_preds = {}
name = self.model.meta["name"]
if (name == "IForest" or name == "OutlierAEGMM"
or name == "Mahalanobis" or name == "SpectralResidual"
or name == "OutlierVAEGMM"):
od_preds = self.model.predict(
X,
# scores used to determine outliers
return_instance_score=ret_instance_score,
)
else:
od_preds = self.model.predict(
X,
outlier_type=outlier_type,
# use 'feature' or 'instance' level
return_feature_score=ret_feature_score,
# scores used to determine outliers
return_instance_score=ret_instance_score,
)
# Register metrics
metrics: List[Dict] = []
_append_outlier_metrcs(metrics, od_preds, "is_outlier")
_append_outlier_metrcs(metrics,
od_preds,
"instance_score",
is_count=False)
# clean result
if ("data" in od_preds and "instance_score" in od_preds["data"]
and od_preds["data"]["instance_score"] is None):
del od_preds["data"]["instance_score"]
if ("data" in od_preds and "feature_score" in od_preds["data"]
and od_preds["data"]["feature_score"] is None):
del od_preds["data"]["feature_score"]
resp_data = json.loads(json.dumps(od_preds, cls=NumpyEncoder))
return ModelResponse(data=resp_data, metrics=metrics)
def process_event(self, inputs: Union[List, Dict],
headers: Dict) -> Optional[ModelResponse]:
"""
Process the event and return Alibi Detect score
Parameters
----------
inputs
Input data
headers
Header options
Returns
-------
Alibi Detect response
"""
logging.info("PROCESSING EVENT.")
logging.info(str(headers))
logging.info("----")
try:
X = np.array(inputs)
except Exception as e:
raise Exception(
"Failed to initialize NumPy array from inputs: %s, %s" %
(e, inputs))
if self.batch is None:
self.batch = X
else:
self.batch = np.concatenate((self.batch, X))
self.batch = cast(np.ndarray, self.batch)
if self.batch.shape[0] >= self.drift_batch_size:
logging.info(
"Running drift detection. Batch size is %d. Needed %d",
self.batch.shape[0],
self.drift_batch_size,
)
cd_preds = self.model.predict(self.batch)
logging.info("Ran drift test")
self.batch = None
output = json.loads(json.dumps(cd_preds, cls=NumpyEncoder))
metrics: List[Dict] = []
drift = output.get("data")
if drift:
_append_drift_metrcs(metrics, drift, "is_drift")
_append_drift_metrcs(metrics, drift, "distance")
_append_drift_metrcs(metrics, drift, "p_val")
_append_drift_metrcs(metrics, drift, "threshold")
return ModelResponse(data=output, metrics=metrics)
else:
logging.info(
"Not running drift detection. Batch size is %d. Need %d",
self.batch.shape[0],
self.drift_batch_size,
)
return None
def getResponse() -> ModelResponse:
return ModelResponse(data={"foo": 1}, metrics=None)
def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Optional[ModelResponse]:
"""
Process the event and return Alibi Detect score
Parameters
----------
inputs
Input data
headers
Header options
Returns
-------
SeldonResponse response
"""
logging.info("PROCESSING Feedback Event.")
logging.info(str(headers))
logging.info("----")
metrics: List[Dict] = []
output: Dict = {}
truth = None
response = None
error = None
if not isinstance(inputs, dict):
raise SeldonMicroserviceException(
f"Data is not a dict: {json.dumps(inputs)}",
status_code=400,
reason="BAD_DATA",
)
if "truth" not in inputs:
raise SeldonMicroserviceException(
f"No truth value provided in: {json.dumps(inputs)}",
status_code=400,
reason="NO_TRUTH_VALUE",
)
else:
truth = inputs["truth"]
# We automatically add any metrics provided in the incoming request
if "metrics" in inputs:
metrics.extend(inputs["metrics"])
# If response is provided then we can perform a comparison
if "response" in inputs:
response = inputs["response"]
elif REQUEST_ID_HEADER_NAME in headers:
# Otherwise if UUID is provided we can fetch from elasticsearch
if not self.elasticsearch_client:
error = "Seldon-Puid provided but elasticsearch client not configured"
else:
try:
seldon_puid = headers.get(REQUEST_ID_HEADER_NAME, "")
seldon_namespace = headers.get(NAMESPACE_HEADER_NAME, "")
# Currently only supports SELDON inference type (not kfserving)
elasticsearch_index = f"inference-log-seldon-{seldon_namespace}-{SELDON_DEPLOYMENT_ID}-{SELDON_PREDICTOR_ID}"
doc = self.elasticsearch_client.get(
index=elasticsearch_index, id=seldon_puid
)
response = (
doc.get("_source", {})
.get("response", None)
.get("instance", None)
)
if not response:
error = f"Elasticsearch index {elasticsearch_index} with id {seldon_puid} did not contain response value"
except NotFoundError:
error = f"Elasticsearch index {elasticsearch_index} with id {seldon_puid} not found"
else:
error = "Neither response nor request Puid provided in headers"
if error:
raise SeldonMicroserviceException(
error, status_code=400, reason="METRICS_SERVER_ERROR"
)
logging.error(f"{truth}, {response}")
metrics_transformed = self.model.transform(truth, response)
metrics.extend(metrics_transformed.metrics)
return ModelResponse(data={}, metrics=metrics)