def test_multiclass_numeric(self):
ad_model = CustomMetricsModel(
"name", "adserver.cm_models.multiclass_numeric.MultiClassNumeric")
ad_model.load()
req = {"truth": [4], "response": 8}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertTrue(
res.metrics[0]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
self.assertTrue(
res.metrics[1]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
req = {"truth": [7], "response": 7}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
req = {"truth": np.array([7]), "response": 7}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
def test_binary(self):
ad_model = CustomMetricsModel(
"name", "adserver.cm_models.binary_metrics.BinaryMetrics")
ad_model.load()
req = {"truth": [0], "response": 1}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 1)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_false_positive")
req = {"truth": [1], "response": 1}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 1)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
def test_multiclass_onehot(self):
ad_model = CustomMetricsModel(
"name", "adserver.cm_models.multiclass_one_hot.MulticlassOneHot")
ad_model.load()
req = {"truth": [0, 0, 1, 0], "response": [0, 0, 0, 1]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertTrue(
res.metrics[0]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
self.assertTrue(
res.metrics[1]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
req = {"truth": [0, 0, 1, 0], "response": [0, 0, 1, 0]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 1)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.1, 0.7]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertTrue(
res.metrics[0]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
self.assertTrue(
res.metrics[1]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.7, 0.1]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 1)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
else:
extra = {}
logging.info("Extra args: %s", extra)
if __name__ == "__main__":
method = AlibiDetectMethod(args.command)
model: Data = None
if method == AlibiDetectMethod.outlier_detector:
model = AlibiDetectOutlierModel(args.model_name, args.storage_uri)
elif method == AlibiDetectMethod.adversarial_detector:
model = AlibiDetectAdversarialDetectionModel(args.model_name,
args.storage_uri)
elif method == AlibiDetectMethod.drift_detector:
model = AlibiDetectConceptDriftModel(args.model_name, args.storage_uri,
**extra)
elif method == AlibiDetectMethod.metrics_server:
model = CustomMetricsModel(args.model_name,
args.storage_uri,
elasticsearch_uri=args.elasticsearch_uri,
**extra)
else:
logging.error("Unknown method %s", args.command)
os._exit(-1)
CEServer(
args.protocol,
args.event_type,
args.event_source,
http_port=args.http_port,
reply_url=args.reply_url,
).start(model)
args, _ = parser.parse_known_args()
argdDict = vars(args).copy()
if "alibi" in argdDict:
extra = vars(args.alibi)
else:
extra = {}
logging.info("Extra args: %s", extra)
if __name__ == "__main__":
method = AlibiDetectMethod(args.command)
model: Data = None
if method == AlibiDetectMethod.outlier_detector:
model = AlibiDetectOutlierModel(args.model_name, args.storage_uri)
elif method == AlibiDetectMethod.adversarial_detector:
model = AlibiDetectAdversarialDetectionModel(args.model_name, args.storage_uri)
elif method == AlibiDetectMethod.drift_detector:
model = AlibiDetectConceptDriftModel(args.model_name, args.storage_uri, **extra)
elif method == AlibiDetectMethod.metrics_server:
model = CustomMetricsModel(args.model_name, args.storage_uri, **extra)
else:
logging.error("Unknown method %s", args.command)
os._exit(-1)
CEServer(
args.protocol,
args.event_type,
args.event_source,
http_port=args.http_port,
reply_url=args.reply_url,
).start(model)
def test_multiclass_onehot(self):
ad_model = CustomMetricsModel(
"name", "adserver.cm_models.multiclass_one_hot.MulticlassOneHot")
ad_model.load()
req = {"truth": [0, 0, 1, 0], "response": [0, 0, 0, 1]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertTrue(
res.metrics[0]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
self.assertTrue(
res.metrics[1]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
req = {"truth": [0, 0, 1, 0], "response": [0, 0, 1, 0]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.1, 0.7]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertTrue(
res.metrics[0]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
self.assertTrue(
res.metrics[1]["key"] in
["seldon_metric_false_positive", "seldon_metric_false_negative"])
req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.7, 0.1]}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
req = {
"truth":
[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478],
"response": [0, 0, 1],
}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
req = {
"truth":
np.array([[
0.0006985194531162841, 0.003668039039435755, 0.9956334415074478
]]),
"response":
np.array([[0, 0, 1]]),
}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
req = {
"truth":
np.array([[0, 0, 1]]),
"response":
np.array([
0.0006985194531162841, 0.003668039039435755, 0.9956334415074478
]),
}
headers = {}
res = ad_model.process_event(req, headers)
self.assertEqual(len(res.metrics), 2)
self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")