def test_multiclass_numeric(self): ad_model = CustomMetricsModel( "name", "adserver.cm_models.multiclass_numeric.MultiClassNumeric") ad_model.load() req = {"truth": [4], "response": 8} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertTrue( res.metrics[0]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) self.assertTrue( res.metrics[1]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) req = {"truth": [7], "response": 7} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive") req = {"truth": np.array([7]), "response": 7} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
def test_binary(self): ad_model = CustomMetricsModel( "name", "adserver.cm_models.binary_metrics.BinaryMetrics") ad_model.load() req = {"truth": [0], "response": 1} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 1) self.assertEqual(res.metrics[0]["key"], "seldon_metric_false_positive") req = {"truth": [1], "response": 1} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 1) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
def test_multiclass_onehot(self): ad_model = CustomMetricsModel( "name", "adserver.cm_models.multiclass_one_hot.MulticlassOneHot") ad_model.load() req = {"truth": [0, 0, 1, 0], "response": [0, 0, 0, 1]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertTrue( res.metrics[0]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) self.assertTrue( res.metrics[1]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) req = {"truth": [0, 0, 1, 0], "response": [0, 0, 1, 0]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 1) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive") req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.1, 0.7]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertTrue( res.metrics[0]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) self.assertTrue( res.metrics[1]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.7, 0.1]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 1) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
def test_multiclass_onehot(self): ad_model = CustomMetricsModel( "name", "adserver.cm_models.multiclass_one_hot.MulticlassOneHot") ad_model.load() req = {"truth": [0, 0, 1, 0], "response": [0, 0, 0, 1]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertTrue( res.metrics[0]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) self.assertTrue( res.metrics[1]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) req = {"truth": [0, 0, 1, 0], "response": [0, 0, 1, 0]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive") req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.1, 0.7]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertTrue( res.metrics[0]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) self.assertTrue( res.metrics[1]["key"] in ["seldon_metric_false_positive", "seldon_metric_false_negative"]) req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.7, 0.1]} headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive") req = { "truth": [0.0006985194531162841, 0.003668039039435755, 0.9956334415074478], "response": [0, 0, 1], } headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive") req = { "truth": np.array([[ 0.0006985194531162841, 0.003668039039435755, 0.9956334415074478 ]]), "response": np.array([[0, 0, 1]]), } headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive") req = { "truth": np.array([[0, 0, 1]]), "response": np.array([ 0.0006985194531162841, 0.003668039039435755, 0.9956334415074478 ]), } headers = {} res = ad_model.process_event(req, headers) self.assertEqual(len(res.metrics), 2) self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")