def test_multiclass_numeric(self):
        ad_model = CustomMetricsModel(
            "name", "adserver.cm_models.multiclass_numeric.MultiClassNumeric")
        ad_model.load()
        req = {"truth": [4], "response": 8}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertTrue(
            res.metrics[0]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])
        self.assertTrue(
            res.metrics[1]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])

        req = {"truth": [7], "response": 7}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")

        req = {"truth": np.array([7]), "response": 7}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
Example #2
0
    def test_binary(self):
        ad_model = CustomMetricsModel(
            "name", "adserver.cm_models.binary_metrics.BinaryMetrics")
        ad_model.load()
        req = {"truth": [0], "response": 1}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 1)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_false_positive")

        req = {"truth": [1], "response": 1}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 1)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
Example #3
0
    def test_multiclass_onehot(self):
        ad_model = CustomMetricsModel(
            "name", "adserver.cm_models.multiclass_one_hot.MulticlassOneHot")
        ad_model.load()

        req = {"truth": [0, 0, 1, 0], "response": [0, 0, 0, 1]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertTrue(
            res.metrics[0]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])
        self.assertTrue(
            res.metrics[1]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])

        req = {"truth": [0, 0, 1, 0], "response": [0, 0, 1, 0]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 1)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")

        req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.1, 0.7]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertTrue(
            res.metrics[0]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])
        self.assertTrue(
            res.metrics[1]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])

        req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.7, 0.1]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 1)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")
Example #4
0
else:
    extra = {}
logging.info("Extra args: %s", extra)

if __name__ == "__main__":
    method = AlibiDetectMethod(args.command)
    model: Data = None
    if method == AlibiDetectMethod.outlier_detector:
        model = AlibiDetectOutlierModel(args.model_name, args.storage_uri)
    elif method == AlibiDetectMethod.adversarial_detector:
        model = AlibiDetectAdversarialDetectionModel(args.model_name,
                                                     args.storage_uri)
    elif method == AlibiDetectMethod.drift_detector:
        model = AlibiDetectConceptDriftModel(args.model_name, args.storage_uri,
                                             **extra)
    elif method == AlibiDetectMethod.metrics_server:
        model = CustomMetricsModel(args.model_name,
                                   args.storage_uri,
                                   elasticsearch_uri=args.elasticsearch_uri,
                                   **extra)
    else:
        logging.error("Unknown method %s", args.command)
        os._exit(-1)
    CEServer(
        args.protocol,
        args.event_type,
        args.event_source,
        http_port=args.http_port,
        reply_url=args.reply_url,
    ).start(model)
Example #5
0
args, _ = parser.parse_known_args()

argdDict = vars(args).copy()
if "alibi" in argdDict:
    extra = vars(args.alibi)
else:
    extra = {}
logging.info("Extra args: %s", extra)

if __name__ == "__main__":
    method = AlibiDetectMethod(args.command)
    model: Data = None
    if method == AlibiDetectMethod.outlier_detector:
        model = AlibiDetectOutlierModel(args.model_name, args.storage_uri)
    elif method == AlibiDetectMethod.adversarial_detector:
        model = AlibiDetectAdversarialDetectionModel(args.model_name, args.storage_uri)
    elif method == AlibiDetectMethod.drift_detector:
        model = AlibiDetectConceptDriftModel(args.model_name, args.storage_uri, **extra)
    elif method == AlibiDetectMethod.metrics_server:
        model = CustomMetricsModel(args.model_name, args.storage_uri, **extra)
    else:
        logging.error("Unknown method %s", args.command)
        os._exit(-1)
    CEServer(
        args.protocol,
        args.event_type,
        args.event_source,
        http_port=args.http_port,
        reply_url=args.reply_url,
    ).start(model)
    def test_multiclass_onehot(self):
        ad_model = CustomMetricsModel(
            "name", "adserver.cm_models.multiclass_one_hot.MulticlassOneHot")
        ad_model.load()

        req = {"truth": [0, 0, 1, 0], "response": [0, 0, 0, 1]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertTrue(
            res.metrics[0]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])
        self.assertTrue(
            res.metrics[1]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])

        req = {"truth": [0, 0, 1, 0], "response": [0, 0, 1, 0]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")

        req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.1, 0.7]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertTrue(
            res.metrics[0]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])
        self.assertTrue(
            res.metrics[1]["key"] in
            ["seldon_metric_false_positive", "seldon_metric_false_negative"])

        req = {"truth": [0.1, 0.2, 0.7, 0.1], "response": [0.1, 0.2, 0.7, 0.1]}
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")

        req = {
            "truth":
            [0.0006985194531162841, 0.003668039039435755, 0.9956334415074478],
            "response": [0, 0, 1],
        }
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")

        req = {
            "truth":
            np.array([[
                0.0006985194531162841, 0.003668039039435755, 0.9956334415074478
            ]]),
            "response":
            np.array([[0, 0, 1]]),
        }
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")

        req = {
            "truth":
            np.array([[0, 0, 1]]),
            "response":
            np.array([
                0.0006985194531162841, 0.003668039039435755, 0.9956334415074478
            ]),
        }
        headers = {}
        res = ad_model.process_event(req, headers)
        self.assertEqual(len(res.metrics), 2)
        self.assertEqual(res.metrics[0]["key"], "seldon_metric_true_positive")