def test_seldon_metrics_send_feedback(cls): user_object = cls() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/send-feedback?json={"reward": 42}') assert rv.status_code == 200 data = seldon_metrics.data[os.getpid()] expected_tags = {"method": FEEDBACK_METRIC_METHOD_TAG} tags_key = SeldonMetrics._generate_tags_key(expected_tags) assert data["COUNTER", "seldon_api_model_feedback_reward", tags_key] == { "value": 42.0, "tags": expected_tags, } rv = client.get('/send-feedback?json={"reward": 42}') assert rv.status_code == 200 data = seldon_metrics.data[os.getpid()] expected_tags = {"method": FEEDBACK_METRIC_METHOD_TAG} tags_key = SeldonMetrics._generate_tags_key(expected_tags) assert data["COUNTER", "seldon_api_model_feedback_reward", tags_key] == { "value": 84.0, "tags": expected_tags, }
def verify_seldon_metrics(data, mycounter_value, histogram_entries, method): expected_base_tags = {"method": method} base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags) expected_custom_tags = {"mytag": "mytagvalue", "method": method} custom_tags_key = SeldonMetrics._generate_tags_key(expected_custom_tags) assert data["GAUGE", "mygauge", base_tags_key]["value"] == 100 assert data["GAUGE", "customtag", custom_tags_key]["value"] == 200 assert data["GAUGE", "customtag", custom_tags_key]["tags"] == expected_custom_tags assert data["COUNTER", "mycounter", base_tags_key]["value"] == mycounter_value assert np.allclose( np.histogram(histogram_entries, BINS)[0], data["TIMER", "mytimer", base_tags_key]["value"][0], ) assert np.allclose(data["TIMER", "mytimer", base_tags_key]["value"][1], np.sum(histogram_entries)) expected_distinct_tags_one = {"mytag": "mytagvalue-1", "method": method} distinct_tags_one_key = SeldonMetrics._generate_tags_key( expected_distinct_tags_one) assert data["GAUGE", "distincttag", distinct_tags_one_key]["value"] == 200 assert (data["GAUGE", "distincttag", distinct_tags_one_key]["tags"] == expected_distinct_tags_one) expected_distinct_tags_two = {"mytag": "mytagvalue-2", "method": method} distinct_tags_two_key = SeldonMetrics._generate_tags_key( expected_distinct_tags_two) assert data["GAUGE", "distincttag", distinct_tags_two_key]["value"] == 200 assert (data["GAUGE", "distincttag", distinct_tags_two_key]["tags"] == expected_distinct_tags_two)
def test_generate_tags_key(): # initializing two different kinds of dictionary insertion_order = OrderedDict({"b": "b", "a": "a"}) sorted_order = {"a": "a", "b": "b"} # assert the items in the list differ based on order assert list(insertion_order.items()) != list(sorted_order.items()) insertion_order_tag = SeldonMetrics._generate_tags_key(insertion_order) sorted_order_tag = SeldonMetrics._generate_tags_key(sorted_order) # same tag generated irrespective of order assert insertion_order_tag == sorted_order_tag
def test_seldon_runtime_data_send_feedback(cls): user_object = cls() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/send-feedback?json={"reward": 42}') assert rv.status_code == 200 j = json.loads(rv.data) assert j["meta"]["tags"] == EXPECTED_TAGS data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 1, [0.0202], FEEDBACK_METRIC_METHOD_TAG) expected_base_tags = {"method": FEEDBACK_METRIC_METHOD_TAG} base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags) assert data["COUNTER", "seldon_api_model_feedback_reward", base_tags_key] == { "value": 42.0, "tags": expected_base_tags, } rv = client.get('/send-feedback?json={"reward": 42}') assert rv.status_code == 200 data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 2, [0.0202, 0.0202], FEEDBACK_METRIC_METHOD_TAG) assert data["COUNTER", "seldon_api_model_feedback_reward", base_tags_key] == { "value": 84.0, "tags": expected_base_tags, }
def verify_seldon_metrics(data, mycounter_value, histogram_entries, method): expected_base_tags = {"method": method} base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags) expected_custom_tags = {"mytag": "mytagvalue", "method": method} custom_tags_key = SeldonMetrics._generate_tags_key(expected_custom_tags) expected_tags_key = f"method-{method}_mytag-mytagvalue" assert custom_tags_key == expected_tags_key assert data["GAUGE", "runtime_gauge", base_tags_key]["value"] == 42 assert data["GAUGE", "mygauge", base_tags_key]["value"] == 100 assert data["GAUGE", "customtag", custom_tags_key]["value"] == 200 assert data["GAUGE", "customtag", custom_tags_key]["tags"] == expected_custom_tags assert data["COUNTER", "mycounter", base_tags_key]["value"] == mycounter_value assert np.allclose( np.histogram(histogram_entries, BINS)[0], data["TIMER", "mytimer", base_tags_key]["value"][0], ) assert np.allclose( data["TIMER", "mytimer", base_tags_key]["value"][1], np.sum(histogram_entries) )