def test_seldon_metrics_send_feedback(cls):
user_object = cls()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/send-feedback?json={"reward": 42}')
assert rv.status_code == 200
data = seldon_metrics.data[os.getpid()]
expected_tags = {"method": FEEDBACK_METRIC_METHOD_TAG}
tags_key = SeldonMetrics._generate_tags_key(expected_tags)
assert data["COUNTER", "seldon_api_model_feedback_reward", tags_key] == {
"value": 42.0,
"tags": expected_tags,
}
rv = client.get('/send-feedback?json={"reward": 42}')
assert rv.status_code == 200
data = seldon_metrics.data[os.getpid()]
expected_tags = {"method": FEEDBACK_METRIC_METHOD_TAG}
tags_key = SeldonMetrics._generate_tags_key(expected_tags)
assert data["COUNTER", "seldon_api_model_feedback_reward", tags_key] == {
"value": 84.0,
"tags": expected_tags,
}
def verify_seldon_metrics(data, mycounter_value, histogram_entries, method):
expected_base_tags = {"method": method}
base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags)
expected_custom_tags = {"mytag": "mytagvalue", "method": method}
custom_tags_key = SeldonMetrics._generate_tags_key(expected_custom_tags)
assert data["GAUGE", "mygauge", base_tags_key]["value"] == 100
assert data["GAUGE", "customtag", custom_tags_key]["value"] == 200
assert data["GAUGE", "customtag",
custom_tags_key]["tags"] == expected_custom_tags
assert data["COUNTER", "mycounter",
base_tags_key]["value"] == mycounter_value
assert np.allclose(
np.histogram(histogram_entries, BINS)[0],
data["TIMER", "mytimer", base_tags_key]["value"][0],
)
assert np.allclose(data["TIMER", "mytimer", base_tags_key]["value"][1],
np.sum(histogram_entries))
expected_distinct_tags_one = {"mytag": "mytagvalue-1", "method": method}
distinct_tags_one_key = SeldonMetrics._generate_tags_key(
expected_distinct_tags_one)
assert data["GAUGE", "distincttag", distinct_tags_one_key]["value"] == 200
assert (data["GAUGE", "distincttag",
distinct_tags_one_key]["tags"] == expected_distinct_tags_one)
expected_distinct_tags_two = {"mytag": "mytagvalue-2", "method": method}
distinct_tags_two_key = SeldonMetrics._generate_tags_key(
expected_distinct_tags_two)
assert data["GAUGE", "distincttag", distinct_tags_two_key]["value"] == 200
assert (data["GAUGE", "distincttag",
distinct_tags_two_key]["tags"] == expected_distinct_tags_two)
def test_generate_tags_key():
# initializing two different kinds of dictionary
insertion_order = OrderedDict({"b": "b", "a": "a"})
sorted_order = {"a": "a", "b": "b"}
# assert the items in the list differ based on order
assert list(insertion_order.items()) != list(sorted_order.items())
insertion_order_tag = SeldonMetrics._generate_tags_key(insertion_order)
sorted_order_tag = SeldonMetrics._generate_tags_key(sorted_order)
# same tag generated irrespective of order
assert insertion_order_tag == sorted_order_tag
def test_seldon_runtime_data_send_feedback(cls):
user_object = cls()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/send-feedback?json={"reward": 42}')
assert rv.status_code == 200
j = json.loads(rv.data)
assert j["meta"]["tags"] == EXPECTED_TAGS
data = seldon_metrics.data[os.getpid()]
verify_seldon_metrics(data, 1, [0.0202], FEEDBACK_METRIC_METHOD_TAG)
expected_base_tags = {"method": FEEDBACK_METRIC_METHOD_TAG}
base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags)
assert data["COUNTER", "seldon_api_model_feedback_reward", base_tags_key] == {
"value": 42.0,
"tags": expected_base_tags,
}
rv = client.get('/send-feedback?json={"reward": 42}')
assert rv.status_code == 200
data = seldon_metrics.data[os.getpid()]
verify_seldon_metrics(data, 2, [0.0202, 0.0202], FEEDBACK_METRIC_METHOD_TAG)
assert data["COUNTER", "seldon_api_model_feedback_reward", base_tags_key] == {
"value": 84.0,
"tags": expected_base_tags,
}
def verify_seldon_metrics(data, mycounter_value, histogram_entries, method):
expected_base_tags = {"method": method}
base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags)
expected_custom_tags = {"mytag": "mytagvalue", "method": method}
custom_tags_key = SeldonMetrics._generate_tags_key(expected_custom_tags)
expected_tags_key = f"method-{method}_mytag-mytagvalue"
assert custom_tags_key == expected_tags_key
assert data["GAUGE", "runtime_gauge", base_tags_key]["value"] == 42
assert data["GAUGE", "mygauge", base_tags_key]["value"] == 100
assert data["GAUGE", "customtag", custom_tags_key]["value"] == 200
assert data["GAUGE", "customtag", custom_tags_key]["tags"] == expected_custom_tags
assert data["COUNTER", "mycounter", base_tags_key]["value"] == mycounter_value
assert np.allclose(
np.histogram(histogram_entries, BINS)[0],
data["TIMER", "mytimer", base_tags_key]["value"][0],
)
assert np.allclose(
data["TIMER", "mytimer", base_tags_key]["value"][1], np.sum(histogram_entries)
)
