def test_store_endpoint_update_existing(db: Session, client: TestClient):
access_key = _get_access_key()
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(access_key=access_key, endpoint=endpoint)
kv_record_before_update = ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert kv_record_before_update.status.state is None
endpoint_dict = endpoint.dict()
endpoint_dict["status"]["state"] = "testing...testing...1 2 1 2"
response = client.put(
url=f"/api/projects/{endpoint.metadata.project}/model-endpoints/{endpoint.metadata.uid}",
headers={"X-V3io-Session-Key": access_key},
json=endpoint_dict,
)
assert response.status_code == 204
kv_record_after_update = ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert kv_record_after_update.status.state == "testing...testing...1 2 1 2"
def test_deserialize_endpoint_from_kv():
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(_get_access_key(), endpoint)
endpoint_from_kv = ModelEndpoints.get_endpoint(
access_key=_get_access_key(),
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert endpoint.metadata.uid == endpoint_from_kv.metadata.uid
def test_get_endpoint_metrics(db: Session, client: TestClient):
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=TEST_PROJECT, kind=EVENTS
)
_, container, path = parse_model_endpoint_store_prefix(path)
frames = get_frames_client(
token=_get_access_key(), container=container, address=config.v3io_framesd,
)
start = datetime.utcnow()
for i in range(5):
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(_get_access_key(), endpoint)
frames.create(backend="tsdb", table=path, rate="10/m", if_exists=1)
total = 0
dfs = []
for i in range(10):
count = randint(1, 10)
total += count
data = {
"predictions_per_second_count_1s": count,
"endpoint_id": endpoint.metadata.uid,
"timestamp": start - timedelta(minutes=10 - i),
}
df = pd.DataFrame(data=[data])
dfs.append(df)
frames.write(
backend="tsdb",
table=path,
dfs=dfs,
index_cols=["timestamp", "endpoint_id"],
)
response = client.get(
url=f"/api/projects/{TEST_PROJECT}/model-endpoints/{endpoint.metadata.uid}?metric=predictions_per_second_count_1s", # noqa
headers={"X-V3io-Session-Key": _get_access_key()},
)
endpoint = ModelEndpoint(**response.json())
assert len(endpoint.status.metrics) > 0
predictions_per_second = endpoint.status.metrics[
"predictions_per_second_count_1s"
]
assert predictions_per_second.name == "predictions_per_second_count_1s"
response_total = sum((m[1] for m in predictions_per_second.values))
assert total == response_total
def test_get_endpoint_metric_function():
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=TEST_PROJECT, kind=EVENTS
)
_, container, path = parse_model_endpoint_store_prefix(path)
frames = get_frames_client(
token=_get_access_key(), container=container, address=config.v3io_framesd,
)
start = datetime.utcnow()
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(_get_access_key(), endpoint)
frames.create(backend="tsdb", table=path, rate="10/m", if_exists=1)
total = 0
dfs = []
for i in range(10):
count = randint(1, 10)
total += count
data = {
"predictions_per_second_count_1s": count,
"endpoint_id": endpoint.metadata.uid,
"timestamp": start - timedelta(minutes=10 - i),
}
df = pd.DataFrame(data=[data])
dfs.append(df)
frames.write(
backend="tsdb", table=path, dfs=dfs, index_cols=["timestamp", "endpoint_id"],
)
endpoint_metrics = get_endpoint_metrics(
access_key=_get_access_key(),
project=TEST_PROJECT,
endpoint_id=endpoint.metadata.uid,
metrics=["predictions_per_second_count_1s"],
)
assert "predictions_per_second_count_1s" in endpoint_metrics
actual_values = endpoint_metrics["predictions_per_second_count_1s"].values
assert len(actual_values) == 10
assert sum(map(lambda t: t[1], actual_values)) == total
def test_list_endpoints(db: Session, client: TestClient):
endpoints_in = [_mock_random_endpoint("testing") for _ in range(5)]
for endpoint in endpoints_in:
write_endpoint_to_kv(_get_access_key(), endpoint)
response = client.get(
url=f"/api/projects/{TEST_PROJECT}/model-endpoints",
headers={"X-V3io-Session-Key": _get_access_key()},
)
endpoints_out = [ModelEndpoint(**e) for e in response.json()["endpoints"]]
in_endpoint_ids = set(map(lambda e: e.metadata.uid, endpoints_in))
out_endpoint_ids = set(map(lambda e: e.metadata.uid, endpoints_out))
endpoints_intersect = in_endpoint_ids.intersection(out_endpoint_ids)
assert len(endpoints_intersect) == 5
def test_list_endpoints_filter(db: Session, client: TestClient):
access_key = _get_access_key()
for i in range(5):
endpoint_details = _mock_random_endpoint()
if i < 1:
endpoint_details.spec.model = "filterme"
if i < 2:
endpoint_details.spec.function_uri = "test/filterme"
if i < 4:
endpoint_details.metadata.labels = {"filtermex": "1", "filtermey": "2"}
write_endpoint_to_kv(_get_access_key(), endpoint_details)
filter_model = client.get(
f"/api/projects/{TEST_PROJECT}/model-endpoints/?model=filterme",
headers={"X-V3io-Session-Key": access_key},
)
assert len(filter_model.json()["endpoints"]) == 1
filter_labels = client.get(
f"/api/projects/{TEST_PROJECT}/model-endpoints/?label=filtermex=1",
headers={"X-V3io-Session-Key": access_key},
)
assert len(filter_labels.json()["endpoints"]) == 4
filter_labels = client.get(
f"/api/projects/{TEST_PROJECT}/model-endpoints/?label=filtermex=1&label=filtermey=2",
headers={"X-V3io-Session-Key": access_key},
)
assert len(filter_labels.json()["endpoints"]) == 4
filter_labels = client.get(
f"/api/projects/{TEST_PROJECT}/model-endpoints/?label=filtermey=2",
headers={"X-V3io-Session-Key": access_key},
)
assert len(filter_labels.json()["endpoints"]) == 4
def test_grafana_list_endpoints(db: Session, client: TestClient):
endpoints_in = [_mock_random_endpoint("active") for _ in range(5)]
for endpoint in endpoints_in:
write_endpoint_to_kv(_get_access_key(), endpoint)
response = client.post(
url="/api/grafana-proxy/model-endpoints/query",
headers={"X-V3io-Session-Key": _get_access_key()},
json={
"targets": [{
"target":
f"project={TEST_PROJECT};target_endpoint=list_endpoints"
}]
},
)
response_json = response.json()
if not response_json:
fail(f"Empty response, expected list of dictionaries. {response_json}")
response_json = response_json[0]
if not response_json:
fail(
f"Empty dictionary, expected dictionary with 'columns', 'rows' and 'type' fields. {response_json}"
)
if "columns" not in response_json:
fail(f"Missing 'columns' key in response dictionary. {response_json}")
if "rows" not in response_json:
fail(f"Missing 'rows' key in response dictionary. {response_json}")
if "type" not in response_json:
fail(f"Missing 'type' key in response dictionary. {response_json}")
assert len(response_json["rows"]) == 5
def test_clear_endpoint(db: Session, client: TestClient):
access_key = _get_access_key()
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(access_key, endpoint)
kv_record = ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert kv_record
response = client.delete(
url=f"/api/projects/{kv_record.metadata.project}/model-endpoints/{endpoint.metadata.uid}",
headers={"X-V3io-Session-Key": access_key},
)
assert response.status_code == 204
with pytest.raises(MLRunNotFoundError):
ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)