def clear_endpoint_record(request: Request, project: str, endpoint_id: str):
"""
Clears endpoint record from KV by endpoint_id
"""
access_key = get_access_key(request)
ModelEndpoints.clear_endpoint_record(access_key, project, endpoint_id)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def test_store_endpoint_update_existing(db: Session, client: TestClient):
access_key = _get_access_key()
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(access_key=access_key, endpoint=endpoint)
kv_record_before_update = ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert kv_record_before_update.status.state is None
endpoint_dict = endpoint.dict()
endpoint_dict["status"]["state"] = "testing...testing...1 2 1 2"
response = client.put(
url=f"/api/projects/{endpoint.metadata.project}/model-endpoints/{endpoint.metadata.uid}",
headers={"X-V3io-Session-Key": access_key},
json=endpoint_dict,
)
assert response.status_code == 204
kv_record_after_update = ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert kv_record_after_update.status.state == "testing...testing...1 2 1 2"
def create_or_patch(
request: Request,
project: str,
endpoint_id: str,
model_endpoint: ModelEndpoint,
auth_verifier: mlrun.api.api.deps.AuthVerifier = Depends(
mlrun.api.api.deps.AuthVerifier),
db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
) -> Response:
"""
Either create or updates the kv record of a given ModelEndpoint object
"""
access_key = get_access_key(request.headers)
if project != model_endpoint.metadata.project:
raise MLRunConflictError(
f"Can't store endpoint of project {model_endpoint.metadata.project} into project {project}"
)
if endpoint_id != model_endpoint.metadata.uid:
raise MLRunConflictError(
f"Mismatch between endpoint_id {endpoint_id} and ModelEndpoint.metadata.uid {model_endpoint.metadata.uid}."
f"\nMake sure the supplied function_uri, and model are configured as intended"
)
ModelEndpoints.create_or_patch(
db_session=db_session,
access_key=access_key,
model_endpoint=model_endpoint,
leader_session=auth_verifier.auth_info.session,
)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def delete_endpoint_record(
request: Request, project: str, endpoint_id: str
) -> Response:
"""
Clears endpoint record from KV by endpoint_id
"""
access_key = get_access_key(request.headers)
ModelEndpoints.delete_endpoint_record(
access_key=access_key, project=project, endpoint_id=endpoint_id,
)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def grafana_overall_feature_analysis(body: Dict[str, Any],
query_parameters: Dict[str, str],
access_key: str):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
endpoint = ModelEndpoints.get_endpoint(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
feature_analysis=True,
)
table = GrafanaTable(columns=[
GrafanaNumberColumn(text="tvd_sum"),
GrafanaNumberColumn(text="tvd_mean"),
GrafanaNumberColumn(text="hellinger_sum"),
GrafanaNumberColumn(text="hellinger_mean"),
GrafanaNumberColumn(text="kld_sum"),
GrafanaNumberColumn(text="kld_mean"),
])
if endpoint.status.drift_measures:
table.add_row(
endpoint.status.drift_measures.get("tvd_sum"),
endpoint.status.drift_measures.get("tvd_mean"),
endpoint.status.drift_measures.get("hellinger_sum"),
endpoint.status.drift_measures.get("hellinger_mean"),
endpoint.status.drift_measures.get("kld_sum"),
endpoint.status.drift_measures.get("kld_mean"),
)
return [table]
def test_deserialize_endpoint_from_kv():
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(_get_access_key(), endpoint)
endpoint_from_kv = ModelEndpoints.get_endpoint(
access_key=_get_access_key(),
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert endpoint.metadata.uid == endpoint_from_kv.metadata.uid
def create_or_patch(
request: Request, project: str, endpoint_id: str, model_endpoint: ModelEndpoint
) -> Response:
"""
Either create or updates the kv record of a given ModelEndpoint object
"""
access_key = get_access_key(request.headers)
if project != model_endpoint.metadata.project:
raise MLRunConflictError(
f"Can't store endpoint of project {model_endpoint.metadata.project} into project {project}"
)
if endpoint_id != model_endpoint.metadata.uid:
raise MLRunConflictError(
f"Mismatch between endpoint_id {endpoint_id} and ModelEndpoint.metadata.uid {model_endpoint.metadata.uid}."
f"\nMake sure the supplied function_uri, and model are configured as intended"
)
ModelEndpoints.create_or_patch(
access_key=access_key, model_endpoint=model_endpoint,
)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def grafana_incoming_features(body: Dict[str, Any],
query_parameters: Dict[str,
str], access_key: str):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
start = body.get("rangeRaw", {}).get("from", "now-1h")
end = body.get("rangeRaw", {}).get("to", "now")
endpoint = ModelEndpoints.get_endpoint(access_key=access_key,
project=project,
endpoint_id=endpoint_id)
time_series = []
feature_names = endpoint.spec.feature_names
if not feature_names:
logger.warn(
"'feature_names' is either missing or not initialized in endpoint record",
endpoint_id=endpoint.metadata.uid,
)
return time_series
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind=EVENTS)
_, container, path = parse_model_endpoint_store_prefix(path)
client = get_frames_client(
token=access_key,
address=config.v3io_framesd,
container=container,
)
data: pd.DataFrame = client.read(
backend="tsdb",
table=path,
columns=feature_names,
filter=f"endpoint_id=='{endpoint_id}'",
start=start,
end=end,
)
data.drop(["endpoint_id"], axis=1, inplace=True, errors="ignore")
data.index = data.index.astype(np.int64) // 10**6
for feature, indexed_values in data.to_dict().items():
target = GrafanaTimeSeriesTarget(target=feature)
for index, value in indexed_values.items():
data_point = GrafanaDataPoint(value=float(value), timestamp=index)
target.add_data_point(data_point)
time_series.append(target)
return time_series
def grafana_individual_feature_analysis(body: Dict[str, Any],
query_parameters: Dict[str, str],
access_key: str):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
endpoint = ModelEndpoints.get_endpoint(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
feature_analysis=True,
)
# Load JSON data from KV, make sure not to fail if a field is missing
feature_stats = endpoint.status.feature_stats or {}
current_stats = endpoint.status.current_stats or {}
drift_measures = endpoint.status.drift_measures or {}
table = GrafanaTable(columns=[
GrafanaColumn(text="feature_name", type="string"),
GrafanaColumn(text="actual_min", type="number"),
GrafanaColumn(text="actual_mean", type="number"),
GrafanaColumn(text="actual_max", type="number"),
GrafanaColumn(text="expected_min", type="number"),
GrafanaColumn(text="expected_mean", type="number"),
GrafanaColumn(text="expected_max", type="number"),
GrafanaColumn(text="tvd", type="number"),
GrafanaColumn(text="hellinger", type="number"),
GrafanaColumn(text="kld", type="number"),
])
for feature, base_stat in feature_stats.items():
current_stat = current_stats.get(feature, {})
drift_measure = drift_measures.get(feature, {})
table.add_row(
feature,
current_stat.get("min"),
current_stat.get("mean"),
current_stat.get("max"),
base_stat.get("min"),
base_stat.get("mean"),
base_stat.get("max"),
drift_measure.get("tvd"),
drift_measure.get("hellinger"),
drift_measure.get("kld"),
)
return [table]
def test_clear_endpoint(db: Session, client: TestClient):
access_key = _get_access_key()
endpoint = _mock_random_endpoint()
write_endpoint_to_kv(access_key, endpoint)
kv_record = ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
assert kv_record
response = client.delete(
url=f"/api/projects/{kv_record.metadata.project}/model-endpoints/{endpoint.metadata.uid}",
headers={"X-V3io-Session-Key": access_key},
)
assert response.status_code == 204
with pytest.raises(MLRunNotFoundError):
ModelEndpoints.get_endpoint(
access_key=access_key,
project=endpoint.metadata.project,
endpoint_id=endpoint.metadata.uid,
)
def get_endpoint(
request: Request,
project: str,
endpoint_id: str,
start: str = Query(default="now-1h"),
end: str = Query(default="now"),
metrics: List[str] = Query([], alias="metric"),
features: bool = Query(default=False),
):
access_key = get_access_key(request)
return ModelEndpoints.get_endpoint(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
metrics=metrics,
start=start,
end=end,
features=features,
)
def list_endpoints(
request: Request,
project: str,
model: Optional[str] = Query(None),
function: Optional[str] = Query(None),
tag: Optional[str] = Query(None),
labels: List[str] = Query([], alias="label"),
start: str = Query(default="now-1h"),
end: str = Query(default="now"),
metrics: List[str] = Query([], alias="metric"),
):
"""
Returns a list of endpoints of type 'ModelEndpoint', supports filtering by model, function, tag and labels.
Lables can be used to filter on the existance of a label:
`api/projects/{project}/model-endpoints/?label=mylabel`
Or on the value of a given label:
`api/projects/{project}/model-endpoints/?label=mylabel=1`
Multiple labels can be queried in a single request by either using `&` seperator:
`api/projects/{project}/model-endpoints/?label=mylabel=1&label=myotherlabel=2`
Or by using a `,` (comma) seperator:
`api/projects/{project}/model-endpoints/?label=mylabel=1,myotherlabel=2`
"""
access_key = get_access_key(request)
endpoint_list = ModelEndpoints.list_endpoints(
access_key=access_key,
project=project,
model=model,
function=function,
tag=tag,
labels=labels,
metrics=metrics,
start=start,
end=end,
)
return ModelEndpointStateList(endpoints=endpoint_list)
def list_endpoints(
request: Request,
project: str,
model: Optional[str] = Query(None),
function: Optional[str] = Query(None),
labels: List[str] = Query([], alias="label"),
start: str = Query(default="now-1h"),
end: str = Query(default="now"),
metrics: List[str] = Query([], alias="metric"),
) -> ModelEndpointList:
"""
Returns a list of endpoints of type 'ModelEndpoint', supports filtering by model, function, tag and labels.
Labels can be used to filter on the existence of a label:
api/projects/{project}/model-endpoints/?label=mylabel
Or on the value of a given label:
api/projects/{project}/model-endpoints/?label=mylabel=1
Multiple labels can be queried in a single request by either using "&" separator:
api/projects/{project}/model-endpoints/?label=mylabel=1&label=myotherlabel=2
Or by using a "," (comma) separator:
api/projects/{project}/model-endpoints/?label=mylabel=1,myotherlabel=2
"""
access_key = get_access_key(request.headers)
endpoints = ModelEndpoints.list_endpoints(
access_key=access_key,
project=project,
model=model,
function=function,
labels=labels,
metrics=metrics,
start=start,
end=end,
)
return endpoints
def test_grafana_incoming_features(db: Session, client: TestClient):
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=TEST_PROJECT, kind=EVENTS)
_, container, path = parse_model_endpoint_store_prefix(path)
frames = get_frames_client(
token=_get_access_key(),
container=container,
address=config.v3io_framesd,
)
frames.create(backend="tsdb", table=path, rate="10/m", if_exists=1)
start = datetime.utcnow()
endpoints = [_mock_random_endpoint() for _ in range(5)]
for e in endpoints:
e.spec.feature_names = ["f0", "f1", "f2", "f3"]
for endpoint in endpoints:
ModelEndpoints.create_or_patch(_get_access_key(), endpoint)
total = 0
dfs = []
for i in range(10):
count = randint(1, 10)
total += count
data = {
"f0": i,
"f1": i + 1,
"f2": i + 2,
"f3": i + 3,
"endpoint_id": endpoint.metadata.uid,
"timestamp": start - timedelta(minutes=10 - i),
}
df = pd.DataFrame(data=[data])
dfs.append(df)
frames.write(
backend="tsdb",
table=path,
dfs=dfs,
index_cols=["timestamp", "endpoint_id"],
)
for endpoint in endpoints:
response = client.post(
url="/api/grafana-proxy/model-endpoints/query",
headers={"X-V3io-Session-Key": _get_access_key()},
json={
"targets": [{
"target":
f"project={TEST_PROJECT};endpoint_id={endpoint.metadata.uid};target_endpoint=incoming_features" # noqa
}]
},
)
response = response.json()
targets = [t["target"] for t in response]
assert targets == ["f0", "f1", "f2", "f3"]
lens = [t["datapoints"] for t in response]
assert all(map(lambda l: len(l) == 10, lens))
def grafana_list_endpoints(body: Dict[str, Any], query_parameters: Dict[str,
str],
access_key: str) -> List[GrafanaTable]:
project = query_parameters.get("project")
# Filters
model = query_parameters.get("model", None)
function = query_parameters.get("function", None)
tag = query_parameters.get("tag", None)
labels = query_parameters.get("labels", "")
labels = labels.split(",") if labels else []
# Metrics to include
metrics = query_parameters.get("metrics", "")
metrics = metrics.split(",") if metrics else []
# Time range for metrics
start = body.get("rangeRaw", {}).get("start", "now-1h")
end = body.get("rangeRaw", {}).get("end", "now")
endpoint_list: List[ModelEndpointState] = ModelEndpoints.list_endpoints(
access_key=access_key,
project=project,
model=model,
function=function,
tag=tag,
labels=labels,
metrics=metrics,
start=start,
end=end,
)
columns = [
GrafanaColumn(text="endpoint_id", type="string"),
GrafanaColumn(text="endpoint_function", type="string"),
GrafanaColumn(text="endpoint_model", type="string"),
GrafanaColumn(text="endpoint_model_class", type="string"),
GrafanaColumn(text="endpoint_tag", type="string"),
GrafanaColumn(text="first_request", type="time"),
GrafanaColumn(text="last_request", type="time"),
GrafanaColumn(text="accuracy", type="number"),
GrafanaColumn(text="error_count", type="number"),
GrafanaColumn(text="drift_status", type="number"),
]
metric_columns = []
found_metrics = set()
for endpoint_state in endpoint_list:
if endpoint_state.metrics:
for key in endpoint_state.metrics.keys():
if key not in found_metrics:
found_metrics.add(key)
metric_columns.append(
GrafanaColumn(text=key, type="number"))
columns = columns + metric_columns
rows = []
for endpoint_state in endpoint_list:
row = [
endpoint_state.endpoint.id,
endpoint_state.endpoint.spec.function,
endpoint_state.endpoint.spec.model,
endpoint_state.endpoint.spec.model_class,
endpoint_state.endpoint.metadata.tag,
endpoint_state.first_request,
endpoint_state.last_request,
endpoint_state.accuracy,
endpoint_state.error_count,
endpoint_state.drift_status,
]
if metric_columns and endpoint_state.metrics:
for metric_column in metric_columns:
row.append(endpoint_state.metrics[metric_column.text])
rows.append(row)
return [GrafanaTable(columns=columns, rows=rows)]