def grafana_overall_feature_analysis(body: Dict[str, Any],
query_parameters: Dict[str, str],
access_key: str):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
endpoint = ModelEndpoints.get_endpoint(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
feature_analysis=True,
)
table = GrafanaTable(columns=[
GrafanaNumberColumn(text="tvd_sum"),
GrafanaNumberColumn(text="tvd_mean"),
GrafanaNumberColumn(text="hellinger_sum"),
GrafanaNumberColumn(text="hellinger_mean"),
GrafanaNumberColumn(text="kld_sum"),
GrafanaNumberColumn(text="kld_mean"),
])
if endpoint.status.drift_measures:
table.add_row(
endpoint.status.drift_measures.get("tvd_sum"),
endpoint.status.drift_measures.get("tvd_mean"),
endpoint.status.drift_measures.get("hellinger_sum"),
endpoint.status.drift_measures.get("hellinger_mean"),
endpoint.status.drift_measures.get("kld_sum"),
endpoint.status.drift_measures.get("kld_mean"),
)
return [table]
def grafana_individual_feature_analysis(
body: Dict[str, Any],
query_parameters: Dict[str, str],
auth_info: mlrun.api.schemas.AuthInfo,
):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
project,
endpoint_id,
mlrun.api.schemas.AuthorizationAction.read,
auth_info,
)
endpoint = mlrun.api.crud.ModelEndpoints().get_endpoint(
auth_info=auth_info,
project=project,
endpoint_id=endpoint_id,
feature_analysis=True,
)
# Load JSON data from KV, make sure not to fail if a field is missing
feature_stats = endpoint.status.feature_stats or {}
current_stats = endpoint.status.current_stats or {}
drift_measures = endpoint.status.drift_measures or {}
table = GrafanaTable(columns=[
GrafanaColumn(text="feature_name", type="string"),
GrafanaColumn(text="actual_min", type="number"),
GrafanaColumn(text="actual_mean", type="number"),
GrafanaColumn(text="actual_max", type="number"),
GrafanaColumn(text="expected_min", type="number"),
GrafanaColumn(text="expected_mean", type="number"),
GrafanaColumn(text="expected_max", type="number"),
GrafanaColumn(text="tvd", type="number"),
GrafanaColumn(text="hellinger", type="number"),
GrafanaColumn(text="kld", type="number"),
])
for feature, base_stat in feature_stats.items():
current_stat = current_stats.get(feature, {})
drift_measure = drift_measures.get(feature, {})
table.add_row(
feature,
current_stat.get("min"),
current_stat.get("mean"),
current_stat.get("max"),
base_stat.get("min"),
base_stat.get("mean"),
base_stat.get("max"),
drift_measure.get("tvd"),
drift_measure.get("hellinger"),
drift_measure.get("kld"),
)
return [table]
def grafana_individual_feature_analysis(body: Dict[str, Any],
query_parameters: Dict[str, str],
access_key: str):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
endpoint = ModelEndpoints.get_endpoint(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
feature_analysis=True,
)
# Load JSON data from KV, make sure not to fail if a field is missing
feature_stats = endpoint.status.feature_stats or {}
current_stats = endpoint.status.current_stats or {}
drift_measures = endpoint.status.drift_measures or {}
table = GrafanaTable(columns=[
GrafanaColumn(text="feature_name", type="string"),
GrafanaColumn(text="actual_min", type="number"),
GrafanaColumn(text="actual_mean", type="number"),
GrafanaColumn(text="actual_max", type="number"),
GrafanaColumn(text="expected_min", type="number"),
GrafanaColumn(text="expected_mean", type="number"),
GrafanaColumn(text="expected_max", type="number"),
GrafanaColumn(text="tvd", type="number"),
GrafanaColumn(text="hellinger", type="number"),
GrafanaColumn(text="kld", type="number"),
])
for feature, base_stat in feature_stats.items():
current_stat = current_stats.get(feature, {})
drift_measure = drift_measures.get(feature, {})
table.add_row(
feature,
current_stat.get("min"),
current_stat.get("mean"),
current_stat.get("max"),
base_stat.get("min"),
base_stat.get("mean"),
base_stat.get("max"),
drift_measure.get("tvd"),
drift_measure.get("hellinger"),
drift_measure.get("kld"),
)
return [table]
def grafana_overall_feature_analysis(
body: Dict[str, Any],
query_parameters: Dict[str, str],
auth_info: mlrun.api.schemas.AuthInfo,
):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
project,
endpoint_id,
mlrun.api.schemas.AuthorizationAction.read,
auth_info,
)
endpoint = mlrun.api.crud.ModelEndpoints().get_endpoint(
auth_info=auth_info,
project=project,
endpoint_id=endpoint_id,
feature_analysis=True,
)
table = GrafanaTable(columns=[
GrafanaNumberColumn(text="tvd_sum"),
GrafanaNumberColumn(text="tvd_mean"),
GrafanaNumberColumn(text="hellinger_sum"),
GrafanaNumberColumn(text="hellinger_mean"),
GrafanaNumberColumn(text="kld_sum"),
GrafanaNumberColumn(text="kld_mean"),
])
if endpoint.status.drift_measures:
table.add_row(
endpoint.status.drift_measures.get("tvd_sum"),
endpoint.status.drift_measures.get("tvd_mean"),
endpoint.status.drift_measures.get("hellinger_sum"),
endpoint.status.drift_measures.get("hellinger_mean"),
endpoint.status.drift_measures.get("kld_sum"),
endpoint.status.drift_measures.get("kld_mean"),
)
return [table]
def grafana_endpoint_features(body: Dict[str, Any],
query_parameters: Dict[str,
str], access_key: str):
endpoint_id = query_parameters.get("endpoint_id")
project = query_parameters.get("project")
start = body.get("rangeRaw", {}).get("start", "now-1h")
end = body.get("rangeRaw", {}).get("end", "now")
frames_client = get_frames_client(
token=access_key,
address=config.v3io_framesd,
container=config.model_endpoint_monitoring.container,
)
results = frames_client.read(
"tsdb",
f"{project}/endpoint-features",
filter=f"endpoint_id=='{endpoint_id}'",
start=start,
end=end,
)
if results.empty:
return []
results.drop(["endpoint_id", "prediction"], inplace=True, axis=1)
columns = [
GrafanaColumn(text="feature_name", type="string"),
GrafanaColumn(text="actual_min", type="number"),
GrafanaColumn(text="actual_mean", type="number"),
GrafanaColumn(text="actual_max", type="number"),
GrafanaColumn(text="expected_min", type="number"),
GrafanaColumn(text="expected_mean", type="number"),
GrafanaColumn(text="expected_max", type="number"),
]
rows = []
if not results.empty:
describes = results.describe().to_dict()
for feature, describe in describes.items():
rows.append([
feature,
describe["min"],
describe["mean"],
describe["max"],
None, # features.get(feature, {}).get("min", None),
None, # features.get(feature, {}).get("mean", None),
None, # features.get(feature, {}).get("max", None),
])
return [GrafanaTable(columns=columns, rows=rows)]
def grafana_list_endpoints(
body: Dict[str, Any],
query_parameters: Dict[str, str],
auth_info: mlrun.api.schemas.AuthInfo,
) -> List[GrafanaTable]:
project = query_parameters.get("project")
# Filters
model = query_parameters.get("model", None)
function = query_parameters.get("function", None)
labels = query_parameters.get("labels", "")
labels = labels.split(",") if labels else []
# Metrics to include
metrics = query_parameters.get("metrics", "")
metrics = metrics.split(",") if metrics else []
# Time range for metrics
start = body.get("rangeRaw", {}).get("start", "now-1h")
end = body.get("rangeRaw", {}).get("end", "now")
if project:
mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
project,
mlrun.api.schemas.AuthorizationAction.read,
auth_info,
)
endpoint_list = mlrun.api.crud.ModelEndpoints().list_endpoints(
auth_info=auth_info,
project=project,
model=model,
function=function,
labels=labels,
metrics=metrics,
start=start,
end=end,
)
allowed_endpoints = mlrun.api.utils.auth.verifier.AuthVerifier(
).filter_project_resources_by_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
endpoint_list.endpoints,
lambda _endpoint: (
_endpoint.metadata.project,
_endpoint.metadata.uid,
),
auth_info,
)
endpoint_list.endpoints = allowed_endpoints
columns = [
GrafanaColumn(text="endpoint_id", type="string"),
GrafanaColumn(text="endpoint_function", type="string"),
GrafanaColumn(text="endpoint_model", type="string"),
GrafanaColumn(text="endpoint_model_class", type="string"),
GrafanaColumn(text="first_request", type="time"),
GrafanaColumn(text="last_request", type="time"),
GrafanaColumn(text="accuracy", type="number"),
GrafanaColumn(text="error_count", type="number"),
GrafanaColumn(text="drift_status", type="number"),
]
metric_columns = []
found_metrics = set()
for endpoint in endpoint_list.endpoints:
if endpoint.status.metrics is not None:
for key in endpoint.status.metrics.keys():
if key not in found_metrics:
found_metrics.add(key)
metric_columns.append(
GrafanaColumn(text=key, type="number"))
columns = columns + metric_columns
table = GrafanaTable(columns=columns)
for endpoint in endpoint_list.endpoints:
row = [
endpoint.metadata.uid,
endpoint.spec.function_uri,
endpoint.spec.model,
endpoint.spec.model_class,
endpoint.status.first_request,
endpoint.status.last_request,
endpoint.status.accuracy,
endpoint.status.error_count,
endpoint.status.drift_status,
]
if endpoint.status.metrics is not None and metric_columns:
for metric_column in metric_columns:
row.append(endpoint.status.metrics[metric_column.text])
table.add_row(*row)
return [table]
def grafana_list_endpoints(body: Dict[str, Any], query_parameters: Dict[str,
str],
access_key: str) -> List[GrafanaTable]:
project = query_parameters.get("project")
# Filters
model = query_parameters.get("model", None)
function = query_parameters.get("function", None)
tag = query_parameters.get("tag", None)
labels = query_parameters.get("labels", "")
labels = labels.split(",") if labels else []
# Metrics to include
metrics = query_parameters.get("metrics", "")
metrics = metrics.split(",") if metrics else []
# Time range for metrics
start = body.get("rangeRaw", {}).get("start", "now-1h")
end = body.get("rangeRaw", {}).get("end", "now")
endpoint_list: List[ModelEndpointState] = ModelEndpoints.list_endpoints(
access_key=access_key,
project=project,
model=model,
function=function,
tag=tag,
labels=labels,
metrics=metrics,
start=start,
end=end,
)
columns = [
GrafanaColumn(text="endpoint_id", type="string"),
GrafanaColumn(text="endpoint_function", type="string"),
GrafanaColumn(text="endpoint_model", type="string"),
GrafanaColumn(text="endpoint_model_class", type="string"),
GrafanaColumn(text="endpoint_tag", type="string"),
GrafanaColumn(text="first_request", type="time"),
GrafanaColumn(text="last_request", type="time"),
GrafanaColumn(text="accuracy", type="number"),
GrafanaColumn(text="error_count", type="number"),
GrafanaColumn(text="drift_status", type="number"),
]
metric_columns = []
found_metrics = set()
for endpoint_state in endpoint_list:
if endpoint_state.metrics:
for key in endpoint_state.metrics.keys():
if key not in found_metrics:
found_metrics.add(key)
metric_columns.append(
GrafanaColumn(text=key, type="number"))
columns = columns + metric_columns
rows = []
for endpoint_state in endpoint_list:
row = [
endpoint_state.endpoint.id,
endpoint_state.endpoint.spec.function,
endpoint_state.endpoint.spec.model,
endpoint_state.endpoint.spec.model_class,
endpoint_state.endpoint.metadata.tag,
endpoint_state.first_request,
endpoint_state.last_request,
endpoint_state.accuracy,
endpoint_state.error_count,
endpoint_state.drift_status,
]
if metric_columns and endpoint_state.metrics:
for metric_column in metric_columns:
row.append(endpoint_state.metrics[metric_column.text])
rows.append(row)
return [GrafanaTable(columns=columns, rows=rows)]