def delete_endpoint_record(access_key: str, project: str,
endpoint_id: str):
"""
Deletes the KV record of a given model endpoint, project and endpoint_id are used for lookup
:param access_key: V3IO access key for managing user permissions
:param project: The name of the project
:param endpoint_id: The id of the endpoint
"""
logger.info("Clearing model endpoint table", endpoint_id=endpoint_id)
client = get_v3io_client(endpoint=config.v3io_api)
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind=ENDPOINTS)
_, container, path = parse_model_endpoint_store_prefix(path)
client.kv.delete(
container=container,
table_path=path,
key=endpoint_id,
access_key=access_key,
)
logger.info("Model endpoint table cleared", endpoint_id=endpoint_id)
def delete_endpoint_record(
self,
auth_info: mlrun.api.schemas.AuthInfo,
project: str,
endpoint_id: str,
access_key: str,
):
"""
Deletes the KV record of a given model endpoint, project and endpoint_id are used for lookup
:param auth_info: The required auth information for doing the deletion
:param project: The name of the project
:param endpoint_id: The id of the endpoint
:param access_key: access key with permission to delete
"""
logger.info("Clearing model endpoint table", endpoint_id=endpoint_id)
client = get_v3io_client(endpoint=config.v3io_api)
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind=mlrun.api.schemas.ModelMonitoringStoreKinds.ENDPOINTS
)
_, container, path = parse_model_endpoint_store_prefix(path)
client.kv.delete(
container=container,
table_path=path,
key=endpoint_id,
access_key=access_key,
)
logger.info("Model endpoint table cleared", endpoint_id=endpoint_id)
def clear_endpoint_record(project: str, endpoint_id: str):
"""
Clears endpoint record from KV by endpoint_id
"""
_verify_endpoint(project, endpoint_id)
logger.info("Clearing model endpoint table", endpoint_id=endpoint_id)
get_v3io_client().kv.delete(
container=config.httpdb.model_endpoint_monitoring.container,
table_path=ENDPOINTS_TABLE_PATH,
key=endpoint_id,
)
logger.info("Model endpoint table deleted", endpoint_id=endpoint_id)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def clear_endpoint_record(access_key: str, project: str, endpoint_id: str):
"""
Clears the KV data of a given model endpoint
:param access_key: V3IO access key for managing user permissions
:param project: The name of the project
:param endpoint_id: The id of the endpoint
"""
verify_endpoint(project, endpoint_id)
logger.info("Clearing model endpoint table", endpoint_id=endpoint_id)
client = get_v3io_client(endpoint=config.v3io_api)
client.kv.delete(
container=config.model_endpoint_monitoring.container,
table_path=f"{project}/{ENDPOINTS_TABLE_PATH}",
key=endpoint_id,
access_key=access_key,
)
logger.info("Model endpoint table deleted", endpoint_id=endpoint_id)
async def delete_endpoint_record(access_key: str, project: str, endpoint_id: str):
"""
Deletes the KV record of a given model endpoint, project nad endpoint_id are used for lookup
:param access_key: V3IO access key for managing user permissions
:param project: The name of the project
:param endpoint_id: The id of the endpoint
"""
logger.info("Clearing model endpoint table", endpoint_id=endpoint_id)
client = get_v3io_client(endpoint=config.v3io_api)
await run_in_threadpool(
client.kv.delete,
container=config.model_endpoint_monitoring.container,
table_path=f"{project}/{ENDPOINTS_TABLE_PATH}",
key=endpoint_id,
access_key=access_key,
)
logger.info("Model endpoint table cleared", endpoint_id=endpoint_id)
def deploy_model_monitoring_batch_processing(
project: str,
model_monitoring_access_key: str,
db_session,
auth_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring batch processing function [{project}]"
)
function_list = get_db().list_functions(
session=db_session, name="model-monitoring-batch", project=project
)
if function_list:
logger.info(
f"Detected model monitoring batch processing function [{project}] already deployed"
)
return
logger.info(f"Deploying model monitoring batch processing function [{project}]")
fn: KubejobRuntime = mlrun.import_function(
f"hub://model_monitoring_batch:{config.model_endpoint_monitoring.batch_processing_function_branch}"
)
fn.set_db_connection(get_run_db_instance(db_session))
fn.metadata.project = project
fn.apply(mlrun.mount_v3io())
fn.set_env_from_secret(
"MODEL_MONITORING_ACCESS_KEY",
mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project),
Secrets().generate_model_monitoring_secret_key(
"MODEL_MONITORING_ACCESS_KEY"
),
)
# Needs to be a member of the project and have access to project data path
fn.metadata.credentials.access_key = model_monitoring_access_key
function_uri = fn.save(versioned=True)
function_uri = function_uri.replace("db://", "")
task = mlrun.new_task(name="model-monitoring-batch", project=project)
task.spec.function = function_uri
data = {
"task": task.to_dict(),
"schedule": "0 */1 * * *",
}
_submit_run(db_session=db_session, auth_info=auth_info, data=data)
def deploy_model_monitoring_stream_processing(
project: str,
model_monitoring_access_key: str,
db_session,
auto_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring stream processing function [{project}]"
)
try:
get_nuclio_deploy_status(
name="model-monitoring-stream", project=project, tag=""
)
logger.info(
f"Detected model monitoring stream processing function [{project}] already deployed"
)
return
except DeployError:
logger.info(
f"Deploying model monitoring stream processing function [{project}]"
)
fn = get_model_monitoring_stream_processing_function(project)
fn.metadata.project = project
stream_path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind="stream"
)
fn.add_v3io_stream_trigger(
stream_path=stream_path, name="monitoring_stream_trigger"
)
fn.set_env_from_secret(
"MODEL_MONITORING_ACCESS_KEY",
mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project),
Secrets().generate_model_monitoring_secret_key(
"MODEL_MONITORING_ACCESS_KEY"
),
)
fn.metadata.credentials.access_key = model_monitoring_access_key
fn.set_env("MODEL_MONITORING_PARAMETERS", json.dumps({"project": project}))
fn.apply(mlrun.mount_v3io())
_build_function(db_session=db_session, auth_info=auto_info, function=fn)
def deploy_model_monitoring_stream_processing(
project: str,
model_monitoring_access_key: str,
auto_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring stream processing function [{project}]"
)
try:
get_nuclio_deploy_status(
name="model-monitoring-stream", project=project, tag=""
)
logger.info(
f"Detected model monitoring stream processing function [{project}] already deployed"
)
return
except DeployError:
logger.info(
f"Deploying model monitoring stream processing function [{project}]"
)
fn = get_model_monitoring_stream_processing_function(project)
fn.metadata.project = project
stream_path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind="stream"
)
fn.add_v3io_stream_trigger(
stream_path=stream_path, name="monitoring_stream_trigger"
)
fn.set_env("MODEL_MONITORING_ACCESS_KEY", model_monitoring_access_key)
fn.set_env("MLRUN_AUTH_SESSION", model_monitoring_access_key)
fn.set_env("MODEL_MONITORING_PARAMETERS", json.dumps({"project": project}))
fn.apply(mlrun.mount_v3io())
deploy_nuclio_function(fn, auth_info=auto_info)
def create_or_patch(
db_session: Session,
access_key: str,
model_endpoint: ModelEndpoint,
leader_session: Optional[str] = None,
):
"""
Creates or patch a KV record with the given model_endpoint record
:param access_key: V3IO access key for managing user permissions
:param model_endpoint: An object representing a model endpoint
"""
if model_endpoint.spec.model_uri or model_endpoint.status.feature_stats:
logger.info(
"Getting feature metadata",
project=model_endpoint.metadata.project,
model=model_endpoint.spec.model,
function=model_endpoint.spec.function_uri,
model_uri=model_endpoint.spec.model_uri,
)
# If model artifact was supplied, grab model meta data from artifact
if model_endpoint.spec.model_uri:
logger.info(
"Getting model object, inferring column names and collecting feature stats"
)
run_db = mlrun.api.api.utils.get_run_db_instance(
db_session, leader_session)
model_obj: ModelArtifact = (
mlrun.datastore.store_resources.get_store_resource(
model_endpoint.spec.model_uri, db=run_db))
if not model_endpoint.status.feature_stats and hasattr(
model_obj, "feature_stats"):
model_endpoint.status.feature_stats = model_obj.feature_stats
if not model_endpoint.spec.label_names and hasattr(
model_obj, "outputs"):
model_label_names = [
_clean_feature_name(f.name) for f in model_obj.outputs
]
model_endpoint.spec.label_names = model_label_names
if not model_endpoint.spec.algorithm and hasattr(
model_obj, "algorithm"):
model_endpoint.spec.algorithm = model_obj.algorithm
# If feature_stats was either populated by model_uri or by manual input, make sure to keep the names
# of the features. If feature_names was supplied, replace the names set in feature_stats, otherwise - make
# sure to keep a clean version of the names
if model_endpoint.status.feature_stats:
logger.info("Feature stats found, cleaning feature names")
if model_endpoint.spec.feature_names:
if len(model_endpoint.status.feature_stats) != len(
model_endpoint.spec.feature_names):
raise MLRunInvalidArgumentError(
f"feature_stats and feature_names have a different number of names, while expected to match"
f"feature_stats({len(model_endpoint.status.feature_stats)}), "
f"feature_names({len(model_endpoint.spec.feature_names)})"
)
clean_feature_stats = {}
clean_feature_names = []
for i, (feature, stats) in enumerate(
model_endpoint.status.feature_stats.items()):
if model_endpoint.spec.feature_names:
clean_name = _clean_feature_name(
model_endpoint.spec.feature_names[i])
else:
clean_name = _clean_feature_name(feature)
clean_feature_stats[clean_name] = stats
clean_feature_names.append(clean_name)
model_endpoint.status.feature_stats = clean_feature_stats
model_endpoint.spec.feature_names = clean_feature_names
logger.info(
"Done preparing feature names and stats",
feature_names=model_endpoint.spec.feature_names,
)
# If none of the above was supplied, feature names will be assigned on first contact with the model monitoring
# system
logger.info("Updating model endpoint",
endpoint_id=model_endpoint.metadata.uid)
write_endpoint_to_kv(
access_key=access_key,
endpoint=model_endpoint,
update=True,
)
logger.info("Model endpoint updated",
endpoint_id=model_endpoint.metadata.uid)
return model_endpoint
def get_endpoint(
access_key: str,
project: str,
endpoint_id: str,
metrics: Optional[List[str]] = None,
start: str = "now-1h",
end: str = "now",
feature_analysis: bool = False,
) -> ModelEndpoint:
"""
Returns a ModelEndpoint object with additional metrics and feature related data.
:param access_key: V3IO access key for managing user permissions
:param project: The name of the project
:param endpoint_id: The id of the model endpoint
:param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
:param start: The start time of the metrics
:param end: The end time of the metrics
:param feature_analysis: When True, the base feature statistics and current feature statistics will be added to
the output of the resulting object
"""
logger.info(
"Getting model endpoint record from kv",
endpoint_id=endpoint_id,
)
client = get_v3io_client(endpoint=config.v3io_api)
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind=ENDPOINTS)
_, container, path = parse_model_endpoint_store_prefix(path)
endpoint = client.kv.get(
container=container,
table_path=path,
key=endpoint_id,
access_key=access_key,
raise_for_status=RaiseForStatus.never,
)
endpoint = endpoint.output.item
if not endpoint:
raise MLRunNotFoundError(f"Endpoint {endpoint_id} not found")
labels = endpoint.get("labels")
feature_names = endpoint.get("feature_names")
feature_names = _json_loads_if_not_none(feature_names)
label_names = endpoint.get("label_names")
label_names = _json_loads_if_not_none(label_names)
feature_stats = endpoint.get("feature_stats")
feature_stats = _json_loads_if_not_none(feature_stats)
current_stats = endpoint.get("current_stats")
current_stats = _json_loads_if_not_none(current_stats)
drift_measures = endpoint.get("drift_measures")
drift_measures = _json_loads_if_not_none(drift_measures)
monitor_configuration = endpoint.get("monitor_configuration")
monitor_configuration = _json_loads_if_not_none(monitor_configuration)
endpoint = ModelEndpoint(
metadata=ModelEndpointMetadata(
project=endpoint.get("project"),
labels=_json_loads_if_not_none(labels),
uid=endpoint_id,
),
spec=ModelEndpointSpec(
function_uri=endpoint.get("function_uri"),
model=endpoint.get("model"),
model_class=endpoint.get("model_class") or None,
model_uri=endpoint.get("model_uri") or None,
feature_names=feature_names or None,
label_names=label_names or None,
stream_path=endpoint.get("stream_path") or None,
algorithm=endpoint.get("algorithm") or None,
monitor_configuration=monitor_configuration or None,
active=endpoint.get("active") or None,
),
status=ModelEndpointStatus(
state=endpoint.get("state") or None,
feature_stats=feature_stats or None,
current_stats=current_stats or None,
first_request=endpoint.get("first_request") or None,
last_request=endpoint.get("last_request") or None,
accuracy=endpoint.get("accuracy") or None,
error_count=endpoint.get("error_count") or None,
drift_status=endpoint.get("drift_status") or None,
),
)
if feature_analysis and feature_names:
endpoint_features = get_endpoint_features(
feature_names=feature_names,
feature_stats=feature_stats,
current_stats=current_stats,
)
if endpoint_features:
endpoint.status.features = endpoint_features
endpoint.status.drift_measures = drift_measures
if metrics:
endpoint_metrics = get_endpoint_metrics(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
start=start,
end=end,
metrics=metrics,
)
if endpoint_metrics:
endpoint.status.metrics = endpoint_metrics
return endpoint
def list_endpoints(
access_key: str,
project: str,
model: Optional[str] = None,
function: Optional[str] = None,
labels: Optional[List[str]] = None,
metrics: Optional[List[str]] = None,
start: str = "now-1h",
end: str = "now",
) -> ModelEndpointList:
"""
Returns a list of ModelEndpointState objects. Each object represents the current state of a model endpoint.
This functions supports filtering by the following parameters:
1) model
2) function
3) labels
By default, when no filters are applied, all available endpoints for the given project will be listed.
In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
and depends on the 'start' and 'end' parameters. By default, when the metrics parameter is None, no metrics are
added to the output of this function.
:param access_key: V3IO access key for managing user permissions
:param project: The name of the project
:param model: The name of the model to filter by
:param function: The name of the function to filter by
:param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a label
(i.e. list("key==value")) or by looking for the existence of a given key (i.e. "key")
:param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
:param start: The start time of the metrics
:param end: The end time of the metrics
"""
logger.info(
"Listing endpoints",
project=project,
model=model,
function=function,
labels=labels,
metrics=metrics,
start=start,
end=end,
)
client = get_v3io_client(endpoint=config.v3io_api)
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind=ENDPOINTS)
_, container, path = parse_model_endpoint_store_prefix(path)
cursor = client.kv.new_cursor(
container=container,
table_path=path,
access_key=access_key,
filter_expression=build_kv_cursor_filter_expression(
project, function, model, labels),
attribute_names=["endpoint_id"],
)
endpoint_list = ModelEndpointList(endpoints=[])
while True:
item = cursor.next_item()
if item is None:
break
endpoint_id = item["endpoint_id"]
endpoint = ModelEndpoints.get_endpoint(
access_key=access_key,
project=project,
endpoint_id=endpoint_id,
metrics=metrics,
start=start,
end=end,
)
endpoint_list.endpoints.append(endpoint)
return endpoint_list
def list_endpoints(
self,
auth_info: mlrun.api.schemas.AuthInfo,
project: str,
model: Optional[str] = None,
function: Optional[str] = None,
labels: Optional[List[str]] = None,
metrics: Optional[List[str]] = None,
start: str = "now-1h",
end: str = "now",
top_level: Optional[bool] = False,
uids: Optional[List[str]] = None,
) -> ModelEndpointList:
"""
Returns a list of ModelEndpointState objects. Each object represents the current state of a model endpoint.
This functions supports filtering by the following parameters:
1) model
2) function
3) labels
4) top level
5) uids
By default, when no filters are applied, all available endpoints for the given project will be listed.
In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
and depends on the 'start' and 'end' parameters. By default, when the metrics parameter is None, no metrics are
added to the output of this function.
:param access_key: V3IO access key for managing user permissions
:param project: The name of the project
:param model: The name of the model to filter by
:param function: The name of the function to filter by
:param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a label
(i.e. list("key==value")) or by looking for the existence of a given key (i.e. "key")
:param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
:param start: The start time of the metrics
:param end: The end time of the metrics
:param top_level: if True will return only routers and endpoint that are NOT children of any router
:param uids: will return ModelEndpointList of endpoints with uid in uids
"""
logger.info(
"Listing endpoints",
project=project,
model=model,
function=function,
labels=labels,
metrics=metrics,
start=start,
end=end,
top_level=top_level,
uids=uids,
)
endpoint_list = ModelEndpointList(endpoints=[])
if uids is None:
client = get_v3io_client(endpoint=config.v3io_api)
path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project,
kind=mlrun.api.schemas.ModelMonitoringStoreKinds.ENDPOINTS,
)
_, container, path = parse_model_endpoint_store_prefix(path)
cursor = client.kv.new_cursor(
container=container,
table_path=path,
access_key=auth_info.data_session,
filter_expression=self.build_kv_cursor_filter_expression(
project, function, model, labels, top_level,
),
attribute_names=["endpoint_id"],
raise_for_status=RaiseForStatus.never,
)
try:
items = cursor.all()
except Exception:
return endpoint_list
uids = [item["endpoint_id"] for item in items]
for endpoint_id in uids:
endpoint = self.get_endpoint(
auth_info=auth_info,
project=project,
endpoint_id=endpoint_id,
metrics=metrics,
start=start,
end=end,
)
endpoint_list.endpoints.append(endpoint)
return endpoint_list
async def create_or_patch(access_key: str, model_endpoint: ModelEndpoint):
"""
Creates or updates a KV record with the given model_endpoint record
:param access_key: V3IO access key for managing user permissions
:param model_endpoint: An object representing a model endpoint
"""
if model_endpoint.spec.model_uri or model_endpoint.status.feature_stats:
logger.info(
"Getting feature metadata",
project=model_endpoint.metadata.project,
model=model_endpoint.spec.model,
function=model_endpoint.spec.function_uri,
model_uri=model_endpoint.spec.model_uri,
)
# If model artifact was supplied but feature_stats was not, grab model artifact and get feature_stats
if model_endpoint.spec.model_uri and not model_endpoint.status.feature_stats:
logger.info(
"Getting model object, inferring column names and collecting feature stats"
)
model_obj = await run_in_threadpool(
get_model, model_endpoint.spec.model_uri
)
model_endpoint.status.feature_stats = model_obj[1].feature_stats
# If feature_stats was either populated by model_uri or by manual input, make sure to keep the names
# of the features. If feature_names was supplied, replace the names set in feature_stats, otherwise - make
# sure to keep a clean version of the names
if model_endpoint.status.feature_stats:
logger.info("Feature stats found, cleaning feature names")
if model_endpoint.spec.feature_names:
if len(model_endpoint.status.feature_stats) != len(
model_endpoint.spec.feature_names
):
raise MLRunInvalidArgumentError(
f"feature_stats and feature_names have a different number of names, while expected to match"
f"feature_stats({len(model_endpoint.status.feature_stats)}), "
f"feature_names({len(model_endpoint.spec.feature_names)})"
)
clean_feature_stats = {}
clean_feature_names = []
for i, (feature, stats) in enumerate(
model_endpoint.status.feature_stats.items()
):
if model_endpoint.spec.feature_names:
clean_name = _clean_feature_name(
model_endpoint.spec.feature_names[i]
)
else:
clean_name = _clean_feature_name(feature)
clean_feature_stats[clean_name] = stats
clean_feature_names.append(clean_name)
model_endpoint.status.feature_stats = clean_feature_stats
model_endpoint.spec.feature_names = clean_feature_names
logger.info(
"Done preparing feature names and stats",
feature_names=model_endpoint.spec.feature_names,
)
# If none of the above was supplied, feature names will be assigned on first contact with the model monitoring
# system
logger.info("Updating model endpoint", endpoint_id=model_endpoint.metadata.uid)
await write_endpoint_to_kv(
access_key=access_key, endpoint=model_endpoint, update=True,
)
logger.info("Model endpoint updated", endpoint_id=model_endpoint.metadata.uid)
return model_endpoint
