def deploy_model_monitoring_stream_processing(
project: str,
model_monitoring_access_key: str,
db_session,
auto_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring stream processing function [{project}]"
)
try:
get_nuclio_deploy_status(
name="model-monitoring-stream", project=project, tag=""
)
logger.info(
f"Detected model monitoring stream processing function [{project}] already deployed"
)
return
except DeployError:
logger.info(
f"Deploying model monitoring stream processing function [{project}]"
)
fn = get_model_monitoring_stream_processing_function(project)
fn.metadata.project = project
stream_path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind="stream"
)
fn.add_v3io_stream_trigger(
stream_path=stream_path, name="monitoring_stream_trigger"
)
fn.set_env_from_secret(
"MODEL_MONITORING_ACCESS_KEY",
mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project),
Secrets().generate_model_monitoring_secret_key(
"MODEL_MONITORING_ACCESS_KEY"
),
)
fn.metadata.credentials.access_key = model_monitoring_access_key
fn.set_env("MODEL_MONITORING_PARAMETERS", json.dumps({"project": project}))
fn.apply(mlrun.mount_v3io())
_build_function(db_session=db_session, auth_info=auto_info, function=fn)
def deploy_model_monitoring_stream_processing(
project: str,
model_monitoring_access_key: str,
auto_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring stream processing function [{project}]"
)
try:
get_nuclio_deploy_status(
name="model-monitoring-stream", project=project, tag=""
)
logger.info(
f"Detected model monitoring stream processing function [{project}] already deployed"
)
return
except DeployError:
logger.info(
f"Deploying model monitoring stream processing function [{project}]"
)
fn = get_model_monitoring_stream_processing_function(project)
fn.metadata.project = project
stream_path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind="stream"
)
fn.add_v3io_stream_trigger(
stream_path=stream_path, name="monitoring_stream_trigger"
)
fn.set_env("MODEL_MONITORING_ACCESS_KEY", model_monitoring_access_key)
fn.set_env("MLRUN_AUTH_SESSION", model_monitoring_access_key)
fn.set_env("MODEL_MONITORING_PARAMETERS", json.dumps({"project": project}))
fn.apply(mlrun.mount_v3io())
deploy_nuclio_function(fn, auth_info=auto_info)
def build_status(
name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: bool = True,
last_log_timestamp: float = 0.0,
verbose: bool = False,
db_session: Session = Depends(deps.get_db_session),
):
fn = get_db().get_function(db_session, name, project, tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND.value,
name=name,
project=project,
tag=tag)
# nuclio deploy status
if fn.get("kind") in RuntimeKinds.nuclio_runtimes():
(
state,
address,
nuclio_name,
last_log_timestamp,
text,
) = get_nuclio_deploy_status(name,
project,
tag,
last_log_timestamp=last_log_timestamp,
verbose=verbose)
if state == "ready":
logger.info("Nuclio function deployed successfully", name=name)
if state == "error":
logger.error(f"Nuclio deploy error, {text}", name=name)
update_in(fn, "status.nuclio_name", nuclio_name)
update_in(fn, "status.state", state)
update_in(fn, "status.address", address)
versioned = False
if state == "ready":
# Versioned means the version will be saved in the DB forever, we don't want to spam
# the DB with intermediate or unusable versions, only successfully deployed versions
versioned = True
get_db().store_function(db_session,
fn,
name,
project,
tag,
versioned=versioned)
return Response(
content=text,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"x-mlrun-last-timestamp": str(last_log_timestamp),
"x-mlrun-address": address,
"x-mlrun-name": nuclio_name,
},
)
# job deploy status
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(
content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
logger.info("get pod {} status".format(pod))
state = get_k8s().get_pod_status(pod)
logger.info("pod state={}".format(state))
if state == "succeeded":
logger.info("build completed successfully")
state = "ready"
if state in ["failed", "error"]:
logger.error("build {}, watch the build pod logs: {}".format(
state, pod))
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == "ready":
update_in(fn, "spec.image", image)
versioned = False
if state == "ready":
versioned = True
get_db().store_function(db_session,
fn,
name,
project,
tag,
versioned=versioned)
return Response(
content=out,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
def build_status(
name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: bool = True,
last_log_timestamp: float = 0.0,
verbose: bool = False,
auth_info: mlrun.api.schemas.AuthInfo = Depends(
deps.authenticate_request),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.function,
project or mlrun.mlconf.default_project,
name,
# store since with the current mechanism we update the status (and store the function) in the DB when a client
# query for the status
mlrun.api.schemas.AuthorizationAction.store,
auth_info,
)
fn = mlrun.api.crud.Functions().get_function(db_session, name, project,
tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND.value,
name=name,
project=project,
tag=tag)
# nuclio deploy status
if fn.get("kind") in RuntimeKinds.nuclio_runtimes():
(
state,
address,
nuclio_name,
last_log_timestamp,
text,
status,
) = get_nuclio_deploy_status(
name,
project,
tag,
last_log_timestamp=last_log_timestamp,
verbose=verbose,
auth_info=auth_info,
)
if state == "ready":
logger.info("Nuclio function deployed successfully", name=name)
if state in ["error", "unhealthy"]:
logger.error(f"Nuclio deploy error, {text}", name=name)
internal_invocation_urls = status.get("internalInvocationUrls", [])
external_invocation_urls = status.get("externalInvocationUrls", [])
# on earlier versions of mlrun, address used to represent the nodePort external invocation url
# now that functions can be not exposed (using service_type clusterIP) this no longer relevant
# and hence, for BC it would be filled with the external invocation url first item
# or completely empty.
address = external_invocation_urls[
0] if external_invocation_urls else ""
update_in(fn, "status.nuclio_name", nuclio_name)
update_in(fn, "status.internal_invocation_urls",
internal_invocation_urls)
update_in(fn, "status.external_invocation_urls",
external_invocation_urls)
update_in(fn, "status.state", state)
update_in(fn, "status.address", address)
versioned = False
if state == "ready":
# Versioned means the version will be saved in the DB forever, we don't want to spam
# the DB with intermediate or unusable versions, only successfully deployed versions
versioned = True
mlrun.api.crud.Functions().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
)
return Response(
content=text,
media_type="text/plain",
headers={
"x-mlrun-function-status":
state,
"x-mlrun-last-timestamp":
str(last_log_timestamp),
"x-mlrun-address":
address,
"x-mlrun-internal-invocation-urls":
",".join(internal_invocation_urls),
"x-mlrun-external-invocation-urls":
",".join(external_invocation_urls),
"x-mlrun-name":
nuclio_name,
},
)
# job deploy status
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(
content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
logger.info(f"get pod {pod} status")
state = get_k8s().get_pod_status(pod)
logger.info(f"pod state={state}")
if state == "succeeded":
logger.info("build completed successfully")
state = mlrun.api.schemas.FunctionState.ready
if state in ["failed", "error"]:
logger.error(f"build {state}, watch the build pod logs: {pod}")
state = mlrun.api.schemas.FunctionState.error
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == mlrun.api.schemas.FunctionState.ready:
update_in(fn, "spec.image", image)
versioned = False
if state == mlrun.api.schemas.FunctionState.ready:
versioned = True
mlrun.api.crud.Functions().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
)
return Response(
content=out,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
def build_status(
name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: bool = True,
last_log_timestamp: float = 0.0,
verbose: bool = False,
auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier),
db_session: Session = Depends(deps.get_db_session),
):
fn = get_db().get_function(db_session, name, project, tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND.value,
name=name,
project=project,
tag=tag)
# nuclio deploy status
if fn.get("kind") in RuntimeKinds.nuclio_runtimes():
(
state,
address,
nuclio_name,
last_log_timestamp,
text,
status,
) = get_nuclio_deploy_status(name,
project,
tag,
last_log_timestamp=last_log_timestamp,
verbose=verbose)
if state == "ready":
logger.info("Nuclio function deployed successfully", name=name)
if state in ["error", "unhealthy"]:
logger.error(f"Nuclio deploy error, {text}", name=name)
# internal / external invocation urls were added on nuclio 1.6.x
# and hence, it might be empty
# to backward compatible with older nuclio versions, we use hard-coded default values
internal_invocation_urls = status.get(
"internalInvocationUrls",
[resolve_function_internal_invocation_url(name)])
external_invocation_urls = status.get("externalInvocationUrls",
[address] if address else [])
# on nuclio > 1.6.x we get the external invocation url on the status block
if external_invocation_urls and not address:
address = external_invocation_urls[0]
update_in(fn, "status.nuclio_name", nuclio_name)
update_in(fn, "status.internal_invocation_urls",
internal_invocation_urls)
update_in(fn, "status.external_invocation_urls",
external_invocation_urls)
update_in(fn, "status.state", state)
update_in(fn, "status.address", address)
versioned = False
if state == "ready":
# Versioned means the version will be saved in the DB forever, we don't want to spam
# the DB with intermediate or unusable versions, only successfully deployed versions
versioned = True
get_db().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
leader_session=auth_verifier.auth_info.session,
)
return Response(
content=text,
media_type="text/plain",
headers={
"x-mlrun-function-status":
state,
"x-mlrun-last-timestamp":
str(last_log_timestamp),
"x-mlrun-address":
address,
"x-mlrun-internal-invocation-urls":
",".join(internal_invocation_urls),
"x-mlrun-external-invocation-urls":
",".join(external_invocation_urls),
"x-mlrun-name":
nuclio_name,
},
)
# job deploy status
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(
content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
logger.info(f"get pod {pod} status")
state = get_k8s().get_pod_status(pod)
logger.info(f"pod state={state}")
if state == "succeeded":
logger.info("build completed successfully")
state = mlrun.api.schemas.FunctionState.ready
if state in ["failed", "error"]:
logger.error(f"build {state}, watch the build pod logs: {pod}")
state = mlrun.api.schemas.FunctionState.error
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == mlrun.api.schemas.FunctionState.ready:
update_in(fn, "spec.image", image)
versioned = False
if state == mlrun.api.schemas.FunctionState.ready:
versioned = True
get_db().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
leader_session=auth_verifier.auth_info.session,
)
return Response(
content=out,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
