def store_function(
self, session, function, name, project="", tag="", versioned=False
):
project = project or config.default_project
self._create_project_if_not_exists(session, project)
tag = tag or get_in(function, "metadata.tag") or "latest"
hash_key = fill_function_hash(function, tag)
# clear tag from object in case another function will "take" that tag
update_in(function, "metadata.tag", "")
# versioned means whether we want to version this function object so that it will queryable by its hash key
# to enable that we set the uid to the hash key so it will have a unique record (Unique constraint of function
# is the set (project, name, uid))
# when it's not enabled it means we want to have one unique function object for the set (project, name, tag)
# that will be reused on every store function (cause we don't want to version each version e.g. create a new
# record) so we set the uid to be unversioned-{tag}
if versioned:
uid = hash_key
else:
uid = f'unversioned-{tag}'
updated = datetime.now(timezone.utc)
update_in(function, "metadata.updated", updated)
fn = self._get_function(session, name, project, uid)
if not fn:
fn = Function(name=name, project=project, uid=uid,)
fn.updated = updated
labels = get_in(function, "metadata.labels", {})
update_labels(fn, labels)
fn.struct = function
self._upsert(session, fn)
self.tag_objects_v2(session, [fn], project, tag)
return hash_key
def _pretty_print_jobs(self, items: typing.List):
print(f"{'status':10} {'name':20} {'start':21} end")
for i in items:
status = self._get_job_launcher_status(i)
name = get_in(i, "metadata.name", "")
start_time = get_in(i, "status.startTime", "")
end_time = get_in(i, "status.completionTime", "")
print(f"{status:10} {name:20} {start_time:21} {end_time}")
def build_status(name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: str = "on",
db_session: Session = Depends(deps.get_db_session)):
logs = strtobool(logs)
fn = get_db().get_function(db_session, name, project, tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND,
name=name,
project=project,
tag=tag)
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod
})
logger.info("get pod {} status".format(pod))
state = get_k8s().get_pod_status(pod)
logger.info("pod state={}".format(state))
if state == "succeeded":
logger.info("build completed successfully")
state = "ready"
if state in ["failed", "error"]:
logger.error("build {}, watch the build pod logs: {}".format(
state, pod))
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == "ready":
update_in(fn, "spec.image", image)
get_db().store_function(db_session, fn, name, project, tag)
return Response(content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod
})
def build_status():
name = request.args.get('name', '')
project = request.args.get('project', '')
tag = request.args.get('tag', '')
offset = int(request.args.get('offset', '0'))
logs = strtobool(request.args.get('logs', 'on'))
fn = _db.get_function(name, project, tag)
if not fn:
return json_error(HTTPStatus.NOT_FOUND,
name=name,
project=project,
tag=tag)
state = get_in(fn, 'status.state', '')
pod = get_in(fn, 'status.build_pod', '')
image = get_in(fn, 'spec.build.image', '')
out = b''
if not pod:
if state == 'ready':
image = image or get_in(fn, 'spec.image')
return Response(out,
mimetype='text/plain',
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod
})
logger.info('get pod {} status'.format(pod))
state = _k8s.get_pod_status(pod)
logger.info('pod state={}'.format(state))
if state == 'succeeded':
logger.info('build completed successfully')
state = 'ready'
if state in ['failed', 'error']:
logger.error('build {}, watch the build pod logs: {}'.format(
state, pod))
if logs and state != 'pending':
resp = _k8s.logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, 'status.state', state)
if state == 'ready':
update_in(fn, 'spec.image', image)
_db.store_function(fn, name, project, tag)
return Response(out,
mimetype='text/plain',
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod
})
def _pretty_print_jobs(self, items: typing.List):
print('{:10} {:20} {:21} {}'.format('status', 'name', 'start', 'end'))
for i in items:
print('{:10} {:20} {:21} {}'.format(
self._get_job_launcher_status(i),
get_in(i, 'metadata.name', ''),
get_in(i, 'status.startTime', ''),
get_in(i, 'status.completionTime', ''),
))
def _pretty_print_jobs(self, items: typing.List):
print("{:10} {:20} {:21} {}".format("status", "name", "start", "end"))
for i in items:
print("{:10} {:20} {:21} {}".format(
self._get_job_launcher_status(i),
get_in(i, "metadata.name", ""),
get_in(i, "status.startTime", ""),
get_in(i, "status.completionTime", ""),
))
def _extract_input_data(input_path, body):
if input_path:
if not hasattr(body, "__getitem__"):
raise TypeError(
"input_path parameter supports only dict-like event bodies")
return get_in(body, input_path)
return body
def get_log(
db_session: Session,
project: str,
uid: str,
size: int = -1,
offset: int = 0,
source: LogSources = LogSources.AUTO,
):
out = b""
log_file = log_path(project, uid)
status = None
if log_file.exists() and source in [
LogSources.AUTO, LogSources.PERSISTENCY
]:
with log_file.open("rb") as fp:
fp.seek(offset)
out = fp.read(size)
status = ""
elif source in [LogSources.AUTO, LogSources.K8S]:
data = get_db().read_run(db_session, uid, project)
if not data:
log_and_raise(HTTPStatus.NOT_FOUND, project=project, uid=uid)
status = get_in(data, "status.state", "")
if get_k8s():
pods = get_k8s().get_logger_pods(uid)
if pods:
pod, new_status = list(pods.items())[0]
new_status = new_status.lower()
# TODO: handle in cron/tracking
if new_status != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
if status == "running":
now = now_date().isoformat()
update_in(data, "status.last_update", now)
if new_status == "failed":
update_in(data, "status.state", "error")
update_in(data, "status.error",
"error, check logs")
get_db().store_run(db_session, data, uid,
project)
if new_status == "succeeded":
update_in(data, "status.state", "completed")
get_db().store_run(db_session, data, uid,
project)
status = new_status
elif status == "running":
update_in(data, "status.state", "error")
update_in(data, "status.error",
"pod not found, maybe terminated")
get_db().store_run(db_session, data, uid, project)
status = "failed"
return out, status
def _get_job_launcher_status(self, resp: typing.List) -> str:
launcher_status = get_in(resp, 'status.replicaStatuses.Launcher')
if launcher_status is None:
return ''
for status in ['active', 'failed', 'succeeded']:
if launcher_status.get(status, 0) == 1:
return status
return ''
def _get_job_launcher_status(self, resp: typing.List) -> str:
launcher_status = get_in(resp, "status.replicaStatuses.Launcher")
if launcher_status is None:
return ""
for status in ["active", "failed", "succeeded"]:
if launcher_status.get(status, 0) == 1:
return status
return ""
def _submit(data):
task = data.get('task')
function = data.get('function')
url = data.get('functionUrl')
if not url and task:
url = get_in(task, 'spec.function')
if not (function or url) or not task:
return json_error(
HTTPStatus.BAD_REQUEST,
reason='bad JSON, need to include function/url and task objects',
)
# TODO: block exec for function['kind'] in ['', 'local] (must be a
# remote/container runtime)
try:
if function:
fn = new_function(runtime=function)
else:
if '://' in url:
fn = import_function(url=url)
else:
project, name, tag = parse_function_uri(url)
runtime = _db.get_function(name, project, tag)
if not runtime:
return json_error(
HTTPStatus.BAD_REQUEST,
reason='runtime error: function {} not found'.format(
url),
)
fn = new_function(runtime=runtime)
fn.set_db_connection(_db, True)
logger.info('func:\n{}'.format(fn.to_yaml()))
# fn.spec.rundb = 'http://mlrun-api:8080'
schedule = data.get('schedule')
if schedule:
args = (task, )
job_id = _scheduler.add(schedule, fn, args)
_db.save_schedule(data)
resp = {'schedule': schedule, 'id': job_id}
else:
resp = fn.run(task, watch=False)
logger.info('resp: %s', resp.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
return json_error(
HTTPStatus.BAD_REQUEST,
reason='runtime error: {}'.format(err),
)
if not isinstance(resp, dict):
resp = resp.to_dict()
return jsonify(ok=True, data=resp)
def parse_submit_run_body(data):
task = data.get("task")
function_dict = data.get("function")
function_url = data.get("functionUrl")
if not function_url and task:
function_url = get_in(task, "spec.function")
if not (function_dict or function_url) or not task:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="bad JSON, need to include function/url and task objects",
)
return function_dict, function_url, task
def delete_job(self, name, namespace=None):
mpi_group, mpi_version, mpi_plural = self._get_crd_info()
k8s = self._get_k8s()
namespace = k8s.resolve_namespace(namespace)
try:
# delete the mpi job
body = client.V1DeleteOptions()
resp = k8s.crdapi.delete_namespaced_custom_object(
mpi_group, mpi_version, namespace, mpi_plural, name, body)
deletion_status = get_in(resp, "status", "unknown")
logger.info(f"del status: {deletion_status}")
except client.rest.ApiException as exc:
print(f"Exception when deleting MPIJob: {exc}")
def delete_job(self, name, namespace=None):
mpi_group, mpi_version, mpi_plural = self._get_crd_info()
k8s = self._get_k8s()
namespace = k8s.resolve_namespace(namespace)
try:
# delete the mpi job
body = client.V1DeleteOptions()
resp = k8s.crdapi.delete_namespaced_custom_object(
mpi_group, mpi_version, namespace, mpi_plural, name, body)
logger.info('del status: {}'.format(
get_in(resp, 'status', 'unknown')))
except client.rest.ApiException as e:
print("Exception when deleting MPIJob: %s" % e)
def get_log(project, uid):
size = int(request.args.get('size', '-1'))
offset = int(request.args.get('offset', '0'))
out = b''
log_file = log_path(project, uid)
if log_file.exists():
with log_file.open('rb') as fp:
fp.seek(offset)
out = fp.read(size)
status = ''
else:
data = _db.read_run(uid, project)
if not data:
return json_error(HTTPStatus.NOT_FOUND,
project=project, uid=uid)
status = get_in(data, 'status.state', '')
if _k8s:
pods = _k8s.get_logger_pods(uid)
if pods:
pod, new_status = list(pods.items())[0]
new_status = new_status.lower()
# TODO: handle in cron/tracking
if new_status != 'pending':
resp = _k8s.logs(pod)
if resp:
out = resp.encode()[offset:]
if status == 'running':
now = now_date().isoformat()
update_in(data, 'status.last_update', now)
if new_status == 'failed':
update_in(data, 'status.state', 'error')
update_in(
data, 'status.error', 'error, check logs')
_db.store_run(data, uid, project)
if new_status == 'succeeded':
update_in(data, 'status.state', 'completed')
_db.store_run(data, uid, project)
status = new_status
elif status == 'running':
update_in(data, 'status.state', 'error')
update_in(
data, 'status.error', 'pod not found, maybe terminated')
_db.store_run(data, uid, project)
status = 'failed'
return Response(out, mimetype='text/plain',
headers={"pod_status": status})
def _parse_submit_run_body(db_session: Session,
auth_info: mlrun.api.schemas.AuthInfo, data):
task = data.get("task")
function_dict = data.get("function")
function_url = data.get("functionUrl")
if not function_url and task:
function_url = get_in(task, "spec.function")
if not (function_dict or function_url) or not task:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="bad JSON, need to include function/url and task objects",
)
# TODO: block exec for function["kind"] in ["", "local] (must be a
# remote/container runtime)
if function_dict and not function_url:
function = new_function(runtime=function_dict)
else:
if "://" in function_url:
function = import_function(url=function_url,
project=task.get("metadata",
{}).get("project"))
else:
project, name, tag, hash_key = parse_versioned_object_uri(
function_url)
function_record = get_db().get_function(db_session, name, project,
tag, hash_key)
if not function_record:
log_and_raise(
HTTPStatus.NOT_FOUND.value,
reason=f"runtime error: function {function_url} not found",
)
function = new_function(runtime=function_record)
if function_dict:
# The purpose of the function dict is to enable the user to override configurations of the existing function
# without modifying it - to do that we're creating a function object from the request function dict and
# assign values from it to the main function object
function = enrich_function_from_dict(function, function_dict)
# if auth given in request ensure the function pod will have these auth env vars set, otherwise the job won't
# be able to communicate with the api
ensure_function_has_auth_set(function, auth_info)
return function, task
def _submit_mpijob(self, job, namespace=None):
mpi_group, mpi_version, mpi_plural = self._get_crd_info()
k8s = self._get_k8s()
namespace = k8s.resolve_namespace(namespace)
try:
resp = k8s.crdapi.create_namespaced_custom_object(
mpi_group,
mpi_version,
namespace=namespace,
plural=mpi_plural,
body=job)
name = get_in(resp, 'metadata.name', 'unknown')
logger.info('MpiJob {} created'.format(name))
return resp
except client.rest.ApiException as e:
logger.error("Exception when creating MPIJob: %s" % e)
raise RunError("Exception when creating MPIJob: %s" % e)
def _submit_mpijob(self, job, namespace=None):
mpi_group, mpi_version, mpi_plural = self._get_crd_info()
k8s = self._get_k8s()
namespace = k8s.resolve_namespace(namespace)
try:
resp = k8s.crdapi.create_namespaced_custom_object(
mpi_group,
mpi_version,
namespace=namespace,
plural=mpi_plural,
body=job,
)
name = get_in(resp, "metadata.name", "unknown")
logger.info(f"MpiJob {name} created")
return resp
except client.rest.ApiException as exc:
logger.error(f"Exception when creating MPIJob: {exc}")
raise RunError(f"Exception when creating MPIJob: {exc}")
def build_status(
name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: bool = True,
last_log_timestamp: float = 0.0,
verbose: bool = False,
db_session: Session = Depends(deps.get_db_session),
):
fn = get_db().get_function(db_session, name, project, tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND.value,
name=name,
project=project,
tag=tag)
# nuclio deploy status
if fn.get("kind") in RuntimeKinds.nuclio_runtimes():
(
state,
address,
nuclio_name,
last_log_timestamp,
text,
) = get_nuclio_deploy_status(name,
project,
tag,
last_log_timestamp=last_log_timestamp,
verbose=verbose)
if state == "ready":
logger.info("Nuclio function deployed successfully", name=name)
if state == "error":
logger.error(f"Nuclio deploy error, {text}", name=name)
update_in(fn, "status.nuclio_name", nuclio_name)
update_in(fn, "status.state", state)
update_in(fn, "status.address", address)
versioned = False
if state == "ready":
# Versioned means the version will be saved in the DB forever, we don't want to spam
# the DB with intermediate or unusable versions, only successfully deployed versions
versioned = True
get_db().store_function(db_session,
fn,
name,
project,
tag,
versioned=versioned)
return Response(
content=text,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"x-mlrun-last-timestamp": str(last_log_timestamp),
"x-mlrun-address": address,
"x-mlrun-name": nuclio_name,
},
)
# job deploy status
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(
content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
logger.info("get pod {} status".format(pod))
state = get_k8s().get_pod_status(pod)
logger.info("pod state={}".format(state))
if state == "succeeded":
logger.info("build completed successfully")
state = "ready"
if state in ["failed", "error"]:
logger.error("build {}, watch the build pod logs: {}".format(
state, pod))
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == "ready":
update_in(fn, "spec.image", image)
versioned = False
if state == "ready":
versioned = True
get_db().store_function(db_session,
fn,
name,
project,
tag,
versioned=versioned)
return Response(
content=out,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
def _get_job_launcher_status(self, resp: typing.List) -> str:
return get_in(resp, "status.launcherStatus")
def build_status(
name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: bool = True,
last_log_timestamp: float = 0.0,
verbose: bool = False,
auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier),
db_session: Session = Depends(deps.get_db_session),
):
fn = get_db().get_function(db_session, name, project, tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND.value,
name=name,
project=project,
tag=tag)
# nuclio deploy status
if fn.get("kind") in RuntimeKinds.nuclio_runtimes():
(
state,
address,
nuclio_name,
last_log_timestamp,
text,
status,
) = get_nuclio_deploy_status(name,
project,
tag,
last_log_timestamp=last_log_timestamp,
verbose=verbose)
if state == "ready":
logger.info("Nuclio function deployed successfully", name=name)
if state in ["error", "unhealthy"]:
logger.error(f"Nuclio deploy error, {text}", name=name)
# internal / external invocation urls were added on nuclio 1.6.x
# and hence, it might be empty
# to backward compatible with older nuclio versions, we use hard-coded default values
internal_invocation_urls = status.get(
"internalInvocationUrls",
[resolve_function_internal_invocation_url(name)])
external_invocation_urls = status.get("externalInvocationUrls",
[address] if address else [])
# on nuclio > 1.6.x we get the external invocation url on the status block
if external_invocation_urls and not address:
address = external_invocation_urls[0]
update_in(fn, "status.nuclio_name", nuclio_name)
update_in(fn, "status.internal_invocation_urls",
internal_invocation_urls)
update_in(fn, "status.external_invocation_urls",
external_invocation_urls)
update_in(fn, "status.state", state)
update_in(fn, "status.address", address)
versioned = False
if state == "ready":
# Versioned means the version will be saved in the DB forever, we don't want to spam
# the DB with intermediate or unusable versions, only successfully deployed versions
versioned = True
get_db().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
leader_session=auth_verifier.auth_info.session,
)
return Response(
content=text,
media_type="text/plain",
headers={
"x-mlrun-function-status":
state,
"x-mlrun-last-timestamp":
str(last_log_timestamp),
"x-mlrun-address":
address,
"x-mlrun-internal-invocation-urls":
",".join(internal_invocation_urls),
"x-mlrun-external-invocation-urls":
",".join(external_invocation_urls),
"x-mlrun-name":
nuclio_name,
},
)
# job deploy status
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(
content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
logger.info(f"get pod {pod} status")
state = get_k8s().get_pod_status(pod)
logger.info(f"pod state={state}")
if state == "succeeded":
logger.info("build completed successfully")
state = mlrun.api.schemas.FunctionState.ready
if state in ["failed", "error"]:
logger.error(f"build {state}, watch the build pod logs: {pod}")
state = mlrun.api.schemas.FunctionState.error
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == mlrun.api.schemas.FunctionState.ready:
update_in(fn, "spec.image", image)
versioned = False
if state == mlrun.api.schemas.FunctionState.ready:
versioned = True
get_db().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
leader_session=auth_verifier.auth_info.session,
)
return Response(
content=out,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
def build_status(
name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: bool = True,
last_log_timestamp: float = 0.0,
verbose: bool = False,
auth_info: mlrun.api.schemas.AuthInfo = Depends(
deps.authenticate_request),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.function,
project or mlrun.mlconf.default_project,
name,
# store since with the current mechanism we update the status (and store the function) in the DB when a client
# query for the status
mlrun.api.schemas.AuthorizationAction.store,
auth_info,
)
fn = mlrun.api.crud.Functions().get_function(db_session, name, project,
tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND.value,
name=name,
project=project,
tag=tag)
# nuclio deploy status
if fn.get("kind") in RuntimeKinds.nuclio_runtimes():
(
state,
address,
nuclio_name,
last_log_timestamp,
text,
status,
) = get_nuclio_deploy_status(
name,
project,
tag,
last_log_timestamp=last_log_timestamp,
verbose=verbose,
auth_info=auth_info,
)
if state == "ready":
logger.info("Nuclio function deployed successfully", name=name)
if state in ["error", "unhealthy"]:
logger.error(f"Nuclio deploy error, {text}", name=name)
internal_invocation_urls = status.get("internalInvocationUrls", [])
external_invocation_urls = status.get("externalInvocationUrls", [])
# on earlier versions of mlrun, address used to represent the nodePort external invocation url
# now that functions can be not exposed (using service_type clusterIP) this no longer relevant
# and hence, for BC it would be filled with the external invocation url first item
# or completely empty.
address = external_invocation_urls[
0] if external_invocation_urls else ""
update_in(fn, "status.nuclio_name", nuclio_name)
update_in(fn, "status.internal_invocation_urls",
internal_invocation_urls)
update_in(fn, "status.external_invocation_urls",
external_invocation_urls)
update_in(fn, "status.state", state)
update_in(fn, "status.address", address)
versioned = False
if state == "ready":
# Versioned means the version will be saved in the DB forever, we don't want to spam
# the DB with intermediate or unusable versions, only successfully deployed versions
versioned = True
mlrun.api.crud.Functions().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
)
return Response(
content=text,
media_type="text/plain",
headers={
"x-mlrun-function-status":
state,
"x-mlrun-last-timestamp":
str(last_log_timestamp),
"x-mlrun-address":
address,
"x-mlrun-internal-invocation-urls":
",".join(internal_invocation_urls),
"x-mlrun-external-invocation-urls":
",".join(external_invocation_urls),
"x-mlrun-name":
nuclio_name,
},
)
# job deploy status
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(
content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
logger.info(f"get pod {pod} status")
state = get_k8s().get_pod_status(pod)
logger.info(f"pod state={state}")
if state == "succeeded":
logger.info("build completed successfully")
state = mlrun.api.schemas.FunctionState.ready
if state in ["failed", "error"]:
logger.error(f"build {state}, watch the build pod logs: {pod}")
state = mlrun.api.schemas.FunctionState.error
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == mlrun.api.schemas.FunctionState.ready:
update_in(fn, "spec.image", image)
versioned = False
if state == mlrun.api.schemas.FunctionState.ready:
versioned = True
mlrun.api.crud.Functions().store_function(
db_session,
fn,
name,
project,
tag,
versioned=versioned,
)
return Response(
content=out,
media_type="text/plain",
headers={
"x-mlrun-function-status": state,
"function_status": state,
"function_image": image,
"builder_pod": pod,
},
)
def _parse_submit_run_body(db_session: Session, data):
task = data.get("task")
function_dict = data.get("function")
function_url = data.get("functionUrl")
if not function_url and task:
function_url = get_in(task, "spec.function")
if not (function_dict or function_url) or not task:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="bad JSON, need to include function/url and task objects",
)
# TODO: block exec for function["kind"] in ["", "local] (must be a
# remote/container runtime)
if function_dict and not function_url:
function = new_function(runtime=function_dict)
else:
if "://" in function_url:
function = import_function(url=function_url)
else:
project, name, tag, hash_key = parse_versioned_object_uri(
function_url)
function_record = get_db().get_function(db_session, name, project,
tag, hash_key)
if not function_record:
log_and_raise(
HTTPStatus.NOT_FOUND.value,
reason="runtime error: function {} not found".format(
function_url),
)
function = new_function(runtime=function_record)
if function_dict:
# The purpose of the function dict is to enable the user to override configurations of the existing function
# without modifying it - to do that we're creating a function object from the request function dict and
# assign values from it to the main function object
override_function = new_function(runtime=function_dict,
kind=function.kind)
for attribute in [
"volumes",
"volume_mounts",
"env",
"resources",
"image_pull_policy",
"replicas",
]:
override_value = getattr(override_function.spec, attribute,
None)
if override_value:
if attribute == "env":
for env_dict in override_value:
function.set_env(env_dict["name"],
env_dict["value"])
elif attribute == "volumes":
function.spec.update_vols_and_mounts(
override_value, [])
elif attribute == "volume_mounts":
# volume mounts don't have a well defined identifier (like name for volume) so we can't merge,
# only override
function.spec.volume_mounts = override_value
elif attribute == "resources":
# don't override it there are limits and requests but both are empty
if override_value.get("limits",
{}) or override_value.get(
"requests", {}):
setattr(function.spec, attribute, override_value)
else:
setattr(function.spec, attribute, override_value)
return function, task
def submit(db_session: Session, data):
task = data.get("task")
function = data.get("function")
url = data.get("functionUrl")
if not url and task:
url = get_in(task, "spec.function")
if not (function or url) or not task:
log_and_raise(
HTTPStatus.BAD_REQUEST,
reason="bad JSON, need to include function/url and task objects")
# TODO: block exec for function["kind"] in ["", "local] (must be a
# remote/container runtime)
resp = None
try:
if function and not url:
fn = new_function(runtime=function)
else:
if "://" in url:
fn = import_function(url=url)
else:
project, name, tag, hash_key = parse_function_uri(url)
runtime = get_db().get_function(db_session, name, project, tag,
hash_key)
if not runtime:
log_and_raise(
HTTPStatus.BAD_REQUEST,
reason="runtime error: function {} not found".format(
url))
fn = new_function(runtime=runtime)
if function:
fn2 = new_function(runtime=function)
for attr in [
"volumes", "volume_mounts", "env", "resources",
"image_pull_policy", "replicas"
]:
val = getattr(fn2.spec, attr, None)
if val:
setattr(fn.spec, attr, val)
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db, True)
logger.info("func:\n{}".format(fn.to_yaml()))
# fn.spec.rundb = "http://mlrun-api:8080"
schedule = data.get("schedule")
if schedule:
args = (task, )
job_id = get_scheduler().add(schedule, fn, args)
get_db().store_schedule(db_session, data)
resp = {"schedule": schedule, "id": job_id}
else:
resp = fn.run(task, watch=False)
logger.info("resp: %s", resp.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST,
reason="runtime error: {}".format(err))
if not isinstance(resp, dict):
resp = resp.to_dict()
return {
"data": resp,
}
def run_start_time(run):
ts = get_in(run, "status.start_time", "")
if not ts:
return None
return parser.parse(ts)
def run_labels(run) -> dict:
return get_in(run, "metadata.labels", {})
def run_state(run):
return get_in(run, "status.state", "")
