def store_function( self, session, function, name, project="", tag="", versioned=False ): project = project or config.default_project self._create_project_if_not_exists(session, project) tag = tag or get_in(function, "metadata.tag") or "latest" hash_key = fill_function_hash(function, tag) # clear tag from object in case another function will "take" that tag update_in(function, "metadata.tag", "") # versioned means whether we want to version this function object so that it will queryable by its hash key # to enable that we set the uid to the hash key so it will have a unique record (Unique constraint of function # is the set (project, name, uid)) # when it's not enabled it means we want to have one unique function object for the set (project, name, tag) # that will be reused on every store function (cause we don't want to version each version e.g. create a new # record) so we set the uid to be unversioned-{tag} if versioned: uid = hash_key else: uid = f'unversioned-{tag}' updated = datetime.now(timezone.utc) update_in(function, "metadata.updated", updated) fn = self._get_function(session, name, project, uid) if not fn: fn = Function(name=name, project=project, uid=uid,) fn.updated = updated labels = get_in(function, "metadata.labels", {}) update_labels(fn, labels) fn.struct = function self._upsert(session, fn) self.tag_objects_v2(session, [fn], project, tag) return hash_key
def _pretty_print_jobs(self, items: typing.List): print(f"{'status':10} {'name':20} {'start':21} end") for i in items: status = self._get_job_launcher_status(i) name = get_in(i, "metadata.name", "") start_time = get_in(i, "status.startTime", "") end_time = get_in(i, "status.completionTime", "") print(f"{status:10} {name:20} {start_time:21} {end_time}")
def build_status(name: str = "", project: str = "", tag: str = "", offset: int = 0, logs: str = "on", db_session: Session = Depends(deps.get_db_session)): logs = strtobool(logs) fn = get_db().get_function(db_session, name, project, tag) if not fn: log_and_raise(HTTPStatus.NOT_FOUND, name=name, project=project, tag=tag) state = get_in(fn, "status.state", "") pod = get_in(fn, "status.build_pod", "") image = get_in(fn, "spec.build.image", "") out = b"" if not pod: if state == "ready": image = image or get_in(fn, "spec.image") return Response(content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod }) logger.info("get pod {} status".format(pod)) state = get_k8s().get_pod_status(pod) logger.info("pod state={}".format(state)) if state == "succeeded": logger.info("build completed successfully") state = "ready" if state in ["failed", "error"]: logger.error("build {}, watch the build pod logs: {}".format( state, pod)) if logs and state != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] update_in(fn, "status.state", state) if state == "ready": update_in(fn, "spec.image", image) get_db().store_function(db_session, fn, name, project, tag) return Response(content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod })
def build_status(): name = request.args.get('name', '') project = request.args.get('project', '') tag = request.args.get('tag', '') offset = int(request.args.get('offset', '0')) logs = strtobool(request.args.get('logs', 'on')) fn = _db.get_function(name, project, tag) if not fn: return json_error(HTTPStatus.NOT_FOUND, name=name, project=project, tag=tag) state = get_in(fn, 'status.state', '') pod = get_in(fn, 'status.build_pod', '') image = get_in(fn, 'spec.build.image', '') out = b'' if not pod: if state == 'ready': image = image or get_in(fn, 'spec.image') return Response(out, mimetype='text/plain', headers={ "function_status": state, "function_image": image, "builder_pod": pod }) logger.info('get pod {} status'.format(pod)) state = _k8s.get_pod_status(pod) logger.info('pod state={}'.format(state)) if state == 'succeeded': logger.info('build completed successfully') state = 'ready' if state in ['failed', 'error']: logger.error('build {}, watch the build pod logs: {}'.format( state, pod)) if logs and state != 'pending': resp = _k8s.logs(pod) if resp: out = resp.encode()[offset:] update_in(fn, 'status.state', state) if state == 'ready': update_in(fn, 'spec.image', image) _db.store_function(fn, name, project, tag) return Response(out, mimetype='text/plain', headers={ "function_status": state, "function_image": image, "builder_pod": pod })
def _pretty_print_jobs(self, items: typing.List): print('{:10} {:20} {:21} {}'.format('status', 'name', 'start', 'end')) for i in items: print('{:10} {:20} {:21} {}'.format( self._get_job_launcher_status(i), get_in(i, 'metadata.name', ''), get_in(i, 'status.startTime', ''), get_in(i, 'status.completionTime', ''), ))
def _pretty_print_jobs(self, items: typing.List): print("{:10} {:20} {:21} {}".format("status", "name", "start", "end")) for i in items: print("{:10} {:20} {:21} {}".format( self._get_job_launcher_status(i), get_in(i, "metadata.name", ""), get_in(i, "status.startTime", ""), get_in(i, "status.completionTime", ""), ))
def _extract_input_data(input_path, body): if input_path: if not hasattr(body, "__getitem__"): raise TypeError( "input_path parameter supports only dict-like event bodies") return get_in(body, input_path) return body
def get_log( db_session: Session, project: str, uid: str, size: int = -1, offset: int = 0, source: LogSources = LogSources.AUTO, ): out = b"" log_file = log_path(project, uid) status = None if log_file.exists() and source in [ LogSources.AUTO, LogSources.PERSISTENCY ]: with log_file.open("rb") as fp: fp.seek(offset) out = fp.read(size) status = "" elif source in [LogSources.AUTO, LogSources.K8S]: data = get_db().read_run(db_session, uid, project) if not data: log_and_raise(HTTPStatus.NOT_FOUND, project=project, uid=uid) status = get_in(data, "status.state", "") if get_k8s(): pods = get_k8s().get_logger_pods(uid) if pods: pod, new_status = list(pods.items())[0] new_status = new_status.lower() # TODO: handle in cron/tracking if new_status != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] if status == "running": now = now_date().isoformat() update_in(data, "status.last_update", now) if new_status == "failed": update_in(data, "status.state", "error") update_in(data, "status.error", "error, check logs") get_db().store_run(db_session, data, uid, project) if new_status == "succeeded": update_in(data, "status.state", "completed") get_db().store_run(db_session, data, uid, project) status = new_status elif status == "running": update_in(data, "status.state", "error") update_in(data, "status.error", "pod not found, maybe terminated") get_db().store_run(db_session, data, uid, project) status = "failed" return out, status
def _get_job_launcher_status(self, resp: typing.List) -> str: launcher_status = get_in(resp, 'status.replicaStatuses.Launcher') if launcher_status is None: return '' for status in ['active', 'failed', 'succeeded']: if launcher_status.get(status, 0) == 1: return status return ''
def _get_job_launcher_status(self, resp: typing.List) -> str: launcher_status = get_in(resp, "status.replicaStatuses.Launcher") if launcher_status is None: return "" for status in ["active", "failed", "succeeded"]: if launcher_status.get(status, 0) == 1: return status return ""
def _submit(data): task = data.get('task') function = data.get('function') url = data.get('functionUrl') if not url and task: url = get_in(task, 'spec.function') if not (function or url) or not task: return json_error( HTTPStatus.BAD_REQUEST, reason='bad JSON, need to include function/url and task objects', ) # TODO: block exec for function['kind'] in ['', 'local] (must be a # remote/container runtime) try: if function: fn = new_function(runtime=function) else: if '://' in url: fn = import_function(url=url) else: project, name, tag = parse_function_uri(url) runtime = _db.get_function(name, project, tag) if not runtime: return json_error( HTTPStatus.BAD_REQUEST, reason='runtime error: function {} not found'.format( url), ) fn = new_function(runtime=runtime) fn.set_db_connection(_db, True) logger.info('func:\n{}'.format(fn.to_yaml())) # fn.spec.rundb = 'http://mlrun-api:8080' schedule = data.get('schedule') if schedule: args = (task, ) job_id = _scheduler.add(schedule, fn, args) _db.save_schedule(data) resp = {'schedule': schedule, 'id': job_id} else: resp = fn.run(task, watch=False) logger.info('resp: %s', resp.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) return json_error( HTTPStatus.BAD_REQUEST, reason='runtime error: {}'.format(err), ) if not isinstance(resp, dict): resp = resp.to_dict() return jsonify(ok=True, data=resp)
def parse_submit_run_body(data): task = data.get("task") function_dict = data.get("function") function_url = data.get("functionUrl") if not function_url and task: function_url = get_in(task, "spec.function") if not (function_dict or function_url) or not task: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="bad JSON, need to include function/url and task objects", ) return function_dict, function_url, task
def delete_job(self, name, namespace=None): mpi_group, mpi_version, mpi_plural = self._get_crd_info() k8s = self._get_k8s() namespace = k8s.resolve_namespace(namespace) try: # delete the mpi job body = client.V1DeleteOptions() resp = k8s.crdapi.delete_namespaced_custom_object( mpi_group, mpi_version, namespace, mpi_plural, name, body) deletion_status = get_in(resp, "status", "unknown") logger.info(f"del status: {deletion_status}") except client.rest.ApiException as exc: print(f"Exception when deleting MPIJob: {exc}")
def delete_job(self, name, namespace=None): mpi_group, mpi_version, mpi_plural = self._get_crd_info() k8s = self._get_k8s() namespace = k8s.resolve_namespace(namespace) try: # delete the mpi job body = client.V1DeleteOptions() resp = k8s.crdapi.delete_namespaced_custom_object( mpi_group, mpi_version, namespace, mpi_plural, name, body) logger.info('del status: {}'.format( get_in(resp, 'status', 'unknown'))) except client.rest.ApiException as e: print("Exception when deleting MPIJob: %s" % e)
def get_log(project, uid): size = int(request.args.get('size', '-1')) offset = int(request.args.get('offset', '0')) out = b'' log_file = log_path(project, uid) if log_file.exists(): with log_file.open('rb') as fp: fp.seek(offset) out = fp.read(size) status = '' else: data = _db.read_run(uid, project) if not data: return json_error(HTTPStatus.NOT_FOUND, project=project, uid=uid) status = get_in(data, 'status.state', '') if _k8s: pods = _k8s.get_logger_pods(uid) if pods: pod, new_status = list(pods.items())[0] new_status = new_status.lower() # TODO: handle in cron/tracking if new_status != 'pending': resp = _k8s.logs(pod) if resp: out = resp.encode()[offset:] if status == 'running': now = now_date().isoformat() update_in(data, 'status.last_update', now) if new_status == 'failed': update_in(data, 'status.state', 'error') update_in( data, 'status.error', 'error, check logs') _db.store_run(data, uid, project) if new_status == 'succeeded': update_in(data, 'status.state', 'completed') _db.store_run(data, uid, project) status = new_status elif status == 'running': update_in(data, 'status.state', 'error') update_in( data, 'status.error', 'pod not found, maybe terminated') _db.store_run(data, uid, project) status = 'failed' return Response(out, mimetype='text/plain', headers={"pod_status": status})
def _parse_submit_run_body(db_session: Session, auth_info: mlrun.api.schemas.AuthInfo, data): task = data.get("task") function_dict = data.get("function") function_url = data.get("functionUrl") if not function_url and task: function_url = get_in(task, "spec.function") if not (function_dict or function_url) or not task: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="bad JSON, need to include function/url and task objects", ) # TODO: block exec for function["kind"] in ["", "local] (must be a # remote/container runtime) if function_dict and not function_url: function = new_function(runtime=function_dict) else: if "://" in function_url: function = import_function(url=function_url, project=task.get("metadata", {}).get("project")) else: project, name, tag, hash_key = parse_versioned_object_uri( function_url) function_record = get_db().get_function(db_session, name, project, tag, hash_key) if not function_record: log_and_raise( HTTPStatus.NOT_FOUND.value, reason=f"runtime error: function {function_url} not found", ) function = new_function(runtime=function_record) if function_dict: # The purpose of the function dict is to enable the user to override configurations of the existing function # without modifying it - to do that we're creating a function object from the request function dict and # assign values from it to the main function object function = enrich_function_from_dict(function, function_dict) # if auth given in request ensure the function pod will have these auth env vars set, otherwise the job won't # be able to communicate with the api ensure_function_has_auth_set(function, auth_info) return function, task
def _submit_mpijob(self, job, namespace=None): mpi_group, mpi_version, mpi_plural = self._get_crd_info() k8s = self._get_k8s() namespace = k8s.resolve_namespace(namespace) try: resp = k8s.crdapi.create_namespaced_custom_object( mpi_group, mpi_version, namespace=namespace, plural=mpi_plural, body=job) name = get_in(resp, 'metadata.name', 'unknown') logger.info('MpiJob {} created'.format(name)) return resp except client.rest.ApiException as e: logger.error("Exception when creating MPIJob: %s" % e) raise RunError("Exception when creating MPIJob: %s" % e)
def _submit_mpijob(self, job, namespace=None): mpi_group, mpi_version, mpi_plural = self._get_crd_info() k8s = self._get_k8s() namespace = k8s.resolve_namespace(namespace) try: resp = k8s.crdapi.create_namespaced_custom_object( mpi_group, mpi_version, namespace=namespace, plural=mpi_plural, body=job, ) name = get_in(resp, "metadata.name", "unknown") logger.info(f"MpiJob {name} created") return resp except client.rest.ApiException as exc: logger.error(f"Exception when creating MPIJob: {exc}") raise RunError(f"Exception when creating MPIJob: {exc}")
def build_status( name: str = "", project: str = "", tag: str = "", offset: int = 0, logs: bool = True, last_log_timestamp: float = 0.0, verbose: bool = False, db_session: Session = Depends(deps.get_db_session), ): fn = get_db().get_function(db_session, name, project, tag) if not fn: log_and_raise(HTTPStatus.NOT_FOUND.value, name=name, project=project, tag=tag) # nuclio deploy status if fn.get("kind") in RuntimeKinds.nuclio_runtimes(): ( state, address, nuclio_name, last_log_timestamp, text, ) = get_nuclio_deploy_status(name, project, tag, last_log_timestamp=last_log_timestamp, verbose=verbose) if state == "ready": logger.info("Nuclio function deployed successfully", name=name) if state == "error": logger.error(f"Nuclio deploy error, {text}", name=name) update_in(fn, "status.nuclio_name", nuclio_name) update_in(fn, "status.state", state) update_in(fn, "status.address", address) versioned = False if state == "ready": # Versioned means the version will be saved in the DB forever, we don't want to spam # the DB with intermediate or unusable versions, only successfully deployed versions versioned = True get_db().store_function(db_session, fn, name, project, tag, versioned=versioned) return Response( content=text, media_type="text/plain", headers={ "x-mlrun-function-status": state, "x-mlrun-last-timestamp": str(last_log_timestamp), "x-mlrun-address": address, "x-mlrun-name": nuclio_name, }, ) # job deploy status state = get_in(fn, "status.state", "") pod = get_in(fn, "status.build_pod", "") image = get_in(fn, "spec.build.image", "") out = b"" if not pod: if state == "ready": image = image or get_in(fn, "spec.image") return Response( content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod, }, ) logger.info("get pod {} status".format(pod)) state = get_k8s().get_pod_status(pod) logger.info("pod state={}".format(state)) if state == "succeeded": logger.info("build completed successfully") state = "ready" if state in ["failed", "error"]: logger.error("build {}, watch the build pod logs: {}".format( state, pod)) if logs and state != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] update_in(fn, "status.state", state) if state == "ready": update_in(fn, "spec.image", image) versioned = False if state == "ready": versioned = True get_db().store_function(db_session, fn, name, project, tag, versioned=versioned) return Response( content=out, media_type="text/plain", headers={ "x-mlrun-function-status": state, "function_status": state, "function_image": image, "builder_pod": pod, }, )
def _get_job_launcher_status(self, resp: typing.List) -> str: return get_in(resp, "status.launcherStatus")
def build_status( name: str = "", project: str = "", tag: str = "", offset: int = 0, logs: bool = True, last_log_timestamp: float = 0.0, verbose: bool = False, auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier), db_session: Session = Depends(deps.get_db_session), ): fn = get_db().get_function(db_session, name, project, tag) if not fn: log_and_raise(HTTPStatus.NOT_FOUND.value, name=name, project=project, tag=tag) # nuclio deploy status if fn.get("kind") in RuntimeKinds.nuclio_runtimes(): ( state, address, nuclio_name, last_log_timestamp, text, status, ) = get_nuclio_deploy_status(name, project, tag, last_log_timestamp=last_log_timestamp, verbose=verbose) if state == "ready": logger.info("Nuclio function deployed successfully", name=name) if state in ["error", "unhealthy"]: logger.error(f"Nuclio deploy error, {text}", name=name) # internal / external invocation urls were added on nuclio 1.6.x # and hence, it might be empty # to backward compatible with older nuclio versions, we use hard-coded default values internal_invocation_urls = status.get( "internalInvocationUrls", [resolve_function_internal_invocation_url(name)]) external_invocation_urls = status.get("externalInvocationUrls", [address] if address else []) # on nuclio > 1.6.x we get the external invocation url on the status block if external_invocation_urls and not address: address = external_invocation_urls[0] update_in(fn, "status.nuclio_name", nuclio_name) update_in(fn, "status.internal_invocation_urls", internal_invocation_urls) update_in(fn, "status.external_invocation_urls", external_invocation_urls) update_in(fn, "status.state", state) update_in(fn, "status.address", address) versioned = False if state == "ready": # Versioned means the version will be saved in the DB forever, we don't want to spam # the DB with intermediate or unusable versions, only successfully deployed versions versioned = True get_db().store_function( db_session, fn, name, project, tag, versioned=versioned, leader_session=auth_verifier.auth_info.session, ) return Response( content=text, media_type="text/plain", headers={ "x-mlrun-function-status": state, "x-mlrun-last-timestamp": str(last_log_timestamp), "x-mlrun-address": address, "x-mlrun-internal-invocation-urls": ",".join(internal_invocation_urls), "x-mlrun-external-invocation-urls": ",".join(external_invocation_urls), "x-mlrun-name": nuclio_name, }, ) # job deploy status state = get_in(fn, "status.state", "") pod = get_in(fn, "status.build_pod", "") image = get_in(fn, "spec.build.image", "") out = b"" if not pod: if state == "ready": image = image or get_in(fn, "spec.image") return Response( content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod, }, ) logger.info(f"get pod {pod} status") state = get_k8s().get_pod_status(pod) logger.info(f"pod state={state}") if state == "succeeded": logger.info("build completed successfully") state = mlrun.api.schemas.FunctionState.ready if state in ["failed", "error"]: logger.error(f"build {state}, watch the build pod logs: {pod}") state = mlrun.api.schemas.FunctionState.error if logs and state != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] update_in(fn, "status.state", state) if state == mlrun.api.schemas.FunctionState.ready: update_in(fn, "spec.image", image) versioned = False if state == mlrun.api.schemas.FunctionState.ready: versioned = True get_db().store_function( db_session, fn, name, project, tag, versioned=versioned, leader_session=auth_verifier.auth_info.session, ) return Response( content=out, media_type="text/plain", headers={ "x-mlrun-function-status": state, "function_status": state, "function_image": image, "builder_pod": pod, }, )
def build_status( name: str = "", project: str = "", tag: str = "", offset: int = 0, logs: bool = True, last_log_timestamp: float = 0.0, verbose: bool = False, auth_info: mlrun.api.schemas.AuthInfo = Depends( deps.authenticate_request), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.auth.verifier.AuthVerifier( ).query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.function, project or mlrun.mlconf.default_project, name, # store since with the current mechanism we update the status (and store the function) in the DB when a client # query for the status mlrun.api.schemas.AuthorizationAction.store, auth_info, ) fn = mlrun.api.crud.Functions().get_function(db_session, name, project, tag) if not fn: log_and_raise(HTTPStatus.NOT_FOUND.value, name=name, project=project, tag=tag) # nuclio deploy status if fn.get("kind") in RuntimeKinds.nuclio_runtimes(): ( state, address, nuclio_name, last_log_timestamp, text, status, ) = get_nuclio_deploy_status( name, project, tag, last_log_timestamp=last_log_timestamp, verbose=verbose, auth_info=auth_info, ) if state == "ready": logger.info("Nuclio function deployed successfully", name=name) if state in ["error", "unhealthy"]: logger.error(f"Nuclio deploy error, {text}", name=name) internal_invocation_urls = status.get("internalInvocationUrls", []) external_invocation_urls = status.get("externalInvocationUrls", []) # on earlier versions of mlrun, address used to represent the nodePort external invocation url # now that functions can be not exposed (using service_type clusterIP) this no longer relevant # and hence, for BC it would be filled with the external invocation url first item # or completely empty. address = external_invocation_urls[ 0] if external_invocation_urls else "" update_in(fn, "status.nuclio_name", nuclio_name) update_in(fn, "status.internal_invocation_urls", internal_invocation_urls) update_in(fn, "status.external_invocation_urls", external_invocation_urls) update_in(fn, "status.state", state) update_in(fn, "status.address", address) versioned = False if state == "ready": # Versioned means the version will be saved in the DB forever, we don't want to spam # the DB with intermediate or unusable versions, only successfully deployed versions versioned = True mlrun.api.crud.Functions().store_function( db_session, fn, name, project, tag, versioned=versioned, ) return Response( content=text, media_type="text/plain", headers={ "x-mlrun-function-status": state, "x-mlrun-last-timestamp": str(last_log_timestamp), "x-mlrun-address": address, "x-mlrun-internal-invocation-urls": ",".join(internal_invocation_urls), "x-mlrun-external-invocation-urls": ",".join(external_invocation_urls), "x-mlrun-name": nuclio_name, }, ) # job deploy status state = get_in(fn, "status.state", "") pod = get_in(fn, "status.build_pod", "") image = get_in(fn, "spec.build.image", "") out = b"" if not pod: if state == "ready": image = image or get_in(fn, "spec.image") return Response( content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod, }, ) logger.info(f"get pod {pod} status") state = get_k8s().get_pod_status(pod) logger.info(f"pod state={state}") if state == "succeeded": logger.info("build completed successfully") state = mlrun.api.schemas.FunctionState.ready if state in ["failed", "error"]: logger.error(f"build {state}, watch the build pod logs: {pod}") state = mlrun.api.schemas.FunctionState.error if logs and state != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] update_in(fn, "status.state", state) if state == mlrun.api.schemas.FunctionState.ready: update_in(fn, "spec.image", image) versioned = False if state == mlrun.api.schemas.FunctionState.ready: versioned = True mlrun.api.crud.Functions().store_function( db_session, fn, name, project, tag, versioned=versioned, ) return Response( content=out, media_type="text/plain", headers={ "x-mlrun-function-status": state, "function_status": state, "function_image": image, "builder_pod": pod, }, )
def _parse_submit_run_body(db_session: Session, data): task = data.get("task") function_dict = data.get("function") function_url = data.get("functionUrl") if not function_url and task: function_url = get_in(task, "spec.function") if not (function_dict or function_url) or not task: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="bad JSON, need to include function/url and task objects", ) # TODO: block exec for function["kind"] in ["", "local] (must be a # remote/container runtime) if function_dict and not function_url: function = new_function(runtime=function_dict) else: if "://" in function_url: function = import_function(url=function_url) else: project, name, tag, hash_key = parse_versioned_object_uri( function_url) function_record = get_db().get_function(db_session, name, project, tag, hash_key) if not function_record: log_and_raise( HTTPStatus.NOT_FOUND.value, reason="runtime error: function {} not found".format( function_url), ) function = new_function(runtime=function_record) if function_dict: # The purpose of the function dict is to enable the user to override configurations of the existing function # without modifying it - to do that we're creating a function object from the request function dict and # assign values from it to the main function object override_function = new_function(runtime=function_dict, kind=function.kind) for attribute in [ "volumes", "volume_mounts", "env", "resources", "image_pull_policy", "replicas", ]: override_value = getattr(override_function.spec, attribute, None) if override_value: if attribute == "env": for env_dict in override_value: function.set_env(env_dict["name"], env_dict["value"]) elif attribute == "volumes": function.spec.update_vols_and_mounts( override_value, []) elif attribute == "volume_mounts": # volume mounts don't have a well defined identifier (like name for volume) so we can't merge, # only override function.spec.volume_mounts = override_value elif attribute == "resources": # don't override it there are limits and requests but both are empty if override_value.get("limits", {}) or override_value.get( "requests", {}): setattr(function.spec, attribute, override_value) else: setattr(function.spec, attribute, override_value) return function, task
def submit(db_session: Session, data): task = data.get("task") function = data.get("function") url = data.get("functionUrl") if not url and task: url = get_in(task, "spec.function") if not (function or url) or not task: log_and_raise( HTTPStatus.BAD_REQUEST, reason="bad JSON, need to include function/url and task objects") # TODO: block exec for function["kind"] in ["", "local] (must be a # remote/container runtime) resp = None try: if function and not url: fn = new_function(runtime=function) else: if "://" in url: fn = import_function(url=url) else: project, name, tag, hash_key = parse_function_uri(url) runtime = get_db().get_function(db_session, name, project, tag, hash_key) if not runtime: log_and_raise( HTTPStatus.BAD_REQUEST, reason="runtime error: function {} not found".format( url)) fn = new_function(runtime=runtime) if function: fn2 = new_function(runtime=function) for attr in [ "volumes", "volume_mounts", "env", "resources", "image_pull_policy", "replicas" ]: val = getattr(fn2.spec, attr, None) if val: setattr(fn.spec, attr, val) run_db = get_run_db_instance(db_session) fn.set_db_connection(run_db, True) logger.info("func:\n{}".format(fn.to_yaml())) # fn.spec.rundb = "http://mlrun-api:8080" schedule = data.get("schedule") if schedule: args = (task, ) job_id = get_scheduler().add(schedule, fn, args) get_db().store_schedule(db_session, data) resp = {"schedule": schedule, "id": job_id} else: resp = fn.run(task, watch=False) logger.info("resp: %s", resp.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST, reason="runtime error: {}".format(err)) if not isinstance(resp, dict): resp = resp.to_dict() return { "data": resp, }
def run_start_time(run): ts = get_in(run, "status.start_time", "") if not ts: return None return parser.parse(ts)
def run_labels(run) -> dict: return get_in(run, "metadata.labels", {})
def run_state(run): return get_in(run, "status.state", "")