def new_model_server(name,
model_class: str,
models: dict = None,
filename='',
protocol='',
image='',
endpoint='',
explainer=False,
workers=8,
canary=None,
handler=None):
f = RemoteRuntime()
if not image:
name, spec, code = nuclio.build_file(filename,
name=name,
handler=serving_handler,
kind='serving')
f.spec.base_spec = spec
elif handler:
f.spec.function_handler = handler
f.metadata.name = name
f.serving(models,
model_class,
protocol,
image=image,
endpoint=endpoint,
explainer=explainer,
workers=workers,
canary=canary)
return f
def deploy(self, source='', dashboard='', project='', tag='',
kind=None):
self.set_config('metadata.labels.mlrun/class', self.kind)
spec = nuclio.ConfigSpec(env=self.spec.env, config=self.spec.config)
spec.cmd = self.spec.build_commands
kind = kind or self.spec.function_kind
project = project or self.metadata.project or 'mlrun'
source = source or self.spec.source
handler = self.spec.function_handler
if self.spec.base_spec:
config = nuclio.config.extend_config(self.spec.base_spec, spec, tag,
self.spec.source)
update_in(config, 'metadata.name', self.metadata.name)
update_in(config, 'spec.volumes', self.spec.volumes)
addr = nuclio.deploy.deploy_config(
config, dashboard, name=self.metadata.name,
project=project, tag=tag, verbose=self.verbose,
create_new=True)
else:
name, config, code = nuclio.build_file(source, name=self.metadata.name,
project=project,
handler=handler,
tag=tag, spec=spec,
kind=kind, verbose=self.verbose)
update_in(config, 'spec.volumes', self.spec.volumes)
addr = deploy_config(config, dashboard_url=dashboard, name=name, project=project,
tag=tag, verbose=self.verbose, create_new=True)
self.spec.command = 'http://{}'.format(addr)
return self.spec.command
def new_v2_model_server(
name,
model_class: str,
models: dict = None,
filename="",
protocol="",
image="",
endpoint="",
workers=8,
canary=None,
):
f = ServingRuntime()
if not image:
name, spec, code = nuclio.build_file(filename,
name=name,
handler="handler",
kind=serving_subkind)
f.spec.base_spec = spec
f.metadata.name = name
f.spec.default_class = model_class
params = None
if protocol:
params = {"protocol": protocol}
if models:
for name, model_path in models.items():
f.add_model(name, model_path=model_path, parameters=params)
f.with_http(workers, host=endpoint, canary=canary)
if image:
f.from_image(image)
return f
def deploy(self, dashboard='', project='', tag='', kind=None):
self.set_config('metadata.labels.mlrun/class', self.kind)
env_dict = {
get_item_name(v): get_item_name(v, 'value')
for v in self.spec.env
}
spec = nuclio.ConfigSpec(env=env_dict, config=self.spec.config)
spec.cmd = self.spec.build.commands or []
project = project or self.metadata.project or 'default'
handler = self.spec.function_handler
if self.spec.no_cache:
spec.set_config('spec.build.noCache', True)
if self.spec.base_spec:
if kind:
raise ValueError('kind cannot be specified on built functions')
config = nuclio.config.extend_config(self.spec.base_spec, spec,
tag,
self.spec.build.code_origin)
update_in(config, 'metadata.name', self.metadata.name)
update_in(config, 'spec.volumes', self.spec.to_nuclio_vol())
logger.info('deploy started')
addr = nuclio.deploy.deploy_config(config,
dashboard,
name=self.metadata.name,
project=project,
tag=tag,
verbose=self.verbose,
create_new=True)
else:
kind = kind if kind is not None else self.spec.function_kind
name, config, code = nuclio.build_file(self.spec.source,
name=self.metadata.name,
project=project,
handler=handler,
tag=tag,
spec=spec,
kind=kind,
verbose=self.verbose)
update_in(config, 'spec.volumes', self.spec.to_nuclio_vol())
addr = deploy_config(config,
dashboard_url=dashboard,
name=name,
project=project,
tag=tag,
verbose=self.verbose,
create_new=True)
self.spec.command = 'http://{}'.format(addr)
return self.spec.command
def with_code(self, from_file='', body=None):
if (not body and not from_file) or (from_file and from_file.endswith('.ipynb')):
from nuclio import build_file
name, spec, code = build_file(from_file)
self.spec.build.inline_code = get_in(spec, 'spec.build.functionSourceCode')
return self
if from_file:
with open(from_file) as fp:
body = fp.read()
self.spec.build.inline_code = b64encode(body.encode('utf-8')).decode('utf-8')
return self
def code_to_function(name='',
filename='',
handler='',
runtime=None,
image=None):
"""convert code or notebook to function object with embedded code
code stored in the function spec and can be refreshed using .with_code()
eliminate the need to build container images everytime we edit the code
:param name: function name
:param filename: blank for current notebook, or path to .py/.ipynb file
:param handler: name of function handler (if not main)
:param runtime: optional, runtime type local, job, dask, mpijob, ..
:param image: optional, container image
:return:
function object
"""
if runtime == 'nuclio':
r = RemoteRuntime()
r.metadata.name = name
return r
from nuclio import build_file
bname, spec, code = build_file(filename, handler=handler)
if runtime is None or runtime in ['', 'local']:
r = LocalRuntime()
elif runtime in runtime_dict:
r = runtime_dict[runtime]()
else:
raise Exception('unsupported runtime ({})'.format(runtime))
h = get_in(spec, 'spec.handler', '').split(':')
r.handler = h[0] if len(h) <= 1 else h[1]
r.metadata = get_in(spec, 'spec.metadata')
r.metadata.name = name or bname or 'mlrun'
r.spec.image = get_in(spec, 'spec.image', image)
build = r.spec.build
build.base_image = get_in(spec, 'spec.build.baseImage')
build.commands = get_in(spec, 'spec.build.commands')
build.functionSourceCode = get_in(spec, 'spec.build.functionSourceCode')
build.image = get_in(spec, 'spec.build.image')
build.secret = get_in(spec, 'spec.build.secret')
if r.kind != 'local':
r.spec.env = get_in(spec, 'spec.env')
for vol in get_in(spec, 'spec.volumes', []):
r.spec.volumes.append(vol.get('volume'))
r.spec.volume_mounts.append(vol.get('volumeMount'))
return r
def make_nuclio_job(filename='', handler='', image=None, secret=None, kind=None):
from nuclio import build_file
name, spec, code = build_file(filename, handler=handler)
r = K8sRuntime()
r.kind = kind or 'job'
h = get_in(spec, 'spec.handler', '').split(':')
r.handler = h[0] if len(h) <= 1 else h[1]
r.metadata = get_in(spec, 'spec.metadata')
r.build.base_image = get_in(spec, 'spec.build.baseImage')
r.build.commands = get_in(spec, 'spec.build.commands')
r.build.inline_code = get_in(spec, 'spec.build.functionSourceCode')
r.build.image = get_in(spec, 'spec.build.image', image)
r.build.secret = get_in(spec, 'spec.build.secret', secret)
r.env = get_in(spec, 'spec.env')
for vol in get_in(spec, 'spec.volumes', []):
r.volumes.append(vol.get('volume'))
r.volume_mounts.append(vol.get('volumeMount'))
return r
def new_v1_model_server(
name,
model_class: str,
models: dict = None,
filename="",
protocol="",
image="",
endpoint="",
workers=8,
canary=None,
):
f = RemoteRuntime()
if not image:
name, spec, code = nuclio.build_file(filename,
name=name,
handler=serving_handler,
kind="serving")
f.spec.base_spec = spec
f.metadata.name = name
if models:
for k, v in models.items():
f.set_env(f"SERVING_MODEL_{k}", v)
if protocol:
f.set_env("TRANSPORT_PROTOCOL", protocol)
if model_class:
f.set_env("MODEL_CLASS", model_class)
f.with_http(workers, host=endpoint, canary=canary)
f.spec.function_kind = "serving"
if image:
f.from_image(image)
return f
def with_code(self, from_file="", body=None, with_doc=True):
"""Update the function code
This function eliminates the need to build container images every time we edit the code
:param from_file: blank for current notebook, or path to .py/.ipynb file
:param body: will use the body as the function code
:param with_doc: update the document of the function parameters
:return: function object
"""
if (not body and not from_file) or (from_file
and from_file.endswith(".ipynb")):
from nuclio import build_file
_, _, body = build_file(from_file)
if from_file:
with open(from_file) as fp:
body = fp.read()
self.spec.build.functionSourceCode = b64encode(
body.encode("utf-8")).decode("utf-8")
if with_doc:
update_function_entry_points(self, body)
return self
def code_to_function(
name: str = "",
project: str = "",
tag: str = "",
filename: str = "",
handler: str = "",
kind: str = "",
image: str = None,
code_output="",
embed_code=True,
description="",
requirements: Union[str, List[str]] = None,
categories: list = None,
labels: dict = None,
with_doc=True,
):
"""convert code or notebook to function object with embedded code
code stored in the function spec and can be refreshed using .with_code()
eliminate the need to build container images every time we edit the code
:param name: function name
:param project: function project (none for 'default')
:param tag: function tag (none for 'latest')
:param filename: blank for current notebook, or path to .py/.ipynb file
:param handler: name of function handler (if not main)
:param kind: optional, runtime type local, job, dask, mpijob, ..
:param image: optional, container image
:param code_output: save the generated code (from notebook) in that path
:param embed_code: embed the source code into the function spec
:param description: function description
:param requirements: python requirements file path or list of packages
:param categories: list of categories (for function marketplace)
:param labels: dict of label names and values to tag the function
:param with_doc: document the function parameters
:return:
function object
"""
filebase, _ = path.splitext(path.basename(filename))
def add_name(origin, name=""):
name = filename or (name + ".ipynb")
if not origin:
return name
return "{}:{}".format(origin, name)
def update_meta(fn):
fn.spec.description = description
fn.metadata.project = project or mlconf.default_project
fn.metadata.tag = tag
fn.metadata.categories = categories
fn.metadata.labels = labels
def resolve_nuclio_subkind(kind):
is_nuclio = kind.startswith("nuclio")
subkind = kind[kind.find(":") +
1:] if is_nuclio and ":" in kind else None
if kind == RuntimeKinds.serving:
is_nuclio = True
subkind = serving_subkind
return is_nuclio, subkind
if (not embed_code and not code_output
and (not filename or filename.endswith(".ipynb"))):
raise ValueError("a valid code file must be specified "
"when not using the embed_code option")
is_nuclio, subkind = resolve_nuclio_subkind(kind)
code_origin = add_name(add_code_metadata(filename), name)
name, spec, code = build_file(filename,
name=name,
handler=handler or "handler",
kind=subkind)
spec_kind = get_in(spec, "kind", "")
if not kind and spec_kind not in ["", "Function"]:
kind = spec_kind.lower()
# if its a nuclio subkind, redo nb parsing
is_nuclio, subkind = resolve_nuclio_subkind(kind)
if is_nuclio:
name, spec, code = build_file(filename,
name=name,
handler=handler or "handler",
kind=subkind)
if code_output:
if code_output == ".":
code_output = name + ".py"
if filename == "" or filename.endswith(".ipynb"):
with open(code_output, "w") as fp:
fp.write(code)
else:
raise ValueError("code_output option is only used with notebooks")
if is_nuclio:
if subkind == serving_subkind:
r = ServingRuntime()
else:
r = RemoteRuntime()
r.spec.function_kind = subkind
if image:
r.spec.image = image
r.spec.default_handler = handler
if embed_code:
update_in(spec, "kind", "Function")
r.spec.base_spec = spec
if with_doc:
update_function_entry_points(r, code)
else:
r.spec.source = filename
r.spec.function_handler = handler
if not name:
raise ValueError("name must be specified")
r.metadata.name = name
r.spec.build.code_origin = code_origin
if requirements:
r.with_requirements(requirements)
update_meta(r)
return r
if kind is None or kind in ["", "Function"]:
raise ValueError("please specify the function kind")
elif kind in ["local"]:
r = LocalRuntime()
elif kind in RuntimeKinds.all():
r = get_runtime_class(kind)()
else:
raise ValueError("unsupported runtime ({})".format(kind))
name, spec, code = build_file(filename, name=name)
if not name:
raise ValueError("name must be specified")
h = get_in(spec, "spec.handler", "").split(":")
r.handler = h[0] if len(h) <= 1 else h[1]
r.metadata = get_in(spec, "spec.metadata")
r.metadata.name = name
r.spec.image = image or get_in(spec, "spec.image", "")
build = r.spec.build
build.code_origin = code_origin
build.base_image = get_in(spec, "spec.build.baseImage")
build.commands = get_in(spec, "spec.build.commands")
build.extra = get_in(spec, "spec.build.extra")
if embed_code:
build.functionSourceCode = get_in(spec,
"spec.build.functionSourceCode")
else:
if code_output:
r.spec.command = code_output
else:
r.spec.command = filename
build.image = get_in(spec, "spec.build.image")
build.secret = get_in(spec, "spec.build.secret")
if requirements:
r.with_requirements(requirements)
if r.kind != "local":
r.spec.env = get_in(spec, "spec.env")
for vol in get_in(spec, "spec.volumes", []):
r.spec.volumes.append(vol.get("volume"))
r.spec.volume_mounts.append(vol.get("volumeMount"))
if with_doc:
update_function_entry_points(r, code)
r.spec.default_handler = handler
update_meta(r)
return r
def code_to_function(
name: str = '',
project: str = '',
tag: str = '',
filename: str = '',
handler: str = '',
kind: str = '',
image: str = None,
code_output='',
embed_code=True,
description='',
categories: list = None,
labels: dict = None,
with_doc=True,
):
"""convert code or notebook to function object with embedded code
code stored in the function spec and can be refreshed using .with_code()
eliminate the need to build container images every time we edit the code
:param name: function name
:param project: function project (none for 'default')
:param tag: function tag (none for 'latest')
:param filename: blank for current notebook, or path to .py/.ipynb file
:param handler: name of function handler (if not main)
:param kind: optional, runtime type local, job, dask, mpijob, ..
:param image: optional, container image
:param code_output: save the generated code (from notebook) in that path
:param embed_code: embed the source code into the function spec
:param description: function description
:param categories: list of categories (for function marketplace)
:param labels: dict of label names and values to tag the function
:param with_doc: document the function parameters
:return:
function object
"""
filebase, _ = path.splitext(path.basename(filename))
def add_name(origin, name=''):
name = filename or (name + '.ipynb')
if not origin:
return name
return '{}:{}'.format(origin, name)
def update_meta(fn):
fn.spec.description = description
fn.metadata.project = project
fn.metadata.tag = tag
fn.metadata.categories = categories
fn.metadata.labels = labels
if (not embed_code and not code_output
and (not filename or filename.endswith('.ipynb'))):
raise ValueError('a valid code file must be specified '
'when not using the embed_code option')
subkind = kind[kind.find(':') + 1:] if kind.startswith('nuclio:') else None
code_origin = add_name(add_code_metadata(filename), name)
name, spec, code = build_file(filename,
name=name,
handler=handler or 'handler',
kind=subkind)
spec_kind = get_in(spec, 'kind', '')
if spec_kind not in ['', 'Function']:
kind = spec_kind.lower()
# if its a nuclio subkind, redo nb parsing
if kind.startswith('nuclio:'):
subkind = kind[kind.find(':') + 1:]
name, spec, code = build_file(filename,
name=name,
handler=handler or 'handler',
kind=subkind)
if code_output:
if code_output == '.':
code_output = name + '.py'
if filename == '' or filename.endswith('.ipynb'):
with open(code_output, 'w') as fp:
fp.write(code)
else:
raise ValueError('code_output option is only used with notebooks')
if kind.startswith('nuclio'):
r = RemoteRuntime()
r.spec.function_kind = subkind
if embed_code:
update_in(spec, 'kind', 'Function')
r.spec.base_spec = spec
if with_doc:
handlers = find_handlers(code)
r.spec.entry_points = {h['name']: as_func(h) for h in handlers}
else:
r.spec.source = filename
r.spec.function_handler = handler
if not name:
raise ValueError('name must be specified')
r.metadata.name = name
r.spec.build.code_origin = code_origin
update_meta(r)
return r
if kind is None or kind in ['', 'Function']:
raise ValueError('please specify the function kind')
elif kind in ['local']:
r = LocalRuntime()
elif kind in RuntimeKinds.all():
r = get_runtime_class(kind)()
else:
raise ValueError('unsupported runtime ({})'.format(kind))
name, spec, code = build_file(filename, name=name)
if not name:
raise ValueError('name must be specified')
h = get_in(spec, 'spec.handler', '').split(':')
r.handler = h[0] if len(h) <= 1 else h[1]
r.metadata = get_in(spec, 'spec.metadata')
r.metadata.name = name
r.spec.image = get_in(spec, 'spec.image', image)
build = r.spec.build
build.code_origin = code_origin
build.base_image = get_in(spec, 'spec.build.baseImage')
build.commands = get_in(spec, 'spec.build.commands')
if embed_code:
build.functionSourceCode = get_in(spec,
'spec.build.functionSourceCode')
else:
if code_output:
r.spec.command = code_output
else:
r.spec.command = filename
build.image = get_in(spec, 'spec.build.image')
build.secret = get_in(spec, 'spec.build.secret')
if r.kind != 'local':
r.spec.env = get_in(spec, 'spec.env')
for vol in get_in(spec, 'spec.volumes', []):
r.spec.volumes.append(vol.get('volume'))
r.spec.volume_mounts.append(vol.get('volumeMount'))
if with_doc:
handlers = find_handlers(code)
r.spec.entry_points = {h['name']: as_func(h) for h in handlers}
r.spec.default_handler = handler
update_meta(r)
return r
def compile_function_config(function: RemoteRuntime):
function.set_config("metadata.labels.mlrun/class", function.kind)
# Add vault configurations to function's pod spec, if vault secret source was added.
# Needs to be here, since it adds env params, which are handled in the next lines.
function.add_secrets_config_to_spec()
env_dict = {get_item_name(v): get_item_name(v, "value") for v in function.spec.env}
for key, value in function._get_runtime_env().items():
env_dict[key] = value
spec = nuclio.ConfigSpec(env=env_dict, config=function.spec.config)
spec.cmd = function.spec.build.commands or []
project = function.metadata.project or "default"
tag = function.metadata.tag
handler = function.spec.function_handler
# In Nuclio >= 1.6.x default serviceType has changed to "ClusterIP".
spec.set_config("spec.serviceType", mlconf.httpdb.nuclio.default_service_type)
if function.spec.readiness_timeout:
spec.set_config("spec.readinessTimeoutSeconds", function.spec.readiness_timeout)
if function.spec.resources:
spec.set_config("spec.resources", function.spec.resources)
if function.spec.no_cache:
spec.set_config("spec.build.noCache", True)
if function.spec.build.functionSourceCode:
spec.set_config(
"spec.build.functionSourceCode", function.spec.build.functionSourceCode
)
if function.spec.replicas:
spec.set_config("spec.minReplicas", function.spec.replicas)
spec.set_config("spec.maxReplicas", function.spec.replicas)
else:
spec.set_config("spec.minReplicas", function.spec.min_replicas)
spec.set_config("spec.maxReplicas", function.spec.max_replicas)
if function.spec.base_spec or function.spec.build.functionSourceCode:
config = function.spec.base_spec
if not config:
# if base_spec was not set (when not using code_to_function) and we have base64 code
# we create the base spec with essential attributes
config = nuclio.config.new_config()
update_in(config, "spec.handler", handler or "main:handler")
config = nuclio.config.extend_config(
config, spec, tag, function.spec.build.code_origin
)
update_in(config, "metadata.name", function.metadata.name)
update_in(config, "spec.volumes", function.spec.generate_nuclio_volumes())
base_image = get_in(config, "spec.build.baseImage") or function.spec.image
if base_image:
update_in(config, "spec.build.baseImage", enrich_image_url(base_image))
logger.info("deploy started")
name = get_fullname(function.metadata.name, project, tag)
function.status.nuclio_name = name
update_in(config, "metadata.name", name)
else:
name, config, code = nuclio.build_file(
function.spec.source,
name=function.metadata.name,
project=project,
handler=handler,
tag=tag,
spec=spec,
kind=function.spec.function_kind,
verbose=function.verbose,
)
update_in(config, "spec.volumes", function.spec.generate_nuclio_volumes())
if function.spec.image:
update_in(
config, "spec.build.baseImage", enrich_image_url(function.spec.image)
)
name = get_fullname(name, project, tag)
function.status.nuclio_name = name
update_in(config, "metadata.name", name)
return name, project, config
def deploy_nuclio_function(function: RemoteRuntime, dashboard="", watch=False):
function.set_config("metadata.labels.mlrun/class", function.kind)
env_dict = {
get_item_name(v): get_item_name(v, "value")
for v in function.spec.env
}
for key, value in function._get_runtime_env().items():
env_dict[key] = value
spec = nuclio.ConfigSpec(env=env_dict, config=function.spec.config)
spec.cmd = function.spec.build.commands or []
project = function.metadata.project or "default"
tag = function.metadata.tag
handler = function.spec.function_handler
# In Nuclio 1.6.0 default serviceType changed to "ClusterIP", make sure we're using NodePort
spec.set_config("spec.serviceType", "NodePort")
if function.spec.readiness_timeout:
spec.set_config("spec.readinessTimeoutSeconds",
function.spec.readiness_timeout)
if function.spec.resources:
spec.set_config("spec.resources", function.spec.resources)
if function.spec.no_cache:
spec.set_config("spec.build.noCache", True)
if function.spec.replicas:
spec.set_config("spec.minReplicas", function.spec.replicas)
spec.set_config("spec.maxReplicas", function.spec.replicas)
else:
spec.set_config("spec.minReplicas", function.spec.min_replicas)
spec.set_config("spec.maxReplicas", function.spec.max_replicas)
dashboard = dashboard or mlconf.nuclio_dashboard_url
if function.spec.base_spec:
config = nuclio.config.extend_config(function.spec.base_spec, spec,
tag,
function.spec.build.code_origin)
update_in(config, "metadata.name", function.metadata.name)
update_in(config, "spec.volumes", function.spec.to_nuclio_vol())
base_image = get_in(config,
"spec.build.baseImage") or function.spec.image
if base_image:
update_in(config, "spec.build.baseImage",
enrich_image_url(base_image))
logger.info("deploy started")
name = get_fullname(function.metadata.name, project, tag)
function.status.nuclio_name = name
update_in(config, "metadata.name", name)
return nuclio.deploy.deploy_config(
config,
dashboard,
name=name,
project=project,
tag=tag,
verbose=function.verbose,
create_new=True,
watch=watch,
)
else:
name, config, code = nuclio.build_file(
function.spec.source,
name=function.metadata.name,
project=project,
handler=handler,
tag=tag,
spec=spec,
kind=function.spec.function_kind,
verbose=function.verbose,
)
update_in(config, "spec.volumes", function.spec.to_nuclio_vol())
if function.spec.image:
update_in(config, "spec.build.baseImage",
enrich_image_url(function.spec.image))
name = get_fullname(name, project, tag)
function.status.nuclio_name = name
update_in(config, "metadata.name", name)
return deploy_config(
config,
dashboard_url=dashboard,
name=name,
project=project,
tag=tag,
verbose=function.verbose,
create_new=True,
watch=watch,
)
def code_to_function(
name: str = "",
project: str = "",
tag: str = "",
filename: str = "",
handler: str = "",
kind: str = "",
image: str = None,
code_output: str = "",
embed_code: bool = True,
description: str = "",
requirements: Union[str, List[str]] = None,
categories: List[str] = None,
labels: Dict[str, str] = None,
with_doc: bool = True,
ignored_tags=None,
) -> Union[MpiRuntimeV1Alpha1, MpiRuntimeV1, RemoteRuntime, ServingRuntime,
DaskCluster, KubejobRuntime, LocalRuntime, Spark2Runtime,
Spark3Runtime, RemoteSparkRuntime, ]:
"""Convenience function to insert code and configure an mlrun runtime.
Easiest way to construct a runtime type object. Provides the most often
used configuration options for all runtimes as parameters.
Instantiated runtimes are considered "functions" in mlrun, but they are
anything from nuclio functions to generic kubernetes pods to spark jobs.
Functions are meant to be focused, and as such limited in scope and size.
Typically a function can be expressed in a single python module with
added support from custom docker images and commands for the environment.
The returned runtime object can be further configured if more
customization is required.
One of the most important parameters is "kind". This is what is used to
specify the chosen runtimes. The options are:
- local: execute a local python or shell script
- job: insert the code into a Kubernetes pod and execute it
- nuclio: insert the code into a real-time serverless nuclio function
- serving: insert code into orchestrated nuclio function(s) forming a DAG
- dask: run the specified python code / script as Dask Distributed job
- mpijob: run distributed Horovod jobs over the MPI job operator
- spark: run distributed Spark job using Spark Kubernetes Operator
- remote-spark: run distributed Spark job on remote Spark service
Learn more about function runtimes here:
https://docs.mlrun.org/en/latest/runtimes/functions.html#function-runtimes
:param name: function name, typically best to use hyphen-case
:param project: project used to namespace the function, defaults to "default"
:param tag: function tag to track multiple versions of the same function, defaults to "latest"
:param filename: path to .py/.ipynb file, defaults to current jupyter notebook
:param handler: The default function handler to call for the job or nuclio function, in batch functions
(job, mpijob, ..) the handler can also be specified in the `.run()` command, when not specified
the entire file will be executed (as main).
for nuclio functions the handler is in the form of module:function, defaults to "main:handler"
:param kind: function runtime type string - nuclio, job, etc. (see docstring for all options)
:param image: base docker image to use for building the function container, defaults to None
:param code_output: specify "." to generate python module from the current jupyter notebook
:param embed_code: indicates whether or not to inject the code directly into the function runtime spec,
defaults to True
:param description: short function description, defaults to ""
:param requirements: list of python packages or pip requirements file path, defaults to None
:param categories: list of categories for mlrun function marketplace, defaults to None
:param labels: immutable name/value pairs to tag the function with useful metadata, defaults to None
:param with_doc: indicates whether to document the function parameters, defaults to True
:param ignored_tags: notebook cells to ignore when converting notebooks to py code (separated by ";")
:return:
pre-configured function object from a mlrun runtime class
example::
import mlrun
# create job function object from notebook code and add doc/metadata
fn = mlrun.code_to_function('file_utils', kind='job',
handler='open_archive', image='mlrun/mlrun',
description = "this function opens a zip archive into a local/mounted folder",
categories = ['fileutils'],
labels = {'author': 'me'})
example::
import mlrun
from pathlib import Path
# create file
Path('mover.py').touch()
# create nuclio function object from python module call mover.py
fn = mlrun.code_to_function('nuclio-mover', kind='nuclio',
filename='mover.py', image='python:3.7',
description = "this function moves files from one system to another",
requirements = ["pandas"],
labels = {'author': 'me'})
"""
filebase, _ = path.splitext(path.basename(filename))
ignored_tags = ignored_tags or mlconf.ignored_notebook_tags
def add_name(origin, name=""):
name = filename or (name + ".ipynb")
if not origin:
return name
return f"{origin}:{name}"
def update_meta(fn):
fn.spec.description = description
fn.metadata.project = project or mlconf.default_project
fn.metadata.tag = tag
fn.metadata.categories = categories
fn.metadata.labels = labels
def resolve_nuclio_subkind(kind):
is_nuclio = kind.startswith("nuclio")
subkind = kind[kind.find(":") +
1:] if is_nuclio and ":" in kind else None
if kind == RuntimeKinds.serving:
is_nuclio = True
subkind = serving_subkind
return is_nuclio, subkind
if (not embed_code and not code_output
and (not filename or filename.endswith(".ipynb"))):
raise ValueError("a valid code file must be specified "
"when not using the embed_code option")
is_nuclio, subkind = resolve_nuclio_subkind(kind)
code_origin = add_name(add_code_metadata(filename), name)
name, spec, code = build_file(
filename,
name=name,
handler=handler or "handler",
kind=subkind,
ignored_tags=ignored_tags,
)
spec_kind = get_in(spec, "kind", "")
if not kind and spec_kind not in ["", "Function"]:
kind = spec_kind.lower()
# if its a nuclio subkind, redo nb parsing
is_nuclio, subkind = resolve_nuclio_subkind(kind)
if is_nuclio:
name, spec, code = build_file(
filename,
name=name,
handler=handler or "handler",
kind=subkind,
ignored_tags=ignored_tags,
)
if code_output:
if code_output == ".":
code_output = name + ".py"
if filename == "" or filename.endswith(".ipynb"):
with open(code_output, "w") as fp:
fp.write(code)
else:
raise ValueError("code_output option is only used with notebooks")
if is_nuclio:
if subkind == serving_subkind:
r = ServingRuntime()
else:
r = RemoteRuntime()
r.spec.function_kind = subkind
if image:
r.spec.image = image
r.spec.default_handler = handler
if embed_code:
update_in(spec, "kind", "Function")
r.spec.base_spec = spec
else:
r.spec.source = filename
r.spec.function_handler = handler
if not name:
raise ValueError("name must be specified")
r.metadata.name = name
r.spec.build.code_origin = code_origin
r.spec.build.origin_filename = filename or (name + ".ipynb")
if requirements:
r.with_requirements(requirements)
update_meta(r)
return r
if kind is None or kind in ["", "Function"]:
raise ValueError("please specify the function kind")
elif kind in RuntimeKinds.all():
r = get_runtime_class(kind)()
else:
raise ValueError(f"unsupported runtime ({kind})")
name, spec, code = build_file(filename,
name=name,
ignored_tags=ignored_tags)
if not name:
raise ValueError("name must be specified")
h = get_in(spec, "spec.handler", "").split(":")
r.handler = h[0] if len(h) <= 1 else h[1]
r.metadata = get_in(spec, "spec.metadata")
r.metadata.name = name
r.spec.image = image or get_in(spec, "spec.image", "")
build = r.spec.build
build.code_origin = code_origin
build.origin_filename = filename or (name + ".ipynb")
build.base_image = get_in(spec, "spec.build.baseImage")
build.commands = get_in(spec, "spec.build.commands")
build.extra = get_in(spec, "spec.build.extra")
if embed_code:
build.functionSourceCode = get_in(spec,
"spec.build.functionSourceCode")
else:
if code_output:
r.spec.command = code_output
else:
r.spec.command = filename
build.image = get_in(spec, "spec.build.image")
build.secret = get_in(spec, "spec.build.secret")
if requirements:
r.with_requirements(requirements)
if r.kind != "local":
r.spec.env = get_in(spec, "spec.env")
for vol in get_in(spec, "spec.volumes", []):
r.spec.volumes.append(vol.get("volume"))
r.spec.volume_mounts.append(vol.get("volumeMount"))
if with_doc:
update_function_entry_points(r, code)
r.spec.default_handler = handler
update_meta(r)
return r
def compile_function_config(function: RemoteRuntime):
labels = function.metadata.labels or {}
labels.update({"mlrun/class": function.kind})
for key, value in labels.items():
function.set_config(f"metadata.labels.{key}", value)
# Add vault configurations to function's pod spec, if vault secret source was added.
# Needs to be here, since it adds env params, which are handled in the next lines.
function.add_secrets_config_to_spec()
env_dict, external_source_env_dict = function.get_nuclio_config_spec_env()
spec = nuclio.ConfigSpec(
env=env_dict,
external_source_env=external_source_env_dict,
config=function.spec.config,
)
spec.cmd = function.spec.build.commands or []
project = function.metadata.project or "default"
tag = function.metadata.tag
handler = function.spec.function_handler
# In Nuclio >= 1.6.x default serviceType has changed to "ClusterIP".
spec.set_config("spec.serviceType",
mlconf.httpdb.nuclio.default_service_type)
if function.spec.readiness_timeout:
spec.set_config("spec.readinessTimeoutSeconds",
function.spec.readiness_timeout)
if function.spec.resources:
spec.set_config("spec.resources", function.spec.resources)
if function.spec.no_cache:
spec.set_config("spec.build.noCache", True)
if function.spec.build.functionSourceCode:
spec.set_config("spec.build.functionSourceCode",
function.spec.build.functionSourceCode)
# don't send node selections if nuclio is not compatible
if validate_nuclio_version_compatibility("1.5.20", "1.6.10"):
if function.spec.node_selector:
spec.set_config("spec.nodeSelector", function.spec.node_selector)
if function.spec.node_name:
spec.set_config("spec.nodeName", function.spec.node_name)
if function.spec.affinity:
spec.set_config("spec.affinity",
function.spec._get_sanitized_affinity())
# don't send default or any priority class name if nuclio is not compatible
if (function.spec.priority_class_name
and validate_nuclio_version_compatibility("1.6.18")
and len(mlconf.get_valid_function_priority_class_names())):
spec.set_config("spec.priorityClassName",
function.spec.priority_class_name)
if function.spec.replicas:
spec.set_config("spec.minReplicas", function.spec.replicas)
spec.set_config("spec.maxReplicas", function.spec.replicas)
else:
spec.set_config("spec.minReplicas", function.spec.min_replicas)
spec.set_config("spec.maxReplicas", function.spec.max_replicas)
if function.spec.service_account:
spec.set_config("spec.serviceAccount", function.spec.service_account)
if function.spec.base_spec or function.spec.build.functionSourceCode:
config = function.spec.base_spec
if not config:
# if base_spec was not set (when not using code_to_function) and we have base64 code
# we create the base spec with essential attributes
config = nuclio.config.new_config()
update_in(config, "spec.handler", handler or "main:handler")
config = nuclio.config.extend_config(config, spec, tag,
function.spec.build.code_origin)
update_in(config, "metadata.name", function.metadata.name)
update_in(config, "spec.volumes",
function.spec.generate_nuclio_volumes())
base_image = (get_in(config, "spec.build.baseImage")
or function.spec.image or function.spec.build.base_image)
if base_image:
update_in(config, "spec.build.baseImage",
enrich_image_url(base_image))
logger.info("deploy started")
name = get_fullname(function.metadata.name, project, tag)
function.status.nuclio_name = name
update_in(config, "metadata.name", name)
else:
name, config, code = nuclio.build_file(
function.spec.source,
name=function.metadata.name,
project=project,
handler=handler,
tag=tag,
spec=spec,
kind=function.spec.function_kind,
verbose=function.verbose,
)
update_in(config, "spec.volumes",
function.spec.generate_nuclio_volumes())
base_image = function.spec.image or function.spec.build.base_image
if base_image:
update_in(
config,
"spec.build.baseImage",
enrich_image_url(base_image),
)
name = get_fullname(name, project, tag)
function.status.nuclio_name = name
update_in(config, "metadata.name", name)
return name, project, config
def deploy(self, dashboard='', project='', tag='', kind=None):
def get_fullname(config, name, project, tag):
if project:
name = '{}-{}'.format(project, name)
if tag:
name = '{}-{}'.format(name, tag)
update_in(config, 'metadata.name', name)
return name
self.set_config('metadata.labels.mlrun/class', self.kind)
env_dict = {
get_item_name(v): get_item_name(v, 'value')
for v in self.spec.env
}
spec = nuclio.ConfigSpec(env=env_dict, config=self.spec.config)
spec.cmd = self.spec.build.commands or []
project = project or self.metadata.project or 'default'
handler = self.spec.function_handler
if self.spec.no_cache:
spec.set_config('spec.build.noCache', True)
if self.spec.replicas:
spec.set_config('spec.minReplicas', self.spec.replicas)
spec.set_config('spec.maxReplicas', self.spec.replicas)
else:
spec.set_config('spec.minReplicas', self.spec.min_replicas)
spec.set_config('spec.maxReplicas', self.spec.max_replicas)
dashboard = get_auth_filled_platform_dashboard_url(dashboard)
if self.spec.base_spec:
if kind:
raise ValueError('kind cannot be specified on built functions')
config = nuclio.config.extend_config(self.spec.base_spec, spec,
tag,
self.spec.build.code_origin)
update_in(config, 'metadata.name', self.metadata.name)
update_in(config, 'spec.volumes', self.spec.to_nuclio_vol())
base_image = get_in(config, 'spec.build.baseImage')
if base_image:
update_in(config, 'spec.build.baseImage',
tag_image(base_image))
logger.info('deploy started')
name = get_fullname(config, self.metadata.name, project, tag)
addr = nuclio.deploy.deploy_config(
config,
dashboard,
name=name,
project=project,
tag=tag,
verbose=self.verbose,
create_new=True,
)
else:
kind = kind if kind is not None else self.spec.function_kind
name, config, code = nuclio.build_file(
self.spec.source,
name=self.metadata.name,
project=project,
handler=handler,
tag=tag,
spec=spec,
kind=kind,
verbose=self.verbose,
)
update_in(config, 'spec.volumes', self.spec.to_nuclio_vol())
name = get_fullname(config, name, project, tag)
addr = deploy_config(
config,
dashboard_url=dashboard,
name=name,
project=project,
tag=tag,
verbose=self.verbose,
create_new=True,
)
self.spec.command = 'http://{}'.format(addr)
self.status.nuclio_name = name
if addr:
self.status.state = 'ready'
self.status.address = addr
self.save()
return self.spec.command
