def deploy_function(function: DaskCluster, secrets=None):
try:
from dask_kubernetes import KubeCluster, make_pod_spec
from dask.distributed import Client, default_client
from kubernetes_asyncio import client
import dask
except ImportError as e:
print('missing dask or dask_kubernetes, please run '
'"pip install dask distributed dask_kubernetes", %s', e)
raise e
spec = function.spec
meta = function.metadata
spec.remote = True
image = function.full_image_path() or 'daskdev/dask:latest'
env = spec.env
namespace = meta.namespace or config.namespace
if spec.extra_pip:
env.append(spec.extra_pip)
pod_labels = get_resource_labels(function)
args = ['dask-worker', "--nthreads", str(spec.nthreads)]
if spec.args:
args += spec.args
container = client.V1Container(name='base',
image=image,
env=env,
args=args,
image_pull_policy=spec.image_pull_policy,
volume_mounts=spec.volume_mounts,
resources=spec.resources)
pod_spec = client.V1PodSpec(containers=[container],
restart_policy='Never',
volumes=spec.volumes,
service_account=spec.service_account)
if spec.image_pull_secret:
pod_spec.image_pull_secrets = [
client.V1LocalObjectReference(name=spec.image_pull_secret)]
pod = client.V1Pod(metadata=client.V1ObjectMeta(namespace=namespace,
labels=pod_labels),
#annotations=meta.annotation),
spec=pod_spec)
svc_temp = dask.config.get("kubernetes.scheduler-service-template")
if spec.service_type or spec.node_port:
if spec.node_port:
spec.service_type = 'NodePort'
svc_temp['spec']['ports'][1]['nodePort'] = spec.node_port
update_in(svc_temp, 'spec.type', spec.service_type)
norm_name = normalize_name(meta.name)
dask.config.set({"kubernetes.scheduler-service-template": svc_temp,
'kubernetes.name': 'mlrun-' + norm_name + '-{uuid}'})
cluster = KubeCluster(
pod, deploy_mode='remote',
namespace=namespace,
scheduler_timeout=spec.scheduler_timeout)
logger.info('cluster {} started at {}'.format(
cluster.name, cluster.scheduler_address
))
function.status.scheduler_address = cluster.scheduler_address
function.status.cluster_name = cluster.name
if spec.service_type == 'NodePort':
ports = cluster.scheduler.service.spec.ports
function.status.node_ports = {'scheduler': ports[0].node_port,
'dashboard': ports[1].node_port}
if spec.replicas:
cluster.scale(spec.replicas)
else:
cluster.adapt(minimum=spec.min_replicas,
maximum=spec.max_replicas)
return cluster
ds.coords['longitude'] = ds_lonlat['longitude']
ds.time[-1]
ds.streamflow
ds.streamflow[:, 0].hvplot()
import warnings
warnings.filterwarnings("ignore")
from dask.distributed import Client, progress, LocalCluster
from dask_kubernetes import KubeCluster
cluster = KubeCluster()
cluster.scale(25)
cluster
client = Client(cluster)
var = 'streamflow'
ds[var].nbytes / 1e9
var_mean = ds[var].mean(dim='time').persist()
progress(var_mean)
df = var_mean.to_pandas().to_frame()
df = df.assign(latitude=df['latitude'].values)
df = df.assign(longitude=df['longitude'].values)
df.rename(columns={0: "transport"}, inplace=True)
class DaskCluster(KubejobRuntime):
kind = 'dask'
_is_nested = False
def __init__(self, spec=None,
metadata=None):
super().__init__(spec, metadata)
self._cluster = None
self.spec.build.base_image = self.spec.build.base_image or 'daskdev/dask:latest'
self.set_label('mlrun/class', self.kind)
@property
def spec(self) -> DaskSpec:
return self._spec
@spec.setter
def spec(self, spec):
self._spec = self._verify_dict(spec, 'spec', DaskSpec)
def to_pod(self):
image = self._image_path() or 'daskdev/dask:latest'
env = self.spec.env
namespace = self.metadata.namespace or config.namespace
if self.spec.extra_pip:
env.append(self.spec.extra_pip)
container = client.V1Container(name='base',
image=image,
env=env,
command=None,
args=self.spec.args,
image_pull_policy=self.spec.image_pull_policy,
volume_mounts=self.spec.volume_mounts,
resources=self.spec.resources)
pod_spec = client.V1PodSpec(containers=[container],
restart_policy='Never',
volumes=self.spec.volumes,
service_account=self.spec.service_account)
meta = client.V1ObjectMeta(namespace=namespace,
labels=self.metadata.labels,
annotations=self.metadata.annotations)
pod = client.V1Pod(metadata=meta, spec=pod_spec)
return pod
@property
def initialized(self):
return True if self._cluster else False
def cluster(self, scale=0):
if not self._cluster:
try:
from dask_kubernetes import KubeCluster
from dask.distributed import Client
except ImportError as e:
print('missing dask_kubernetes, please run "pip install dask_kubernetes"')
raise e
self._cluster = KubeCluster(self.to_pod())
if not scale:
self._cluster.adapt()
else:
self._cluster.scale(scale)
Client(self._cluster)
return self._cluster
@property
def client(self):
from dask.distributed import Client, default_client
try:
return default_client()
except ValueError:
if self._cluster:
return Client(self._cluster)
return Client()
def close(self):
from dask.distributed import Client, default_client, as_completed
try:
client = default_client()
client.close()
except ValueError:
pass
if self._cluster:
self._cluster.close()
def _run(self, runobj: RunObject, execution):
handler = runobj.spec.handler
self._force_handler(handler)
from dask import delayed
if self.spec.rundb:
# todo: remote dask via k8s spec env
environ['MLRUN_DBPATH'] = self.spec.rundb
arg_list = get_func_arg(handler, runobj, execution)
try:
task = delayed(handler)(*arg_list)
out = task.compute()
except Exception as e:
err = str(e)
execution.set_state(error=err)
if out:
execution.log_result('return', out)
return execution.to_dict()
def _run_many(self, tasks, execution, runobj: RunObject):
handler = runobj.spec.handler
self._force_handler(handler)
futures = []
contexts = []
tasks = list(tasks)
for task in tasks:
ctx = MLClientCtx.from_dict(task.to_dict(),
self.spec.rundb,
autocommit=True)
args = get_func_arg(handler, task, ctx)
resp = self.client.submit(handler, *args)
futures.append(resp)
contexts.append(ctx)
resps = self.client.gather(futures)
results = RunList()
for r, c, t in zip(resps, contexts, tasks):
if r:
c.log_result('return', r)
# todo: handle task errors
resp = self._post_run(task=t)
results.append(resp)
print(resps)
return results
def deploy_function(function: DaskCluster, secrets=None):
# TODO: why is this here :|
try:
from dask_kubernetes import KubeCluster, make_pod_spec # noqa: F401
from dask.distributed import Client, default_client # noqa: F401
from kubernetes_asyncio import client
import dask
except ImportError as e:
print(
"missing dask or dask_kubernetes, please run "
'"pip install dask distributed dask_kubernetes", %s',
e,
)
raise e
spec = function.spec
meta = function.metadata
spec.remote = True
image = function.full_image_path() or "daskdev/dask:latest"
env = spec.env
namespace = meta.namespace or config.namespace
if spec.extra_pip:
env.append(spec.extra_pip)
pod_labels = get_resource_labels(function, scrape_metrics=False)
args = ["dask-worker", "--nthreads", str(spec.nthreads)]
memory_limit = spec.resources.get("limits", {}).get("memory")
if memory_limit:
args.extend(["--memory-limit", str(memory_limit)])
if spec.args:
args.extend(spec.args)
container = client.V1Container(
name="base",
image=image,
env=env,
args=args,
image_pull_policy=spec.image_pull_policy,
volume_mounts=spec.volume_mounts,
resources=spec.resources,
)
pod_spec = client.V1PodSpec(
containers=[container],
restart_policy="Never",
volumes=spec.volumes,
service_account=spec.service_account,
)
if spec.image_pull_secret:
pod_spec.image_pull_secrets = [
client.V1LocalObjectReference(name=spec.image_pull_secret)
]
pod = client.V1Pod(
metadata=client.V1ObjectMeta(namespace=namespace, labels=pod_labels),
# annotations=meta.annotation),
spec=pod_spec,
)
svc_temp = dask.config.get("kubernetes.scheduler-service-template")
if spec.service_type or spec.node_port:
if spec.node_port:
spec.service_type = "NodePort"
svc_temp["spec"]["ports"][1]["nodePort"] = spec.node_port
update_in(svc_temp, "spec.type", spec.service_type)
norm_name = normalize_name(meta.name)
dask.config.set(
{
"kubernetes.scheduler-service-template": svc_temp,
"kubernetes.name": "mlrun-" + norm_name + "-{uuid}",
}
)
cluster = KubeCluster(
pod,
deploy_mode="remote",
namespace=namespace,
scheduler_timeout=spec.scheduler_timeout,
)
logger.info(
"cluster {} started at {}".format(cluster.name, cluster.scheduler_address)
)
function.status.scheduler_address = cluster.scheduler_address
function.status.cluster_name = cluster.name
if spec.service_type == "NodePort":
ports = cluster.scheduler.service.spec.ports
function.status.node_ports = {
"scheduler": ports[0].node_port,
"dashboard": ports[1].node_port,
}
if spec.replicas:
cluster.scale(spec.replicas)
else:
cluster.adapt(minimum=spec.min_replicas, maximum=spec.max_replicas)
return cluster
