def _create_pod(self,
task_type,
task_idx,
command=None,
args=None,
sidecar_args_fn=None,
resources=None,
restart_policy='Never'):
job_name = self.pod_manager.get_job_name(task_type=task_type, task_idx=task_idx)
sidecar_args = sidecar_args_fn(pod_id=job_name)
labels = self.pod_manager.get_labels(task_type=task_type, task_idx=task_idx)
volumes, volume_mounts = self.get_pod_volumes()
pod = self.pod_manager.get_pod(task_type=task_type,
task_idx=task_idx,
volume_mounts=volume_mounts,
volumes=volumes,
command=command,
args=args,
sidecar_args=sidecar_args,
resources=resources,
restart_policy=restart_policy)
pod_resp, _ = self.create_or_update_pod(name=job_name, data=pod)
service = services.get_service(namespace=self.namespace,
name=job_name,
labels=labels,
ports=self.pod_manager.ports,
target_ports=self.pod_manager.ports)
service_resp, _ = self.create_or_update_service(name=job_name, data=service)
return {
'pod': pod_resp.to_dict(),
'service': service_resp.to_dict()
}
def start_tensorboard(self, image, resources=None):
ports = [self.request_tensorboard_port()]
target_ports = [6006]
volumes, volume_mounts = K8SSpawner.get_pod_volumes()
outputs_path = get_project_outputs_path(project_name=self.project_name)
deployment = deployments.get_deployment(
namespace=self.namespace,
name=self.NOTEBOOK_APP,
project_name=self.project_name,
project_uuid=self.project_uuid,
volume_mounts=volume_mounts,
volumes=volumes,
image=image,
command=["/bin/sh", "-c"],
args=["tensorboard --logdir={} --port=6006".format(outputs_path)],
ports=target_ports,
resources=resources,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
deployment_name = constants.DEPLOYMENT_NAME.format(
project_uuid=self.project_uuid, name=self.NOTEBOOK_APP)
deployment_labels = deployments.get_labels(
name=self.NOTEBOOK_APP,
project_name=self.project_name,
project_uuid=self.project_uuid,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
self.create_or_update_deployment(name=deployment_name, data=deployment)
service = services.get_service(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
ports=ports,
target_ports=target_ports,
service_type=self._get_service_type())
self.create_or_update_service(name=deployment_name, data=service)
if self._use_ingress():
annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS)
paths = [{
'path':
'/tensorboard/{}'.format(self.project_name.replace('.', '/')),
'backend': {
'serviceName': deployment_name,
'servicePort': ports[0]
}
}]
ingress = ingresses.get_ingress(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
annotations=annotations,
paths=paths)
self.create_or_update_ingress(name=deployment_name, data=ingress)
def start_notebook(self, image, resources=None):
ports = [self.request_notebook_port()]
target_ports = [8888]
volumes, volume_mounts = K8SSpawner.get_pod_volumes()
deployment_name = constants.DEPLOYMENT_NAME.format(
project_uuid=self.project_uuid, name=self.NOTEBOOK_APP)
notebook_url = self._get_proxy_url(
namespace=self.namespace,
job_name=self.NOTEBOOK_APP,
deployment_name=deployment_name,
port=ports[0])
deployment = deployments.get_deployment(
namespace=self.namespace,
name=self.NOTEBOOK_APP,
project_name=self.project_name,
project_uuid=self.project_uuid,
volume_mounts=volume_mounts,
volumes=volumes,
image=image,
command=["/bin/sh", "-c"],
args=["jupyter notebook "
"--no-browser "
"--port=8888 "
"--ip=0.0.0.0 "
"--allow-root "
"--NotebookApp.token='' "
"--NotebookApp.trust_xheaders=True "
"--NotebookApp.base_url={} ".format(notebook_url)],
ports=target_ports,
resources=resources,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
deployment_labels = deployments.get_labels(name=self.NOTEBOOK_APP,
project_name=self.project_name,
project_uuid=self.project_uuid,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
self.create_or_update_deployment(name=deployment_name, data=deployment)
service = services.get_service(
namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
ports=ports,
target_ports=target_ports,
service_type=self._get_service_type())
self.create_or_update_service(name=deployment_name, data=service)
if self._use_ingress():
annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS)
paths = [{
'path': '/notebook/{}'.format(self.project_name.replace('.', '/')),
'backend': {
'serviceName': deployment_name,
'servicePort': ports[0]
}
}]
ingress = ingresses.get_ingress(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
annotations=annotations,
paths=paths)
self.create_or_update_ingress(name=deployment_name, data=ingress)