def test_get_pod_volumes(self):
settings.DATA_CLAIM_NAME = 'test-claim-data'
settings.OUTPUTS_CLAIM_NAME = 'test-claim-outputs'
volumes, volume_mounts = get_pod_volumes()
assert len(volumes) == 2
assert volumes[0].persistent_volume_claim.claim_name == 'test-claim-data'
assert volumes[1].persistent_volume_claim.claim_name == 'test-claim-outputs'
settings.DATA_CLAIM_NAME = 'test-claim-data'
settings.OUTPUTS_CLAIM_NAME = 'test-claim-outputs'
settings.EXTRA_PERSISTENCES = [{
'mountPath': '/storage/1',
'existingClaim': 'test-claim-extra-1'
}, {
'mountPath': '/storage/2',
'hostPath': '/root/test'
}]
volumes, volume_mounts = get_pod_volumes()
assert len(volumes) == 4
assert volumes[0].persistent_volume_claim.claim_name == 'test-claim-data'
assert volumes[1].persistent_volume_claim.claim_name == 'test-claim-outputs'
assert volumes[2].persistent_volume_claim.claim_name == 'test-claim-extra-1'
assert volumes[3].host_path.path == '/root/test'
assert volume_mounts[2].mount_path == '/storage/1'
assert volume_mounts[3].mount_path == '/storage/2'
def start_tensorboard(self, image, resources=None, node_selectors=None):
ports = [self.request_tensorboard_port()]
target_ports = [self.PORT]
volumes, volume_mounts = get_pod_volumes()
outputs_path = get_project_outputs_path(project_name=self.project_name)
deployment = deployments.get_deployment(
namespace=self.namespace,
app=settings.APP_LABELS_TENSORBOARD,
name=self.TENSORBOARD_JOB_NAME,
project_name=self.project_name,
project_uuid=self.project_uuid,
volume_mounts=volume_mounts,
volumes=volumes,
image=image,
command=["/bin/sh", "-c"],
args=[
"tensorboard --logdir={} --port={}".format(
outputs_path, self.PORT)
],
ports=target_ports,
container_name=settings.CONTAINER_NAME_PLUGIN_JOB,
resources=resources,
node_selector=node_selectors,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
deployment_name = constants.DEPLOYMENT_NAME.format(
project_uuid=self.project_uuid, name=self.TENSORBOARD_JOB_NAME)
deployment_labels = deployments.get_labels(
app=settings.APP_LABELS_TENSORBOARD,
project_name=self.project_name,
project_uuid=self.project_uuid,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
self.create_or_update_deployment(name=deployment_name, data=deployment)
service = services.get_service(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
ports=ports,
target_ports=target_ports,
service_type=self._get_service_type())
self.create_or_update_service(name=deployment_name, data=service)
if self._use_ingress():
annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS)
paths = [{
'path':
'/tensorboard/{}'.format(self.project_name.replace('.', '/')),
'backend': {
'serviceName': deployment_name,
'servicePort': ports[0]
}
}]
ingress = ingresses.get_ingress(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
annotations=annotations,
paths=paths)
self.create_or_update_ingress(name=deployment_name, data=ingress)
def _create_job(self,
task_type,
task_idx,
add_service,
command=None,
args=None,
sidecar_args_fn=None,
env_vars=None,
resources=None,
node_selector=None,
restart_policy='Never'):
job_name = self.pod_manager.get_job_name(task_type=task_type,
task_idx=task_idx)
sidecar_args = sidecar_args_fn(pod_id=job_name)
labels = self.pod_manager.get_labels(task_type=task_type,
task_idx=task_idx)
volumes, volume_mounts = get_pod_volumes()
pod = self.pod_manager.get_pod(task_type=task_type,
task_idx=task_idx,
volume_mounts=volume_mounts,
volumes=volumes,
env_vars=env_vars,
command=command,
args=args,
sidecar_args=sidecar_args,
resources=resources,
node_selector=node_selector,
restart_policy=restart_policy)
pod_resp, _ = self.create_or_update_pod(name=job_name, data=pod)
service = services.get_service(namespace=self.namespace,
name=job_name,
labels=labels,
ports=self.pod_manager.ports,
target_ports=self.pod_manager.ports)
results = {'pod': pod_resp.to_dict()}
if add_service:
service_resp, _ = self.create_or_update_service(name=job_name,
data=service)
results['service'] = service_resp.to_dict()
return results
def start_notebook(self, image, resources=None):
ports = [self.request_notebook_port()]
target_ports = [self.PORT]
volumes, volume_mounts = get_pod_volumes()
code_volume, code_volume_mount = self.get_notebook_code_volume()
volumes.append(code_volume)
volume_mounts.append(code_volume_mount)
deployment_name = constants.DEPLOYMENT_NAME.format(
project_uuid=self.project_uuid, name=self.NOTEBOOK_JOB_NAME)
notebook_token = self.get_notebook_token()
notebook_url = self._get_proxy_url(
namespace=self.namespace,
job_name=self.NOTEBOOK_JOB_NAME,
deployment_name=deployment_name,
port=ports[0])
notebook_dir = get_project_repos_path(self.project_name)
notebook_dir = '{}/{}'.format(notebook_dir, notebook_dir.split('/')[-1])
deployment = deployments.get_deployment(
namespace=self.namespace,
app=settings.APP_LABELS_NOTEBOOK,
name=self.NOTEBOOK_JOB_NAME,
project_name=self.project_name,
project_uuid=self.project_uuid,
volume_mounts=volume_mounts,
volumes=volumes,
image=image,
command=["/bin/sh", "-c"],
args=[
"jupyter notebook "
"--no-browser "
"--port={port} "
"--ip=0.0.0.0 "
"--allow-root "
"--NotebookApp.token={token} "
"--NotebookApp.trust_xheaders=True "
"--NotebookApp.base_url={base_url} "
"--NotebookApp.notebook_dir={notebook_dir} ".format(
port=self.PORT,
token=notebook_token,
base_url=notebook_url,
notebook_dir=notebook_dir)],
ports=target_ports,
container_name=settings.CONTAINER_NAME_PLUGIN_JOB,
resources=resources,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
deployment_labels = deployments.get_labels(app=settings.APP_LABELS_NOTEBOOK,
project_name=self.project_name,
project_uuid=self.project_uuid,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
self.create_or_update_deployment(name=deployment_name, data=deployment)
service = services.get_service(
namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
ports=ports,
target_ports=target_ports,
service_type=self._get_service_type())
self.create_or_update_service(name=deployment_name, data=service)
if self._use_ingress():
annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS)
paths = [{
'path': '/notebook/{}'.format(self.project_name.replace('.', '/')),
'backend': {
'serviceName': deployment_name,
'servicePort': ports[0]
}
}]
ingress = ingresses.get_ingress(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
annotations=annotations,
paths=paths)
self.create_or_update_ingress(name=deployment_name, data=ingress)