def get_labels(self, task_type, task_idx, job_uuid):
        labels = self.resource_manager.get_labels(task_type=task_type,
                                                  task_idx=task_idx,
                                                  job_uuid=job_uuid)

        return get_labels(default_labels=labels,
                          labels=self.spec.master_labels)
    def start_dockerizer(self,
                         secret_refs=None,
                         config_map_refs=None,
                         resources=None,
                         labels=None,
                         annotations=None,
                         node_selector=None,
                         affinity=None,
                         tolerations=None,
                         reconcile_url=None,
                         max_restarts=None):
        volumes, volume_mounts = get_docker_volumes()
        context_volumes, context_mounts = get_build_context_volumes()
        volumes += context_volumes
        volume_mounts += context_mounts
        registry_auth_volumes, registry_auth_mounts = self.get_docker_credentials_volumes()
        volumes += registry_auth_volumes
        volume_mounts += registry_auth_mounts

        resource_name = self.resource_manager.get_resource_name()
        command, args = self.get_pod_command_args()
        init_command, init_args = self.get_init_command_args()

        labels = get_labels(default_labels=self.resource_manager.labels, labels=labels)
        pod = self.resource_manager.get_pod(
            resource_name=resource_name,
            volume_mounts=volume_mounts,
            volumes=volumes,
            labels=labels,
            env_vars=self.get_env_vars(),
            command=command,
            args=args,
            init_command=init_command,
            init_args=init_args,
            init_env_vars=self.get_init_env_vars(),
            persistence_outputs=None,
            persistence_data=None,
            outputs_refs_jobs=None,
            outputs_refs_experiments=None,
            secret_refs=secret_refs,
            config_map_refs=config_map_refs,
            resources=resources,
            annotations=annotations,
            ephemeral_token=None,
            node_selector=node_selector,
            affinity=affinity,
            tolerations=tolerations,
            init_context_mounts=context_mounts,
            reconcile_url=reconcile_url,
            max_restarts=max_restarts,
            restart_policy=get_pod_restart_policy(max_restarts))

        pod_resp, _ = self.create_or_update_pod(name=resource_name, body=pod, reraise=True)
        return pod_resp.to_dict()
Exemple #3
0
 def get_labels(self, task_type, task_idx, job_uuid):
     labels = self.resource_manager.get_labels(task_type=task_type,
                                               task_idx=task_idx,
                                               job_uuid=job_uuid)
     return get_labels(default_labels=labels,
                       labels=self.labels.get(task_type, {}).get(task_idx))
    def start_tensorboard(self,
                          outputs_path,
                          persistence_outputs,
                          outputs_specs=None,
                          outputs_refs_jobs=None,
                          outputs_refs_experiments=None,
                          resources=None,
                          labels=None,
                          annotations=None,
                          node_selector=None,
                          affinity=None,
                          tolerations=None,
                          max_restarts=None,
                          reconcile_url=None):
        ports = [self.request_tensorboard_port()]
        target_ports = [self.port]
        volumes, volume_mounts = get_pod_outputs_volume(persistence_outputs)
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_refs_jobs,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_specs,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_refs_experiments,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts

        # Add volumes for persistence outputs secrets
        stores_secrets = get_stores_secrets(specs=outputs_specs)
        self.validate_stores_secrets_keys(stores_secrets=stores_secrets)
        secrets_volumes, secrets_volume_mounts = self.get_stores_secrets_volumes(
            stores_secrets=stores_secrets)
        volumes += secrets_volumes
        volume_mounts += secrets_volume_mounts

        resource_name = self.resource_manager.get_resource_name()
        tensorboard_url = self._get_proxy_url(namespace=self.namespace,
                                              job_name=TENSORBOARD_JOB_NAME,
                                              deployment_name=resource_name)
        # Get persistence outputs secrets auth commands
        command_args = self.get_stores_secrets_command_args(
            stores_secrets=stores_secrets)
        command_args.append("tensorboard "
                            "--logdir={log_dir} "
                            "--port={port} "
                            "--path_prefix={path_prefix}".format(
                                log_dir=outputs_path,
                                port=self.port,
                                path_prefix=tensorboard_url))
        args = [' && '.join(command_args)]
        command = ["/bin/sh", "-c"]

        labels = get_labels(default_labels=self.resource_manager.labels,
                            labels=labels)
        deployment = self.resource_manager.get_deployment(
            resource_name=resource_name,
            volume_mounts=volume_mounts,
            volumes=volumes,
            labels=labels,
            env_vars=None,
            command=command,
            args=args,
            persistence_outputs=persistence_outputs,
            outputs_refs_jobs=outputs_refs_jobs,
            outputs_refs_experiments=outputs_refs_experiments,
            resources=resources,
            annotations=annotations,
            ephemeral_token=None,
            node_selector=node_selector,
            affinity=affinity,
            tolerations=tolerations,
            ports=target_ports,
            reconcile_url=reconcile_url,
            max_restarts=max_restarts,
            restart_policy=get_deployment_restart_policy(max_restarts))

        dep_resp, _ = self.create_or_update_deployment(name=resource_name,
                                                       body=deployment,
                                                       reraise=True)
        service = services.get_service(
            namespace=self.namespace,
            name=resource_name,
            labels=self.resource_manager.get_labels(),
            ports=ports,
            target_ports=target_ports,
            service_type=self._get_service_type())
        service_resp, _ = self.create_or_update_service(name=resource_name,
                                                        body=service,
                                                        reraise=True)
        results = {
            'deployment': dep_resp.to_dict(),
            'service': service_resp.to_dict()
        }

        if self._use_ingress():
            annotations = json.loads(conf.get(K8S_INGRESS_ANNOTATIONS))
            paths = [{
                'path':
                '/tensorboards/{}'.format(self.project_name.replace('.', '/')),
                'backend': {
                    'serviceName': resource_name,
                    'servicePort': ports[0]
                }
            }]
            ingress = ingresses.get_ingress(
                namespace=self.namespace,
                name=resource_name,
                labels=self.resource_manager.get_labels(),
                annotations=annotations,
                paths=paths)
            self.create_or_update_ingress(name=resource_name,
                                          body=ingress,
                                          reraise=True)

        return results
    def start_notebook(self,
                       persistence_outputs=None,
                       persistence_data=None,
                       outputs_refs_jobs=None,
                       outputs_refs_experiments=None,
                       resources=None,
                       labels=None,
                       annotations=None,
                       secret_refs=None,
                       config_map_refs=None,
                       node_selector=None,
                       affinity=None,
                       tolerations=None,
                       backend=None,
                       max_restarts=None,
                       reconcile_url=None,
                       mount_code_in_notebooks=False):
        ports = [self.request_notebook_port()]
        target_ports = [self.port]
        volumes, volume_mounts = get_pod_volumes(persistence_outputs=persistence_outputs,
                                                 persistence_data=persistence_data)
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_refs_jobs,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_refs_experiments,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts
        shm_volumes, shm_volume_mounts = get_shm_volumes()
        volumes += shm_volumes
        volume_mounts += shm_volume_mounts

        context_volumes, context_mounts = get_auth_context_volumes()
        volumes += context_volumes
        volume_mounts += context_mounts

        if mount_code_in_notebooks:
            code_volume, code_volume_mount = self.get_notebook_code_volume()
            volumes.append(code_volume)
            volume_mounts.append(code_volume_mount)

        resource_name = self.resource_manager.get_resource_name()
        args = self.get_notebook_args(deployment_name=resource_name,
                                      mount_code_in_notebooks=mount_code_in_notebooks,
                                      backend=backend)
        command = ["/bin/sh", "-c"]
        labels = get_labels(default_labels=self.resource_manager.labels, labels=labels)
        deployment = self.resource_manager.get_deployment(
            resource_name=resource_name,
            volume_mounts=volume_mounts,
            volumes=volumes,
            labels=labels,
            env_vars=None,
            command=command,
            args=args,
            init_env_vars=self.get_init_env_vars(),
            persistence_outputs=persistence_outputs,
            persistence_data=persistence_data,
            outputs_refs_jobs=outputs_refs_jobs,
            outputs_refs_experiments=outputs_refs_experiments,
            secret_refs=secret_refs,
            config_map_refs=config_map_refs,
            resources=resources,
            annotations=annotations,
            ephemeral_token=None,
            node_selector=node_selector,
            affinity=affinity,
            tolerations=tolerations,
            ports=target_ports,
            init_context_mounts=context_mounts,
            reconcile_url=reconcile_url,
            max_restarts=max_restarts,
            restart_policy=get_deployment_restart_policy(max_restarts))
        dep_resp, _ = self.create_or_update_deployment(name=resource_name,
                                                       body=deployment,
                                                       reraise=True)
        service = services.get_service(
            namespace=self.namespace,
            name=resource_name,
            labels=self.resource_manager.get_labels(),
            ports=ports,
            target_ports=target_ports,
            service_type=self._get_service_type())
        service_resp, _ = self.create_or_update_service(name=resource_name,
                                                        body=service,
                                                        reraise=True)
        results = {'deployment': dep_resp.to_dict(), 'service': service_resp.to_dict()}

        if self._use_ingress():
            annotations = json.loads(conf.get(K8S_INGRESS_ANNOTATIONS))
            paths = [{
                'path': '/notebooks/{}'.format(self.project_name.replace('.', '/')),
                'backend': {
                    'serviceName': resource_name,
                    'servicePort': ports[0]
                }
            }]
            ingress = ingresses.get_ingress(namespace=self.namespace,
                                            name=resource_name,
                                            labels=self.resource_manager.get_labels(),
                                            annotations=annotations,
                                            paths=paths)
            self.create_or_update_ingress(name=resource_name, body=ingress, reraise=True)
        return results
Exemple #6
0
    def start_job(self,
                  container_cmd_callback,
                  persistence_outputs=None,
                  persistence_data=None,
                  outputs_refs_jobs=None,
                  outputs_refs_experiments=None,
                  resources=None,
                  labels=None,
                  annotations=None,
                  secret_refs=None,
                  config_map_refs=None,
                  node_selector=None,
                  affinity=None,
                  tolerations=None,
                  reconcile_url=None,
                  max_restarts=None):
        # Set and validate volumes
        volumes, volume_mounts = get_pod_volumes(
            persistence_outputs=persistence_outputs,
            persistence_data=persistence_data)
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_refs_jobs,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts
        refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes(
            outputs_refs=outputs_refs_experiments,
            persistence_outputs=persistence_outputs)
        volumes += refs_volumes
        volume_mounts += refs_volume_mounts
        shm_volumes, shm_volume_mounts = get_shm_volumes()
        volumes += shm_volumes
        volume_mounts += shm_volume_mounts

        context_volumes, context_mounts = get_auth_context_volumes()
        volumes += context_volumes
        volume_mounts += context_mounts

        command, args = container_cmd_callback()
        resource_name = self.resource_manager.get_resource_name()
        labels = get_labels(default_labels=self.resource_manager.labels,
                            labels=labels)
        pod = self.resource_manager.get_pod(
            resource_name=resource_name,
            volume_mounts=volume_mounts,
            volumes=volumes,
            labels=labels,
            env_vars=None,
            command=command,
            args=args,
            init_env_vars=self.get_init_env_vars(),
            persistence_outputs=persistence_outputs,
            persistence_data=persistence_data,
            outputs_refs_jobs=outputs_refs_jobs,
            outputs_refs_experiments=outputs_refs_experiments,
            secret_refs=secret_refs,
            config_map_refs=config_map_refs,
            resources=resources,
            annotations=annotations,
            ephemeral_token=None,
            node_selector=node_selector,
            affinity=affinity,
            tolerations=tolerations,
            init_context_mounts=context_mounts,
            reconcile_url=reconcile_url,
            max_restarts=max_restarts,
            restart_policy=get_pod_restart_policy(max_restarts))
        pod_resp, _ = self.create_or_update_pod(name=resource_name,
                                                body=pod,
                                                reraise=True)

        return pod_resp.to_dict()