def get_pod_container(self, volume_mounts, env_vars=None, command=None, args=None, persistence_outputs=None, persistence_data=None, outputs_refs_jobs=None, outputs_refs_experiments=None, secret_refs=None, configmap_refs=None, resources=None, ephemeral_token=None): """Pod job container for task.""" assert self.cluster_def is not None # Env vars preparations env_vars = to_list(env_vars, check_none=True) outputs_path = get_experiment_outputs_path( persistence_outputs=persistence_outputs, experiment_name=self.experiment_name, original_name=self.original_name, cloning_strategy=self.cloning_strategy) env_vars += get_job_env_vars( persistence_outputs=persistence_outputs, outputs_path=outputs_path, persistence_data=persistence_data, log_level=self.log_level, logs_path=get_experiment_logs_path(self.experiment_name, temp=False), outputs_refs_jobs=outputs_refs_jobs, outputs_refs_experiments=outputs_refs_experiments, ephemeral_token=ephemeral_token, ) env_vars += [ get_env_var(name=constants.CONFIG_MAP_CLUSTER_KEY_NAME, value=json.dumps(self.cluster_def)), get_env_var(name=constants.CONFIG_MAP_DECLARATIONS_KEY_NAME, value=self.declarations), get_env_var(name=constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME, value=json.dumps(self.experiment_labels)), ] env_vars += get_resources_env_vars(resources=resources) # Env from configmap and secret refs env_from = get_pod_env_from(secret_refs=secret_refs, configmap_refs=configmap_refs) ports = [ client.V1ContainerPort(container_port=port) for port in self.ports ] return client.V1Container(name=self.job_container_name, image=self.job_docker_image, command=command, args=args, ports=ports, env=env_vars, env_from=env_from, resources=get_resources(resources), volume_mounts=volume_mounts)
def test_get_pod_env_from(self): K8SSecretFactory(name='secret1', k8s_ref='secret1') K8SSecretFactory(name='secret2', k8s_ref='secret2', items=['key1', 'key2']) K8SConfigMapFactory(name='config1', k8s_ref='config1') K8SConfigMapFactory(name='config2', k8s_ref='config2', items=['key1', 'key2']) env_from = get_pod_env_from(secret_refs=['secret1', 'secret2'], config_map_refs=['config1', 'config2']) assert len(env_from) == 4
def get_pod_container(self, volume_mounts, persistence_outputs, persistence_data, outputs_refs_jobs=None, outputs_refs_experiments=None, secret_refs=None, configmap_refs=None, env_vars=None, command=None, args=None, resources=None): """Pod job container for task.""" # Env vars preparation env_vars = to_list(env_vars, check_none=True) logs_path = stores.get_job_logs_path(job_name=self.job_name, temp=False) outputs_path = stores.get_job_outputs_path( persistence=persistence_outputs, job_name=self.job_name) env_vars += get_job_env_vars( log_level=self.log_level, persistence_outputs=persistence_outputs, outputs_path=outputs_path, persistence_data=persistence_data, logs_path=logs_path, outputs_refs_jobs=outputs_refs_jobs, outputs_refs_experiments=outputs_refs_experiments) env_vars += [ get_env_var(name=constants.CONFIG_MAP_JOB_INFO_KEY_NAME, value=json.dumps(self.labels)), ] env_vars += get_resources_env_vars(resources=resources) # Env from configmap and secret refs env_from = get_pod_env_from(secret_refs=secret_refs, configmap_refs=configmap_refs) ports = [ client.V1ContainerPort(container_port=port) for port in self.ports ] return client.V1Container(name=self.job_container_name, image=self.job_docker_image, command=command, args=args, ports=ports or None, env=env_vars, env_from=env_from, resources=get_resources(resources), volume_mounts=volume_mounts)
def get_pod_container(self, volume_mounts, persistence_outputs=None, persistence_data=None, outputs_refs_jobs=None, outputs_refs_experiments=None, secret_refs=None, config_map_refs=None, env_vars=None, command=None, args=None, resources=None, ports=None, ephemeral_token=None): """Pod job container for task.""" self._pod_container_checks() # Env vars preparations env_vars = to_list(env_vars, check_none=True) env_vars += self._get_container_pod_env_vars( persistence_outputs=persistence_outputs, persistence_data=persistence_data, outputs_refs_jobs=outputs_refs_jobs, outputs_refs_experiments=outputs_refs_experiments, ephemeral_token=ephemeral_token) env_vars += get_resources_env_vars(resources=resources) env_vars += get_kv_env_vars(self._get_kv_env_vars(None)) # Env from config_map and secret refs env_from = get_pod_env_from(secret_refs=secret_refs, config_map_refs=config_map_refs) def get_ports(): _ports = to_list(ports) if ports else [] return [ client.V1ContainerPort(container_port=port) for port in _ports ] or None return client.V1Container(name=self.job_container_name, image=self.job_docker_image, command=command, args=args, ports=get_ports(), env=env_vars, env_from=env_from, resources=get_resources(resources), volume_mounts=volume_mounts)
def start_notebook(self, image, persistence_outputs=None, persistence_data=None, outputs_refs_jobs=None, outputs_refs_experiments=None, resources=None, secret_refs=None, configmap_refs=None, node_selector=None, affinity=None, tolerations=None, allow_commits=False): ports = [self.request_notebook_port()] target_ports = [self.PORT] volumes, volume_mounts = get_pod_volumes( persistence_outputs=persistence_outputs, persistence_data=persistence_data) refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes( outputs_refs=outputs_refs_jobs, persistence_outputs=persistence_outputs) volumes += refs_volumes volume_mounts += refs_volume_mounts refs_volumes, refs_volume_mounts = get_pod_refs_outputs_volumes( outputs_refs=outputs_refs_experiments, persistence_outputs=persistence_outputs) volumes += refs_volumes volume_mounts += refs_volume_mounts shm_volumes, shm_volume_mounts = get_shm_volumes() volumes += shm_volumes volume_mounts += shm_volume_mounts env_vars = get_job_env_vars( persistence_outputs=persistence_outputs, outputs_path=get_notebook_job_outputs_path( persistence_outputs=persistence_outputs, notebook_job=self.job_name), persistence_data=persistence_data, outputs_refs_jobs=outputs_refs_jobs, outputs_refs_experiments=outputs_refs_experiments) secret_refs = validate_secret_refs(secret_refs) configmap_refs = validate_configmap_refs(configmap_refs) env_from = get_pod_env_from(secret_refs=secret_refs, configmap_refs=configmap_refs) code_volume, code_volume_mount = self.get_notebook_code_volume() volumes.append(code_volume) volume_mounts.append(code_volume_mount) deployment_name = JOB_NAME_FORMAT.format(name=NOTEBOOK_JOB_NAME, job_uuid=self.job_uuid) node_selector = get_node_selector( node_selector=node_selector, default_node_selector=settings.NODE_SELECTOR_EXPERIMENTS) affinity = get_affinity(affinity=affinity, default_affinity=settings.AFFINITY_EXPERIMENTS) tolerations = get_tolerations( tolerations=tolerations, default_tolerations=settings.TOLERATIONS_EXPERIMENTS) deployment = deployments.get_deployment( namespace=self.namespace, app=settings.APP_LABELS_NOTEBOOK, name=NOTEBOOK_JOB_NAME, project_name=self.project_name, project_uuid=self.project_uuid, job_name=self.job_name, job_uuid=self.job_uuid, volume_mounts=volume_mounts, volumes=volumes, image=image, command=["/bin/sh", "-c"], args=self.get_notebook_args(deployment_name=deployment_name, ports=ports, allow_commits=allow_commits), ports=target_ports, container_name=settings.CONTAINER_NAME_PLUGIN_JOB, env_vars=env_vars, env_from=env_from, resources=resources, node_selector=node_selector, affinity=affinity, tolerations=tolerations, role=settings.ROLE_LABELS_DASHBOARD, type=settings.TYPE_LABELS_RUNNER, service_account_name=settings.K8S_SERVICE_ACCOUNT_EXPERIMENTS) deployment_labels = deployments.get_labels( app=settings.APP_LABELS_NOTEBOOK, project_name=self.project_name, project_uuid=self.project_uuid, job_name=self.job_name, job_uuid=self.job_uuid, role=settings.ROLE_LABELS_DASHBOARD, type=settings.TYPE_LABELS_RUNNER) dep_resp, _ = self.create_or_update_deployment(name=deployment_name, data=deployment) service = services.get_service(namespace=self.namespace, name=deployment_name, labels=deployment_labels, ports=ports, target_ports=target_ports, service_type=self._get_service_type()) service_resp, _ = self.create_or_update_service(name=deployment_name, data=service) results = { 'deployment': dep_resp.to_dict(), 'service': service_resp.to_dict() } if self._use_ingress(): annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS) paths = [{ 'path': '/notebook/{}'.format(self.project_name.replace('.', '/')), 'backend': { 'serviceName': deployment_name, 'servicePort': ports[0] } }] ingress = ingresses.get_ingress(namespace=self.namespace, name=deployment_name, labels=deployment_labels, annotations=annotations, paths=paths) self.create_or_update_ingress(name=deployment_name, data=ingress) return results
def test_get_pod_env_from(self): env_from = get_pod_env_from(secret_refs=['secret1', 'secret2'], configmap_refs=['config1', 'config2']) assert len(env_from) == 4