def get_volume_mounts( self, docker_volumes: Sequence[DockerVolume], aws_ebs_volumes: Sequence[AwsEbsVolume], persistent_volumes: Sequence[PersistentVolume], ) -> Sequence[V1VolumeMount]: return [ V1VolumeMount( mount_path=docker_volume['containerPath'], name=self.get_docker_volume_name(docker_volume), read_only=self.read_only_mode(docker_volume), ) for docker_volume in docker_volumes ] + [ V1VolumeMount( mount_path=aws_ebs_volume['container_path'], name=self.get_aws_ebs_volume_name(aws_ebs_volume), read_only=self.read_only_mode(aws_ebs_volume), ) for aws_ebs_volume in aws_ebs_volumes ] + [ V1VolumeMount( mount_path=volume['container_path'], name=self.get_persistent_volume_name(volume), read_only=self.read_only_mode(volume), ) for volume in persistent_volumes ]
def test_get_volume_mounts(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_volume_name', autospec=True, return_value='some-volume', ): mock_volumes = [ { 'hostPath': '/nail/blah', 'containerPath': '/nail/foo' }, { 'hostPath': '/nail/thing', 'containerPath': '/nail/bar', 'mode': 'RW' }, ] expected_volumes = [ V1VolumeMount( mount_path='/nail/foo', name='some-volume', read_only=True, ), V1VolumeMount( mount_path='/nail/bar', name='some-volume', read_only=False, ), ] assert self.deployment.get_volume_mounts( mock_volumes) == expected_volumes
def __init__(self) -> None: metadata = V1ObjectMeta(name="postgres", labels={"app": "postgres"}) label_selector = V1LabelSelector(match_labels={"app": "postgres"}) env = [V1EnvVar(name="POSTGRES_HOST_AUTH_METHOD", value="trust")] ports = [V1ContainerPort(container_port=5432, name="sql")] volume_mounts = [ V1VolumeMount(name="data", mount_path="/data"), V1VolumeMount( name="postgres-init", mount_path="/docker-entrypoint-initdb.d" ), ] volume_config = V1ConfigMapVolumeSource( name="postgres-init", ) volumes = [V1Volume(name="postgres-init", config_map=volume_config)] container = V1Container( name="postgres", image="postgres:14.3", env=env, ports=ports, volume_mounts=volume_mounts, ) pod_spec = V1PodSpec(containers=[container], volumes=volumes) template_spec = V1PodTemplateSpec(metadata=metadata, spec=pod_spec) claim_templates = [ V1PersistentVolumeClaim( metadata=V1ObjectMeta(name="data"), spec=V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], resources=V1ResourceRequirements(requests={"storage": "1Gi"}), ), ) ] self.stateful_set = V1StatefulSet( api_version="apps/v1", kind="StatefulSet", metadata=metadata, spec=V1StatefulSetSpec( service_name="postgres", replicas=1, selector=label_selector, template=template_spec, volume_claim_templates=claim_templates, ), )
def setUp(self): super().setUp() self.cluster_dict = getExampleClusterDefinition() self.cluster_object = V1MongoClusterConfiguration(**self.cluster_dict) self.name = self.cluster_object.metadata.name self.namespace = self.cluster_object.metadata.namespace self.stateful_set = V1beta1StatefulSet( metadata=self._createMeta(self.name), spec=V1beta1StatefulSetSpec( replicas=3, service_name=self.name, template=V1PodTemplateSpec( metadata=V1ObjectMeta(labels=KubernetesResources. createDefaultLabels(self.name)), spec=V1PodSpec(containers=[ V1Container( name="mongodb", env=[ V1EnvVar(name="POD_IP", value_from=V1EnvVarSource( field_ref=V1ObjectFieldSelector( api_version="v1", field_path="status.podIP"))) ], command=[ "mongod", "--replSet", self.name, "--bind_ip", "0.0.0.0", "--smallfiles", "--noprealloc" ], image="mongo:3.6.4", ports=[ V1ContainerPort(name="mongodb", container_port=27017, protocol="TCP") ], volume_mounts=[ V1VolumeMount(name="mongo-storage", read_only=False, mount_path="/data/db") ], resources=V1ResourceRequirements(limits={ "cpu": "100m", "memory": "64Mi" }, requests={ "cpu": "100m", "memory": "64Mi" })) ])), volume_claim_templates=[ V1PersistentVolumeClaim( metadata=V1ObjectMeta(name="mongo-storage"), spec=V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], resources=V1ResourceRequirements( requests={"storage": "30Gi"}))) ], ), )
def define_configmap(self, data): """This returns a k8s configmap using the data from the new-workflow POST. """ with start_action(action_type="define_configmap"): ni_cmd = data["command"] idkey = list_digest(ni_cmd) cm_name = "command.{}.json".format(idkey) k8s_vol = V1Volume( name="noninteractive-command", config_map=V1ConfigMapVolumeSource(name=cm_name), ) k8s_mt = V1VolumeMount( name="noninteractive-command", mount_path=("/opt/lsst/software/jupyterlab/" + "noninteractive/command/"), read_only=True, ) self.cmd_vol = k8s_vol self.cmd_mt = k8s_mt # Now the configmap cm_data = {} cm_data.update(data) del cm_data["image"] del cm_data["size"] jd = json.dumps(data, sort_keys=True, indent=4) k8s_configmap = V1ConfigMap( metadata=V1ObjectMeta(name=cm_name), data={"command.json": json.dumps(data)}, ) self.log.debug("Created configmap '{}': {}".format(cm_name, jd)) self.cfg_map = k8s_configmap
def __init__(self, name, mount, fs_type, image, monitors, pool, secret_name, sub_path, user="******", read_only=False): self.mount = V1VolumeMount(name=name, mount_path=mount, read_only=read_only, sub_path=sub_path) self.volume = V1Volume( name=name, rbd=V1RBDVolumeSource( fs_type=fs_type, image=image, monitors=monitors.split(","), pool=pool, secret_ref=V1LocalObjectReference(secret_name), read_only=read_only, user=user))
def create_volumes(task_volumes) -> Tuple[List[V1Volume], List[V1VolumeMount]]: index = 0 mounts = [] volumes = [] for target, volume in task_volumes.items(): index += 1 name = volume.get('name', f'volume{index}') for source_type, VolumeSource in VOLUME_SOURCES.items(): if source_type not in volume: continue volume_config = volume[source_type] volumes.append( V1Volume(**{ 'name': name, source_type: VolumeSource(**volume_config), })) mounts.append( V1VolumeMount( name=name, read_only=volume.get('read_only', False), mount_path=target, )) return volumes, mounts
def __init__(self, name, mount, claim_name, read_only=False): self.mount = V1VolumeMount(name=name, mount_path=mount, read_only=read_only) self.volume = V1Volume( name=name, persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( claim_name=claim_name))
def get_volume_mounts(self, volumes: Sequence[DockerVolume]) -> Sequence[V1VolumeMount]: volume_mounts = [] for volume in volumes: volume_mounts.append( V1VolumeMount( mount_path=volume['containerPath'], name=self.get_sanitised_volume_name(volume['containerPath']), read_only=True if volume.get('mode', 'RO') == 'RO' else False, ), ) return volume_mounts
def _create_flush_job( batch_api: BatchV1Api, command: List[str], env: List[V1EnvVar], image: str, name: str, namespace: str, service_account_name: str, ) -> V1Job: logger.info(f"creating job: {name}") try: return batch_api.create_namespaced_job( namespace=namespace, body=V1Job( api_version="batch/v1", kind="Job", metadata=V1ObjectMeta(name=name, namespace=namespace), spec=V1JobSpec( template=V1PodTemplateSpec( spec=V1PodSpec( containers=[ V1Container( image=image, command=command, name="flush", volume_mounts=[ V1VolumeMount(mount_path="/data", name="queue") ], env=env, ) ], restart_policy="OnFailure", volumes=[ V1Volume( name="queue", persistent_volume_claim=( V1PersistentVolumeClaimVolumeSource( claim_name=name ) ), ) ], service_account_name=service_account_name, ) ) ), ), ) except ApiException as e: if e.reason == CONFLICT and json.loads(e.body)["reason"] == ALREADY_EXISTS: logger.info(f"using existing job: {name}") return batch_api.read_namespaced_job(name, namespace) raise
def get_kubernetes_volume_mounts( volumes: PVector['DockerVolume']) -> List[V1VolumeMount]: """ Given a list of volume mounts, return a list corresponding to the Kubernetes objects representing these mounts. """ return [ V1VolumeMount( mount_path=volume["container_path"], name=get_sanitised_volume_name(f"host--{volume['host_path']}", length_limit=63), read_only=volume.get("mode", "RO") == "RO", ) for volume in volumes ]
def volume_pipeline(): op1 = op1_op() op1.add_volume( V1Volume(name='gcp-credentials', secret=V1SecretVolumeSource(secret_name='user-gcp-sa'))) op1.container.add_volume_mount( V1VolumeMount(mount_path='/secret/gcp-credentials', name='gcp-credentials')) op1.container.add_env_variable( V1EnvVar(name='GOOGLE_APPLICATION_CREDENTIALS', value='/secret/gcp-credentials/user-gcp-sa.json')) op1.container.add_env_variable(V1EnvVar(name='Foo', value='bar')) op2 = op2_op(op1.output)
def core_config_mount(self, name, config_map, key, target_path, read_only=True): if name not in self.core_config_volumes: self.core_config_volumes[name] = V1Volume( name=name, config_map=V1ConfigMapVolumeSource( name=config_map, optional=False ) ) self.core_config_mounts[target_path] = V1VolumeMount( name=name, mount_path=target_path, sub_path=key, read_only=read_only )
def _createStatefulSet(self) -> V1beta1StatefulSet: return V1beta1StatefulSet( metadata=self._createMeta(self.name), spec=V1beta1StatefulSetSpec( replicas=3, service_name=self.name, template=V1PodTemplateSpec( metadata=V1ObjectMeta(labels=KubernetesResources. createDefaultLabels(self.name)), spec=V1PodSpec(containers=[ V1Container( name="mongodb", env=[ V1EnvVar(name="POD_IP", value_from=V1EnvVarSource( field_ref=V1ObjectFieldSelector( api_version="v1", field_path="status.podIP"))) ], command=[ "mongod", "--wiredTigerCacheSizeGB", "0.25", "--replSet", self.name, "--bind_ip", "0.0.0.0", "--smallfiles", "--noprealloc" ], image="mongo:3.6.4", ports=[ V1ContainerPort(name="mongodb", container_port=27017, protocol="TCP") ], volume_mounts=[ V1VolumeMount(name="mongo-storage", read_only=False, mount_path="/data/db") ], resources=self._createResourceLimits()) ])), volume_claim_templates=[ V1PersistentVolumeClaim( metadata=V1ObjectMeta(name="mongo-storage"), spec=V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], resources=V1ResourceRequirements( requests={"storage": "30Gi"}))) ], ), )
def volume_pipeline(): op1 = dsl.ContainerOp(name='download', image='google/cloud-sdk', command=['sh', '-c'], arguments=['ls | tee /tmp/results.txt'], file_outputs={'downloaded': '/tmp/results.txt'}) \ .add_volume(V1Volume(name='gcp-credentials', secret=V1SecretVolumeSource(secret_name='user-gcp-sa'))) \ .add_volume_mount(V1VolumeMount(mount_path='/secret/gcp-credentials', name='gcp-credentials')) \ .add_env_variable(V1EnvVar(name='GOOGLE_APPLICATION_CREDENTIALS', value='/secret/gcp-credentials/user-gcp-sa.json')) \ .add_env_variable(V1EnvVar(name='Foo', value='bar')) op2 = dsl.ContainerOp(name='echo', image='library/bash', command=['sh', '-c'], arguments=['echo %s' % op1.output])
def start_stateful_container(self, service_name, container_name, spec, labels): # Setup PVC deployment_name = service_name + '-' + container_name mounts, volumes = [], [] for volume_name, volume_spec in spec.volumes.items(): mount_name = deployment_name + volume_name # Check if the PVC exists, create if not self._ensure_pvc(mount_name, volume_spec.storage_class, volume_spec.capacity) # Create the volume info volumes.append(V1Volume( name=mount_name, persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(mount_name) )) mounts.append(V1VolumeMount(mount_path=volume_spec.mount_path, name=mount_name)) self._create_deployment(service_name, deployment_name, spec.container, 30, 1, labels, volumes=volumes, mounts=mounts)
def create_cluster(self, spec, cluster_management, namespace_name, volume_claim_name): count = int(spec[FRAMEWORK_RESOURCES][FRAMEWORK_DPU_COUNT]) version = str(spec[FRAMEWORK_VERSION]) image = "tensorflow/tensorflow:" + version + "-gpu" ### v1_api = cluster_management.kube_api api_response_list = [] for i in range(count): body = kubernetes.client.V1Pod() body.api_version = "v1" body.kind = "Pod" meta = V1ObjectMeta() meta.generate_name = "tensorflow-" body.metadata = meta uuid = str(uuid4()) container = V1Container(name=uuid, image=image) pod_spec = V1PodSpec(containers=[container]) container_mounts = V1VolumeMount( mount_path=GLUSTER_DEFAULT_MOUNT_PATH, name=CONTAINER_VOLUME_PREFIX) container.volume_mounts = [container_mounts] compute_resource = V1ResourceRequirements() compute_resource.limits = {"nvidia.com/gpu": 1} compute_resource.requests = {"nvidia.com/gpu": 1} container.resources = compute_resource claim = V1PersistentVolumeClaimVolumeSource( claim_name=volume_claim_name) volume_claim = V1Volume(name=CONTAINER_VOLUME_PREFIX, persistent_volume_claim=claim) volume_claim.persistent_volume_claim = claim pod_spec.volumes = [volume_claim] body.spec = pod_spec try: api_response = v1_api.create_namespaced_pod( namespace_name, body) except ApiException as e: raise Exception( "Exception when calling CoreV1Api->create_namespaced_pod: %s\n" % e) api_response_list.append(api_response) return api_response_list
def __init__(self, name, mount, monitors, secret_name, fs_path, sub_path, user="******", read_only=False): self.mount = V1VolumeMount(name=name, mount_path=mount, read_only=read_only, sub_path=sub_path) self.volume = V1Volume( name=name, cephfs=V1CephFSVolumeSource( monitors=monitors.split(","), path=fs_path, secret_ref=V1LocalObjectReference(secret_name), read_only=read_only, user=user))
def _create_volumes(self, service_name): volumes, mounts = [], [] # Attach the mount that provides the config file volumes.extend(self.config_volumes.values()) mounts.extend(self.config_mounts.values()) # Attach the mount that provides the update volumes.append(V1Volume( name='update-directory', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( claim_name=FILE_UPDATE_VOLUME, read_only=True ), )) mounts.append(V1VolumeMount( name='update-directory', mount_path=CONTAINER_UPDATE_DIRECTORY, sub_path=service_name, read_only=True, )) return volumes, mounts
def perform_cloud_ops(): # set GOOGLE_APPLICATION_CREDENTIALS env to credentials file # set GOOGLE_CLOUD_PROJECT env to project id domain = os.getenv('DOMAIN') assert domain logger.info(f'using domain: {domain}') static_ip = os.getenv('STATIC_IP') assert static_ip logger.info(f'using static IP: {static_ip}') admin_email = os.getenv('ADMIN_EMAIL') assert admin_email logger.info(f'using ACME admin email: {admin_email}') oauth_client_id = os.getenv('OAUTH_CLIENT_ID') assert oauth_client_id logger.info(f'using oauth client id: {oauth_client_id}') oauth_client_secret = os.getenv('OAUTH_CLIENT_SECRET') assert oauth_client_secret logger.info(f'using oauth client secret: {oauth_client_secret}') oauth_secret = os.getenv('OAUTH_SECRET') assert oauth_secret logger.info(f'using oauth secret: {oauth_secret}') oauth_domain = os.getenv('OAUTH_DOMAIN') assert oauth_domain logger.info(f'using domain: {oauth_domain}') django_secret_key = os.getenv('DJANGO_SECRET_KEY') assert django_secret_key logger.info(f'using DJANGO_SECRET_KEY: {django_secret_key}') credentials, project = google.auth.default() gcloud_client = container_v1.ClusterManagerClient(credentials=credentials) scan_clusters(gcloud_client, project) # FIXME add the k8s cert to a trust store urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) auth_gcloud_k8s(credentials) api_core_v1 = client.CoreV1Api() api_apps_v1 = client.AppsV1Api() api_storage_v1 = client.StorageV1Api() api_custom = client.CustomObjectsApi() api_extensions_v1_beta1 = client.ExtensionsV1beta1Api() api_ext_v1_beta1 = client.ApiextensionsV1beta1Api() api_rbac_auth_v1_b1 = client.RbacAuthorizationV1beta1Api() ensure_traefik(api_core_v1, api_ext_v1_beta1, api_apps_v1, api_custom, api_rbac_auth_v1_b1, admin_email, domain, static_ip, oauth_client_id, oauth_client_secret, oauth_domain, oauth_secret) with open(os.getenv('GOOGLE_APPLICATION_CREDENTIALS'), 'rb') as f: gcloud_credentials_b64 = b64encode(f.read()).decode('UTF-8') ensure_secret(api=api_core_v1, name='webui-credentials', namespace='default', secret=V1Secret( metadata=client.V1ObjectMeta(name='webui-credentials'), data={'gcloud-credentials': gcloud_credentials_b64})) webui_volume_paths = [ ('data', '/opt/nipyapi/data', '20Gi', 'standard'), ] webui_volume_mounts = [ V1VolumeMount(name=path[0], mount_path=path[1]) for path in webui_volume_paths ] webui_volume_mounts.append( V1VolumeMount(name='webui-credentials', mount_path='/root/webui', read_only=True)) dind_volume_paths = [ ('docker', '/var/lib/docker', '200Gi', 'standard'), ] dind_volume_mounts = [ V1VolumeMount(name=path[0], mount_path=path[1]) for path in dind_volume_paths ] shared_volume_mounts = [ V1VolumeMount(name='dind-socket', mount_path='/var/run-shared') ] ensure_statefulset_with_containers( api_apps_v1=api_apps_v1, name='admin', namespace='default', replicas=1, containers=[ V1Container( name='webui', image='aichrist/nipyapi-ds:latest', env=[ # FIXME use k8s secrets for these values V1EnvVar(name='DOMAIN', value=domain), V1EnvVar(name='STATIC_IP', value=static_ip), V1EnvVar(name='ADMIN_EMAIL', value=admin_email), V1EnvVar(name='OAUTH_CLIENT_ID', value=oauth_client_id), V1EnvVar(name='OAUTH_CLIENT_SECRET', value=oauth_client_secret), V1EnvVar(name='OAUTH_SECRET', value=oauth_secret), V1EnvVar(name='OAUTH_DOMAIN', value=oauth_domain), V1EnvVar(name='DJANGO_SECRET_KEY', value=django_secret_key), V1EnvVar(name='GOOGLE_APPLICATION_CREDENTIALS', value='/root/webui/gcloud_credentials.json'), V1EnvVar(name='CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE', value='/root/webui/gcloud_credentials.json'), V1EnvVar(name='GOOGLE_CLOUD_PROJECT', value=os.getenv('GOOGLE_CLOUD_PROJECT')), V1EnvVar(name='DOCKER_HOST', value='unix:///var/run-shared/docker.sock'), ], ports=[V1ContainerPort(container_port=8000)], volume_mounts=webui_volume_mounts + shared_volume_mounts), V1Container( name='dind', image='docker:19-dind', security_context=V1SecurityContext(privileged=True), command=[ 'dockerd', '-H', 'unix:///var/run-shared/docker.sock' ], volume_mounts=dind_volume_mounts + shared_volume_mounts) ], volumes=[ V1Volume(name='dind-socket', empty_dir={}), V1Volume(name='webui-credentials', projected=V1ProjectedVolumeSource(sources=[ V1VolumeProjection(secret=V1SecretProjection( name='webui-credentials', items=[ V1KeyToPath(key='gcloud-credentials', path='gcloud_credentials.json') ])) ])) ], volume_paths=webui_volume_paths + dind_volume_paths) ensure_ingress_routed_svc(api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname='admin', name='admin', target_name='admin', namespace='default', port_name='web', svc_port=80, target_port=8000) reg_volume_paths = [ ('database', '/opt/nifi-registry/nifi-registry-current/database', '10Gi', 'standard'), ('flow-storage', '/opt/nifi-registry/nifi-registry-current/flow_storage', '20Gi', 'standard'), ] reg_volume_mounts = [ V1VolumeMount(name=path[0], mount_path=path[1]) for path in reg_volume_paths ] ensure_statefulset_with_containers( api_apps_v1=api_apps_v1, name='registry', namespace='default', replicas=1, containers=[ V1Container(name='registry', image='apache/nifi-registry:latest', env=[ V1EnvVar(name='NIFI_REGISTRY_WEB_HTTP_PORT', value='19090'), ], ports=[V1ContainerPort(container_port=19090)], volume_mounts=reg_volume_mounts), ], init_containers=[ V1Container( name='init-permissions', image='busybox', command=[ 'sh', '-c', 'chown -R 1000:1000 /opt/nifi-registry/nifi-registry-current' ], volume_mounts=[ V1VolumeMount(name=path[0], mount_path=path[1]) for path in reg_volume_paths ]) ], volumes=[], volume_paths=reg_volume_paths) ensure_ingress_routed_svc(api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname='registry', name='registry', target_name='registry', namespace='default', port_name='web', svc_port=80, target_port=19090) perform_nifi_ops(api_apps_v1, api_core_v1, api_custom, domain) perform_build_ops_bg() perform_mirror_ops_bg()
def create_nifi_instances(api_apps_v1, api_core_v1, api_custom, domain): for instance in NifiInstance.objects.filter(state='PENDING_CREATE'): instance.state = 'CREATING' instance.save() port_name = 'web' instance.state = 'CREATE_FAILED' try: namespace = 'default' if instance.namespace is not None and instance.namespace != 'default': namespace = instance.namespace ensure_namespace(api_core_v1, namespace) else: instance.namespace = 'default' # deploy nifi nifi_volume_paths = [ ('db-repo', '/opt/nifi/nifi-current/database_repository', '20Gi', 'standard'), ('flowfile-repo', '/opt/nifi/nifi-current/flowfile_repository', '20Gi', 'standard'), ('provenance-repo', '/opt/nifi/nifi-current/provenance_repository', '20Gi', 'standard'), ('content-repo', '/opt/nifi/nifi-current/content_repository', '20Gi', 'standard'), ] ensure_statefulset_with_containers( api_apps_v1=api_apps_v1, name=instance.hostname, namespace=namespace, replicas=1, containers=[ V1Container(name='nifi', image=instance.image, env=[ V1EnvVar(name='NIFI_WEB_HTTP_HOST', value='0.0.0.0') ], ports=[V1ContainerPort(container_port=8080)], volume_mounts=[ V1VolumeMount(name=path[0], mount_path=path[1]) for path in nifi_volume_paths ]) ], init_containers=[ V1Container(name='init-permissions', image='busybox', command=[ 'sh', '-c', 'chown -R 1000:1000 /opt/nifi/nifi-current' ], volume_mounts=[ V1VolumeMount(name=path[0], mount_path=path[1]) for path in nifi_volume_paths ]) ], volume_paths=nifi_volume_paths) ensure_ingress_routed_svc(api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname=instance.hostname, name=instance.hostname, target_name=instance.hostname, namespace=namespace, port_name=port_name, svc_port=80, target_port=8080) # deploy mongo if instance.deploy_mongo: mongo_volume_paths = [ ('db', '/data/db', '20Gi', 'standard'), ] ensure_statefulset_with_containers( api_apps_v1=api_apps_v1, name='mongo', namespace=namespace, replicas=1, containers=[ V1Container( name='mongo', image='mongo', env=[ V1EnvVar(name='MONGO_INITDB_ROOT_USERNAME', value='admin'), V1EnvVar(name='MONGO_INITDB_ROOT_PASSWORD', value='admin') ], ports=[ V1ContainerPort(name='mongo', container_port=27017) ], volume_mounts=[ V1VolumeMount(name=path[0], mount_path=path[1]) for path in mongo_volume_paths ]) ], volume_paths=mongo_volume_paths) ensure_service(api=api_core_v1, service=V1Service( api_version="v1", metadata=V1ObjectMeta(name='mongo'), spec=V1ServiceSpec( type='ClusterIP', ports=[ V1ServicePort(protocol='TCP', port=27017, name='mongo', target_port=27017), ], selector={'app': 'mongo'})), name='mongo', namespace=namespace) ensure_single_container_deployment( api_apps_v1, V1Container( name='mongo-express', image='mongo-express', env=[ V1EnvVar(name='ME_CONFIG_MONGODB_ADMINUSERNAME', value='admin'), V1EnvVar(name='ME_CONFIG_MONGODB_ADMINPASSWORD', value='admin') ], ports=[V1ContainerPort(container_port=8000)]), 'mongo-express', instance.namespace) ensure_ingress_routed_svc(api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname="mongo-" + instance.hostname, name="mongo-" + instance.hostname, target_name="mongo-express", namespace=namespace, port_name=port_name, svc_port=80, target_port=8081) if instance.deploy_kafka: # deploy zookeeper ensure_single_container_deployment( api_apps_v1, V1Container(name='zookeeper', image='wurstmeister/zookeeper', env=[], ports=[V1ContainerPort(container_port=2181)]), 'zookeeper', instance.namespace) ensure_service(api=api_core_v1, service=V1Service( api_version="v1", metadata=V1ObjectMeta(name='zookeeper'), spec=V1ServiceSpec( type='ClusterIP', ports=[ V1ServicePort(protocol='TCP', port=2181, name='zookeeper', target_port=2181), ], selector={'app': 'zookeeper'})), name='zookeeper', namespace=namespace) # deploy kafka ensure_single_container_deployment( api_apps_v1, V1Container(name='kafka', image='wurstmeister/kafka', env=[ V1EnvVar(name='KAFKA_ADVERTISED_HOST_NAME', value='kafka'), V1EnvVar(name='KAFKA_ZOOKEEPER_CONNECT', value='zookeeper:2181'), V1EnvVar(name='KAFKA_PORT', value='9092') ], ports=[V1ContainerPort(container_port=9092)]), 'kafka', instance.namespace) ensure_service(api=api_core_v1, service=V1Service( api_version="v1", metadata=V1ObjectMeta(name='kafka'), spec=V1ServiceSpec( type='ClusterIP', ports=[ V1ServicePort(protocol='TCP', port=9092, name='kafka', target_port=9092), ], selector={'app': 'kafka'})), name='kafka', namespace=namespace) if instance.deploy_prometheus: # deploy prometheus ensure_single_container_deployment( api_apps_v1, V1Container(name='prometheus', image='prom/prometheus', env=[], ports=[V1ContainerPort(container_port=9090)]), 'prometheus', instance.namespace) ensure_ingress_routed_svc(api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname="prometheus-" + instance.hostname, name="prometheus", target_name="prometheus", namespace=namespace, port_name=port_name, svc_port=9090, target_port=9090) if instance.deploy_jupyter: # deploy jupyter instance.jupyter_token = str(uuid.uuid1()) ensure_single_container_deployment( api_apps_v1, V1Container( name='jupyter', image='jupyter/datascience-notebook', command=[ 'start-notebook.sh', '--NotebookApp.token=' + instance.jupyter_token ], env=[], ports=[V1ContainerPort(container_port=8888)]), 'jupyter', instance.namespace) ensure_ingress_routed_svc(api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname="jupyter-" + instance.hostname, name="jupyter", target_name="jupyter", namespace=namespace, port_name=port_name, svc_port=8888, target_port=8888) # deploy custom instance types custom_instances = Instance.objects.filter(parent=instance) for ci in custom_instances: inst_type: InstanceType = ci.instance_type env_vars = [ V1EnvVar(name=e.name, value=e.default_value) for e in InstanceTypeEnvVar.objects.filter( instance_type=inst_type) ] ports = [ V1ContainerPort(container_port=p.internal) for p in InstanceTypePort.objects.filter( instance_type=inst_type) ] ensure_single_container_deployment( api_apps_v1, V1Container(name=inst_type.container_name, image=inst_type.image, env=env_vars, ports=ports), inst_type.container_name, instance.namespace) for svc in InstanceTypeIngressRoutedService.objects.filter( instance_type=inst_type): ensure_ingress_routed_svc( api_core_v1=api_core_v1, api_custom=api_custom, domain=domain, hostname=svc.service_name + '-' + instance.hostname, name=svc.service_name + '-' + instance.hostname, target_name=inst_type.container_name, namespace=namespace, port_name=port_name, svc_port=svc.svc_port, target_port=svc.target_port) instance.state = 'RUNNING' finally: instance.save()
def __init__(self, name, mount, path, read_only=True): self.mount = V1VolumeMount(name=name, mount_path=mount, read_only=read_only) self.volume = V1Volume(name=name, host_path=V1HostPathVolumeSource(path=path))
def __init__(self, name, mount, read_only=False): self.mount = V1VolumeMount(name=name, mount_path=mount, read_only=read_only) self.volume = V1Volume(name=name, empty_dir=V1EmptyDirVolumeSource())
def createsvc(self, deploy, port, imagename, namespace, envvar, nameapp, service, pvc, volumename, datadir): bservice = client.V1Service() smeta = V1ObjectMeta() dcmeta = V1ObjectMeta() pmt = V1ObjectMeta() sspec = client.V1ServiceSpec() bdc = openshift.client.V1DeploymentConfig() dcspec = openshift.client.V1DeploymentConfigSpec() strategy = openshift.client.V1DeploymentStrategy() rollingparams = openshift.client.V1RollingDeploymentStrategyParams() podtemp = client.V1PodTemplateSpec() podspec = client.V1PodSpec() container = client.V1Container() idname = nameapp + "-" + deploy smeta.name = idname # !!! smeta.namespace = namespace smeta.labels = {"label": idname, "bundle": service + "-" + nameapp} sspec.selector = {"label": idname} sspec.ports = [] for l in range(0, len(port)): p = client.V1ServicePort() p.name = "{port}-{tcp}".format(**port[l]) p.protocol = "TCP" p.port = port[l]['tcp'] p.target_port = "{port}-{tcp}".format(**port[l]) sspec.ports.append(p) if port[l]['route'] == 'yes': self.createroute(p.target_port, idname, namespace, service, nameapp, l) bservice.api_version = 'v1' bservice.kind = 'Service' bservice.metadata = smeta bservice.spec = sspec bservice.api_version = 'v1' # DeploymentConfig dcmeta.labels = {"label": idname, "bundle": service + "-" + nameapp} dcmeta.name = idname dcmeta.namespace = namespace rollingparams.interval_seconds = 1 strategy.labels = {"label": idname, "bundle": service + "-" + nameapp} strategy.type = 'Rolling' strategy.rolling_params = rollingparams container.image = imagename container.name = idname container.env = [] if pvc: vm = V1VolumeMount() vm.mount_path = datadir vm.name = volumename container.volume_mounts = [vm] else: pass for key in envvar: v = client.V1EnvVar() v.name = key v.value = envvar[key] container.env.append(v) container.ports = [] for o in range(0, len(port)): p = client.V1ContainerPort() p.name = ("{port}-{tcp}".format(**port[o])) p.protocol = "TCP" p.container_port = port[o]['tcp'] container.ports.append(p) pmt.labels = {"label": idname, "bundle": service + "-" + nameapp} pmt.name = idname podspec.containers = [container] if pvc: vol = V1Volume() pvcname = V1PersistentVolumeClaimVolumeSource() volgfs = V1GlusterfsVolumeSource() pvcname.claim_name = volumename volgfs.endpoints = volumename volgfs.path = datadir #vol.glusterfs = volgfs vol.name = volumename vol.persistent_volume_claim = pvcname podspec.volumes = [vol] else: pass podtemp.metadata = pmt podtemp.spec = podspec dcspec.replicas = 1 dcspec.selector = {"label": idname} dcspec.template = podtemp dcspec.strategy = strategy bdc.api_version = 'v1' bdc.spec = dcspec bdc.metadata = dcmeta bdc.kind = 'DeploymentConfig' try: self.k1.create_namespaced_service(namespace=namespace, body=bservice, pretty='true') except ApiException as e: print("Exception when calling OapiApi->create_service: %s\n" % e) try: self.o1.create_namespaced_deployment_config(namespace=namespace, body=bdc, pretty='true') except ApiException as e: print("Exception when calling OapiApi->create_dc: %s\n" % e)
def apply_pod_profile(self, username, pod, profile, gpu_types, default_mount_path, gpu_mode=None, selected_gpu_type="ALL"): api_client = kubernetes.client.ApiClient() pod.metadata.labels['jupyterhub.opendatahub.io/user'] = escape( username) profile_volumes = profile.get('volumes') if profile_volumes: for volume in profile_volumes: volume_name = re.sub('[^a-zA-Z0-9\.]', '-', volume['name']).lower() read_only = volume['persistentVolumeClaim'].get('readOnly') pvc = V1PersistentVolumeClaimVolumeSource( volume['persistentVolumeClaim']['claimName'], read_only=read_only) mount_path = self.generate_volume_path(volume.get('mountPath'), default_mount_path, volume_name) pod.spec.volumes.append( V1Volume(name=volume_name, persistent_volume_claim=pvc)) pod.spec.containers[0].volume_mounts.append( V1VolumeMount(name=volume_name, mount_path=mount_path)) profile_environment = profile.get('env') if profile_environment: # Kept for backwards compatibility with simplified env var definitions if isinstance(profile_environment, dict): for k, v in profile['env'].items(): update = False for e in pod.spec.containers[0].env: if e.name == k: e.value = v update = True break if not update: pod.spec.containers[0].env.append(V1EnvVar(k, v)) elif isinstance(profile_environment, list): for i in profile_environment: r = type("Response", (), {}) r.data = json.dumps(i) env_var = api_client.deserialize(r, V1EnvVar) pod.spec.containers[0].env.append(env_var) resource_var = None resource_json = type("Response", (), {}) resource_json.data = json.dumps(profile.get('resources')) resource_var = api_client.deserialize(resource_json, V1ResourceRequirements) if resource_var: pod.spec.containers[0].resources = resource_var mem_limit = resource_var.limits.get('memory', '') if mem_limit: pod.spec.containers[0].env.append( V1EnvVar(name='MEM_LIMIT', value=self.get_mem_limit(mem_limit))) for c in pod.spec.containers: update = False if type(c) is dict: env = c['env'] else: env = c.env for e in env: if type(e) is dict: if e['name'] == _JUPYTERHUB_USER_NAME_ENV: e['value'] = username update = True break else: if e.name == _JUPYTERHUB_USER_NAME_ENV: e.value = username update = True break if not update: env.append(V1EnvVar(_JUPYTERHUB_USER_NAME_ENV, username)) self.apply_gpu_config(gpu_mode, profile, gpu_types, pod, selected_gpu_type) node_tolerations = profile.get('node_tolerations', []) node_affinity = profile.get('node_affinity', {}) self.apply_pod_schedulers(node_tolerations, node_affinity, pod) return pod
def start_stateful_container(self, service_name: str, container_name: str, spec, labels: dict[str, str], change_key: str): # Setup PVC deployment_name = self._dependency_name(service_name, container_name) mounts, volumes = [], [] for volume_name, volume_spec in spec.volumes.items(): mount_name = f'{deployment_name}-{volume_name}' # Check if the PVC exists, create if not self._ensure_pvc(mount_name, volume_spec.storage_class, volume_spec.capacity, deployment_name) # Create the volume info volumes.append(V1Volume( name=mount_name, persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(mount_name) )) mounts.append(V1VolumeMount(mount_path=volume_spec.mount_path, name=mount_name)) # Read the key being used for the deployment instance or generate a new one instance_key = uuid.uuid4().hex try: old_deployment = self.apps_api.read_namespaced_deployment(deployment_name, self.namespace) for container in old_deployment.spec.template.spec.containers: for env in container.env: if env.name == 'AL_INSTANCE_KEY': instance_key = env.value break except ApiException as error: if error.status != 404: raise # Setup the deployment itself labels['container'] = container_name spec.container.environment.append({'name': 'AL_INSTANCE_KEY', 'value': instance_key}) self._create_deployment(service_name, deployment_name, spec.container, 30, 1, labels, volumes=volumes, mounts=mounts, core_mounts=spec.run_as_core, change_key=change_key) # Setup a service to direct to the deployment try: service = self.api.read_namespaced_service(deployment_name, self.namespace) service.metadata.labels = labels service.spec.selector = labels service.spec.ports = [V1ServicePort(port=int(_p)) for _p in spec.container.ports] self.api.patch_namespaced_service(deployment_name, self.namespace, service) except ApiException as error: if error.status != 404: raise service = V1Service( metadata=V1ObjectMeta(name=deployment_name, labels=labels), spec=V1ServiceSpec( cluster_ip='None', selector=labels, ports=[V1ServicePort(port=int(_p)) for _p in spec.container.ports] ) ) self.api.create_namespaced_service(self.namespace, service) # Add entries to the environment variable list to point to this container self._service_limited_env[service_name][f'{container_name}_host'] = deployment_name self._service_limited_env[service_name][f'{container_name}_key'] = instance_key if spec.container.ports: self._service_limited_env[service_name][f'{container_name}_port'] = spec.container.ports[0]
("host--hello_world", 0, "host--hello--world"), ("host--hello_world" + ("a" * 60), 63, "host--hello--worldaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--1090"), ("a/b/c/d/e/f/g/h/i/j/k/l", 63, "aslash-bslash-cslash-dslash-eslash-fslash-gslash-hslash-i--646b"), )) def test_get_sanitised_volume_name(name, length_limit, expected): assert get_sanitised_volume_name(name, length_limit) == expected @pytest.mark.parametrize("volumes,expected", ( (v({ "container_path": "/a", "host_path": "/b", "mode": "RO" }), [V1VolumeMount(mount_path="/a", name="host--slash-b", read_only=True) ]), (v( { "container_path": "/a", "host_path": "/b", "mode": "RO" }, { "container_path": "/b", "host_path": "/a/b/cd/e/f/g/h/u/j/k/l", "mode": "RW" }, ), [ V1VolumeMount(mount_path="/a", name="host--slash-b", read_only=True), V1VolumeMount(
def run(provider, provider_kwargs, cluster=None, job=None, storage=None): # TODO, temp fix s3 = storage["s3"] _validate_fields( provider=provider_kwargs, cluster=cluster, job=job, storage=storage, s3=s3 ) _required_run_arguments(provider_kwargs, cluster, job, storage, s3) response = {"job": {}} if "name" not in job["meta"] or not job["meta"]["name"]: since_epoch = int(time.time()) job["meta"]["name"] = "{}-{}".format(JOB_DEFAULT_NAME, since_epoch) if "bucket_name" not in s3 or not s3["bucket_name"]: s3["bucket_name"] = job["meta"]["name"] container_engine_client = new_client( ContainerEngineClient, composite_class=ContainerEngineClientCompositeOperations, name=provider_kwargs["profile"]["name"], ) compute_cluster = get_cluster_by_name( container_engine_client, provider_kwargs["profile"]["compartment_id"], name=cluster["name"], ) if not compute_cluster: response["msg"] = "Failed to find a cluster with name: {}".format( cluster["name"] ) return False, response refreshed = refresh_kube_config( compute_cluster.id, name=provider_kwargs["profile"]["name"] ) if not refreshed: response["msg"] = "Failed to refresh the kubernetes config" return False, response node_manager = NodeManager() if not node_manager.discover(): response["msg"] = "Failed to discover any nodes to schedule jobs on" return False, response node = node_manager.select() if not node: response["msg"] = "Failed to select a node to schedule on" return False, response # Ensure we have the newest config scheduler = KubenetesScheduler() jobio_args = [ "jobio", "run", ] jobio_args.extend(job["commands"]) jobio_args.extend(["--job-meta-name", job["meta"]["name"]]) if "output_path" in job: jobio_args.extend( ["--job-output-path", job["output_path"],] ) if "capture" in job and job["capture"]: jobio_args.append("--job-capture") if "debug" in job["meta"]: jobio_args.append("--job-meta-debug") if "env_override" in job["meta"]: jobio_args.append("--job-meta-env-override") # Maintained by the pod volumes = [] # Maintained by the container volume_mounts = [] # Environment to pass to the container envs = [] # Prepare config for the scheduler scheduler_config = {} if storage and storage["enable"]: validate_dict_values(storage, required_storage_fields, throw=True) jobio_args.append("--storage-enable") # Means that results should be exported to the specified storage # Create kubernetes secrets core_api = client.CoreV1Api() # storage_api = client.StorageV1Api() # Storage endpoint credentials secret (Tied to a profile and job) secret_profile_name = "{}-{}-{}".format( STORAGE_CREDENTIALS_NAME, s3["name"], job["meta"]["name"] ) try: storage_credentials_secret = core_api.read_namespaced_secret( secret_profile_name, KUBERNETES_NAMESPACE ) except ApiException: storage_credentials_secret = None # volumes secret_volume_source = V1SecretVolumeSource(secret_name=secret_profile_name) secret_volume = V1Volume(name=secret_profile_name, secret=secret_volume_source) volumes.append(secret_volume) # Where the storage credentials should be mounted # in the compute unit secret_mount = V1VolumeMount( name=secret_profile_name, mount_path=storage["credentials_path"], read_only=True, ) volume_mounts.append(secret_mount) if s3: validate_dict_values(s3, required_staging_values, verbose=True, throw=True) jobio_args.append("--storage-s3") # S3 storage # Look for s3 credentials and config files s3_config = load_aws_config( s3["config_file"], s3["credentials_file"], profile_name=s3["name"], ) s3_config["endpoint_url"] = storage["endpoint"] if not storage_credentials_secret: secret_data = dict( aws_access_key_id=s3_config["aws_access_key_id"], aws_secret_access_key=s3_config["aws_secret_access_key"], ) secret_metadata = V1ObjectMeta(name=secret_profile_name) secrets_config = dict(metadata=secret_metadata, string_data=secret_data) scheduler_config.update(dict(secret_kwargs=secrets_config)) # If `access_key` # TODO, unify argument endpoint, with s3 config endpoint' s3_resource = boto3.resource("s3", **s3_config) bucket = bucket_exists(s3_resource.meta.client, s3["bucket_name"]) if not bucket: bucket = s3_resource.create_bucket( Bucket=s3["bucket_name"], CreateBucketConfiguration={ "LocationConstraint": s3_config["region_name"] }, ) if "upload_path" in storage and storage["upload_path"]: # Upload local path to the bucket as designated input for the job uploaded = None if os.path.exists(storage["upload_path"]): if os.path.isdir(storage["upload_path"]): uploaded = upload_directory_to_s3( s3_resource.meta.client, storage["upload_path"], s3["bucket_name"], s3_prefix=s3["bucket_input_prefix"], ) elif os.path.isfile(storage["upload_path"]): s3_path = os.path.basename(storage["upload_path"]) if s3["bucket_input_prefix"]: s3_path = os.path.join(s3["bucket_input_prefix"], s3_path) # Upload uploaded = upload_to_s3( s3_resource.meta.client, storage["upload_path"], s3_path, s3["bucket_name"], ) if not uploaded: response[ "msg" ] = "Failed to local path: {} in the upload folder to s3".format( storage["upload_path"] ) return False, response jobio_args.extend( [ "--s3-region-name", s3_config["region_name"], "--storage-secrets-dir", storage["credentials_path"], "--storage-endpoint", storage["endpoint"], "--storage-input-path", storage["input_path"], "--storage-output-path", storage["output_path"], "--bucket-name", s3["bucket_name"], "--bucket-input-prefix", s3["bucket_input_prefix"], "--bucket-output-prefix", s3["bucket_output_prefix"], ] ) # Provide a way to allow pod specific output prefixes field_ref = client.V1ObjectFieldSelector(field_path="metadata.name") env_var_source = client.V1EnvVarSource(field_ref=field_ref) # HACK, Set the output prefix in the bucket to the name of the pod env_output_prefix = client.V1EnvVar( name="JOBIO_BUCKET_OUTPUT_PREFIX", value_from=env_var_source ) envs.append(env_output_prefix) if scheduler_config: prepared = scheduler.prepare(**scheduler_config) if not prepared: response["msg"] = "Failed to prepare the scheduler" return False, response container_spec = dict( name=job["meta"]["name"], image=cluster["image"], env=envs, args=jobio_args, volume_mounts=volume_mounts, ) # If the working directory does not exist inside the container # It will set permissions where it will be unable to expand the # s3 bucket if the user doesn't have root permissions if "working_dir" in job: container_spec.update({"working_dir": job["working_dir"]}) # If the container requires a specific set of resources resources = {} if "min_cores" in job: resources["requests"] = {"cpu": job["min_cores"]} if "max_cores" in job: resources["limits"] = {"cpu": job["max_cores"]} if "min_memory" in job: resources["requests"].update({"memory": job["min_memory"]}) if "max_memory" in job: resources["limits"].update({"memory": job["max_memory"]}) if resources: resource_req = client.V1ResourceRequirements(**resources) container_spec.update({"resources": resource_req}) # args=jobio_args, pod_spec = dict(node_name=node.metadata.name, volumes=volumes, dns_policy="Default") job_spec = dict( backoff_limit=2, parallelism=job["meta"]["num_parallel"], completions=job["meta"]["num_jobs"], ) task = dict( container_kwargs=container_spec, pod_spec_kwargs=pod_spec, job_spec_kwargs=job_spec, ) job = scheduler.submit(**task) if not job: response["msg"] = "Failed to submit the job" return False, response response["job"] = job response["msg"] = "Job submitted" return True, response
def generate_secrets_server_deployment( secrets_server_config: SecretsServerConfig, ): service_name = secrets_server_config.service_name() secret_name = secrets_server_config.secrets().concourse_secret_name() # We need to ensure that the labels and selectors match for both the deployment and the service, # therefore we base them on the configured service name. labels = {'app': service_name} return V1Deployment( kind='Deployment', metadata=V1ObjectMeta(name=service_name, labels=labels), spec=V1DeploymentSpec( replicas=1, selector=V1LabelSelector(match_labels=labels), template=V1PodTemplateSpec( metadata=V1ObjectMeta(labels=labels), spec=V1PodSpec(containers=[ V1Container( image='eu.gcr.io/gardener-project/cc/job-image:latest', image_pull_policy='IfNotPresent', name='secrets-server', resources=V1ResourceRequirements( requests={ 'cpu': '50m', 'memory': '50Mi' }, limits={ 'cpu': '50m', 'memory': '50Mi' }, ), command=['bash'], args=[ '-c', ''' # chdir to secrets dir; create if absent mkdir -p /secrets && cd /secrets # make Kubernetes serviceaccount secrets available by default cp -r /var/run/secrets/kubernetes.io/serviceaccount serviceaccount # store Kubernetes service endpoint env as file for consumer env | grep KUBERNETES_SERVICE > serviceaccount/env # launch secrets server serving secrets dir contents on all IFs python3 -m http.server 8080 ''' ], ports=[ V1ContainerPort(container_port=8080), ], liveness_probe=V1Probe( tcp_socket=V1TCPSocketAction(port=8080), initial_delay_seconds=10, period_seconds=10, ), volume_mounts=[ V1VolumeMount( name=secret_name, mount_path='/secrets/concourse-secrets', read_only=True, ), ], ), ], node_selector={ "worker.garden.sapcloud.io/group": "cc-control" }, volumes=[ V1Volume(name=secret_name, secret=V1SecretVolumeSource( secret_name=secret_name, )) ]))))
def launch(self, name, docker_config: DockerConfig, mounts, env, blocking: bool = True): name = (self.prefix + 'update-' + name.lower()).replace('_', '-') # If we have been given a username or password for the registry, we have to # update it, if we haven't been, make sure its been cleaned up in the system # so we don't leave passwords lying around pull_secret_name = f'{name}-job-pull-secret' use_pull_secret = False try: # Check if there is already a username/password defined for this job current_pull_secret = self.api.read_namespaced_secret(pull_secret_name, self.namespace, _request_timeout=API_TIMEOUT) except ApiException as error: if error.status != 404: raise current_pull_secret = None if docker_config.registry_username or docker_config.registry_password: use_pull_secret = True # Build the secret we want to make new_pull_secret = V1Secret( metadata=V1ObjectMeta(name=pull_secret_name, namespace=self.namespace), type='kubernetes.io/dockerconfigjson', string_data={ '.dockerconfigjson': create_docker_auth_config( image=docker_config.image, username=docker_config.registry_username, password=docker_config.registry_password, ) } ) # Send it to the server if current_pull_secret: self.api.replace_namespaced_secret(pull_secret_name, namespace=self.namespace, body=new_pull_secret, _request_timeout=API_TIMEOUT) else: self.api.create_namespaced_secret(namespace=self.namespace, body=new_pull_secret, _request_timeout=API_TIMEOUT) elif current_pull_secret: # If there is a password set in kubernetes, but not in our configuration clear it out self.api.delete_namespaced_secret(pull_secret_name, self.namespace, _request_timeout=API_TIMEOUT) try: self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace, propagation_policy='Background', _request_timeout=API_TIMEOUT) while True: self.batch_api.read_namespaced_job(namespace=self.namespace, name=name, _request_timeout=API_TIMEOUT) time.sleep(1) except ApiException: pass volumes = [] volume_mounts = [] for index, mnt in enumerate(mounts): volumes.append(V1Volume( name=f'mount-{index}', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( claim_name=mnt['volume'], read_only=False ), )) volume_mounts.append(V1VolumeMount( name=f'mount-{index}', mount_path=mnt['dest_path'], sub_path=mnt['source_path'], read_only=False, )) if CONFIGURATION_CONFIGMAP: volumes.append(V1Volume( name='mount-configuration', config_map=V1ConfigMapVolumeSource( name=CONFIGURATION_CONFIGMAP ), )) volume_mounts.append(V1VolumeMount( name='mount-configuration', mount_path='/etc/assemblyline/config.yml', sub_path="config", read_only=True, )) section = 'service' labels = { 'app': 'assemblyline', 'section': section, 'privilege': 'core', 'component': 'update-script', } labels.update(self.extra_labels) metadata = V1ObjectMeta( name=name, labels=labels ) environment_variables = [V1EnvVar(name=_e.name, value=_e.value) for _e in docker_config.environment] environment_variables.extend([V1EnvVar(name=k, value=v) for k, v in env.items()]) environment_variables.extend([V1EnvVar(name=k, value=os.environ[k]) for k in INHERITED_VARIABLES if k in os.environ]) environment_variables.append(V1EnvVar(name="LOG_LEVEL", value=self.log_level)) cores = docker_config.cpu_cores memory = docker_config.ram_mb memory_min = min(docker_config.ram_mb_min, memory) container = V1Container( name=name, image=docker_config.image, command=docker_config.command, env=environment_variables, image_pull_policy='Always', volume_mounts=volume_mounts, resources=V1ResourceRequirements( limits={'cpu': cores, 'memory': f'{memory}Mi'}, requests={'cpu': cores / 4, 'memory': f'{memory_min}Mi'}, ) ) pod = V1PodSpec( volumes=volumes, restart_policy='Never', containers=[container], priority_class_name=self.priority_class, ) if use_pull_secret: pod.image_pull_secrets = [V1LocalObjectReference(name=pull_secret_name)] job = V1Job( metadata=metadata, spec=V1JobSpec( backoff_limit=1, completions=1, template=V1PodTemplateSpec( metadata=metadata, spec=pod ) ) ) status = self.batch_api.create_namespaced_job(namespace=self.namespace, body=job, _request_timeout=API_TIMEOUT).status if blocking: try: while not (status.failed or status.succeeded): time.sleep(3) status = self.batch_api.read_namespaced_job(namespace=self.namespace, name=name, _request_timeout=API_TIMEOUT).status self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace, propagation_policy='Background', _request_timeout=API_TIMEOUT) except ApiException as error: if error.status != 404: raise