def get_pod_spec(name, ssh_key="", container="wshand/cutter:latest"): pod = client.V1Pod() pod.api_version = "v1" labels = {"app": "lawliet-env", "app-specific": get_pod_name(name)} pod.metadata = client.V1ObjectMeta(name=get_pod_name(name), labels=labels) ports = [ client.V1ContainerPort(container_port=22), client.V1ContainerPort(container_port=6080) ] container = client.V1Container( name=get_pod_name(name), image=container, image_pull_policy="Always", #command=["/bin/bash"], #args=["-c", "echo '%s' > ~/.ssh/authorized_keys && service ssh start; mkdir -p /dev/net && mknod /dev/net/tun c 10 200 && chmod 0666 /dev/net/tun; /start.sh" % ssh_key], ports=ports, security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(add=["NET_ADMIN"]))) context = client.V1PodSecurityContext(sysctls=[ client.V1Sysctl(name="net.ipv6.conf.all.disable_ipv6", value="0") ]) logging.debug("made context") pod.spec = client.V1PodSpec(containers=[container], security_context=context) return pod
def _make_container(self, player_id): return client.V1Container( env=[ client.V1EnvVar(name="DATA_URL", value="%s/player/%d" % (self.game_url, player_id)), client.V1EnvVar(name="PORT", value="5000"), ], name="aimmo-game-worker", image="ocadotechnology/aimmo-game-worker:%s" % os.environ.get("IMAGE_SUFFIX", "latest"), ports=[ client.V1ContainerPort(container_port=5000, protocol="TCP") ], resources=client.V1ResourceRequirements( limits={ "cpu": "10m", "memory": "64Mi" }, requests={ "cpu": "6m", "memory": "32Mi" }, ), security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(drop=["all"], add=["NET_BIND_SERVICE"])), )
def _make_container(self, player_id): return client.V1Container( env=[ client.V1EnvVar(name='DATA_URL', value='%s/player/%d' % (self.game_url, player_id)), client.V1EnvVar(name='PORT', value='5000') ], name='aimmo-game-worker', image='ocadotechnology/aimmo-game-worker:%s' % os.environ.get('IMAGE_SUFFIX', 'latest'), ports=[ client.V1ContainerPort(container_port=5000, protocol='TCP') ], resources=client.V1ResourceRequirements(limits={ 'cpu': '10m', 'memory': '64Mi' }, requests={ 'cpu': '6m', 'memory': '32Mi' }), security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(drop=['all'], add=['NET_BIND_SERVICE'])))
def create_iperf_deploy(self, name): self.apps_v1_api.create_namespaced_deployment( namespace='default', body=client.V1Deployment( api_version="apps/v1", kind="Deployment", metadata=client.V1ObjectMeta( name=name, namespace="default", labels={'app': name} ), spec=client.V1DeploymentSpec( replicas=10, selector=client.V1LabelSelector(match_labels={'app': name}), template=client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={'app': name}), spec=client.V1PodSpec( containers=[ client.V1Container( name=name, tty=True, image="zhuangweikang/k8stc:latest", image_pull_policy="IfNotPresent", security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(add=["NET_ADMIN"])), resources=client.V1ResourceRequirements( limits={"cpu": "100m", "memory": "1Gi"}, requests={"cpu": "100m", "memory": "1Gi"}) ) ] ) ) ) ) )
def create_iperf(self, name='server'): self.core_v1_api.create_namespaced_pod( namespace="default", body=client.V1Pod( api_version="v1", kind="Pod", metadata=client.V1ObjectMeta( name=name, namespace="default" ), spec=client.V1PodSpec( containers=[ client.V1Container( name="iperf", tty=True, image="zhuangweikang/k8stc:latest", image_pull_policy="IfNotPresent", security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(add=["NET_ADMIN"])), resources=client.V1ResourceRequirements( limits={"cpu": "100m", "memory": "1Gi"}, requests={"cpu": "100m", "memory": "1Gi"}) ) ], restart_policy="Never" ) ), async_req=False)
if not cloud_metadata.get('enabled', False): # Use iptables to block access to cloud metadata by default network_tools_image_name = get_config( 'singleuser.network-tools.image.name') network_tools_image_tag = get_config('singleuser.network-tools.image.tag') ip_block_container = client.V1Container( name="block-cloud-metadata", image=f"{network_tools_image_name}:{network_tools_image_tag}", command=[ 'iptables', '-A', 'OUTPUT', '-d', cloud_metadata.get('ip', '169.254.169.254'), '-j', 'DROP' ], security_context=client.V1SecurityContext( privileged=True, run_as_user=0, capabilities=client.V1Capabilities(add=['NET_ADMIN']))) c.KubeSpawner.singleuser_init_containers.append(ip_block_container) scheduler_strategy = get_config('singleuser.scheduler-strategy', 'spread') if scheduler_strategy == 'pack': # FIXME: Support setting affinity directly in KubeSpawner c.KubeSpawner.singleuser_extra_pod_config = { 'affinity': { 'podAffinity': { 'preferredDuringSchedulingIgnoredDuringExecution': [{ 'weight': 100, 'podAffinityTerm': { 'labelSelector': { 'matchExpressions': [{
def create_deployment_old(config_file): """ Create IBM Spectrum Scale CSI Operator deployment object in operator namespace using deployment_operator_image_for_crd and deployment_driver_image_for_crd parameters from config.json file Args: param1: config_file - configuration json file Returns: None Raises: Raises an exception on kubernetes client api failure and asserts """ deployment_apps_api_instance = client.AppsV1Api() deployment_labels = { "app.kubernetes.io/instance": "ibm-spectrum-scale-csi-operator", "app.kubernetes.io/managed-by": "ibm-spectrum-scale-csi-operator", "app.kubernetes.io/name": "ibm-spectrum-scale-csi-operator", "product": "ibm-spectrum-scale-csi", "release": "ibm-spectrum-scale-csi-operator" } deployment_annotations = { "productID": "ibm-spectrum-scale-csi-operator", "productName": "IBM Spectrum Scale CSI Operator", "productVersion": "2.0.0" } deployment_metadata = client.V1ObjectMeta( name="ibm-spectrum-scale-csi-operator", labels=deployment_labels, namespace=namespace_value) deployment_selector = client.V1LabelSelector( match_labels={ "app.kubernetes.io/name": "ibm-spectrum-scale-csi-operator" }) podtemplate_metadata = client.V1ObjectMeta( labels=deployment_labels, annotations=deployment_annotations) pod_affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( required_during_scheduling_ignored_during_execution=client. V1NodeSelector(node_selector_terms=[ client.V1NodeSelectorTerm(match_expressions=[ client.V1NodeSelectorRequirement(key="beta.kubernetes.io/arch", operator="Exists") ]) ]))) ansible_pod_container = client.V1Container( image=config_file["deployment_operator_image_for_crd"], command=[ "/usr/local/bin/ao-logs", "/tmp/ansible-operator/runner", "stdout" ], liveness_probe=client.V1Probe( _exec=client.V1ExecAction(command=["/health_check.sh"]), initial_delay_seconds=10, period_seconds=30), readiness_probe=client.V1Probe( _exec=client.V1ExecAction(command=["/health_check.sh"]), initial_delay_seconds=3, period_seconds=1), name="ansible", image_pull_policy="IfNotPresent", security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(drop=["ALL"])), volume_mounts=[ client.V1VolumeMount(mount_path="/tmp/ansible-operator/runner", name="runner", read_only=True) ], env=[ client.V1EnvVar( name="CSI_DRIVER_IMAGE", value=config_file["deployment_driver_image_for_crd"]) ]) operator_pod_container = client.V1Container( image=config_file["deployment_operator_image_for_crd"], name="operator", image_pull_policy="IfNotPresent", liveness_probe=client.V1Probe( _exec=client.V1ExecAction(command=["/health_check.sh"]), initial_delay_seconds=10, period_seconds=30), readiness_probe=client.V1Probe( _exec=client.V1ExecAction(command=["/health_check.sh"]), initial_delay_seconds=3, period_seconds=1), security_context=client.V1SecurityContext( capabilities=client.V1Capabilities(drop=["ALL"])), env=[ client.V1EnvVar(name="WATCH_NAMESPACE", value_from=client.V1EnvVarSource( field_ref=client.V1ObjectFieldSelector( field_path="metadata.namespace"))), client.V1EnvVar(name="POD_NAME", value_from=client.V1EnvVarSource( field_ref=client.V1ObjectFieldSelector( field_path="metadata.name"))), client.V1EnvVar(name="OPERATOR_NAME", value="ibm-spectrum-scale-csi-operator"), client.V1EnvVar( name="CSI_DRIVER_IMAGE", value=config_file["deployment_driver_image_for_crd"]) ], volume_mounts=[ client.V1VolumeMount(mount_path="/tmp/ansible-operator/runner", name="runner") ]) pod_spec = client.V1PodSpec( affinity=pod_affinity, containers=[ansible_pod_container, operator_pod_container], service_account_name="ibm-spectrum-scale-csi-operator", volumes=[ client.V1Volume( empty_dir=client.V1EmptyDirVolumeSource(medium="Memory"), name="runner") ]) podtemplate_spec = client.V1PodTemplateSpec(metadata=podtemplate_metadata, spec=pod_spec) deployment_spec = client.V1DeploymentSpec(replicas=1, selector=deployment_selector, template=podtemplate_spec) body_dep = client.V1Deployment(kind='Deployment', api_version='apps/v1', metadata=deployment_metadata, spec=deployment_spec) try: LOGGER.info("creating deployment for operator") deployment_apps_api_response = deployment_apps_api_instance.create_namespaced_deployment( namespace=namespace_value, body=body_dep) LOGGER.debug(str(deployment_apps_api_response)) except ApiException as e: LOGGER.error( f"Exception when calling RbacAuthorizationV1Api->create_namespaced_deployment: {e}" ) assert False
def dispatch(job=None, namespace='default', debug=False, **kwargs): logger.info(f"starting {job['id']}") from kubernetes import client, config config.load_incluster_config() coreV1 = client.CoreV1Api() batchV1 = client.BatchV1Api() batchV1.create_namespaced_job( namespace=namespace, body=client.V1Job( api_version='batch/v1', kind='Job', metadata=client.V1ObjectMeta( name=job['id'], annotations={ f"container.apparmor.security.beta.kubernetes.io/appyter-{job['id']}": 'unconfined' }, ), spec=client.V1JobSpec( template=client.V1PodTemplateSpec(spec=client.V1PodSpec( restart_policy='Never', containers=[ client.V1Container( name=f"appyter-{job['id']}", image=job['image'], command=[ 'appyter', 'orchestration', 'job', json.dumps(job) ], security_context=client.V1SecurityContext( privileged=True, capabilities=client.V1Capabilities( add=['SYS_ADMIN'], ), ), volume_mounts=[ client.V1VolumeMount( name='fuse', mount_path='/dev/fuse', ) ], ), ], volumes=[ client.V1Volume( name='fuse', host_path=client.V1HostPathVolumeSource( path='/dev/fuse', )), ], ), ), backoff_limit=1, ), ), ) # for event in endless_watch(batchV1.list_namespaced_job, namespace, label_selector=f"job-name={job['id']}"): logger.debug(str(event)) event_type = event['type'] event_job = event['object'] if event_type == 'MODIFIED': if event_job.status.succeeded or event_job.status.failed: break # if not debug: batchV1.delete_namespaced_job(job['id'], namespace) # delete associated pod(s) for event_pod in coreV1.list_namespaced_pod( namespace, label_selector=f"job-name={job['id']}", watch=False).items: coreV1.delete_namespaced_pod(event_pod.metadata.name, namespace) # logger.info(f"{job['id']} completed")
def test_sanitize_k8s_container_attribute(self): # test cases for implicit type sanitization(conversion) op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'], arguments=['echo test | tee /tmp/message.txt'], file_outputs={'merged': '/tmp/message.txt'}) op.container \ .add_volume_mount(k8s_client.V1VolumeMount( mount_path='/secret/gcp-credentials', name='gcp-credentials')) \ .add_env_variable(k8s_client.V1EnvVar( name=80, value=80)) \ .add_env_variable(k8s_client.V1EnvVar( name=80, value_from=k8s_client.V1EnvVarSource( config_map_key_ref=k8s_client.V1ConfigMapKeySelector(key=80, name=8080, optional='False'), field_ref=k8s_client.V1ObjectFieldSelector(api_version=80, field_path=8080), resource_field_ref=k8s_client.V1ResourceFieldSelector(container_name=80, divisor=8080, resource=8888), secret_key_ref=k8s_client.V1SecretKeySelector(key=80, name=8080, optional='False') ) )) \ .add_env_from(k8s_client.V1EnvFromSource( config_map_ref=k8s_client.V1ConfigMapEnvSource(name=80, optional='True'), prefix=999 )) \ .add_env_from(k8s_client.V1EnvFromSource( secret_ref=k8s_client.V1SecretEnvSource(name=80, optional='True'), prefix=888 )) \ .add_volume_mount(k8s_client.V1VolumeMount( mount_path=111, mount_propagation=222, name=333, read_only='False', sub_path=444, sub_path_expr=555 )) \ .add_volume_devices(k8s_client.V1VolumeDevice( device_path=111, name=222 )) \ .add_port(k8s_client.V1ContainerPort( container_port='8080', host_ip=111, host_port='8888', name=222, protocol=333 )) \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation='True', capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]), privileged='False', proc_mount=111, read_only_root_filesystem='False', run_as_group='222', run_as_non_root='True', run_as_user='******', se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44), windows_options=k8s_client.V1WindowsSecurityContextOptions( gmsa_credential_spec=11, gmsa_credential_spec_name=22) )) \ .set_stdin(stdin='False') \ .set_stdin_once(stdin_once='False') \ .set_termination_message_path(termination_message_path=111) \ .set_tty(tty='False') \ .set_readiness_probe(readiness_probe=k8s_client.V1Probe( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), failure_threshold='111', http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), initial_delay_seconds='222', period_seconds='333', success_threshold='444', tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'), timeout_seconds='777' )) \ .set_liveness_probe(liveness_probe=k8s_client.V1Probe( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), failure_threshold='111', http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), initial_delay_seconds='222', period_seconds='333', success_threshold='444', tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'), timeout_seconds='777' )) \ .set_lifecycle(lifecycle=k8s_client.V1Lifecycle( post_start=k8s_client.V1Handler( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666') ), pre_stop=k8s_client.V1Handler( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666') ) )) sanitize_k8s_object(op.container) for e in op.container.env: self.assertIsInstance(e.name, str) if e.value: self.assertIsInstance(e.value, str) if e.value_from: if e.value_from.config_map_key_ref: self.assertIsInstance(e.value_from.config_map_key_ref.key, str) if e.value_from.config_map_key_ref.name: self.assertIsInstance(e.value_from.config_map_key_ref.name, str) if e.value_from.config_map_key_ref.optional: self.assertIsInstance(e.value_from.config_map_key_ref.optional, bool) if e.value_from.field_ref: self.assertIsInstance(e.value_from.field_ref.field_path, str) if e.value_from.field_ref.api_version: self.assertIsInstance(e.value_from.field_ref.api_version, str) if e.value_from.resource_field_ref: self.assertIsInstance(e.value_from.resource_field_ref.resource, str) if e.value_from.resource_field_ref.container_name: self.assertIsInstance(e.value_from.resource_field_ref.container_name, str) if e.value_from.resource_field_ref.divisor: self.assertIsInstance(e.value_from.resource_field_ref.divisor, str) if e.value_from.secret_key_ref: self.assertIsInstance(e.value_from.secret_key_ref.key, str) if e.value_from.secret_key_ref.name: self.assertIsInstance(e.value_from.secret_key_ref.name, str) if e.value_from.secret_key_ref.optional: self.assertIsInstance(e.value_from.secret_key_ref.optional, bool) for e in op.container.env_from: if e.prefix: self.assertIsInstance(e.prefix, str) if e.config_map_ref: if e.config_map_ref.name: self.assertIsInstance(e.config_map_ref.name, str) if e.config_map_ref.optional: self.assertIsInstance(e.config_map_ref.optional, bool) if e.secret_ref: if e.secret_ref.name: self.assertIsInstance(e.secret_ref.name, str) if e.secret_ref.optional: self.assertIsInstance(e.secret_ref.optional, bool) for e in op.container.volume_mounts: if e.mount_path: self.assertIsInstance(e.mount_path, str) if e.mount_propagation: self.assertIsInstance(e.mount_propagation, str) if e.name: self.assertIsInstance(e.name, str) if e.read_only: self.assertIsInstance(e.read_only, bool) if e.sub_path: self.assertIsInstance(e.sub_path, str) if e.sub_path_expr: self.assertIsInstance(e.sub_path_expr, str) for e in op.container.volume_devices: if e.device_path: self.assertIsInstance(e.device_path, str) if e.name: self.assertIsInstance(e.name, str) for e in op.container.ports: if e.container_port: self.assertIsInstance(e.container_port, int) if e.host_ip: self.assertIsInstance(e.host_ip, str) if e.host_port: self.assertIsInstance(e.host_port, int) if e.name: self.assertIsInstance(e.name, str) if e.protocol: self.assertIsInstance(e.protocol, str) if op.container.security_context: e = op.container.security_context if e.allow_privilege_escalation: self.assertIsInstance(e.allow_privilege_escalation, bool) if e.capabilities: for a in e.capabilities.add: self.assertIsInstance(a, str) for d in e.capabilities.drop: self.assertIsInstance(d, str) if e.privileged: self.assertIsInstance(e.privileged, bool) if e.proc_mount: self.assertIsInstance(e.proc_mount, str) if e.read_only_root_filesystem: self.assertIsInstance(e.read_only_root_filesystem, bool) if e.run_as_group: self.assertIsInstance(e.run_as_group, int) if e.run_as_non_root: self.assertIsInstance(e.run_as_non_root, bool) if e.run_as_user: self.assertIsInstance(e.run_as_user, int) if e.se_linux_options: if e.se_linux_options.level: self.assertIsInstance(e.se_linux_options.level, str) if e.se_linux_options.role: self.assertIsInstance(e.se_linux_options.role, str) if e.se_linux_options.type: self.assertIsInstance(e.se_linux_options.type, str) if e.se_linux_options.user: self.assertIsInstance(e.se_linux_options.user, str) if e.windows_options: if e.windows_options.gmsa_credential_spec: self.assertIsInstance(e.windows_options.gmsa_credential_spec, str) if e.windows_options.gmsa_credential_spec_name: self.assertIsInstance(e.windows_options.gmsa_credential_spec_name, str) if op.container.stdin: self.assertIsInstance(op.container.stdin, bool) if op.container.stdin_once: self.assertIsInstance(op.container.stdin_once, bool) if op.container.termination_message_path: self.assertIsInstance(op.container.termination_message_path, str) if op.container.tty: self.assertIsInstance(op.container.tty, bool) for e in [op.container.readiness_probe, op.container.liveness_probe]: if e: if e._exec: for c in e._exec.command: self.assertIsInstance(c, str) if e.failure_threshold: self.assertIsInstance(e.failure_threshold, int) if e.http_get: if e.http_get.host: self.assertIsInstance(e.http_get.host, str) if e.http_get.http_headers: for h in e.http_get.http_headers: if h.name: self.assertIsInstance(h.name, str) if h.value: self.assertIsInstance(h.value, str) if e.http_get.path: self.assertIsInstance(e.http_get.path, str) if e.http_get.port: self.assertIsInstance(e.http_get.port, (str, int)) if e.http_get.scheme: self.assertIsInstance(e.http_get.scheme, str) if e.initial_delay_seconds: self.assertIsInstance(e.initial_delay_seconds, int) if e.period_seconds: self.assertIsInstance(e.period_seconds, int) if e.success_threshold: self.assertIsInstance(e.success_threshold, int) if e.tcp_socket: if e.tcp_socket.host: self.assertIsInstance(e.tcp_socket.host, str) if e.tcp_socket.port: self.assertIsInstance(e.tcp_socket.port, (str, int)) if e.timeout_seconds: self.assertIsInstance(e.timeout_seconds, int) if op.container.lifecycle: for e in [op.container.lifecycle.post_start, op.container.lifecycle.pre_stop]: if e: if e._exec: for c in e._exec.command: self.assertIsInstance(c, str) if e.http_get: if e.http_get.host: self.assertIsInstance(e.http_get.host, str) if e.http_get.http_headers: for h in e.http_get.http_headers: if h.name: self.assertIsInstance(h.name, str) if h.value: self.assertIsInstance(h.value, str) if e.http_get.path: self.assertIsInstance(e.http_get.path, str) if e.http_get.port: self.assertIsInstance(e.http_get.port, (str, int)) if e.http_get.scheme: self.assertIsInstance(e.http_get.scheme, str) if e.tcp_socket: if e.tcp_socket.host: self.assertIsInstance(e.tcp_socket.host, str) if e.tcp_socket.port: self.assertIsInstance(e.tcp_socket.port, (str, int)) # test cases for checking value after sanitization check_value_op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'], arguments=['echo test | tee /tmp/message.txt'], file_outputs={'merged': '/tmp/message.txt'}) check_value_op.container \ .add_env_variable(k8s_client.V1EnvVar( name=80, value=8080)) \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation='true', capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]), privileged='false', proc_mount=111, read_only_root_filesystem='False', run_as_group='222', run_as_non_root='True', run_as_user='******', se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44), windows_options=k8s_client.V1WindowsSecurityContextOptions( gmsa_credential_spec=11, gmsa_credential_spec_name=22) )) sanitize_k8s_object(check_value_op.container) self.assertEqual(check_value_op.container.env[0].name, '80') self.assertEqual(check_value_op.container.env[0].value, '8080') self.assertEqual(check_value_op.container.security_context.allow_privilege_escalation, True) self.assertEqual(check_value_op.container.security_context.capabilities.add[0], '11') self.assertEqual(check_value_op.container.security_context.capabilities.add[1], '22') self.assertEqual(check_value_op.container.security_context.capabilities.drop[0], '33') self.assertEqual(check_value_op.container.security_context.capabilities.drop[1], '44') self.assertEqual(check_value_op.container.security_context.privileged, False) self.assertEqual(check_value_op.container.security_context.proc_mount, '111') self.assertEqual(check_value_op.container.security_context.read_only_root_filesystem, False) self.assertEqual(check_value_op.container.security_context.run_as_group, 222) self.assertEqual(check_value_op.container.security_context.run_as_non_root, True) self.assertEqual(check_value_op.container.security_context.run_as_user, 333) self.assertEqual(check_value_op.container.security_context.se_linux_options.level, '11') self.assertEqual(check_value_op.container.security_context.se_linux_options.role, '22') self.assertEqual(check_value_op.container.security_context.se_linux_options.type, '33') self.assertEqual(check_value_op.container.security_context.se_linux_options.user, '44') self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec, '11') self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec_name, '22') # test cases for exception with self.assertRaises(ValueError, msg='Invalid boolean string 2. Should be boolean.'): exception_op = dsl.ContainerOp(name='echo', image='image') exception_op.container \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation=1 )) sanitize_k8s_object(exception_op.container) with self.assertRaises(ValueError, msg='Invalid boolean string Test. Should be "true" or "false".'): exception_op = dsl.ContainerOp(name='echo', image='image') exception_op.container \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation='Test' )) sanitize_k8s_object(exception_op.container) with self.assertRaises(ValueError, msg='Invalid test. Should be integer.'): exception_op = dsl.ContainerOp(name='echo', image='image') exception_op.container \ .set_security_context(k8s_client.V1SecurityContext( run_as_group='test', )) sanitize_k8s_object(exception_op.container)
ip_block_container = client.V1Container( name="block-cloud-metadata", image=f"{network_tools_image_name}:{network_tools_image_tag}", command=[ "iptables", "-A", "OUTPUT", "-d", cloud_metadata.get("ip", "169.254.169.254"), "-j", "DROP", ], security_context=client.V1SecurityContext( privileged=True, run_as_user=0, capabilities=client.V1Capabilities(add=["NET_ADMIN"]), ), resources=network_tools_resources, ) c.KubeSpawner.init_containers.append(ip_block_container) if get_config("debug.enabled", False): c.JupyterHub.log_level = "DEBUG" c.Spawner.debug = True # load potentially seeded secrets # # NOTE: ConfigurableHTTPProxy.auth_token is set through an environment variable # that is set using the chart managed secret. c.JupyterHub.cookie_secret = get_secret_value(
def deployment_object(self, instance_uuid, cnf_yaml, service_uuid, vim_uuid): """ CNF modeling method. This build a deployment object in kubernetes instance_uuid: k8s deployment name cnf_yaml: CNF Descriptor in yaml format """ t0 = time.time() LOG.debug("CNFD: {}".format(cnf_yaml)) container_list = [] pod_volume_list = [] deployment_k8s = None privileged = False node_selector = {} host_network = False if "cloudnative_deployment_units" in cnf_yaml: cdu = cnf_yaml.get('cloudnative_deployment_units') for cdu_obj in cdu: env_vars = env_from = cpu = memory = huge_pages = gpu = sr_iov = resources = volume_mounts = None port_list = [] environment = [] capabilities_list = [] cdu_id = cdu_obj.get('id') image = cdu_obj.get('image') cdu_conex = cdu_obj.get('connection_points') container_name = cdu_id config_map_id = cdu_id if cdu_obj.get('parameters'): env_vars = cdu_obj['parameters'].get('env') volume_mounts = cdu_obj['parameters'].get('volume_mounts') capabilities_list = cdu_obj['parameters'].get('capabilities') if cdu_obj['parameters'].get('privileged'): privileged = cdu_obj['parameters'].get('privileged') if cdu_obj['parameters'].get('host_network'): privileged = cdu_obj['parameters']['host_network'] if not isinstance(capabilities_list, list): capabilities_list = [] if cdu_obj['parameters'].get('node_selector'): node_selector = cdu_obj['parameters']['node_selector'] if cdu_obj.get('resource_requirements'): gpu = cdu_obj['resource_requirements'].get('gpu') cpu = cdu_obj['resource_requirements'].get('cpu') memory = cdu_obj['resource_requirements'].get('memory') sr_iov = cdu_obj['resource_requirements'].get('sr-iov') huge_pages = cdu_obj['resource_requirements'].get('huge-pages') if cdu_conex: for po in cdu_conex: port = po.get('port') port_name = po.get('id') protocol = "TCP" if po.get("protocol"): protocol = po["protocol"] port_list.append(client.V1ContainerPort(container_port = port, name = port_name, protocol=protocol)) limits = {} requests = {} if gpu: LOG.debug("Features requested: {}".format(gpu)) # gpu_type can be amd or nvidia for gpu_type, amount in gpu.items(): limits["{}.com/gpu".format(gpu_type)] = amount if cpu: # TODO pass if memory: # TODO pass if sr_iov: # TODO pass if huge_pages: # TODO pass resources = client.V1ResourceRequirements(limits=limits, requests=requests) # Environment variables from descriptor if env_vars: LOG.debug("Configmap: {}".format(config_map_id)) KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid, vim_uuid, namespace = "default") else: env_vars = {"sonata": "rules"} LOG.debug("Configmap: {}".format(config_map_id)) KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid, vim_uuid, namespace = "default") env_from = client.V1EnvFromSource(config_map_ref = client.V1ConfigMapEnvSource(name = config_map_id, optional = False)) # Default static environment variables environment.append(client.V1EnvVar(name="instance_uuid", value=instance_uuid)) environment.append(client.V1EnvVar(name="service_uuid", value=service_uuid)) environment.append(client.V1EnvVar(name="container_name", value=container_name)) environment.append(client.V1EnvVar(name="vendor", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('vendor')))) environment.append(client.V1EnvVar(name="name", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('name')))) environment.append(client.V1EnvVar(name="version", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('version')))) image_pull_policy = KubernetesWrapperEngine.check_connection(self) # Volume mounts container_volume_mount_list = [] if volume_mounts: LOG.debug("volume mounts: {}".format(volume_mounts)) # Create the specification of volumes for volume_mounts_item in volume_mounts: if volume_mounts_item.get('id') and volume_mounts_item.get('location'): if volume_mounts_item.get('persistent'): volumes = client.V1Volume(name=volume_mounts_item['id'], host_path=client.V1HostPathVolumeSource(path='/mnt/data', type='DirectoryOrCreate' )) else: volumes = client.V1Volume(name=volume_mounts_item['id'], empty_dir=client.V1EmptyDirVolumeSource(medium='' )) if volumes not in pod_volume_list: pod_volume_list.append(volumes) container_volume_mount = client.V1VolumeMount(name=volume_mounts_item['id'], mount_path=volume_mounts_item['location'] ) container_volume_mount_list.append(container_volume_mount) LOG.debug("Security capabilities: {}, privileged: {} applied to {}".format(capabilities_list, privileged, container_name)) sec_context = client.V1SecurityContext(privileged=privileged, capabilities=client.V1Capabilities(add=capabilities_list)) # Configureate Pod template container container = client.V1Container( env = environment, name = container_name, resources = resources, image = image, image_pull_policy = image_pull_policy, ports = port_list, env_from = [env_from], volume_mounts = container_volume_mount_list, security_context=sec_context) container_list.append(container) else: return deployment_k8s # Create and configurate a spec section deployment_label = ("{}-{}-{}-{}".format(cnf_yaml.get("vendor"), cnf_yaml.get("name"), cnf_yaml.get("version"), instance_uuid.split("-")[0])).replace(".", "-") template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={'deployment': deployment_label, 'instance_uuid': cnf_yaml['instance_uuid'], 'service_uuid': service_uuid, 'sp': "sonata", 'descriptor_uuid': cnf_yaml['uuid']} ), spec=client.V1PodSpec(containers=container_list, volumes=pod_volume_list, node_selector=node_selector, host_network=host_network)) selector=client.V1LabelSelector(match_labels={'deployment': deployment_label, 'instance_uuid': cnf_yaml['instance_uuid'], 'service_uuid': service_uuid, 'sp': "sonata", 'descriptor_uuid': cnf_yaml['uuid']} ) # Create the specification of deployment spec = client.V1DeploymentSpec( replicas=1, template=template, selector=selector) # Instantiate the deployment object deployment_k8s = client.V1Deployment( api_version="apps/v1", kind="Deployment", metadata=client.V1ObjectMeta(name=deployment_label, labels={'deployment': deployment_label, 'instance_uuid': cnf_yaml['instance_uuid'], 'service_uuid': service_uuid, 'sp': "sonata", 'descriptor_uuid': cnf_yaml['uuid']} ), spec=spec) LOG.info("Deployment object: {}".format(deployment_k8s)) LOG.info("CreatingDeploymentObject-time: {} ms".format(int((time.time() - t0)* 1000))) return deployment_k8s
def create_job(self, image, command, name, resources_dict, mountdir=None): """ Create and return a new job instance. """ number_of_workers = resources_dict.get('number_of_workers') cpu_limit = str(resources_dict.get('cpu_limit')) + 'm' memory_limit = str(resources_dict.get('memory_limit')) + 'Mi' gpu_limit = resources_dict.get('gpu_limit') # configure pod's containers requests = {'memory': '150Mi', 'cpu': '250m'} limits = {'memory': memory_limit, 'cpu': cpu_limit} env = [] if gpu_limit > 0: # ref: https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ limits['nvidia.com/gpu'] = gpu_limit env = [ k_client.V1EnvVar(name='NVIDIA_VISIBLE_DEVICES', value='all'), k_client.V1EnvVar(name='NVIDIA_DRIVER_CAPABILITIES', value='compute,utility'), k_client.V1EnvVar(name='NVIDIA_REQUIRE_CUDA', value='cuda>=9.0') ], container = k_client.V1Container( name=name, image=image, env=env, command=command.split(' '), security_context=k_client.V1SecurityContext( allow_privilege_escalation=False, capabilities=k_client.V1Capabilities(drop=['ALL'])), resources=k_client.V1ResourceRequirements(limits=limits, requests=requests), volume_mounts=[ k_client.V1VolumeMount(mount_path='/share', name='shared-volume') ]) # configure pod template's spec volumes = [] if self.config.get('STORAGE_TYPE') == 'host': volumes = [ k_client.V1Volume( name='shared-volume', host_path=k_client.V1HostPathVolumeSource(path=mountdir)) ] template = k_client.V1PodTemplateSpec(spec=k_client.V1PodSpec( restart_policy='Never', containers=[container], volumes=volumes)) # configure job's spec spec = k_client.V1JobSpec( parallelism=number_of_workers, completions=number_of_workers, backoff_limit=1, ttl_seconds_after_finished=86400, # 24h active_deadline_seconds=43200, # 12h template=template) # instantiate the job object job = k_client.V1Job(api_version='batch/v1', kind='Job', metadata=k_client.V1ObjectMeta(name=name), spec=spec) return job
def create_job(MODEL): assert MODEL is not None, "model name is None, cannot spawn a new worker" api = client.BatchV1Api() body = client.V1Job(api_version="batch/v1", kind="Job") name = 'speechlab-worker-job-{}-{}'.format(MODEL.lower().replace("_", "-"), id_generator()) body.metadata = client.V1ObjectMeta(namespace=NAMESPACE, name=name) body.status = client.V1JobStatus() template = client.V1PodTemplate() template.template = client.V1PodTemplateSpec() template.template.metadata = client.V1ObjectMeta( annotations={ "prometheus.io/scrape": "true", "prometheus.io/port": "8081" }) azure_file_volume = client.V1AzureFileVolumeSource( read_only=True, secret_name=MODELS_FILESHARE_SECRET, share_name=MODELS_SHARE_NAME) volume = client.V1Volume(name="models-azurefiles", azure_file=azure_file_volume) env_vars = { "AZURE_STORAGE_ACCOUNT": AZURE_STORAGE_ACCOUNT, "AZURE_STORAGE_ACCESS_KEY": AZURE_STORAGE_ACCESS_KEY, "AZURE_CONTAINER": AZURE_CONTAINER, "MASTER": MASTER, "NAMESPACE": NAMESPACE, "RUN_FREQ": "ONCE", "MODEL_DIR": MODEL, # important "MODELS_FILESHARE_SECRET": MODELS_FILESHARE_SECRET, "MODELS_SHARE_NAME": MODELS_SHARE_NAME } env_list = [] if env_vars: for env_name, env_value in env_vars.items(): env_list.append(client.V1EnvVar(name=env_name, value=env_value)) container = client.V1Container( name='{}-c'.format(name), image=IMAGE, image_pull_policy="IfNotPresent", command=[ "/home/appuser/opt/tini", "--", "/home/appuser/opt/start_worker.sh" ], env=env_list, ports=[client.V1ContainerPort(container_port=8081, name="prometheus")], security_context=client.V1SecurityContext( privileged=True, capabilities=client.V1Capabilities(add=["SYS_ADMIN"])), resources=client.V1ResourceRequirements(limits={ "memory": "5G", "cpu": "1" }, requests={ "memory": "5G", "cpu": "1" }), volume_mounts=[ client.V1VolumeMount(mount_path="/home/appuser/opt/models", name="models-azurefiles", read_only=True) ]) template.template.spec = client.V1PodSpec( containers=[container], image_pull_secrets=[{ "name": "azure-cr-secret" }], # reason to use OnFailure https://github.com/kubernetes/kubernetes/issues/20255 restart_policy="OnFailure", volumes=[volume]) # And finaly we can create our V1JobSpec! body.spec = client.V1JobSpec(ttl_seconds_after_finished=100, template=template.template) try: api_response = api.create_namespaced_job(NAMESPACE, body) print("api_response=" + str(api_response)) return True except ApiException as e: logging.exception('error spawning new job') print("Exception when creating a job: %s\n" % e)