def get_pod_spec(name, ssh_key="", container="wshand/cutter:latest"):
pod = client.V1Pod()
pod.api_version = "v1"
labels = {"app": "lawliet-env", "app-specific": get_pod_name(name)}
pod.metadata = client.V1ObjectMeta(name=get_pod_name(name), labels=labels)
ports = [
client.V1ContainerPort(container_port=22),
client.V1ContainerPort(container_port=6080)
]
container = client.V1Container(
name=get_pod_name(name),
image=container,
image_pull_policy="Always",
#command=["/bin/bash"],
#args=["-c", "echo '%s' > ~/.ssh/authorized_keys && service ssh start; mkdir -p /dev/net && mknod /dev/net/tun c 10 200 && chmod 0666 /dev/net/tun; /start.sh" % ssh_key],
ports=ports,
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(add=["NET_ADMIN"])))
context = client.V1PodSecurityContext(sysctls=[
client.V1Sysctl(name="net.ipv6.conf.all.disable_ipv6", value="0")
])
logging.debug("made context")
pod.spec = client.V1PodSpec(containers=[container],
security_context=context)
return pod
def _make_container(self, player_id):
return client.V1Container(
env=[
client.V1EnvVar(name="DATA_URL",
value="%s/player/%d" %
(self.game_url, player_id)),
client.V1EnvVar(name="PORT", value="5000"),
],
name="aimmo-game-worker",
image="ocadotechnology/aimmo-game-worker:%s" %
os.environ.get("IMAGE_SUFFIX", "latest"),
ports=[
client.V1ContainerPort(container_port=5000, protocol="TCP")
],
resources=client.V1ResourceRequirements(
limits={
"cpu": "10m",
"memory": "64Mi"
},
requests={
"cpu": "6m",
"memory": "32Mi"
},
),
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(drop=["all"],
add=["NET_BIND_SERVICE"])),
)
def _make_container(self, player_id):
return client.V1Container(
env=[
client.V1EnvVar(name='DATA_URL',
value='%s/player/%d' %
(self.game_url, player_id)),
client.V1EnvVar(name='PORT', value='5000')
],
name='aimmo-game-worker',
image='ocadotechnology/aimmo-game-worker:%s' %
os.environ.get('IMAGE_SUFFIX', 'latest'),
ports=[
client.V1ContainerPort(container_port=5000, protocol='TCP')
],
resources=client.V1ResourceRequirements(limits={
'cpu': '10m',
'memory': '64Mi'
},
requests={
'cpu': '6m',
'memory': '32Mi'
}),
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(drop=['all'],
add=['NET_BIND_SERVICE'])))
def create_iperf_deploy(self, name):
self.apps_v1_api.create_namespaced_deployment(
namespace='default',
body=client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(
name=name,
namespace="default",
labels={'app': name}
),
spec=client.V1DeploymentSpec(
replicas=10,
selector=client.V1LabelSelector(match_labels={'app': name}),
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={'app': name}),
spec=client.V1PodSpec(
containers=[
client.V1Container(
name=name,
tty=True,
image="zhuangweikang/k8stc:latest",
image_pull_policy="IfNotPresent",
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(add=["NET_ADMIN"])),
resources=client.V1ResourceRequirements(
limits={"cpu": "100m", "memory": "1Gi"},
requests={"cpu": "100m", "memory": "1Gi"})
)
]
)
)
)
)
)
def create_iperf(self, name='server'):
self.core_v1_api.create_namespaced_pod(
namespace="default",
body=client.V1Pod(
api_version="v1",
kind="Pod",
metadata=client.V1ObjectMeta(
name=name,
namespace="default"
),
spec=client.V1PodSpec(
containers=[
client.V1Container(
name="iperf",
tty=True,
image="zhuangweikang/k8stc:latest",
image_pull_policy="IfNotPresent",
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(add=["NET_ADMIN"])),
resources=client.V1ResourceRequirements(
limits={"cpu": "100m", "memory": "1Gi"},
requests={"cpu": "100m", "memory": "1Gi"})
)
],
restart_policy="Never"
)
), async_req=False)
if not cloud_metadata.get('enabled', False):
# Use iptables to block access to cloud metadata by default
network_tools_image_name = get_config(
'singleuser.network-tools.image.name')
network_tools_image_tag = get_config('singleuser.network-tools.image.tag')
ip_block_container = client.V1Container(
name="block-cloud-metadata",
image=f"{network_tools_image_name}:{network_tools_image_tag}",
command=[
'iptables', '-A', 'OUTPUT', '-d',
cloud_metadata.get('ip', '169.254.169.254'), '-j', 'DROP'
],
security_context=client.V1SecurityContext(
privileged=True,
run_as_user=0,
capabilities=client.V1Capabilities(add=['NET_ADMIN'])))
c.KubeSpawner.singleuser_init_containers.append(ip_block_container)
scheduler_strategy = get_config('singleuser.scheduler-strategy', 'spread')
if scheduler_strategy == 'pack':
# FIXME: Support setting affinity directly in KubeSpawner
c.KubeSpawner.singleuser_extra_pod_config = {
'affinity': {
'podAffinity': {
'preferredDuringSchedulingIgnoredDuringExecution': [{
'weight': 100,
'podAffinityTerm': {
'labelSelector': {
'matchExpressions': [{
def create_deployment_old(config_file):
"""
Create IBM Spectrum Scale CSI Operator deployment object in operator namespace using
deployment_operator_image_for_crd and deployment_driver_image_for_crd parameters from
config.json file
Args:
param1: config_file - configuration json file
Returns:
None
Raises:
Raises an exception on kubernetes client api failure and asserts
"""
deployment_apps_api_instance = client.AppsV1Api()
deployment_labels = {
"app.kubernetes.io/instance": "ibm-spectrum-scale-csi-operator",
"app.kubernetes.io/managed-by": "ibm-spectrum-scale-csi-operator",
"app.kubernetes.io/name": "ibm-spectrum-scale-csi-operator",
"product": "ibm-spectrum-scale-csi",
"release": "ibm-spectrum-scale-csi-operator"
}
deployment_annotations = {
"productID": "ibm-spectrum-scale-csi-operator",
"productName": "IBM Spectrum Scale CSI Operator",
"productVersion": "2.0.0"
}
deployment_metadata = client.V1ObjectMeta(
name="ibm-spectrum-scale-csi-operator",
labels=deployment_labels,
namespace=namespace_value)
deployment_selector = client.V1LabelSelector(
match_labels={
"app.kubernetes.io/name": "ibm-spectrum-scale-csi-operator"
})
podtemplate_metadata = client.V1ObjectMeta(
labels=deployment_labels, annotations=deployment_annotations)
pod_affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity(
required_during_scheduling_ignored_during_execution=client.
V1NodeSelector(node_selector_terms=[
client.V1NodeSelectorTerm(match_expressions=[
client.V1NodeSelectorRequirement(key="beta.kubernetes.io/arch",
operator="Exists")
])
])))
ansible_pod_container = client.V1Container(
image=config_file["deployment_operator_image_for_crd"],
command=[
"/usr/local/bin/ao-logs", "/tmp/ansible-operator/runner", "stdout"
],
liveness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=10,
period_seconds=30),
readiness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=3,
period_seconds=1),
name="ansible",
image_pull_policy="IfNotPresent",
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(drop=["ALL"])),
volume_mounts=[
client.V1VolumeMount(mount_path="/tmp/ansible-operator/runner",
name="runner",
read_only=True)
],
env=[
client.V1EnvVar(
name="CSI_DRIVER_IMAGE",
value=config_file["deployment_driver_image_for_crd"])
])
operator_pod_container = client.V1Container(
image=config_file["deployment_operator_image_for_crd"],
name="operator",
image_pull_policy="IfNotPresent",
liveness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=10,
period_seconds=30),
readiness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=3,
period_seconds=1),
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(drop=["ALL"])),
env=[
client.V1EnvVar(name="WATCH_NAMESPACE",
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path="metadata.namespace"))),
client.V1EnvVar(name="POD_NAME",
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path="metadata.name"))),
client.V1EnvVar(name="OPERATOR_NAME",
value="ibm-spectrum-scale-csi-operator"),
client.V1EnvVar(
name="CSI_DRIVER_IMAGE",
value=config_file["deployment_driver_image_for_crd"])
],
volume_mounts=[
client.V1VolumeMount(mount_path="/tmp/ansible-operator/runner",
name="runner")
])
pod_spec = client.V1PodSpec(
affinity=pod_affinity,
containers=[ansible_pod_container, operator_pod_container],
service_account_name="ibm-spectrum-scale-csi-operator",
volumes=[
client.V1Volume(
empty_dir=client.V1EmptyDirVolumeSource(medium="Memory"),
name="runner")
])
podtemplate_spec = client.V1PodTemplateSpec(metadata=podtemplate_metadata,
spec=pod_spec)
deployment_spec = client.V1DeploymentSpec(replicas=1,
selector=deployment_selector,
template=podtemplate_spec)
body_dep = client.V1Deployment(kind='Deployment',
api_version='apps/v1',
metadata=deployment_metadata,
spec=deployment_spec)
try:
LOGGER.info("creating deployment for operator")
deployment_apps_api_response = deployment_apps_api_instance.create_namespaced_deployment(
namespace=namespace_value, body=body_dep)
LOGGER.debug(str(deployment_apps_api_response))
except ApiException as e:
LOGGER.error(
f"Exception when calling RbacAuthorizationV1Api->create_namespaced_deployment: {e}"
)
assert False
def dispatch(job=None, namespace='default', debug=False, **kwargs):
logger.info(f"starting {job['id']}")
from kubernetes import client, config
config.load_incluster_config()
coreV1 = client.CoreV1Api()
batchV1 = client.BatchV1Api()
batchV1.create_namespaced_job(
namespace=namespace,
body=client.V1Job(
api_version='batch/v1',
kind='Job',
metadata=client.V1ObjectMeta(
name=job['id'],
annotations={
f"container.apparmor.security.beta.kubernetes.io/appyter-{job['id']}":
'unconfined'
},
),
spec=client.V1JobSpec(
template=client.V1PodTemplateSpec(spec=client.V1PodSpec(
restart_policy='Never',
containers=[
client.V1Container(
name=f"appyter-{job['id']}",
image=job['image'],
command=[
'appyter', 'orchestration', 'job',
json.dumps(job)
],
security_context=client.V1SecurityContext(
privileged=True,
capabilities=client.V1Capabilities(
add=['SYS_ADMIN'], ),
),
volume_mounts=[
client.V1VolumeMount(
name='fuse',
mount_path='/dev/fuse',
)
],
),
],
volumes=[
client.V1Volume(
name='fuse',
host_path=client.V1HostPathVolumeSource(
path='/dev/fuse', )),
],
), ),
backoff_limit=1,
),
),
)
#
for event in endless_watch(batchV1.list_namespaced_job,
namespace,
label_selector=f"job-name={job['id']}"):
logger.debug(str(event))
event_type = event['type']
event_job = event['object']
if event_type == 'MODIFIED':
if event_job.status.succeeded or event_job.status.failed:
break
#
if not debug:
batchV1.delete_namespaced_job(job['id'], namespace)
# delete associated pod(s)
for event_pod in coreV1.list_namespaced_pod(
namespace, label_selector=f"job-name={job['id']}",
watch=False).items:
coreV1.delete_namespaced_pod(event_pod.metadata.name, namespace)
#
logger.info(f"{job['id']} completed")
def test_sanitize_k8s_container_attribute(self):
# test cases for implicit type sanitization(conversion)
op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'],
arguments=['echo test | tee /tmp/message.txt'],
file_outputs={'merged': '/tmp/message.txt'})
op.container \
.add_volume_mount(k8s_client.V1VolumeMount(
mount_path='/secret/gcp-credentials',
name='gcp-credentials')) \
.add_env_variable(k8s_client.V1EnvVar(
name=80,
value=80)) \
.add_env_variable(k8s_client.V1EnvVar(
name=80,
value_from=k8s_client.V1EnvVarSource(
config_map_key_ref=k8s_client.V1ConfigMapKeySelector(key=80, name=8080, optional='False'),
field_ref=k8s_client.V1ObjectFieldSelector(api_version=80, field_path=8080),
resource_field_ref=k8s_client.V1ResourceFieldSelector(container_name=80, divisor=8080, resource=8888),
secret_key_ref=k8s_client.V1SecretKeySelector(key=80, name=8080, optional='False')
)
)) \
.add_env_from(k8s_client.V1EnvFromSource(
config_map_ref=k8s_client.V1ConfigMapEnvSource(name=80, optional='True'),
prefix=999
)) \
.add_env_from(k8s_client.V1EnvFromSource(
secret_ref=k8s_client.V1SecretEnvSource(name=80, optional='True'),
prefix=888
)) \
.add_volume_mount(k8s_client.V1VolumeMount(
mount_path=111,
mount_propagation=222,
name=333,
read_only='False',
sub_path=444,
sub_path_expr=555
)) \
.add_volume_devices(k8s_client.V1VolumeDevice(
device_path=111,
name=222
)) \
.add_port(k8s_client.V1ContainerPort(
container_port='8080',
host_ip=111,
host_port='8888',
name=222,
protocol=333
)) \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation='True',
capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]),
privileged='False',
proc_mount=111,
read_only_root_filesystem='False',
run_as_group='222',
run_as_non_root='True',
run_as_user='******',
se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44),
windows_options=k8s_client.V1WindowsSecurityContextOptions(
gmsa_credential_spec=11, gmsa_credential_spec_name=22)
)) \
.set_stdin(stdin='False') \
.set_stdin_once(stdin_once='False') \
.set_termination_message_path(termination_message_path=111) \
.set_tty(tty='False') \
.set_readiness_probe(readiness_probe=k8s_client.V1Probe(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
failure_threshold='111',
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
initial_delay_seconds='222',
period_seconds='333',
success_threshold='444',
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'),
timeout_seconds='777'
)) \
.set_liveness_probe(liveness_probe=k8s_client.V1Probe(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
failure_threshold='111',
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
initial_delay_seconds='222',
period_seconds='333',
success_threshold='444',
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'),
timeout_seconds='777'
)) \
.set_lifecycle(lifecycle=k8s_client.V1Lifecycle(
post_start=k8s_client.V1Handler(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666')
),
pre_stop=k8s_client.V1Handler(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666')
)
))
sanitize_k8s_object(op.container)
for e in op.container.env:
self.assertIsInstance(e.name, str)
if e.value:
self.assertIsInstance(e.value, str)
if e.value_from:
if e.value_from.config_map_key_ref:
self.assertIsInstance(e.value_from.config_map_key_ref.key, str)
if e.value_from.config_map_key_ref.name:
self.assertIsInstance(e.value_from.config_map_key_ref.name, str)
if e.value_from.config_map_key_ref.optional:
self.assertIsInstance(e.value_from.config_map_key_ref.optional, bool)
if e.value_from.field_ref:
self.assertIsInstance(e.value_from.field_ref.field_path, str)
if e.value_from.field_ref.api_version:
self.assertIsInstance(e.value_from.field_ref.api_version, str)
if e.value_from.resource_field_ref:
self.assertIsInstance(e.value_from.resource_field_ref.resource, str)
if e.value_from.resource_field_ref.container_name:
self.assertIsInstance(e.value_from.resource_field_ref.container_name, str)
if e.value_from.resource_field_ref.divisor:
self.assertIsInstance(e.value_from.resource_field_ref.divisor, str)
if e.value_from.secret_key_ref:
self.assertIsInstance(e.value_from.secret_key_ref.key, str)
if e.value_from.secret_key_ref.name:
self.assertIsInstance(e.value_from.secret_key_ref.name, str)
if e.value_from.secret_key_ref.optional:
self.assertIsInstance(e.value_from.secret_key_ref.optional, bool)
for e in op.container.env_from:
if e.prefix:
self.assertIsInstance(e.prefix, str)
if e.config_map_ref:
if e.config_map_ref.name:
self.assertIsInstance(e.config_map_ref.name, str)
if e.config_map_ref.optional:
self.assertIsInstance(e.config_map_ref.optional, bool)
if e.secret_ref:
if e.secret_ref.name:
self.assertIsInstance(e.secret_ref.name, str)
if e.secret_ref.optional:
self.assertIsInstance(e.secret_ref.optional, bool)
for e in op.container.volume_mounts:
if e.mount_path:
self.assertIsInstance(e.mount_path, str)
if e.mount_propagation:
self.assertIsInstance(e.mount_propagation, str)
if e.name:
self.assertIsInstance(e.name, str)
if e.read_only:
self.assertIsInstance(e.read_only, bool)
if e.sub_path:
self.assertIsInstance(e.sub_path, str)
if e.sub_path_expr:
self.assertIsInstance(e.sub_path_expr, str)
for e in op.container.volume_devices:
if e.device_path:
self.assertIsInstance(e.device_path, str)
if e.name:
self.assertIsInstance(e.name, str)
for e in op.container.ports:
if e.container_port:
self.assertIsInstance(e.container_port, int)
if e.host_ip:
self.assertIsInstance(e.host_ip, str)
if e.host_port:
self.assertIsInstance(e.host_port, int)
if e.name:
self.assertIsInstance(e.name, str)
if e.protocol:
self.assertIsInstance(e.protocol, str)
if op.container.security_context:
e = op.container.security_context
if e.allow_privilege_escalation:
self.assertIsInstance(e.allow_privilege_escalation, bool)
if e.capabilities:
for a in e.capabilities.add:
self.assertIsInstance(a, str)
for d in e.capabilities.drop:
self.assertIsInstance(d, str)
if e.privileged:
self.assertIsInstance(e.privileged, bool)
if e.proc_mount:
self.assertIsInstance(e.proc_mount, str)
if e.read_only_root_filesystem:
self.assertIsInstance(e.read_only_root_filesystem, bool)
if e.run_as_group:
self.assertIsInstance(e.run_as_group, int)
if e.run_as_non_root:
self.assertIsInstance(e.run_as_non_root, bool)
if e.run_as_user:
self.assertIsInstance(e.run_as_user, int)
if e.se_linux_options:
if e.se_linux_options.level:
self.assertIsInstance(e.se_linux_options.level, str)
if e.se_linux_options.role:
self.assertIsInstance(e.se_linux_options.role, str)
if e.se_linux_options.type:
self.assertIsInstance(e.se_linux_options.type, str)
if e.se_linux_options.user:
self.assertIsInstance(e.se_linux_options.user, str)
if e.windows_options:
if e.windows_options.gmsa_credential_spec:
self.assertIsInstance(e.windows_options.gmsa_credential_spec, str)
if e.windows_options.gmsa_credential_spec_name:
self.assertIsInstance(e.windows_options.gmsa_credential_spec_name, str)
if op.container.stdin:
self.assertIsInstance(op.container.stdin, bool)
if op.container.stdin_once:
self.assertIsInstance(op.container.stdin_once, bool)
if op.container.termination_message_path:
self.assertIsInstance(op.container.termination_message_path, str)
if op.container.tty:
self.assertIsInstance(op.container.tty, bool)
for e in [op.container.readiness_probe, op.container.liveness_probe]:
if e:
if e._exec:
for c in e._exec.command:
self.assertIsInstance(c, str)
if e.failure_threshold:
self.assertIsInstance(e.failure_threshold, int)
if e.http_get:
if e.http_get.host:
self.assertIsInstance(e.http_get.host, str)
if e.http_get.http_headers:
for h in e.http_get.http_headers:
if h.name:
self.assertIsInstance(h.name, str)
if h.value:
self.assertIsInstance(h.value, str)
if e.http_get.path:
self.assertIsInstance(e.http_get.path, str)
if e.http_get.port:
self.assertIsInstance(e.http_get.port, (str, int))
if e.http_get.scheme:
self.assertIsInstance(e.http_get.scheme, str)
if e.initial_delay_seconds:
self.assertIsInstance(e.initial_delay_seconds, int)
if e.period_seconds:
self.assertIsInstance(e.period_seconds, int)
if e.success_threshold:
self.assertIsInstance(e.success_threshold, int)
if e.tcp_socket:
if e.tcp_socket.host:
self.assertIsInstance(e.tcp_socket.host, str)
if e.tcp_socket.port:
self.assertIsInstance(e.tcp_socket.port, (str, int))
if e.timeout_seconds:
self.assertIsInstance(e.timeout_seconds, int)
if op.container.lifecycle:
for e in [op.container.lifecycle.post_start, op.container.lifecycle.pre_stop]:
if e:
if e._exec:
for c in e._exec.command:
self.assertIsInstance(c, str)
if e.http_get:
if e.http_get.host:
self.assertIsInstance(e.http_get.host, str)
if e.http_get.http_headers:
for h in e.http_get.http_headers:
if h.name:
self.assertIsInstance(h.name, str)
if h.value:
self.assertIsInstance(h.value, str)
if e.http_get.path:
self.assertIsInstance(e.http_get.path, str)
if e.http_get.port:
self.assertIsInstance(e.http_get.port, (str, int))
if e.http_get.scheme:
self.assertIsInstance(e.http_get.scheme, str)
if e.tcp_socket:
if e.tcp_socket.host:
self.assertIsInstance(e.tcp_socket.host, str)
if e.tcp_socket.port:
self.assertIsInstance(e.tcp_socket.port, (str, int))
# test cases for checking value after sanitization
check_value_op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'],
arguments=['echo test | tee /tmp/message.txt'],
file_outputs={'merged': '/tmp/message.txt'})
check_value_op.container \
.add_env_variable(k8s_client.V1EnvVar(
name=80,
value=8080)) \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation='true',
capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]),
privileged='false',
proc_mount=111,
read_only_root_filesystem='False',
run_as_group='222',
run_as_non_root='True',
run_as_user='******',
se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44),
windows_options=k8s_client.V1WindowsSecurityContextOptions(
gmsa_credential_spec=11, gmsa_credential_spec_name=22)
))
sanitize_k8s_object(check_value_op.container)
self.assertEqual(check_value_op.container.env[0].name, '80')
self.assertEqual(check_value_op.container.env[0].value, '8080')
self.assertEqual(check_value_op.container.security_context.allow_privilege_escalation, True)
self.assertEqual(check_value_op.container.security_context.capabilities.add[0], '11')
self.assertEqual(check_value_op.container.security_context.capabilities.add[1], '22')
self.assertEqual(check_value_op.container.security_context.capabilities.drop[0], '33')
self.assertEqual(check_value_op.container.security_context.capabilities.drop[1], '44')
self.assertEqual(check_value_op.container.security_context.privileged, False)
self.assertEqual(check_value_op.container.security_context.proc_mount, '111')
self.assertEqual(check_value_op.container.security_context.read_only_root_filesystem, False)
self.assertEqual(check_value_op.container.security_context.run_as_group, 222)
self.assertEqual(check_value_op.container.security_context.run_as_non_root, True)
self.assertEqual(check_value_op.container.security_context.run_as_user, 333)
self.assertEqual(check_value_op.container.security_context.se_linux_options.level, '11')
self.assertEqual(check_value_op.container.security_context.se_linux_options.role, '22')
self.assertEqual(check_value_op.container.security_context.se_linux_options.type, '33')
self.assertEqual(check_value_op.container.security_context.se_linux_options.user, '44')
self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec, '11')
self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec_name, '22')
# test cases for exception
with self.assertRaises(ValueError, msg='Invalid boolean string 2. Should be boolean.'):
exception_op = dsl.ContainerOp(name='echo', image='image')
exception_op.container \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation=1
))
sanitize_k8s_object(exception_op.container)
with self.assertRaises(ValueError, msg='Invalid boolean string Test. Should be "true" or "false".'):
exception_op = dsl.ContainerOp(name='echo', image='image')
exception_op.container \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation='Test'
))
sanitize_k8s_object(exception_op.container)
with self.assertRaises(ValueError, msg='Invalid test. Should be integer.'):
exception_op = dsl.ContainerOp(name='echo', image='image')
exception_op.container \
.set_security_context(k8s_client.V1SecurityContext(
run_as_group='test',
))
sanitize_k8s_object(exception_op.container)
ip_block_container = client.V1Container(
name="block-cloud-metadata",
image=f"{network_tools_image_name}:{network_tools_image_tag}",
command=[
"iptables",
"-A",
"OUTPUT",
"-d",
cloud_metadata.get("ip", "169.254.169.254"),
"-j",
"DROP",
],
security_context=client.V1SecurityContext(
privileged=True,
run_as_user=0,
capabilities=client.V1Capabilities(add=["NET_ADMIN"]),
),
resources=network_tools_resources,
)
c.KubeSpawner.init_containers.append(ip_block_container)
if get_config("debug.enabled", False):
c.JupyterHub.log_level = "DEBUG"
c.Spawner.debug = True
# load potentially seeded secrets
#
# NOTE: ConfigurableHTTPProxy.auth_token is set through an environment variable
# that is set using the chart managed secret.
c.JupyterHub.cookie_secret = get_secret_value(
def deployment_object(self, instance_uuid, cnf_yaml, service_uuid, vim_uuid):
"""
CNF modeling method. This build a deployment object in kubernetes
instance_uuid: k8s deployment name
cnf_yaml: CNF Descriptor in yaml format
"""
t0 = time.time()
LOG.debug("CNFD: {}".format(cnf_yaml))
container_list = []
pod_volume_list = []
deployment_k8s = None
privileged = False
node_selector = {}
host_network = False
if "cloudnative_deployment_units" in cnf_yaml:
cdu = cnf_yaml.get('cloudnative_deployment_units')
for cdu_obj in cdu:
env_vars = env_from = cpu = memory = huge_pages = gpu = sr_iov = resources = volume_mounts = None
port_list = []
environment = []
capabilities_list = []
cdu_id = cdu_obj.get('id')
image = cdu_obj.get('image')
cdu_conex = cdu_obj.get('connection_points')
container_name = cdu_id
config_map_id = cdu_id
if cdu_obj.get('parameters'):
env_vars = cdu_obj['parameters'].get('env')
volume_mounts = cdu_obj['parameters'].get('volume_mounts')
capabilities_list = cdu_obj['parameters'].get('capabilities')
if cdu_obj['parameters'].get('privileged'):
privileged = cdu_obj['parameters'].get('privileged')
if cdu_obj['parameters'].get('host_network'):
privileged = cdu_obj['parameters']['host_network']
if not isinstance(capabilities_list, list):
capabilities_list = []
if cdu_obj['parameters'].get('node_selector'):
node_selector = cdu_obj['parameters']['node_selector']
if cdu_obj.get('resource_requirements'):
gpu = cdu_obj['resource_requirements'].get('gpu')
cpu = cdu_obj['resource_requirements'].get('cpu')
memory = cdu_obj['resource_requirements'].get('memory')
sr_iov = cdu_obj['resource_requirements'].get('sr-iov')
huge_pages = cdu_obj['resource_requirements'].get('huge-pages')
if cdu_conex:
for po in cdu_conex:
port = po.get('port')
port_name = po.get('id')
protocol = "TCP"
if po.get("protocol"):
protocol = po["protocol"]
port_list.append(client.V1ContainerPort(container_port = port, name = port_name, protocol=protocol))
limits = {}
requests = {}
if gpu:
LOG.debug("Features requested: {}".format(gpu))
# gpu_type can be amd or nvidia
for gpu_type, amount in gpu.items():
limits["{}.com/gpu".format(gpu_type)] = amount
if cpu:
# TODO
pass
if memory:
# TODO
pass
if sr_iov:
# TODO
pass
if huge_pages:
# TODO
pass
resources = client.V1ResourceRequirements(limits=limits, requests=requests)
# Environment variables from descriptor
if env_vars:
LOG.debug("Configmap: {}".format(config_map_id))
KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid,
vim_uuid, namespace = "default")
else:
env_vars = {"sonata": "rules"}
LOG.debug("Configmap: {}".format(config_map_id))
KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid,
vim_uuid, namespace = "default")
env_from = client.V1EnvFromSource(config_map_ref = client.V1ConfigMapEnvSource(name = config_map_id,
optional = False))
# Default static environment variables
environment.append(client.V1EnvVar(name="instance_uuid", value=instance_uuid))
environment.append(client.V1EnvVar(name="service_uuid", value=service_uuid))
environment.append(client.V1EnvVar(name="container_name", value=container_name))
environment.append(client.V1EnvVar(name="vendor", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('vendor'))))
environment.append(client.V1EnvVar(name="name", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('name'))))
environment.append(client.V1EnvVar(name="version", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('version'))))
image_pull_policy = KubernetesWrapperEngine.check_connection(self)
# Volume mounts
container_volume_mount_list = []
if volume_mounts:
LOG.debug("volume mounts: {}".format(volume_mounts))
# Create the specification of volumes
for volume_mounts_item in volume_mounts:
if volume_mounts_item.get('id') and volume_mounts_item.get('location'):
if volume_mounts_item.get('persistent'):
volumes = client.V1Volume(name=volume_mounts_item['id'],
host_path=client.V1HostPathVolumeSource(path='/mnt/data', type='DirectoryOrCreate' ))
else:
volumes = client.V1Volume(name=volume_mounts_item['id'],
empty_dir=client.V1EmptyDirVolumeSource(medium='' ))
if volumes not in pod_volume_list:
pod_volume_list.append(volumes)
container_volume_mount = client.V1VolumeMount(name=volume_mounts_item['id'], mount_path=volume_mounts_item['location'] )
container_volume_mount_list.append(container_volume_mount)
LOG.debug("Security capabilities: {}, privileged: {} applied to {}".format(capabilities_list, privileged, container_name))
sec_context = client.V1SecurityContext(privileged=privileged, capabilities=client.V1Capabilities(add=capabilities_list))
# Configureate Pod template container
container = client.V1Container(
env = environment,
name = container_name,
resources = resources,
image = image,
image_pull_policy = image_pull_policy,
ports = port_list,
env_from = [env_from],
volume_mounts = container_volume_mount_list,
security_context=sec_context)
container_list.append(container)
else:
return deployment_k8s
# Create and configurate a spec section
deployment_label = ("{}-{}-{}-{}".format(cnf_yaml.get("vendor"), cnf_yaml.get("name"), cnf_yaml.get("version"),
instance_uuid.split("-")[0])).replace(".", "-")
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={'deployment': deployment_label,
'instance_uuid': cnf_yaml['instance_uuid'],
'service_uuid': service_uuid,
'sp': "sonata",
'descriptor_uuid': cnf_yaml['uuid']}
),
spec=client.V1PodSpec(containers=container_list, volumes=pod_volume_list, node_selector=node_selector,
host_network=host_network))
selector=client.V1LabelSelector(match_labels={'deployment': deployment_label,
'instance_uuid': cnf_yaml['instance_uuid'],
'service_uuid': service_uuid,
'sp': "sonata",
'descriptor_uuid': cnf_yaml['uuid']}
)
# Create the specification of deployment
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector=selector)
# Instantiate the deployment object
deployment_k8s = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=deployment_label,
labels={'deployment': deployment_label,
'instance_uuid': cnf_yaml['instance_uuid'],
'service_uuid': service_uuid,
'sp': "sonata",
'descriptor_uuid': cnf_yaml['uuid']}
),
spec=spec)
LOG.info("Deployment object: {}".format(deployment_k8s))
LOG.info("CreatingDeploymentObject-time: {} ms".format(int((time.time() - t0)* 1000)))
return deployment_k8s
def create_job(self, image, command, name, resources_dict, mountdir=None):
"""
Create and return a new job instance.
"""
number_of_workers = resources_dict.get('number_of_workers')
cpu_limit = str(resources_dict.get('cpu_limit')) + 'm'
memory_limit = str(resources_dict.get('memory_limit')) + 'Mi'
gpu_limit = resources_dict.get('gpu_limit')
# configure pod's containers
requests = {'memory': '150Mi', 'cpu': '250m'}
limits = {'memory': memory_limit, 'cpu': cpu_limit}
env = []
if gpu_limit > 0:
# ref: https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/
limits['nvidia.com/gpu'] = gpu_limit
env = [
k_client.V1EnvVar(name='NVIDIA_VISIBLE_DEVICES', value='all'),
k_client.V1EnvVar(name='NVIDIA_DRIVER_CAPABILITIES',
value='compute,utility'),
k_client.V1EnvVar(name='NVIDIA_REQUIRE_CUDA',
value='cuda>=9.0')
],
container = k_client.V1Container(
name=name,
image=image,
env=env,
command=command.split(' '),
security_context=k_client.V1SecurityContext(
allow_privilege_escalation=False,
capabilities=k_client.V1Capabilities(drop=['ALL'])),
resources=k_client.V1ResourceRequirements(limits=limits,
requests=requests),
volume_mounts=[
k_client.V1VolumeMount(mount_path='/share',
name='shared-volume')
])
# configure pod template's spec
volumes = []
if self.config.get('STORAGE_TYPE') == 'host':
volumes = [
k_client.V1Volume(
name='shared-volume',
host_path=k_client.V1HostPathVolumeSource(path=mountdir))
]
template = k_client.V1PodTemplateSpec(spec=k_client.V1PodSpec(
restart_policy='Never', containers=[container], volumes=volumes))
# configure job's spec
spec = k_client.V1JobSpec(
parallelism=number_of_workers,
completions=number_of_workers,
backoff_limit=1,
ttl_seconds_after_finished=86400, # 24h
active_deadline_seconds=43200, # 12h
template=template)
# instantiate the job object
job = k_client.V1Job(api_version='batch/v1',
kind='Job',
metadata=k_client.V1ObjectMeta(name=name),
spec=spec)
return job
def create_job(MODEL):
assert MODEL is not None, "model name is None, cannot spawn a new worker"
api = client.BatchV1Api()
body = client.V1Job(api_version="batch/v1", kind="Job")
name = 'speechlab-worker-job-{}-{}'.format(MODEL.lower().replace("_", "-"),
id_generator())
body.metadata = client.V1ObjectMeta(namespace=NAMESPACE, name=name)
body.status = client.V1JobStatus()
template = client.V1PodTemplate()
template.template = client.V1PodTemplateSpec()
template.template.metadata = client.V1ObjectMeta(
annotations={
"prometheus.io/scrape": "true",
"prometheus.io/port": "8081"
})
azure_file_volume = client.V1AzureFileVolumeSource(
read_only=True,
secret_name=MODELS_FILESHARE_SECRET,
share_name=MODELS_SHARE_NAME)
volume = client.V1Volume(name="models-azurefiles",
azure_file=azure_file_volume)
env_vars = {
"AZURE_STORAGE_ACCOUNT": AZURE_STORAGE_ACCOUNT,
"AZURE_STORAGE_ACCESS_KEY": AZURE_STORAGE_ACCESS_KEY,
"AZURE_CONTAINER": AZURE_CONTAINER,
"MASTER": MASTER,
"NAMESPACE": NAMESPACE,
"RUN_FREQ": "ONCE",
"MODEL_DIR": MODEL, # important
"MODELS_FILESHARE_SECRET": MODELS_FILESHARE_SECRET,
"MODELS_SHARE_NAME": MODELS_SHARE_NAME
}
env_list = []
if env_vars:
for env_name, env_value in env_vars.items():
env_list.append(client.V1EnvVar(name=env_name, value=env_value))
container = client.V1Container(
name='{}-c'.format(name),
image=IMAGE,
image_pull_policy="IfNotPresent",
command=[
"/home/appuser/opt/tini", "--", "/home/appuser/opt/start_worker.sh"
],
env=env_list,
ports=[client.V1ContainerPort(container_port=8081, name="prometheus")],
security_context=client.V1SecurityContext(
privileged=True,
capabilities=client.V1Capabilities(add=["SYS_ADMIN"])),
resources=client.V1ResourceRequirements(limits={
"memory": "5G",
"cpu": "1"
},
requests={
"memory": "5G",
"cpu": "1"
}),
volume_mounts=[
client.V1VolumeMount(mount_path="/home/appuser/opt/models",
name="models-azurefiles",
read_only=True)
])
template.template.spec = client.V1PodSpec(
containers=[container],
image_pull_secrets=[{
"name": "azure-cr-secret"
}],
# reason to use OnFailure https://github.com/kubernetes/kubernetes/issues/20255
restart_policy="OnFailure",
volumes=[volume])
# And finaly we can create our V1JobSpec!
body.spec = client.V1JobSpec(ttl_seconds_after_finished=100,
template=template.template)
try:
api_response = api.create_namespaced_job(NAMESPACE, body)
print("api_response=" + str(api_response))
return True
except ApiException as e:
logging.exception('error spawning new job')
print("Exception when creating a job: %s\n" % e)
