def test_resource_quota(self): init_test_env(NS) run(f'kubectl create -f quota-cpu-memory.yaml -n {NS}') run(f'kubectl describe quota -n {NS}') # when creating a ResourceQuota is that you will also want to create a LimitRange object alongside it. stdout = run(f'kubectl create -f kubia-manual.yaml -n {NS} 2>&1; true') self.assertIn('must specify limits.cpu,limits.memory,requests.cpu,requests.memory', stdout) run(f'kubectl create -f limits.yaml -n {NS}') run(f'kubectl create -f kubia-manual.yaml -n {NS}') # So having a LimitRange with defaults for those resources can make life a bit easier for people creating pods. ensure_pod_phase('kubia-manual', 'Running', NS)
def test_pre_stop_hook(self): init_test_env(NS) with self.subTest("Using an command hook handler"): run(f'kubectl create -f pre-stop-hook-command.yaml -n {NS}') ensure_pod_phase('pod-with-prestop-hook', 'Running', NS) time.sleep(60) run(f'kubectl describe pod pod-with-prestop-hook -n {NS}') run(f'kubectl get pod pod-with-prestop-hook -n {NS}') with self.subTest("Using an HTTP GET hook handler"): run(f'kubectl create -f pre-stop-hook-httpget.yaml -n {NS}') ensure_pod_phase('pod-with-prestop-http-hook', 'Running', NS) time.sleep(60) run(f'kubectl describe pod pod-with-prestop-http-hook -n {NS}') run(f'kubectl get pod pod-with-prestop-http-hook -n {NS}')
def test_auto_scaling_down_with_pod_disruption_budget(self): init_test_env(NS) run(f"kubectl create pdb kubia-pdb --selector=app=kubia --min-available=2 -n {NS}" ) run(f"kubectl get pdb kubia-pdb -o yaml -n {NS}") # create deployment run(f'kubectl create -f deployment.yaml -n {NS}') ensure_deploy_ready('kubia', NS) ensure_replicas('kubia', 3, 'deploy', NS) # create hpa run(f'kubectl autoscale deployment kubia --cpu-percent=30 --min=1 --max=5 -n {NS}' ) # pdb has nothing to do with hpa ensure_replicas('kubia', 1, 'deploy', NS)
def test_node_affinity(self): ''' Run this testcase under EKS with at least 2 worker nodes ''' init_test_env(NS) nodes = run( f"kubectl get node -o jsonpath='{{.items[*].metadata.name}}'", True).split() for node in nodes: run(f'kubectl label node {node} gpu-', True) # delete label first run(f'kubectl label node {node} availability-zone-', True) run(f'kubectl label node {node} share-type-', True) with self.subTest("Specifying hard node affinity rules"): node = random.choice(nodes) run(f'kubectl label node {node} gpu=true') run(f'kubectl create -f kubia-gpu-nodeaffinity.yaml -n {NS}') ensure_pod_phase('kubia-gpu', 'Running', NS) stdout = run(f'kubectl get pod kubia-gpu -o wide -n {NS}') self.assertIn(node, stdout) with self.subTest("Prioritizing nodes when scheduling a pod"): node1 = nodes[0] node2 = nodes[1] run(f'kubectl label node {node1} availability-zone=zone1', True) run(f'kubectl label node {node1} share-type=dedicated', True) run(f'kubectl label node {node2} availability-zone=zone2', True) run(f'kubectl label node {node2} share-type=shared', True) run(f'kubectl get node -L availability-zone -L share-type') run(f'kubectl create -f preferred-deployment.yaml -n {NS}') ensure_deploy_ready('pref', NS) # Nodes whose 'availability-zone' and 'share-type' labels match the pod's node affinity are ranked the highest. # Next come the 'shared' nodes in 'zone1', then come the 'dedicated' nodes in the other zones, # and at the lowest priority are all the other nodes. run(f'kubectl get po -l app=pref -o wide -n {NS}') node1_num = int( run( f'kubectl get po -l app=pref -o wide -n {NS} | grep {node1} | wc -l', True)) node2_num = int( run( f'kubectl get po -l app=pref -o wide -n {NS} | grep {node2} | wc -l', True)) self.assertGreater(node1_num, node2_num)
def test_creating_pod_with_resource_requested(self): init_test_env(NS) run(f'kubectl create -f requests-pod.yaml -n {NS}') ensure_pod_phase('requests-pod', 'Running', NS) # The Minikube VM, which is where this example is running, has two CPU cores allotted to it. # That's why the process is shown consuming 50% of the whole CPU. run(f'kubectl exec requests-pod -n {NS} -- top -bn1') with self.subTest("Creating a pod that doesn't fit on any node"): run(f"kubectl run requests-pod-2 --image=busybox --restart Never --requests='cpu=800m,memory=20Mi' -n {NS} -- dd if=/dev/zero of=/dev/null") ensure_pod_phase('requests-pod-2', 'Running', NS) time.sleep(5) run(f"kubectl run requests-pod-3 --image=busybox --restart Never --requests='cpu=1,memory=20Mi' -n {NS} -- dd if=/dev/zero of=/dev/null") ensure_pod_phase('requests-pod-3', 'Pending', NS) stdout = run(f"kubectl get po requests-pod-3 -n {NS} -o jsonpath='{{.status.conditions[0].message}}'") self.assertIn("Insufficient cpu", stdout) run(f'kubectl delete po requests-pod-2 -n {NS}') ensure_pod_phase('requests-pod-2', 'Deleted', NS) ensure_pod_phase('requests-pod-3', 'Running', NS)
def test_auto_scaling_based_on_cpu(self): init_test_env(NS) # create deployment first run(f'kubectl create -f deployment.yaml -n {NS}') ensure_deploy_ready('kubia', NS) ensure_replicas('kubia', 3, 'deploy', NS) # create HPA (have to enable Heapster and metrics-server:) # minikube addons enable heapster # minikube addons enable metrics-server run(f'kubectl autoscale deployment kubia --cpu-percent=30 --min=1 --max=5 -n {NS}' ) # show HPA run(f'kubectl get hpa kubia -o yaml -n {NS}') # Because you're running three pods that are currently receiving no requests, # which means their CPU usage should be close to zero, # you should expect the Autoscaler to scale them down to a single pod. ensure_replicas('kubia', 1, 'deploy', NS) # expose pod to service run(f'kubectl expose deployment kubia --port=80 --target-port=8080 -n {NS}' ) # create loadgenerator pod run(f'kubectl run --restart=Never loadgenerator --image=busybox -n {NS} -- sh -c "while true; do wget -O - -q http://kubia.{NS}; done"' ) ensure_pod_phase('loadgenerator', 'Running', NS) # autoscaler increase the number of replicas. ensure_replicas('kubia', 4, 'deploy', NS) run(f'kubectl top pod -n {NS}') run(f'kubectl get hpa -n {NS}') cpu_util_percent = int( run(f"kubectl get hpa kubia -n {NS} -o jsonpath='{{.status.currentCPUUtilizationPercentage}}'" )) self.assertLessEqual(cpu_util_percent / 4, 30)
def test_limit_range(self): init_test_env(NS) with self.subTest("Enforcing limits"): run(f'kubectl create -f limits.yaml -n {NS}') stdout = run(f'kubectl create -f limits-pod-too-big.yaml -n {NS} 2>&1; true') self.assertIn('must be less than or equal to cpu limit', stdout) with self.subTest("Applying default resource requests and limits"): run(f'kubectl create -f kubia-manual.yaml -n {NS}') ensure_pod_phase('kubia-manual', 'Running', NS) default_cpu_request = run(f"kubectl get po kubia-manual -n {NS} -o jsonpath='{{.spec.containers[0].resources.requests.cpu}}'") default_cpu_limit = run(f"kubectl get po kubia-manual -n {NS} -o jsonpath='{{.spec.containers[0].resources.limits.cpu}}'") default_mem_request = run(f"kubectl get po kubia-manual -n {NS} -o jsonpath='{{.spec.containers[0].resources.requests.memory}}'") default_mem_limit = run(f"kubectl get po kubia-manual -n {NS} -o jsonpath='{{.spec.containers[0].resources.limits.memory}}'") with open('limits.yaml', 'rb') as fp: definition = yaml.load(fp, Loader=yaml.Loader) container_limits = [limit for limit in definition['spec']['limits'] if limit['type'] == 'Container'][0] self.assertEqual(default_cpu_request, container_limits['defaultRequest']['cpu']) self.assertEqual(default_cpu_limit, container_limits['default']['cpu']) self.assertEqual(default_mem_request, container_limits['defaultRequest']['memory']) self.assertEqual(default_mem_limit, container_limits['default']['memory'])
def test_pod_affinity(self): ''' Run this testcase under EKS with at least 2 worker nodes ''' init_test_env(NS) # First, deploy the backend pod run(f'kubectl run backend -l app=backend --image busybox -n {NS} -- sleep 999999' ) ensure_deploy_ready('backend', NS) with self.subTest( "Using inter-pod affinity to deploy pods on the same node"): ''' Deploy a backend pod and five frontend pod replicas with pod affinity configured so that they're all deployed on the same node as the backend pod. ''' run(f'kubectl create -f frontend-podaffinity-host.yaml -n {NS}') ensure_deploy_ready('frontend', NS) run(f'kubectl get po -o wide -n {NS}') backend_node = run( f"kubectl get po -l app=backend -n {NS} -o jsonpath='{{.items[*].spec.nodeName}}'", True) frontend_nodes = run( f"kubectl get po -l app=frontend -n {NS} -o jsonpath='{{.items[*].spec.nodeName}}'", True).split() self.assertEqual(len(set(frontend_nodes)), 1) self.assertEqual(frontend_nodes[0], backend_node) # If you now delete the backend pod, the Scheduler will schedule the pod to origin node. # You can confirm the Scheduler takes other pods' pod affinity rules INTO ACCOUNT. old_backend_pod = run( f"kubectl get po -l app=backend -n {NS} -o jsonpath='{{.items[*].metadata.name}}'", True) run(f"kubectl delete pod -l app=backend -n {NS}") ensure_pod_phase(old_backend_pod, 'Deleted', NS) new_backend_node = run( f"kubectl get po -l app=backend -n {NS} -o jsonpath='{{.items[*].spec.nodeName}}'", True) self.assertEqual(backend_node, new_backend_node) with self.subTest("Expressing pod affinity preferences"): run(f'kubectl create -f frontend-podaffinity-preferred-host.yaml -n {NS}' ) ensure_deploy_ready('frontend-pref', NS) run(f"kubectl get pod -l app=frontend-pref -o wide -n {NS}") frontend_pref_nodes = run( f"kubectl get po -l app=frontend-pref -n {NS} -o jsonpath='{{.items[*].spec.nodeName}}'", True).split() backend_node_num = frontend_pref_nodes.count(backend_node) other_node_num = len(frontend_pref_nodes) - backend_node_num # Scheduler will prefer backend_node for frontend-pref pods, # but may schedule pods to other nodes as well. self.assertGreater(backend_node_num, other_node_num) with self.subTest( "Using anti-affinity to spread apart pods of the same deployment" ): init_test_env(NS) # remove test history first run(f'kubectl create -f frontend-podantiaffinity-host.yaml -n {NS}' ) ensure_replicas('frontend-anti', 3, 'deploy', NS) running_pods = run( f"""kubectl get pod -n {NS} -o jsonpath='{{.items[?(@.status.phase=="Running")].metadata.name}}'""", True).split() pending_pods = run( f"""kubectl get pod -n {NS} -o jsonpath='{{.items[?(@.status.phase=="Pending")].metadata.name}}'""", True).split() run(f'kubectl get pod -n {NS} -o wide') node_num = len( run( f"kubectl get node -o jsonpath='{{.items[*].metadata.name}}'", True).split()) # Every node has only one pod, remaining pods are all Pending, # because the Scheduler isn't allowed to schedule them to the same nodes. self.assertEqual(len(running_pods), node_num) self.assertEqual(len(pending_pods), 5 - node_num)
def test_configuring_container_security_context(self): init_test_env(NAME_SPACE) run(f'kubectl run pod-with-defaults --image alpine --restart Never --namespace {NAME_SPACE} -- /bin/sleep 999999' ) ensure_pod_phase("pod-with-defaults", 'Running', NAME_SPACE) with self.subTest("Running a container as a specific user"): run(f'kubectl create -f pod-as-user-guest.yaml -n {NAME_SPACE}') ensure_pod_phase('pod-as-user-guest', 'Running', NAME_SPACE) stdout = run( f'kubectl exec pod-as-user-guest -n {NAME_SPACE} -- id -u') self.assertEqual(stdout, '405') with self.subTest("Running container in privileged mode"): run(f'kubectl create -f pod-privileged.yaml -n {NAME_SPACE}') ensure_pod_phase('pod-privileged', 'Running', NAME_SPACE) stdout1 = run( f'kubectl exec pod-with-defaults -n {NAME_SPACE} -- ls /dev') stdout2 = run( f'kubectl exec pod-privileged -n {NAME_SPACE} -- ls /dev') # the privileged container sees all the host node's devices. This means it can use any device freely. self.assertGreater(len(stdout2.split()), len(stdout1.split())) with self.subTest( "Adding individual kernel capabilities to a container"): stdout = run( f'kubectl exec pod-with-defaults -n {NAME_SPACE} -- date -s "12:00:00" 2>&1' ) self.assertIn("can't", stdout) run(f'kubectl create -f pod-add-settime-capability.yaml -n {NAME_SPACE}' ) ensure_pod_phase('pod-add-settime-capability', 'Running', NAME_SPACE) run(f'kubectl exec pod-add-settime-capability -n {NAME_SPACE} -- date +%T -s "12:00:00"' ) stdout = run( f'kubectl exec pod-add-settime-capability -n {NAME_SPACE} -- date +%T' ) self.assertIn('12:00:0', stdout) run( 'minikube ssh date' ) # node date will be changed, but it can be changed back quickly due to NTP. with self.subTest('Dropping capabilities from a container'): run(f'kubectl create -f pod-drop-chown-capability.yaml -n {NAME_SPACE}' ) ensure_pod_phase('pod-drop-chown-capability', 'Running', NAME_SPACE) stdout = run( f'kubectl exec pod-drop-chown-capability -n {NAME_SPACE} -- chown guest /tmp 2>&1', True) self.assertIn('Operation not permitted', stdout) with self.subTest( "Preventing processes from writing to the container's filesystem" ): run(f'kubectl create -f pod-with-readonly-filesystem.yaml -n {NAME_SPACE}' ) ensure_pod_phase('pod-with-readonly-filesystem', 'Running', NAME_SPACE) stdout = run( f'kubectl exec pod-with-readonly-filesystem -n {NAME_SPACE} -- touch /newfile 2>&1', True) self.assertIn('Read-only', stdout) run(f'kubectl exec pod-with-readonly-filesystem -n {NAME_SPACE} -- touch /volume/newfile' ) stdout = run( f'kubectl exec pod-with-readonly-filesystem -n {NAME_SPACE} -- ls -l /volume' ) self.assertIn('newfile', stdout) with self.subTest( "Sharing volumes when containers run as different users"): run(f'kubectl create -f pod-with-shared-volume-fsgroup.yaml -n {NAME_SPACE}' ) ensure_pod_phase('pod-with-shared-volume-fsgroup', 'Running', NAME_SPACE) run(f'kubectl exec pod-with-shared-volume-fsgroup -n {NAME_SPACE} -c first -- id' ) stdout = run( f'kubectl exec pod-with-shared-volume-fsgroup -n {NAME_SPACE} -c first -- id -G', True) self.assertEqual(stdout.split(), ['0', '555', '666', '777']) stdout = run( f'kubectl exec pod-with-shared-volume-fsgroup -n {NAME_SPACE} -c first -- ls -l / | grep volume' ) self.assertIn('555', stdout) # fsGroup security context property is used when the process cre- ates files IN A VOLUME run(f'kubectl exec pod-with-shared-volume-fsgroup -n {NAME_SPACE} -c first -- touch /tmp/foo' ) stdout = run( f'kubectl exec pod-with-shared-volume-fsgroup -n {NAME_SPACE} -c first -- ls -l /tmp/' ) self.assertNotIn('555', stdout)