def test_get_pod_statuses_single(self, kube): # Mocking kube api - https://github.com/kubernetes-client/python/blob/9b438eed5a4fdab4377515b5a0c62d695dffc354/kubernetes/docs/V1PodList.md kube.core_api.list_namespaced_pod = MagicMock(return_value=V1PodList( api_version='v1', items=[ V1Pod(api_version='v1', kind='Pod', metadata=V1ObjectMeta(name="pod_1"), spec=None, status=V1PodStatus(phase='failed')), V1Pod(api_version='v1', kind='Pod', metadata=V1ObjectMeta(name="pod_2"), spec=None, status=V1PodStatus(phase='pending')), V1Pod(api_version='v1', kind='Pod', metadata=V1ObjectMeta(name="pod_3"), spec=None, status=V1PodStatus(phase='running')) ])) actual = KubeUtil.get_status_of_all_pods(kube, label_selector="label=bar") self.assertEquals(actual, [{ 'pod': 'pod_1', 'phase': 'failed' }, { 'pod': 'pod_2', 'phase': 'pending' }, { 'pod': 'pod_3', 'phase': 'running' }])
def test_job_logs_multiple_pods(self, mock_core_client): namespace = "treesbecomelogs" manager = JobManager( namespace=namespace, signer=Mock(), register=StaticJobDefinitionsRegister() ) job_name = "ahoymatey" pod_name_1, pod_name_2 = "p1", "p2" container_name = "c1" mock_core_client.list_namespaced_pod.return_value.items = [ V1Pod( metadata=V1ObjectMeta(name=pod_name_1), spec=V1PodSpec(containers=[V1Container(name=container_name)]), ), V1Pod( metadata=V1ObjectMeta(name=pod_name_2), spec=V1PodSpec(containers=[V1Container(name=container_name)]), ), ] log_msg = "this is a log" mock_core_client.read_namespaced_pod_log.return_value = log_msg logs = manager.job_logs(job_name) assert logs == { pod_name_1: {container_name: [log_msg]}, pod_name_2: {container_name: [log_msg]}, }
def test_failed_at_benchmark_container_but_sidecar_still_running(): status = V1PodStatus( phase="Running", container_statuses=[ V1ContainerStatus( image="benchmarkai/hello-world", name="benchmark", image_id="", ready=False, restart_count=0, state=V1ContainerState( terminated=CONTAINER_STATE_TERMINATED_AND_FAILED), ), V1ContainerStatus( image="benchmarkai/metrics-pusher", name="sidecar", image_id="", ready=True, restart_count=0, state=V1ContainerState(running=V1ContainerStateRunning()), ), ], ) pod = V1Pod(metadata=V1ObjectMeta(name="pod-name"), status=status) inferrer = SingleNodeStrategyKubernetesStatusInferrer( V1JobStatus(active=1), pods=[pod]) # TODO: Not sure if this is the status we want assert inferrer.status() == BenchmarkJobStatus.RUNNING_AT_MAIN_CONTAINERS
def exec_python(kube_ns, kube_client): """Return a callable to execute Python code in a pod in the test namespace This fixture creates a dedicated pod for executing commands """ # note: this was created when there were only single-user pods running, # but now there's always a hub pod where we could be running, # and the ssl case *must* run from the hub pod for access to certs # Note: we could do without this feature if we always ran pod_name = "kubespawner-test-exec" pod_manifest = V1Pod( metadata={"name": pod_name}, spec=V1PodSpec( containers=[ { "image": "python:3.8", "name": "python", "args": ["/bin/sh", "-c", "while true; do sleep 5; done"], } ], termination_grace_period_seconds=0, ), ) pod = create_resource(kube_client, kube_ns, "pod", pod_manifest) yield partial(_exec_python_in_pod, kube_client, kube_ns, pod_name)
def test_job_logs_not_ready(self, mock_core_client): namespace = "notready" manager = JobManager( namespace=namespace, signer=Mock(), register=StaticJobDefinitionsRegister() ) pod_name = "p" container_name = "c" mock_core_client.list_namespaced_pod.return_value.items = [ V1Pod( metadata=V1ObjectMeta(name=pod_name), spec=V1PodSpec(containers=[V1Container(name=container_name)]), ) ] mock_core_client.read_namespaced_pod_log.side_effect = ApiException( http_resp=Mock( data={ "message": f'container "{container_name}" in pod "{pod_name}" is waiting to start: ContainerCreating' } ) ) # No exception logs = manager.job_logs("whatever") assert logs == {pod_name: {container_name: ["ContainerCreating"]}}
def pod_with_preferred_affinity(): return V1Pod( status=V1PodStatus(phase='Pending', conditions=[ V1PodCondition(status='False', type='PodScheduled', reason='Unschedulable') ]), spec=V1PodSpec( containers=[ V1Container( name='container', resources=V1ResourceRequirements(requests={'cpu': '1.5'})) ], affinity=V1Affinity(node_affinity=V1NodeAffinity( required_during_scheduling_ignored_during_execution= V1NodeSelector(node_selector_terms=[ V1NodeSelectorTerm(match_expressions=[ V1NodeSelectorRequirement( key='clusterman.com/scheduler', operator='Exists') ]) ]), preferred_during_scheduling_ignored_during_execution=[ V1PreferredSchedulingTerm( weight=10, preference=V1NodeSelectorTerm(match_expressions=[ V1NodeSelectorRequirement( key='clusterman.com/pool', operator='In', values=['bar']) ])) ]))))
def test_list(self, batch_p, get_p): batch_p.return_value = V1JobList( items=[V1Pod(metadata=V1ObjectMeta(name='drwho-cate-sdav'))]) get_p.return_value = { 'cubegen_id': 'id', 'status': 'Ready', 'output': ['bla'], 'progress': 100 } res = cubegens.list('drwho') self.assertEqual(1, len(res)) self.assertDictEqual( { 'cubegen_id': 'id', 'status': 'Ready', 'output': ['bla'], 'progress': 100 }, res[0]) batch_p.side_effect = ApiException(401, 'Unauthorized') with self.assertRaises(api.ApiError) as e: cubegens.list(user_id='drwho') self.assertIn("Reason: Unauthorized", str(e.exception)) self.assertIn("401", str(e.exception)) self.assertEqual(400, e.exception.status_code)
def test_list_pods(self, list_all_p, list_p): pod = V1Pod(metadata=V1ObjectMeta(name='test')) list_p.return_value = V1PodList(items=[pod]) list_all_p.return_value = V1PodList(items=[pod]) res = list_pods(namespace='test', label_selector='test') self.assertIsInstance(res, V1PodList) self.assertEqual(1, len(res.items)) res = list_pods(label_selector='test') self.assertIsInstance(res, V1PodList) self.assertEqual(1, len(res.items)) list_p.side_effect = ApiException('Test') list_all_p.side_effect = ApiException('Test') with self.assertRaises(api.ApiError) as e: list_pods(namespace='test') expected = ("Error when listing pods in namespace test: (Test)\n" "Reason: None\n") self.assertEqual(expected, str(e.exception)) self.assertEqual(400, e.exception.status_code) with self.assertRaises(api.ApiError) as e: list_pods() expected = ("Error when listing pods in namespace All: (Test)\n" "Reason: None\n") self.assertEqual(expected, str(e.exception)) self.assertEqual(400, e.exception.status_code)
def mock_cluster_connector(): with mock.patch('clusterman.kubernetes.kubernetes_cluster_connector.kubernetes'), \ mock.patch('clusterman.kubernetes.kubernetes_cluster_connector.staticconf'): mock_cluster_connector = KubernetesClusterConnector( 'kubernetes-test', 'bar') mock_cluster_connector._nodes_by_ip = { '10.10.10.1': KubernetesNode(metadata=V1ObjectMeta(name='node1'), status=V1NodeStatus(allocatable={ 'cpu': '4', 'gpu': 2 }, capacity={ 'cpu': '4', 'gpu': '2' })), '10.10.10.2': KubernetesNode(metadata=V1ObjectMeta(name='node2'), status=V1NodeStatus(allocatable={'cpu': '6.5'}, capacity={'cpu': '8'})) } mock_cluster_connector._pods_by_ip = { '10.10.10.1': [], '10.10.10.2': [ V1Pod(metadata=V1ObjectMeta(name='pod1'), status=V1PodStatus(phase='Running'), spec=V1PodSpec(containers=[ V1Container(name='container1', resources=V1ResourceRequirements( requests={'cpu': '1.5'})) ])), ] } return mock_cluster_connector
def test_delete_unschedulable_pods_raises_server_error(api: MagicMock): api.list_namespaced_pod.return_value = V1PodList(items=[ V1Pod( metadata=V1ObjectMeta( name="web-0", namespace="default", uid="uid-web-0", resource_version="1", owner_references=[V1ObjectReference(kind="StatefulSet")], ), status=V1PodStatus( phase="Pending", conditions=[ V1PodCondition( status="Not Ready", type="False", reason="Unschedulable", message='persistentvolumeclaim "queue-web-0" not found', ) ], ), ), ]) def delete_pod(name, namespace, body): raise ApiException(reason="Server Error") api.delete_namespaced_pod.side_effect = delete_pod with pytest.raises(ApiException): delete_unschedulable_pods(api, "namespace") api.list_namespaced_pod.called_once_with("namespace")
def create_work_pod(self): broker.coreV1.create_namespaced_pod( namespace='nectar', body=V1Pod( metadata=V1ObjectMeta( name=self.pod_name, labels=self.pod_labels() ), spec=V1PodSpec( restart_policy='Never', containers=[ V1Container( name='docker', image='docker:latest', command=["/bin/sh"], args=["-c", self.command()], env=[ V1EnvVar( name='DOCKER_HOST', value=self.daemon_host() ) ] ) ] ) ) )
def test_waiting_for_benchmark_container(): status = V1PodStatus( phase="Pending", container_statuses=[ V1ContainerStatus( image="benchmarkai/hello-world", name="benchmark", image_id="", ready=False, restart_count=0, state=V1ContainerState(waiting=CONTAINER_STATE_WAITING), ), V1ContainerStatus( image="benchmarkai/metrics-pusher", name="sidecar", image_id="", ready=True, restart_count=0, state=V1ContainerState(running=V1ContainerStateRunning()), ), ], ) pod = V1Pod(metadata=V1ObjectMeta(name="pod-name"), status=status) inferrer = SingleNodeStrategyKubernetesStatusInferrer( V1JobStatus(active=1), pods=[pod]) assert inferrer.status( ) == BenchmarkJobStatus.PENDING_AT_BENCHMARK_CONTAINER
def test_failed_at_sidecar_container_and_pod_terminated(): status = V1PodStatus( phase="Failed", container_statuses=[ V1ContainerStatus( image="benchmarkai/hello-world", name="benchmark", image_id="", ready=True, restart_count=0, state=V1ContainerState( terminated=CONTAINER_STATE_TERMINATED_AND_SUCCEEDED), ), V1ContainerStatus( image="benchmarkai/metrics-pusher", name="sidecar", image_id="", ready=True, restart_count=0, state=V1ContainerState( terminated=CONTAINER_STATE_TERMINATED_AND_FAILED), ), ], ) pod = V1Pod(metadata=V1ObjectMeta(name="pod-name"), status=status) inferrer = SingleNodeStrategyKubernetesStatusInferrer( V1JobStatus(active=1), pods=[pod]) assert inferrer.status() == BenchmarkJobStatus.FAILED_AT_SIDECAR_CONTAINER
def pending_pods(): return [ ( V1Pod( metadata=V1ObjectMeta(name='pod1'), status=V1PodStatus( phase='Pending', conditions=[ V1PodCondition(status='False', type='PodScheduled', reason='Unschedulable') ], ), spec=V1PodSpec(containers=[ V1Container( name='container1', resources=V1ResourceRequirements(requests={'cpu': '1.5', 'memory': '150MB'}) ), V1Container( name='container1', resources=V1ResourceRequirements(requests={'cpu': '1.5', 'memory': '350MB'}) ) ]), ), PodUnschedulableReason.InsufficientResources, ), ( V1Pod( metadata=V1ObjectMeta(name='pod2'), status=V1PodStatus( phase='Pending', conditions=[ V1PodCondition(status='False', type='PodScheduled', reason='Unschedulable') ], ), spec=V1PodSpec(containers=[ V1Container( name='container1', resources=V1ResourceRequirements(requests={'cpu': '1.5'}) ), V1Container( name='container1', resources=V1ResourceRequirements(requests={'cpu': '1.5', 'mem': '300MB'}) ) ]), ), PodUnschedulableReason.Unknown, ) ]
def test_pod_scaling_nodes(): condition = V1PodCondition(type="PodScheduled", reason="Unschedulable", status="False") status = V1PodStatus(phase="Pending", conditions=[condition]) inferrer = SingleNodeStrategyKubernetesStatusInferrer( V1JobStatus(active=1), pods=[V1Pod(status=status)]) assert inferrer.status() == BenchmarkJobStatus.PENDING_NODE_SCALING
def from_file(cls, pod_dict): """ expects input in format: { "name": "bonita-webapp-0", "namespace": "default", "containers": { "limits": { "memory": "24Gi" }, "requests": { "cpu": "3", "memory": "12Gi" } }, "initContainers": null }, """ containers = pod_dict['containers'] if containers is not None: c_limits = containers.get('limits') c_requests = containers.get('requests') else: c_limits, c_requests = {}, {} init_containers = pod_dict['initContainers'] if init_containers is not None: ic_limits = containers.get('limits') ic_requests = containers.get('requests') else: ic_limits, ic_requests = {}, {} pod = V1Pod( metadata=V1ObjectMeta( name=pod_dict['name'], namespace=pod_dict['namespace'] ), spec=V1PodSpec( containers=[ V1Container( name='1', resources=V1ResourceRequirements( limits=c_limits, requests=c_requests ) ) ], init_containers=[ V1Container( name='1', resources=V1ResourceRequirements( limits=ic_limits, requests=ic_requests )) ] ) ) return cls.from_k8s(pod)
def running_pod_1(): return V1Pod(metadata=V1ObjectMeta(name='running_pod_1'), status=V1PodStatus(phase='Running', host_ip='10.10.10.2'), spec=V1PodSpec(containers=[ V1Container(name='container1', resources=V1ResourceRequirements( requests={'cpu': '1.5'})) ], node_selector={'clusterman.com/pool': 'bar'}))
def deploy(self, restart_policy="Never"): spec = V1PodSpec(containers=self.containers, node_name=self.node_name, volumes=self.volumes, restart_policy=restart_policy) self.meta.labels = self.target_labels pod = V1Pod(spec=spec, metadata=self.meta) return k8sclient.apiV1.create_namespaced_pod(self.meta.namespace, body=pod)
def test_get_status(self, pod_p): pod_p.return_value = None res = cate.get_status('drwho') self.assertDictEqual({'phase': 'Unknown'}, res) pod_p.return_value = V1Pod(status=V1PodStatus(phase='Running')) res = cate.get_status('drwho') self.assertEqual('Running', res['phase'])
def get_pod(prefix: str, namespace=None, label_selector: str = None): global _CT if _CT == 0: _CT += 1 return None elif _CT == 1: _CT += 1 return V1Pod(metadata=V1ObjectMeta(name='test'), status=V1PodStatus(phase='deas')) else: _CT += 1 return pod
def unevictable_pod(): return V1Pod(metadata=V1ObjectMeta( name='unevictable_pod', annotations={'clusterman.com/safe_to_evict': 'false'}, owner_references=[]), status=V1PodStatus(phase='Running', host_ip='10.10.10.2'), spec=V1PodSpec(containers=[ V1Container(name='container1', resources=V1ResourceRequirements( requests={'cpu': '1.5'})) ]))
def __init__(self) -> None: metadata = V1ObjectMeta(name="testdrive") container = V1Container( name="testdrive", image=self.image("testdrive"), command=["sleep", "infinity"], ) pod_spec = V1PodSpec(containers=[container]) self.pod = V1Pod(metadata=metadata, spec=pod_spec)
def create(self): pod = V1Pod(api_version='v1', metadata=V1ObjectMeta(name=self.pod_name, labels=self.labels()), spec=V1PodSpec(containers=[ V1Container(name="primary", image=self.image(), image_pull_policy="Always") ])) return broker.coreV1.create_namespaced_pod(body=pod, namespace=self.namespace)
def create_test_pod(): core_v1.create_namespaced_pod( "default", V1Pod( metadata=V1ObjectMeta(name=name, ), spec=V1PodSpec(containers=[ V1Container( name="test", image="alpine", tty=True, ) ]), ))
def test_get_pod(self, list_p): pod = V1Pod(metadata=V1ObjectMeta(name='test')) list_p.return_value = V1PodList(items=[pod]) res = k8s.get_pod(namespace='test', prefix='test') self.assertIsInstance(res, V1Pod) self.assertEqual('test', res.metadata.name) list_p.return_value = V1PodList(items=[]) res = k8s.get_pod(namespace='test', prefix='test') self.assertIsNone(res)
def pending_pod(): return V1Pod(metadata=V1ObjectMeta(name='pending_pod', annotations=dict(), owner_references=[]), status=V1PodStatus( phase='Pending', conditions=None, ), spec=V1PodSpec(containers=[ V1Container(name='container2', resources=V1ResourceRequirements( requests={'cpu': '1.5'})) ], node_selector={'clusterman.com/pool': 'bar'}))
def test_kill_pod(self): with setup_kubernetes(None) as cli: core_v1 = client.CoreV1Api(cli) pod = core_v1.create_namespaced_pod( "default", V1Pod( metadata=V1ObjectMeta(generate_name="test-", ), spec=V1PodSpec(containers=[ V1Container( name="test", image="alpine", tty=True, ) ]), )) def remove_test_pod(): try: core_v1.delete_namespaced_pod(pod.metadata.name, pod.metadata.namespace) except ApiException as e: if e.status != 404: raise self.addCleanup(remove_test_pod) output_id, output_data = pod_plugin.kill_pods( KillPodConfig( namespace_pattern=re.compile("^default$"), name_pattern=re.compile("^" + re.escape(pod.metadata.name) + "$"), )) if output_id == "error": self.fail(output_data.error) self.assertIsInstance(output_data, PodKillSuccessOutput) out: PodKillSuccessOutput = output_data self.assertEqual(1, len(out.pods)) pod_list = list(out.pods.values()) self.assertEqual(pod.metadata.name, pod_list[0].name) try: core_v1.read_namespaced_pod(pod_list[0].name, pod_list[0].namespace) self.fail("Killed pod is still present.") except ApiException as e: if e.status != 404: self.fail( "Incorrect API exception encountered: {}".format(e))
def deploy(self, restart_policy="Never", dns_policy="ClusterFirst"): spec = V1PodSpec( containers=self.containers, node_name=self.node_name, volumes=self.volumes, restart_policy=restart_policy, dns_policy=dns_policy, ) if self.meta.labels: self.meta.labels.update(self.target_labels) else: self.meta.labels = self.target_labels pod = V1Pod(spec=spec, metadata=self.meta) return k8sclient.apiV1.create_namespaced_pod(self.meta.namespace, body=pod)
def test_poll_pod_phase(self): pod = V1Pod(metadata=V1ObjectMeta(name='test'), status=V1PodStatus(phase='running')) def get_pod(prefix: str, namespace=None, label_selector: str = None): global _CT if _CT == 0: _CT += 1 return None elif _CT == 1: _CT += 1 return V1Pod(metadata=V1ObjectMeta(name='test'), status=V1PodStatus(phase='deas')) else: _CT += 1 return pod poller.poll_pod_phase(get_pod, prefix='test')
def unschedulable_pod(): return V1Pod(metadata=V1ObjectMeta(name='unschedulable_pod', annotations=dict(), owner_references=[]), status=V1PodStatus(phase='Pending', conditions=[ V1PodCondition(status='False', type='PodScheduled', reason='Unschedulable') ]), spec=V1PodSpec(containers=[ V1Container(name='container2', resources=V1ResourceRequirements( requests={'cpu': '1.5'})) ], node_selector={'clusterman.com/pool': 'bar'}))