def test_get_pod_statuses_single(self, kube):
# Mocking kube api - https://github.com/kubernetes-client/python/blob/9b438eed5a4fdab4377515b5a0c62d695dffc354/kubernetes/docs/V1PodList.md
kube.core_api.list_namespaced_pod = MagicMock(return_value=V1PodList(
api_version='v1',
items=[
V1Pod(api_version='v1',
kind='Pod',
metadata=V1ObjectMeta(name="pod_1"),
spec=None,
status=V1PodStatus(phase='failed')),
V1Pod(api_version='v1',
kind='Pod',
metadata=V1ObjectMeta(name="pod_2"),
spec=None,
status=V1PodStatus(phase='pending')),
V1Pod(api_version='v1',
kind='Pod',
metadata=V1ObjectMeta(name="pod_3"),
spec=None,
status=V1PodStatus(phase='running'))
]))
actual = KubeUtil.get_status_of_all_pods(kube,
label_selector="label=bar")
self.assertEquals(actual, [{
'pod': 'pod_1',
'phase': 'failed'
}, {
'pod': 'pod_2',
'phase': 'pending'
}, {
'pod': 'pod_3',
'phase': 'running'
}])
def test_job_logs_multiple_pods(self, mock_core_client):
namespace = "treesbecomelogs"
manager = JobManager(
namespace=namespace, signer=Mock(), register=StaticJobDefinitionsRegister()
)
job_name = "ahoymatey"
pod_name_1, pod_name_2 = "p1", "p2"
container_name = "c1"
mock_core_client.list_namespaced_pod.return_value.items = [
V1Pod(
metadata=V1ObjectMeta(name=pod_name_1),
spec=V1PodSpec(containers=[V1Container(name=container_name)]),
),
V1Pod(
metadata=V1ObjectMeta(name=pod_name_2),
spec=V1PodSpec(containers=[V1Container(name=container_name)]),
),
]
log_msg = "this is a log"
mock_core_client.read_namespaced_pod_log.return_value = log_msg
logs = manager.job_logs(job_name)
assert logs == {
pod_name_1: {container_name: [log_msg]},
pod_name_2: {container_name: [log_msg]},
}
def test_failed_at_benchmark_container_but_sidecar_still_running():
status = V1PodStatus(
phase="Running",
container_statuses=[
V1ContainerStatus(
image="benchmarkai/hello-world",
name="benchmark",
image_id="",
ready=False,
restart_count=0,
state=V1ContainerState(
terminated=CONTAINER_STATE_TERMINATED_AND_FAILED),
),
V1ContainerStatus(
image="benchmarkai/metrics-pusher",
name="sidecar",
image_id="",
ready=True,
restart_count=0,
state=V1ContainerState(running=V1ContainerStateRunning()),
),
],
)
pod = V1Pod(metadata=V1ObjectMeta(name="pod-name"), status=status)
inferrer = SingleNodeStrategyKubernetesStatusInferrer(
V1JobStatus(active=1), pods=[pod])
# TODO: Not sure if this is the status we want
assert inferrer.status() == BenchmarkJobStatus.RUNNING_AT_MAIN_CONTAINERS
def exec_python(kube_ns, kube_client):
"""Return a callable to execute Python code in a pod in the test namespace
This fixture creates a dedicated pod for executing commands
"""
# note: this was created when there were only single-user pods running,
# but now there's always a hub pod where we could be running,
# and the ssl case *must* run from the hub pod for access to certs
# Note: we could do without this feature if we always ran
pod_name = "kubespawner-test-exec"
pod_manifest = V1Pod(
metadata={"name": pod_name},
spec=V1PodSpec(
containers=[
{
"image": "python:3.8",
"name": "python",
"args": ["/bin/sh", "-c", "while true; do sleep 5; done"],
}
],
termination_grace_period_seconds=0,
),
)
pod = create_resource(kube_client, kube_ns, "pod", pod_manifest)
yield partial(_exec_python_in_pod, kube_client, kube_ns, pod_name)
def test_job_logs_not_ready(self, mock_core_client):
namespace = "notready"
manager = JobManager(
namespace=namespace, signer=Mock(), register=StaticJobDefinitionsRegister()
)
pod_name = "p"
container_name = "c"
mock_core_client.list_namespaced_pod.return_value.items = [
V1Pod(
metadata=V1ObjectMeta(name=pod_name),
spec=V1PodSpec(containers=[V1Container(name=container_name)]),
)
]
mock_core_client.read_namespaced_pod_log.side_effect = ApiException(
http_resp=Mock(
data={
"message": f'container "{container_name}" in pod "{pod_name}" is waiting to start: ContainerCreating'
}
)
)
# No exception
logs = manager.job_logs("whatever")
assert logs == {pod_name: {container_name: ["ContainerCreating"]}}
def pod_with_preferred_affinity():
return V1Pod(
status=V1PodStatus(phase='Pending',
conditions=[
V1PodCondition(status='False',
type='PodScheduled',
reason='Unschedulable')
]),
spec=V1PodSpec(
containers=[
V1Container(
name='container',
resources=V1ResourceRequirements(requests={'cpu': '1.5'}))
],
affinity=V1Affinity(node_affinity=V1NodeAffinity(
required_during_scheduling_ignored_during_execution=
V1NodeSelector(node_selector_terms=[
V1NodeSelectorTerm(match_expressions=[
V1NodeSelectorRequirement(
key='clusterman.com/scheduler', operator='Exists')
])
]),
preferred_during_scheduling_ignored_during_execution=[
V1PreferredSchedulingTerm(
weight=10,
preference=V1NodeSelectorTerm(match_expressions=[
V1NodeSelectorRequirement(
key='clusterman.com/pool',
operator='In',
values=['bar'])
]))
]))))
def test_list(self, batch_p, get_p):
batch_p.return_value = V1JobList(
items=[V1Pod(metadata=V1ObjectMeta(name='drwho-cate-sdav'))])
get_p.return_value = {
'cubegen_id': 'id',
'status': 'Ready',
'output': ['bla'],
'progress': 100
}
res = cubegens.list('drwho')
self.assertEqual(1, len(res))
self.assertDictEqual(
{
'cubegen_id': 'id',
'status': 'Ready',
'output': ['bla'],
'progress': 100
}, res[0])
batch_p.side_effect = ApiException(401, 'Unauthorized')
with self.assertRaises(api.ApiError) as e:
cubegens.list(user_id='drwho')
self.assertIn("Reason: Unauthorized", str(e.exception))
self.assertIn("401", str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_list_pods(self, list_all_p, list_p):
pod = V1Pod(metadata=V1ObjectMeta(name='test'))
list_p.return_value = V1PodList(items=[pod])
list_all_p.return_value = V1PodList(items=[pod])
res = list_pods(namespace='test', label_selector='test')
self.assertIsInstance(res, V1PodList)
self.assertEqual(1, len(res.items))
res = list_pods(label_selector='test')
self.assertIsInstance(res, V1PodList)
self.assertEqual(1, len(res.items))
list_p.side_effect = ApiException('Test')
list_all_p.side_effect = ApiException('Test')
with self.assertRaises(api.ApiError) as e:
list_pods(namespace='test')
expected = ("Error when listing pods in namespace test: (Test)\n"
"Reason: None\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
with self.assertRaises(api.ApiError) as e:
list_pods()
expected = ("Error when listing pods in namespace All: (Test)\n"
"Reason: None\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def mock_cluster_connector():
with mock.patch('clusterman.kubernetes.kubernetes_cluster_connector.kubernetes'), \
mock.patch('clusterman.kubernetes.kubernetes_cluster_connector.staticconf'):
mock_cluster_connector = KubernetesClusterConnector(
'kubernetes-test', 'bar')
mock_cluster_connector._nodes_by_ip = {
'10.10.10.1':
KubernetesNode(metadata=V1ObjectMeta(name='node1'),
status=V1NodeStatus(allocatable={
'cpu': '4',
'gpu': 2
},
capacity={
'cpu': '4',
'gpu': '2'
})),
'10.10.10.2':
KubernetesNode(metadata=V1ObjectMeta(name='node2'),
status=V1NodeStatus(allocatable={'cpu': '6.5'},
capacity={'cpu': '8'}))
}
mock_cluster_connector._pods_by_ip = {
'10.10.10.1': [],
'10.10.10.2': [
V1Pod(metadata=V1ObjectMeta(name='pod1'),
status=V1PodStatus(phase='Running'),
spec=V1PodSpec(containers=[
V1Container(name='container1',
resources=V1ResourceRequirements(
requests={'cpu': '1.5'}))
])),
]
}
return mock_cluster_connector
def test_delete_unschedulable_pods_raises_server_error(api: MagicMock):
api.list_namespaced_pod.return_value = V1PodList(items=[
V1Pod(
metadata=V1ObjectMeta(
name="web-0",
namespace="default",
uid="uid-web-0",
resource_version="1",
owner_references=[V1ObjectReference(kind="StatefulSet")],
),
status=V1PodStatus(
phase="Pending",
conditions=[
V1PodCondition(
status="Not Ready",
type="False",
reason="Unschedulable",
message='persistentvolumeclaim "queue-web-0" not found',
)
],
),
),
])
def delete_pod(name, namespace, body):
raise ApiException(reason="Server Error")
api.delete_namespaced_pod.side_effect = delete_pod
with pytest.raises(ApiException):
delete_unschedulable_pods(api, "namespace")
api.list_namespaced_pod.called_once_with("namespace")
def create_work_pod(self):
broker.coreV1.create_namespaced_pod(
namespace='nectar',
body=V1Pod(
metadata=V1ObjectMeta(
name=self.pod_name,
labels=self.pod_labels()
),
spec=V1PodSpec(
restart_policy='Never',
containers=[
V1Container(
name='docker',
image='docker:latest',
command=["/bin/sh"],
args=["-c", self.command()],
env=[
V1EnvVar(
name='DOCKER_HOST',
value=self.daemon_host()
)
]
)
]
)
)
)
def test_waiting_for_benchmark_container():
status = V1PodStatus(
phase="Pending",
container_statuses=[
V1ContainerStatus(
image="benchmarkai/hello-world",
name="benchmark",
image_id="",
ready=False,
restart_count=0,
state=V1ContainerState(waiting=CONTAINER_STATE_WAITING),
),
V1ContainerStatus(
image="benchmarkai/metrics-pusher",
name="sidecar",
image_id="",
ready=True,
restart_count=0,
state=V1ContainerState(running=V1ContainerStateRunning()),
),
],
)
pod = V1Pod(metadata=V1ObjectMeta(name="pod-name"), status=status)
inferrer = SingleNodeStrategyKubernetesStatusInferrer(
V1JobStatus(active=1), pods=[pod])
assert inferrer.status(
) == BenchmarkJobStatus.PENDING_AT_BENCHMARK_CONTAINER
def test_failed_at_sidecar_container_and_pod_terminated():
status = V1PodStatus(
phase="Failed",
container_statuses=[
V1ContainerStatus(
image="benchmarkai/hello-world",
name="benchmark",
image_id="",
ready=True,
restart_count=0,
state=V1ContainerState(
terminated=CONTAINER_STATE_TERMINATED_AND_SUCCEEDED),
),
V1ContainerStatus(
image="benchmarkai/metrics-pusher",
name="sidecar",
image_id="",
ready=True,
restart_count=0,
state=V1ContainerState(
terminated=CONTAINER_STATE_TERMINATED_AND_FAILED),
),
],
)
pod = V1Pod(metadata=V1ObjectMeta(name="pod-name"), status=status)
inferrer = SingleNodeStrategyKubernetesStatusInferrer(
V1JobStatus(active=1), pods=[pod])
assert inferrer.status() == BenchmarkJobStatus.FAILED_AT_SIDECAR_CONTAINER
def pending_pods():
return [
(
V1Pod(
metadata=V1ObjectMeta(name='pod1'),
status=V1PodStatus(
phase='Pending',
conditions=[
V1PodCondition(status='False', type='PodScheduled', reason='Unschedulable')
],
),
spec=V1PodSpec(containers=[
V1Container(
name='container1',
resources=V1ResourceRequirements(requests={'cpu': '1.5', 'memory': '150MB'})
),
V1Container(
name='container1',
resources=V1ResourceRequirements(requests={'cpu': '1.5', 'memory': '350MB'})
)
]),
),
PodUnschedulableReason.InsufficientResources,
),
(
V1Pod(
metadata=V1ObjectMeta(name='pod2'),
status=V1PodStatus(
phase='Pending',
conditions=[
V1PodCondition(status='False', type='PodScheduled', reason='Unschedulable')
],
),
spec=V1PodSpec(containers=[
V1Container(
name='container1',
resources=V1ResourceRequirements(requests={'cpu': '1.5'})
),
V1Container(
name='container1',
resources=V1ResourceRequirements(requests={'cpu': '1.5', 'mem': '300MB'})
)
]),
),
PodUnschedulableReason.Unknown,
)
]
def test_pod_scaling_nodes():
condition = V1PodCondition(type="PodScheduled",
reason="Unschedulable",
status="False")
status = V1PodStatus(phase="Pending", conditions=[condition])
inferrer = SingleNodeStrategyKubernetesStatusInferrer(
V1JobStatus(active=1), pods=[V1Pod(status=status)])
assert inferrer.status() == BenchmarkJobStatus.PENDING_NODE_SCALING
def from_file(cls, pod_dict):
"""
expects input in format:
{
"name": "bonita-webapp-0",
"namespace": "default",
"containers": {
"limits": {
"memory": "24Gi"
},
"requests": {
"cpu": "3",
"memory": "12Gi"
}
},
"initContainers": null
},
"""
containers = pod_dict['containers']
if containers is not None:
c_limits = containers.get('limits')
c_requests = containers.get('requests')
else:
c_limits, c_requests = {}, {}
init_containers = pod_dict['initContainers']
if init_containers is not None:
ic_limits = containers.get('limits')
ic_requests = containers.get('requests')
else:
ic_limits, ic_requests = {}, {}
pod = V1Pod(
metadata=V1ObjectMeta(
name=pod_dict['name'],
namespace=pod_dict['namespace']
),
spec=V1PodSpec(
containers=[
V1Container(
name='1',
resources=V1ResourceRequirements(
limits=c_limits,
requests=c_requests
)
)
],
init_containers=[
V1Container(
name='1',
resources=V1ResourceRequirements(
limits=ic_limits,
requests=ic_requests
))
]
)
)
return cls.from_k8s(pod)
def running_pod_1():
return V1Pod(metadata=V1ObjectMeta(name='running_pod_1'),
status=V1PodStatus(phase='Running', host_ip='10.10.10.2'),
spec=V1PodSpec(containers=[
V1Container(name='container1',
resources=V1ResourceRequirements(
requests={'cpu': '1.5'}))
],
node_selector={'clusterman.com/pool': 'bar'}))
def deploy(self, restart_policy="Never"):
spec = V1PodSpec(containers=self.containers,
node_name=self.node_name,
volumes=self.volumes,
restart_policy=restart_policy)
self.meta.labels = self.target_labels
pod = V1Pod(spec=spec, metadata=self.meta)
return k8sclient.apiV1.create_namespaced_pod(self.meta.namespace,
body=pod)
def test_get_status(self, pod_p):
pod_p.return_value = None
res = cate.get_status('drwho')
self.assertDictEqual({'phase': 'Unknown'}, res)
pod_p.return_value = V1Pod(status=V1PodStatus(phase='Running'))
res = cate.get_status('drwho')
self.assertEqual('Running', res['phase'])
def get_pod(prefix: str, namespace=None, label_selector: str = None):
global _CT
if _CT == 0:
_CT += 1
return None
elif _CT == 1:
_CT += 1
return V1Pod(metadata=V1ObjectMeta(name='test'), status=V1PodStatus(phase='deas'))
else:
_CT += 1
return pod
def unevictable_pod():
return V1Pod(metadata=V1ObjectMeta(
name='unevictable_pod',
annotations={'clusterman.com/safe_to_evict': 'false'},
owner_references=[]),
status=V1PodStatus(phase='Running', host_ip='10.10.10.2'),
spec=V1PodSpec(containers=[
V1Container(name='container1',
resources=V1ResourceRequirements(
requests={'cpu': '1.5'}))
]))
def __init__(self) -> None:
metadata = V1ObjectMeta(name="testdrive")
container = V1Container(
name="testdrive",
image=self.image("testdrive"),
command=["sleep", "infinity"],
)
pod_spec = V1PodSpec(containers=[container])
self.pod = V1Pod(metadata=metadata, spec=pod_spec)
def create(self):
pod = V1Pod(api_version='v1',
metadata=V1ObjectMeta(name=self.pod_name,
labels=self.labels()),
spec=V1PodSpec(containers=[
V1Container(name="primary",
image=self.image(),
image_pull_policy="Always")
]))
return broker.coreV1.create_namespaced_pod(body=pod,
namespace=self.namespace)
def create_test_pod():
core_v1.create_namespaced_pod(
"default",
V1Pod(
metadata=V1ObjectMeta(name=name, ),
spec=V1PodSpec(containers=[
V1Container(
name="test",
image="alpine",
tty=True,
)
]),
))
def test_get_pod(self, list_p):
pod = V1Pod(metadata=V1ObjectMeta(name='test'))
list_p.return_value = V1PodList(items=[pod])
res = k8s.get_pod(namespace='test', prefix='test')
self.assertIsInstance(res, V1Pod)
self.assertEqual('test', res.metadata.name)
list_p.return_value = V1PodList(items=[])
res = k8s.get_pod(namespace='test', prefix='test')
self.assertIsNone(res)
def pending_pod():
return V1Pod(metadata=V1ObjectMeta(name='pending_pod',
annotations=dict(),
owner_references=[]),
status=V1PodStatus(
phase='Pending',
conditions=None,
),
spec=V1PodSpec(containers=[
V1Container(name='container2',
resources=V1ResourceRequirements(
requests={'cpu': '1.5'}))
],
node_selector={'clusterman.com/pool': 'bar'}))
def test_kill_pod(self):
with setup_kubernetes(None) as cli:
core_v1 = client.CoreV1Api(cli)
pod = core_v1.create_namespaced_pod(
"default",
V1Pod(
metadata=V1ObjectMeta(generate_name="test-", ),
spec=V1PodSpec(containers=[
V1Container(
name="test",
image="alpine",
tty=True,
)
]),
))
def remove_test_pod():
try:
core_v1.delete_namespaced_pod(pod.metadata.name,
pod.metadata.namespace)
except ApiException as e:
if e.status != 404:
raise
self.addCleanup(remove_test_pod)
output_id, output_data = pod_plugin.kill_pods(
KillPodConfig(
namespace_pattern=re.compile("^default$"),
name_pattern=re.compile("^" +
re.escape(pod.metadata.name) +
"$"),
))
if output_id == "error":
self.fail(output_data.error)
self.assertIsInstance(output_data, PodKillSuccessOutput)
out: PodKillSuccessOutput = output_data
self.assertEqual(1, len(out.pods))
pod_list = list(out.pods.values())
self.assertEqual(pod.metadata.name, pod_list[0].name)
try:
core_v1.read_namespaced_pod(pod_list[0].name,
pod_list[0].namespace)
self.fail("Killed pod is still present.")
except ApiException as e:
if e.status != 404:
self.fail(
"Incorrect API exception encountered: {}".format(e))
def deploy(self, restart_policy="Never", dns_policy="ClusterFirst"):
spec = V1PodSpec(
containers=self.containers,
node_name=self.node_name,
volumes=self.volumes,
restart_policy=restart_policy,
dns_policy=dns_policy,
)
if self.meta.labels:
self.meta.labels.update(self.target_labels)
else:
self.meta.labels = self.target_labels
pod = V1Pod(spec=spec, metadata=self.meta)
return k8sclient.apiV1.create_namespaced_pod(self.meta.namespace,
body=pod)
def test_poll_pod_phase(self):
pod = V1Pod(metadata=V1ObjectMeta(name='test'), status=V1PodStatus(phase='running'))
def get_pod(prefix: str, namespace=None, label_selector: str = None):
global _CT
if _CT == 0:
_CT += 1
return None
elif _CT == 1:
_CT += 1
return V1Pod(metadata=V1ObjectMeta(name='test'), status=V1PodStatus(phase='deas'))
else:
_CT += 1
return pod
poller.poll_pod_phase(get_pod, prefix='test')
def unschedulable_pod():
return V1Pod(metadata=V1ObjectMeta(name='unschedulable_pod',
annotations=dict(),
owner_references=[]),
status=V1PodStatus(phase='Pending',
conditions=[
V1PodCondition(status='False',
type='PodScheduled',
reason='Unschedulable')
]),
spec=V1PodSpec(containers=[
V1Container(name='container2',
resources=V1ResourceRequirements(
requests={'cpu': '1.5'}))
],
node_selector={'clusterman.com/pool': 'bar'}))