def _get_pod_for_job(self, job_state):
pods = Pod.objects(self._pykube_api).filter(selector=f"app={job_state.job_id}",
namespace=self.runner_params['k8s_namespace'])
if not pods.response['items']:
return None
pod = Pod(self._pykube_api, pods.response['items'][0])
return pod
def __job_failed_due_to_low_memory(self, job_state):
"""
checks the state of the pod to see if it was killed
for being out of memory (pod status OOMKilled). If that is the case
marks the job for resubmission (resubmit logic is part of destinations).
"""
pods = Pod.objects(self._pykube_api).filter(selector="app=%s" % job_state.job_id,
namespace=self.runner_params['k8s_namespace'])
if not pods.response['items']:
return False
pod = Pod(self._pykube_api, pods.response['items'][0])
if pod.obj['status']['phase'] == "Failed" and \
pod.obj['status']['containerStatuses'][0]['state']['terminated']['reason'] == "OOMKilled":
return True
return False
def __job_pending_due_to_unschedulable_pod(self, job_state):
"""
checks the state of the pod to see if it is unschedulable.
"""
pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace'])
if not pods.response['items']:
return False
pod = Pod(self._pykube_api, pods.response['items'][0])
return is_pod_unschedulable(self._pykube_api, pod, self.runner_params['k8s_namespace'])