def wait_for_job(
self,
name, #pylint: disable=inconsistent-return-statements
namespace=None,
watch=False,
timeout_seconds=600,
polling_interval=30,
status_callback=None):
"""Wait for the specified job to finish.
:param name: Name of the PyTorchJob.
:param namespace: defaults to current or default namespace.
:param timeout_seconds: How long to wait for the job.
:param polling_interval: How often to poll for the status of the job.
:param status_callback: (Optional): Callable. If supplied this callable is
invoked after we poll the job. Callable takes a single argument which
is the job.
:return:
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
if watch:
pytorchjob_watch(name=name,
namespace=namespace,
timeout_seconds=timeout_seconds)
else:
return self.wait_for_condition(name, ["Succeeded", "Failed"],
namespace=namespace,
timeout_seconds=timeout_seconds,
polling_interval=polling_interval,
status_callback=status_callback)
def wait_for_job(self, name,
namespace=None,
timeout_seconds=600,
polling_interval=30,
status_callback=None):
"""Wait for the specified job to finish.
Args:
name: Name of the PyTorchJob.
namespace: defaults to current or default namespace.
timeout_seconds: How long to wait for the job.
polling_interval: How often to poll for the status of the job.
status_callback: (Optional): Callable. If supplied this callable is
invoked after we poll the job. Callable takes a single argument which
is the job.
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
return self.wait_for_condition(
name,
["Succeeded", "Failed"],
namespace=namespace,
timeout_seconds=timeout_seconds,
polling_interval=polling_interval,
status_callback=status_callback)
def get(self, name=None, namespace=None):
"""
Get the pytorchjob
:param name: existing pytorchjob name
:param namespace: defaults to current or default namespace
:return: pytorchjob
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
if name:
try:
return self.api_instance.get_namespaced_custom_object(
constants.PYTORCHJOB_GROUP, constants.PYTORCHJOB_VERSION,
namespace, constants.PYTORCHJOB_PLURAL, name)
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->get_namespaced_custom_object:\
%s\n" % e)
else:
try:
return self.api_instance.list_namespaced_custom_object(
constants.PYTORCHJOB_GROUP, constants.PYTORCHJOB_VERSION,
namespace, constants.PYTORCHJOB_PLURAL)
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->list_namespaced_custom_object:\
%s\n" % e)
def watch(name=None, namespace=None, timeout_seconds=600):
"""Watch the created or patched InferenceService in the specified namespace"""
if namespace is None:
namespace = utils.get_default_target_namespace()
stream = k8s_watch.Watch().stream(
client.CustomObjectsApi().list_namespaced_custom_object,
constants.PYTORCHJOB_GROUP,
constants.PYTORCHJOB_VERSION,
namespace,
constants.PYTORCHJOB_PLURAL,
timeout_seconds=timeout_seconds)
for event in stream:
pytorchjob = event['object']
pytorchjob_name = pytorchjob['metadata']['name']
if name and name != pytorchjob_name:
continue
else:
status = ''
update_time = ''
last_condition = pytorchjob.get('status',
{}).get('conditions', [])[-1]
status = last_condition.get('type', '')
update_time = last_condition.get('lastTransitionTime', '')
tbl(pytorchjob_name, status, update_time)
if name == pytorchjob_name:
if status == 'Succeeded' or status == 'Failed':
break
def get(self, name=None, namespace=None):
"""
Get the pytorchjob
:param name: existing pytorchjob name
:param namespace: defaults to current or default namespace
:return: pytorchjob
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
if name:
thread = self.api_instance.get_namespaced_custom_object(
constants.PYTORCHJOB_GROUP,
constants.PYTORCHJOB_VERSION,
namespace,
constants.PYTORCHJOB_PLURAL,
name,
async_req=True)
pytorchjob = None
try:
pytorchjob = thread.get(constants.APISERVER_TIMEOUT)
except multiprocessing.TimeoutError:
raise RuntimeError("Timeout trying to get PyTorchJob.")
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->get_namespaced_custom_object:\
%s\n" % e)
except Exception as e:
raise RuntimeError(
"There was a problem to get PyTorchJob {0} in namespace {1}. Exception: \
{2} ".format(name, namespace, e))
else:
thread = self.api_instance.list_namespaced_custom_object(
constants.PYTORCHJOB_GROUP,
constants.PYTORCHJOB_VERSION,
namespace,
constants.PYTORCHJOB_PLURAL,
async_req=True)
pytorchjob = None
try:
pytorchjob = thread.get(constants.APISERVER_TIMEOUT)
except multiprocessing.TimeoutError:
raise RuntimeError("Timeout trying to get PyTorchJob.")
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->list_namespaced_custom_object: \
%s\n" % e)
except Exception as e:
raise RuntimeError(
"There was a problem to List PyTorchJob in namespace {0}. \
Exception: {1} ".format(namespace, e))
return pytorchjob
def get_job_status(self, name, namespace=None):
"""Returns PyTorchJob status, such as Running, Failed or Succeeded.
:param name: The PyTorchJob name.
:param namespace: defaults to current or default namespace.
:return: str: PyTorchJob status
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
pytorchjob = self.get(name, namespace=namespace)
last_condition = pytorchjob.get("status", {}).get("conditions", [])[-1]
return last_condition.get("type", "")
def wait_for_condition(self,
name,
expected_condition,
namespace=None,
timeout_seconds=600,
polling_interval=30,
status_callback=None):
"""Waits until any of the specified conditions occur.
:param name: Name of the job.
:param expected_condition: A list of conditions. Function waits until any of the
supplied conditions is reached.
:param namespace: defaults to current or default namespace.
:param timeout_seconds: How long to wait for the job.
:param polling_interval: How often to poll for the status of the job.
:param status_callback: (Optional): Callable. If supplied this callable is
invoked after we poll the job. Callable takes a single argument which
is the job.
:return: Object: PyTorchJob
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
for _ in range(round(timeout_seconds / polling_interval)):
pytorchjob = None
pytorchjob = self.get(name, namespace=namespace)
if pytorchjob:
if status_callback:
status_callback(pytorchjob)
# If we poll the CRD quick enough status won't have been set yet.
conditions = pytorchjob.get("status", {}).get("conditions", [])
# Conditions might have a value of None in status.
conditions = conditions or []
for c in conditions:
if c.get("type", "") in expected_condition:
return pytorchjob
time.sleep(polling_interval)
raise RuntimeError(
"Timeout waiting for PyTorchJob {0} in namespace {1} to enter one of the "
"conditions {2}.".format(name, namespace, expected_condition),
pytorchjob)
def get_pod_names(
self,
name,
namespace=None,
master=False, #pylint: disable=inconsistent-return-statements
replica_type=None,
replica_index=None):
"""
Get pod names of PyTorchJob.
:param name: PyTorchJob name
:param namespace: defaults to current or default namespace.
:param master: Only get pod with label 'job-role: master' pod if True.
:param replica_type: User can specify one of 'master, worker' to only get one type pods.
By default get all type pods.
:param replica_index: User can specfy replica index to get one pod of PyTorchJob.
:return: set: pods name
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
labels = utils.get_labels(name,
master=master,
replica_type=replica_type,
replica_index=replica_index)
try:
resp = self.core_api.list_namespaced_pod(
namespace, label_selector=utils.to_selector(labels))
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n"
% e)
pod_names = []
for pod in resp.items:
if pod.metadata and pod.metadata.name:
pod_names.append(pod.metadata.name)
if not pod_names:
logging.warning(
"Not found Pods of the PyTorchJob %s with the labels %s.",
name, labels)
else:
return set(pod_names)
def get_logs(self,
name,
namespace=None,
master=True,
replica_type=None,
replica_index=None,
follow=False):
"""
Get training logs of the PyTorchJob.
By default only get the logs of Pod that has labels 'job-role: master'.
:param name: PyTorchJob name
:param namespace: defaults to current or default namespace.
:param master: By default get pod with label 'job-role: master' pod if True.
If need to get more Pod Logs, set False.
:param replica_type: User can specify one of 'master, worker' to only get one type pods.
By default get all type pods.
:param replica_index: User can specfy replica index to get one pod of PyTorchJob.
:param follow: Follow the log stream of the pod. Defaults to false.
:return: str: pods logs
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
pod_names = self.get_pod_names(name,
namespace=namespace,
master=master,
replica_type=replica_type,
replica_index=replica_index)
if pod_names:
for pod in pod_names:
try:
pod_logs = self.core_api.read_namespaced_pod_log(
pod, namespace, follow=follow)
logging.info("The logs of Pod %s:\n %s", pod, pod_logs)
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n"
% e)
else:
raise RuntimeError("Not found Pods of the PyTorchJob {} "
"in namespace {}".format(name, namespace))
def delete(self, name, namespace=None):
"""
Delete the pytorchjob
:param name: pytorchjob name
:param namespace: defaults to current or default namespace
:return:
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
try:
return self.custom_api.delete_namespaced_custom_object(
constants.PYTORCHJOB_GROUP, constants.PYTORCHJOB_VERSION,
namespace, constants.PYTORCHJOB_PLURAL, name,
client.V1DeleteOptions())
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->delete_namespaced_custom_object:\
%s\n" % e)
def get(self, name=None, namespace=None, watch=False, timeout_seconds=600): #pylint: disable=inconsistent-return-statements
"""
Get the pytorchjob
:param name: existing pytorchjob name, if not defined, get all pytorchjobs in the namespace.
:param namespace: defaults to current or default namespace
:param watch: Watch the pytorchjob if `True`.
:param timeout_seconds: How long to watch the pytorchjob.
:return: pytorchjob
"""
if namespace is None:
namespace = utils.get_default_target_namespace()
if name:
if watch:
pytorchjob_watch(name=name,
namespace=namespace,
timeout_seconds=timeout_seconds)
else:
thread = self.custom_api.get_namespaced_custom_object(
constants.PYTORCHJOB_GROUP,
constants.PYTORCHJOB_VERSION,
namespace,
constants.PYTORCHJOB_PLURAL,
name,
async_req=True)
pytorchjob = None
try:
pytorchjob = thread.get(constants.APISERVER_TIMEOUT)
except multiprocessing.TimeoutError:
raise RuntimeError("Timeout trying to get PyTorchJob.")
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->get_namespaced_custom_object:\
%s\n" % e)
except Exception as e:
raise RuntimeError(
"There was a problem to get PyTorchJob {0} in namespace {1}. Exception: \
{2} ".format(name, namespace, e))
return pytorchjob
else:
if watch:
pytorchjob_watch(namespace=namespace,
timeout_seconds=timeout_seconds)
else:
thread = self.custom_api.list_namespaced_custom_object(
constants.PYTORCHJOB_GROUP,
constants.PYTORCHJOB_VERSION,
namespace,
constants.PYTORCHJOB_PLURAL,
async_req=True)
pytorchjob = None
try:
pytorchjob = thread.get(constants.APISERVER_TIMEOUT)
except multiprocessing.TimeoutError:
raise RuntimeError("Timeout trying to get PyTorchJob.")
except client.rest.ApiException as e:
raise RuntimeError(
"Exception when calling CustomObjectsApi->list_namespaced_custom_object: \
%s\n" % e)
except Exception as e:
raise RuntimeError(
"There was a problem to List PyTorchJob in namespace {0}. \
Exception: {1} ".format(namespace, e))
return pytorchjob