def print_failure_log_for_run(host: Text, run_id: Text, namespace: Text):
"""Prints logs of failed components of a run.
Prints execution logs for failed componentsusing `logging.info`.
This resembles the behavior of `argo logs` but uses K8s API directly.
Don't print anything if the run was successful.
Args:
host: address of the KFP deployment.
run_id: id of the execution of the pipeline.
namespace: namespace of K8s cluster.
"""
client = kfp.Client(host=host)
run = client.get_run(run_id=run_id)
workflow_manifest = json.loads(run.pipeline_runtime.workflow_manifest)
if kube_utils.PodPhase(workflow_manifest['status']
['phase']) != kube_utils.PodPhase.FAILED:
return
k8s_client = kube_utils.make_core_v1_api()
pods = [
i for i in workflow_manifest['status']['nodes'] if i['type'] == 'Pod'
]
for pod in pods:
if kube_utils.PodPhase(pod['phase']) != kube_utils.PodPhase.FAILED:
continue
display_name = pod['displayName']
pod_id = pod['id']
log = k8s_client.read_namespaced_pod_log(pod_id,
namespace=namespace,
container='main')
for line in log.splitlines():
logging.info('%s:%s', display_name, line)
def __init__(self, model_path: str,
serving_binary: serving_bins.ServingBinary,
serving_spec: infra_validator_pb2.ServingSpec):
"""Create a kubernetes model server runner.
Args:
model_path: An IV-flavored model path. (See model_path_utils.py)
serving_binary: A ServingBinary to run.
serving_spec: A ServingSpec instance.
"""
assert serving_spec.WhichOneof('serving_platform') == 'kubernetes', (
'ServingSpec configuration mismatch.')
self._config = serving_spec.kubernetes
self._model_path = model_path
self._serving_binary = serving_binary
self._serving_spec = serving_spec
self._k8s_core_api = kube_utils.make_core_v1_api()
if not kube_utils.is_inside_kfp():
raise NotImplementedError(
'KubernetesRunner should be running inside KFP.')
self._executor_pod = kube_utils.get_current_kfp_pod(self._k8s_core_api)
self._executor_container = _get_container_or_error(
self._executor_pod,
container_name=kube_utils.ARGO_MAIN_CONTAINER_NAME)
self._namespace = kube_utils.get_kfp_namespace()
self._label_dict = {
_APP_KEY: _MODEL_SERVER_APP_LABEL,
}
# Pod name would be populated once creation request sent.
self._pod_name = None
# Endpoint would be populated once the Pod is running.
self._endpoint = None
def _run_executor(self, execution_id: int,
input_dict: Dict[Text, List[types.Artifact]],
output_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, Any]) -> None:
"""Execute underlying component implementation.
Runs executor container in a Kubernetes Pod and wait until it goes into
`Succeeded` or `Failed` state.
Args:
execution_id: The ID of the execution.
input_dict: Input dict from input key to a list of Artifacts. These are
often outputs of another component in the pipeline and passed to the
component by the orchestration system.
output_dict: Output dict from output key to a list of Artifacts. These are
often consumed by a dependent component.
exec_properties: A dict of execution properties. These are inputs to
pipeline with primitive types (int, string, float) and fully
materialized when a pipeline is constructed. No dependency to other
component or later injection from orchestration systems is necessary or
possible on these values.
Raises:
RuntimeError: when the pod is in `Failed` state or unexpected failure from
Kubernetes API.
"""
container_spec = cast(executor_spec.ExecutorContainerSpec,
self._component_executor_spec)
# Replace container spec with jinja2 template.
container_spec = container_common.resolve_container_template(
container_spec, input_dict, output_dict, exec_properties)
pod_name = self._build_pod_name(execution_id)
# TODO(hongyes): replace the default value from component config.
try:
namespace = kube_utils.get_kfp_namespace()
except RuntimeError:
namespace = 'kubeflow'
pod_manifest = self._build_pod_manifest(pod_name, container_spec)
core_api = kube_utils.make_core_v1_api()
if kube_utils.is_inside_kfp():
launcher_pod = kube_utils.get_current_kfp_pod(core_api)
pod_manifest['spec']['serviceAccount'] = launcher_pod.spec.service_account
pod_manifest['spec'][
'serviceAccountName'] = launcher_pod.spec.service_account_name
pod_manifest['metadata'][
'ownerReferences'] = container_common.to_swagger_dict(
launcher_pod.metadata.owner_references)
else:
pod_manifest['spec']['serviceAccount'] = kube_utils.TFX_SERVICE_ACCOUNT
pod_manifest['spec'][
'serviceAccountName'] = kube_utils.TFX_SERVICE_ACCOUNT
logging.info('Looking for pod "%s:%s".', namespace, pod_name)
resp = kube_utils.get_pod(core_api, pod_name, namespace)
if not resp:
logging.info('Pod "%s:%s" does not exist. Creating it...',
namespace, pod_name)
logging.info('Pod manifest: %s', pod_manifest)
try:
resp = core_api.create_namespaced_pod(
namespace=namespace, body=pod_manifest)
except client.rest.ApiException as e:
raise RuntimeError(
'Failed to created container executor pod!\nReason: %s\nBody: %s' %
(e.reason, e.body))
# Wait up to 300 seconds for the pod to move from pending to another status.
logging.info('Waiting for pod "%s:%s" to start.', namespace, pod_name)
kube_utils.wait_pod(
core_api,
pod_name,
namespace,
exit_condition_lambda=kube_utils.pod_is_not_pending,
condition_description='non-pending status',
timeout_sec=300)
logging.info('Start log streaming for pod "%s:%s".', namespace, pod_name)
try:
logs = core_api.read_namespaced_pod_log(
name=pod_name,
namespace=namespace,
container=kube_utils.ARGO_MAIN_CONTAINER_NAME,
follow=True,
_preload_content=False).stream()
except client.rest.ApiException as e:
raise RuntimeError(
'Failed to stream the logs from the pod!\nReason: %s\nBody: %s' %
(e.reason, e.body))
for log in logs:
logging.info(log.decode().rstrip('\n'))
# Wait indefinitely for the pod to complete.
resp = kube_utils.wait_pod(
core_api,
pod_name,
namespace,
exit_condition_lambda=kube_utils.pod_is_done,
condition_description='done state')
if resp.status.phase == kube_utils.PodPhase.FAILED.value:
raise RuntimeError('Pod "%s:%s" failed with status "%s".' %
(namespace, pod_name, resp.status))
logging.info('Pod "%s:%s" is done.', namespace, pod_name)
def run_as_kubernetes_job(pipeline: tfx_pipeline.Pipeline,
tfx_image: Text) -> None:
"""Submits and runs a TFX pipeline from outside the cluster.
Args:
pipeline: Logical pipeline containing pipeline args and components.
tfx_image: Container image URI for the TFX container.
Raises:
RuntimeError: When an error is encountered running the Kubernetes Job.
"""
# TODO(ccy): Look for alternative serialization schemes once available.
serialized_pipeline = _serialize_pipeline(pipeline)
arguments = [
'--serialized_pipeline',
serialized_pipeline,
'--tfx_image',
tfx_image,
]
batch_api = kube_utils.make_batch_v1_api()
job_name = 'Job_' + pipeline.pipeline_info.run_id
pod_label = kube_utils.sanitize_pod_name(job_name)
container_name = 'pipeline-orchestrator'
job = kube_utils.make_job_object(
name=job_name,
container_image=tfx_image,
command=_ORCHESTRATOR_COMMAND + arguments,
container_name=container_name,
pod_labels={
'job-name': pod_label,
},
service_account_name=kube_utils.TFX_SERVICE_ACCOUNT,
)
try:
batch_api.create_namespaced_job('default', job, pretty=True)
except client.rest.ApiException as e:
raise RuntimeError('Failed to submit job! \nReason: %s\nBody: %s' %
(e.reason, e.body))
# Wait for pod to start.
orchestrator_pods = []
core_api = kube_utils.make_core_v1_api()
start_time = datetime.datetime.utcnow()
# Wait for the kubernetes job to launch a pod.
while not orchestrator_pods and (datetime.datetime.utcnow() - start_time
).seconds < JOB_CREATION_TIMEOUT:
try:
orchestrator_pods = core_api.list_namespaced_pod(
namespace='default',
label_selector='job-name={}'.format(pod_label)).items
except client.rest.ApiException as e:
if e.status != 404:
raise RuntimeError('Unknown error! \nReason: %s\nBody: %s' %
(e.reason, e.body))
time.sleep(1)
# Transient orchestrator should only have 1 pod.
if len(orchestrator_pods) != 1:
raise RuntimeError(
'Expected 1 pod launched by Kubernetes job, found %d' %
len(orchestrator_pods))
orchestrator_pod = orchestrator_pods.pop()
pod_name = orchestrator_pod.metadata.name
absl.logging.info('Waiting for pod "default:%s" to start.', pod_name)
kube_utils.wait_pod(core_api,
pod_name,
'default',
exit_condition_lambda=kube_utils.pod_is_not_pending,
condition_description='non-pending status')
# Stream logs from orchestrator pod.
absl.logging.info('Start log streaming for pod "default:%s".', pod_name)
try:
logs = core_api.read_namespaced_pod_log(
name=pod_name,
namespace='default',
container=container_name,
follow=True,
_preload_content=False).stream()
except client.rest.ApiException as e:
raise RuntimeError(
'Failed to stream the logs from the pod!\nReason: %s\nBody: %s' %
(e.reason, e.body))
for log in logs:
absl.logging.info(log.decode().rstrip('\n'))
resp = kube_utils.wait_pod(core_api,
pod_name,
'default',
exit_condition_lambda=kube_utils.pod_is_done,
condition_description='done state',
exponential_backoff=True)
if resp.status.phase == kube_utils.PodPhase.FAILED.value:
raise RuntimeError('Pod "default:%s" failed with status "%s".' %
(pod_name, resp.status))
def run_executor(
self, execution_info: data_types.ExecutionInfo
) -> execution_result_pb2.ExecutorOutput:
"""Execute underlying component implementation.
Runs executor container in a Kubernetes Pod and wait until it goes into
`Succeeded` or `Failed` state.
Args:
execution_info: All the information that the launcher provides.
Raises:
RuntimeError: when the pod is in `Failed` state or unexpected failure from
Kubernetes API.
Returns:
An ExecutorOutput instance
"""
context = placeholder_utils.ResolutionContext(
exec_info=execution_info,
executor_spec=self._executor_spec,
platform_config=self._platform_config)
container_spec = executor_specs.TemplatedExecutorContainerSpec(
image=self._container_executor_spec.image,
command=[
placeholder_utils.resolve_placeholder_expression(cmd, context)
for cmd in self._container_executor_spec.commands
] or None,
args=[
placeholder_utils.resolve_placeholder_expression(arg, context)
for arg in self._container_executor_spec.args
] or None,
)
pod_name = self._build_pod_name(execution_info)
# TODO(hongyes): replace the default value from component config.
try:
namespace = kube_utils.get_kfp_namespace()
except RuntimeError:
namespace = 'kubeflow'
pod_manifest = self._build_pod_manifest(pod_name, container_spec)
core_api = kube_utils.make_core_v1_api()
if kube_utils.is_inside_kfp():
launcher_pod = kube_utils.get_current_kfp_pod(core_api)
pod_manifest['spec'][
'serviceAccount'] = launcher_pod.spec.service_account
pod_manifest['spec'][
'serviceAccountName'] = launcher_pod.spec.service_account_name
pod_manifest['metadata'][
'ownerReferences'] = container_common.to_swagger_dict(
launcher_pod.metadata.owner_references)
else:
pod_manifest['spec'][
'serviceAccount'] = kube_utils.TFX_SERVICE_ACCOUNT
pod_manifest['spec'][
'serviceAccountName'] = kube_utils.TFX_SERVICE_ACCOUNT
logging.info('Looking for pod "%s:%s".', namespace, pod_name)
resp = kube_utils.get_pod(core_api, pod_name, namespace)
if not resp:
logging.info('Pod "%s:%s" does not exist. Creating it...',
namespace, pod_name)
logging.info('Pod manifest: %s', pod_manifest)
try:
resp = core_api.create_namespaced_pod(namespace=namespace,
body=pod_manifest)
except client.rest.ApiException as e:
raise RuntimeError(
'Failed to created container executor pod!\nReason: %s\nBody: %s'
% (e.reason, e.body))
# Wait up to 300 seconds for the pod to move from pending to another status.
logging.info('Waiting for pod "%s:%s" to start.', namespace, pod_name)
kube_utils.wait_pod(
core_api,
pod_name,
namespace,
exit_condition_lambda=kube_utils.pod_is_not_pending,
condition_description='non-pending status',
timeout_sec=300)
logging.info('Start log streaming for pod "%s:%s".', namespace,
pod_name)
try:
logs = core_api.read_namespaced_pod_log(
name=pod_name,
namespace=namespace,
container=kube_utils.ARGO_MAIN_CONTAINER_NAME,
follow=True,
_preload_content=False).stream()
except client.rest.ApiException as e:
raise RuntimeError(
'Failed to stream the logs from the pod!\nReason: %s\nBody: %s'
% (e.reason, e.body))
for log in logs:
logging.info(log.decode().rstrip('\n'))
# Wait indefinitely for the pod to complete.
resp = kube_utils.wait_pod(
core_api,
pod_name,
namespace,
exit_condition_lambda=kube_utils.pod_is_done,
condition_description='done state')
if resp.status.phase == kube_utils.PodPhase.FAILED.value:
raise RuntimeError('Pod "%s:%s" failed with status "%s".' %
(namespace, pod_name, resp.status))
logging.info('Pod "%s:%s" is done.', namespace, pod_name)
return execution_result_pb2.ExecutorOutput()
