def update_uimetadata(artifact_name,
uimetadata_path=KFP_UI_METADATA_FILE_PATH):
"""Update ui-metadata dictionary with a new web-app entry.
Args:
artifact_name: Name of the artifact
uimetadata_path: path to mlpipeline-ui-metadata.json
"""
try:
outputs = get_current_uimetadata(uimetadata_path,
default_if_not_exist=True)
except json.JSONDecodeError:
log.error("This step will not be able to visualize artifacts in the"
" KFP UI")
return
pod_name = podutils.get_pod_name()
namespace = podutils.get_namespace()
workflow_name = workflowutils.get_workflow_name(pod_name, namespace)
html_artifact_entry = [{
'type':
'web-app',
'storage':
'minio',
'source':
'minio://mlpipeline/artifacts/{}/{}/{}'.format(workflow_name, pod_name,
artifact_name + '.tgz')
}]
outputs['outputs'] += html_artifact_entry
with open(uimetadata_path, "w") as f:
json.dump(outputs, f)
def update_uimetadata(artifact_name,
uimetadata_path='/mlpipeline-ui-metadata.json'):
"""Update ui-metadata dictionary with a new web-app entry.
Args:
artifact_name: Name of the artifact
uimetadata_path: path to mlpipeline-ui-metadata.json
"""
# Default empty ui-metadata dict
outputs = {"outputs": []}
if os.path.exists(uimetadata_path):
try:
outputs = json.loads(open(uimetadata_path, 'r').read())
if not outputs.get('outputs', None):
outputs['outputs'] = []
except json.JSONDecodeError as e:
print("Failed to parse json file {}: {}\n"
"This step will not be able to visualize artifacts in the"
" KFP UI".format(uimetadata_path, e))
pod_name = podutils.get_pod_name()
namespace = podutils.get_namespace()
workflow_name = workflowutils.get_workflow_name(pod_name, namespace)
html_artifact_entry = [{
'type':
'web-app',
'storage':
'minio',
'source':
'minio://mlpipeline/artifacts/{}/{}/{}'.format(workflow_name, pod_name,
artifact_name + '.tgz')
}]
outputs['outputs'] += html_artifact_entry
with open(uimetadata_path, "w") as f:
json.dump(outputs, f)
def is_kfp_step() -> bool:
"""Detect if running inside a KFP step.
The detection involves two steps:
1. Auto-detect if the current Pod is part of an Argo workflow
2. Read one of the annotations that the KFP API Server sets in the
workflow object (one-off runs and recurring ones have different
annotations).
"""
log.info("Checking if running inside a KFP step...")
try:
namespace = podutils.get_namespace()
workflow = workflowutils.get_workflow(
workflowutils.get_workflow_name(podutils.get_pod_name(),
namespace), namespace)
annotations = workflow["metadata"]["annotations"]
try:
_ = annotations[KFP_RUN_NAME_ANNOTATION_KEY]
except KeyError:
_ = annotations[KFP_SWF_NAME_ANNOTATION_KEY]
except Exception:
log.info("Not in a KFP step.")
return False
log.info("Running in a KFP step.")
return True
def detect_run_uuid() -> str:
"""Get the workflow's UUID form inside a pipeline step."""
namespace = podutils.get_namespace()
workflow = workflowutils.get_workflow(
workflowutils.get_workflow_name(podutils.get_pod_name(), namespace),
namespace)
run_uuid = (workflow["metadata"].get("labels",
{}).get(KFP_RUN_ID_LABEL_KEY, None))
# KFP api-server adds run UUID as label to workflows for KFP>=0.1.26.
# Return run UUID if available. Else return workflow UUID to maintain
# backwards compatibility.
return run_uuid or workflow["metadata"]["uid"]
def update_uimetadata(artifact_name,
uimetadata_path=KFP_UI_METADATA_FILE_PATH):
"""Update ui-metadata dictionary with a new web-app entry.
Args:
artifact_name: Name of the artifact
uimetadata_path: path to mlpipeline-ui-metadata.json
"""
log.info("Adding artifact '%s' to KFP UI metadata...", artifact_name)
try:
outputs = get_current_uimetadata(uimetadata_path,
default_if_not_exist=True)
except json.JSONDecodeError:
log.error("This step will not be able to visualize artifacts in the"
" KFP UI")
return
pod_name = podutils.get_pod_name()
namespace = podutils.get_namespace()
workflow_name = workflowutils.get_workflow_name(pod_name, namespace)
html_artifact_entry = [{
'type':
'web-app',
'storage':
'minio',
'source':
'minio://mlpipeline/artifacts/{}/{}/{}'.format(workflow_name, pod_name,
artifact_name + '.tgz')
}]
outputs['outputs'] += html_artifact_entry
try:
utils.ensure_or_create_dir(uimetadata_path)
except RuntimeError:
log.exception(
"Writing to '%s' failed. This step will not be able to"
" visualize artifacts in the KFP UI.", uimetadata_path)
return
with open(uimetadata_path, "w") as f:
json.dump(outputs, f)
log.info("Artifact successfully added")
def get_run_uuid():
"""Get the Workflow's UUID form inside a pipeline step."""
# Retrieve the pod
pod_name = get_pod_name()
namespace = get_namespace()
workflow_name = workflowutils.get_workflow_name(pod_name, namespace)
# Retrieve the Argo workflow
api_group = "argoproj.io"
api_version = "v1alpha1"
co_name = "workflows"
co_client = _get_k8s_custom_objects_client()
workflow = co_client.get_namespaced_custom_object(api_group, api_version,
namespace, co_name,
workflow_name)
run_uuid = workflow["metadata"].get("labels", {}).get(KFP_RUN_ID_LABEL_KEY,
None)
# KFP api-server adds run UUID as label to workflows for KFP>=0.1.26.
# Return run UUID if available. Else return workflow UUID to maintain
# backwards compatibility.
return run_uuid or workflow["metadata"]["uid"]